diff --git a/app/src/main/java/com/quillstudios/pokegoalshelper/ScreenCaptureService.kt b/app/src/main/java/com/quillstudios/pokegoalshelper/ScreenCaptureService.kt index cc87849..351974f 100644 --- a/app/src/main/java/com/quillstudios/pokegoalshelper/ScreenCaptureService.kt +++ b/app/src/main/java/com/quillstudios/pokegoalshelper/ScreenCaptureService.kt @@ -31,6 +31,8 @@ import com.quillstudios.pokegoalshelper.capture.ScreenCaptureManagerImpl import com.quillstudios.pokegoalshelper.ml.MLInferenceEngine import com.quillstudios.pokegoalshelper.ml.YOLOInferenceEngine import com.quillstudios.pokegoalshelper.ml.Detection as MLDetection +import com.quillstudios.pokegoalshelper.data.PokemonDataExtractor +import com.quillstudios.pokegoalshelper.data.PokemonDataExtractorImpl import kotlinx.coroutines.runBlocking import org.opencv.android.Utils import org.opencv.core.* @@ -59,6 +61,8 @@ data class PokemonInfo( val species: String?, val primaryType: String?, val secondaryType: String?, + val teraType: String?, + val isShiny: Boolean, val stamps: List, val labels: List, val marks: List, @@ -120,6 +124,9 @@ class ScreenCaptureService : Service() { // ML inference engine private var mlInferenceEngine: MLInferenceEngine? = null + + // Pokemon data extractor + private var pokemonDataExtractor: PokemonDataExtractor? = null private lateinit var screenCaptureManager: ScreenCaptureManager private var detectionOverlay: DetectionOverlay? = null @@ -169,6 +176,10 @@ class ScreenCaptureService : Service() { } }.start() + // Initialize Pokemon data extractor + pokemonDataExtractor = PokemonDataExtractorImpl(this) + PGHLog.i(TAG, "✅ Pokemon data extractor initialized") + // Initialize MVC components detectionController = DetectionController(mlInferenceEngine!!) detectionController.setDetectionRequestCallback { triggerManualDetection() } @@ -349,6 +360,9 @@ class ScreenCaptureService : Service() { } val (screenWidth, screenHeight) = screenDimensions + // Configure Pokemon data extractor with screen dimensions + pokemonDataExtractor?.configureRegions(android.util.Size(screenWidth, screenHeight)) + val planes = image.planes val buffer = planes[0].buffer val pixelStride = planes[0].pixelStride @@ -500,10 +514,12 @@ class ScreenCaptureService : Service() { val matCopy = Mat() mat.copyTo(matCopy) - // Process in background thread + // Process in background thread with coroutines ocrExecutor.submit { try { - val pokemonInfo = extractPokemonInfoFromYOLO(matCopy, detections) + val pokemonInfo = runBlocking { + extractPokemonInfoFromYOLO(matCopy, detections) + } // Post results back to main thread handler.post { @@ -540,112 +556,24 @@ class ScreenCaptureService : Service() { } } - private fun extractPokemonInfoFromYOLO(mat: Mat, detections: List): PokemonInfo? { - try { - PGHLog.i(TAG, "🎯 Extracting Pokemon info from ${detections.size} YOLO detections") - - // Group detections by type - val detectionMap = detections.groupBy { it.className } - - // Extract key information using YOLO bounding boxes (async OCR calls) - val ocrResults = mutableMapOf() - val latch = CountDownLatch(7) // Wait for 7 OCR operations - - // Submit all OCR tasks in parallel - submitOCRTask("nickname", mat, detectionMap["pokemon_nickname"]?.firstOrNull(), ocrResults, latch) - submitOCRTask("species", mat, detectionMap["pokemon_species"]?.firstOrNull(), ocrResults, latch) - submitOCRTask("nature", mat, detectionMap["nature_name"]?.firstOrNull(), ocrResults, latch) - submitOCRTask("ability", mat, detectionMap["ability_name"]?.firstOrNull(), ocrResults, latch) - submitOCRTask("otName", mat, detectionMap["original_trainer_name"]?.firstOrNull(), ocrResults, latch) - submitOCRTask("otId", mat, detectionMap["original_trainder_number"]?.firstOrNull(), ocrResults, latch) - // For level, prioritize wider bounding boxes (more likely to contain full level text) - val levelDetection = detectionMap["pokemon_level"]?.maxByOrNull { it.boundingBox.width } - submitLevelOCRTask("level", mat, levelDetection, ocrResults, latch) + private suspend fun extractPokemonInfoFromYOLO(mat: Mat, detections: List): PokemonInfo? { + return try { + PGHLog.i(TAG, "🎯 Delegating Pokemon extraction to PokemonDataExtractor") - // Wait for all OCR tasks to complete (max timeout) - val completed = latch.await(OCR_TASK_TIMEOUT_SECONDS, TimeUnit.SECONDS) - if (!completed) { - PGHLog.w(TAG, "⏱️ Some OCR tasks timed out after ${OCR_TASK_TIMEOUT_SECONDS} seconds") + // Convert MLDetection to Detection for the extractor + val extractorDetections = detections.map { mlDetection -> + com.quillstudios.pokegoalshelper.ml.Detection( + className = mlDetection.className, + confidence = mlDetection.confidence, + boundingBox = mlDetection.boundingBox + ) } - // Extract results - val nickname = ocrResults["nickname"] - val level = ocrResults["level"]?.toIntOrNull() - val species = ocrResults["species"] - val nature = ocrResults["nature"] - val ability = ocrResults["ability"] - val otName = ocrResults["otName"] - val otId = ocrResults["otId"] - - // Extract stats (multiple detections) - val stats = extractStatsFromDetections(mat, detectionMap) - - // Extract moves (multiple detections) - val moves = extractMovesFromDetections(mat, detectionMap) - - // Detect gender - val gender = when { - detectionMap["gender_icon_male"]?.isNotEmpty() == true -> "Male" - detectionMap["gender_icon_female"]?.isNotEmpty() == true -> "Female" - else -> null - } - - // Detect pokeball type - val pokeballType = detectPokeballTypeFromDetections(detectionMap) - - // Detect types - val types = extractTypesFromDetections(mat, detectionMap) - - // Detect shiny status - val isShiny = detectionMap["shiny_icon"]?.isNotEmpty() == true - - // Detect tera type - val teraType = detectTeraTypeFromDetections(detectionMap) - - PGHLog.i(TAG, "📊 YOLO extraction summary:") - PGHLog.i(TAG, " Nickname: '${nickname ?: "null"}'") - PGHLog.i(TAG, " Level: ${level ?: "null"}") - PGHLog.i(TAG, " Species: '${species ?: "null"}'") - PGHLog.i(TAG, " Nature: '${nature ?: "null"}'") - PGHLog.i(TAG, " Ability: '${ability ?: "null"}'") - PGHLog.i(TAG, " Gender: '${gender ?: "null"}'") - PGHLog.i(TAG, " Pokeball: '${pokeballType ?: "null"}'") - PGHLog.i(TAG, " Types: ${types}") - PGHLog.i(TAG, " Tera: '${teraType ?: "null"}'") - PGHLog.i(TAG, " Shiny: $isShiny") - - if (nickname.isNullOrBlank() && species.isNullOrBlank() && level == null) { - PGHLog.w(TAG, "⚠️ No essential Pokemon data found with YOLO detection") - return null - } - - return PokemonInfo( - pokeballType = pokeballType, - nickname = nickname, - gender = gender, - level = level, - language = detectLanguage(nickname, species), - gameSource = detectGameSourceFromDetections(detectionMap), - isFavorited = false, // TODO: implement favorite detection - nationalDexNumber = extractDexNumber(species), - species = species, - primaryType = types.getOrNull(0), - secondaryType = types.getOrNull(1), - stamps = emptyList(), // TODO: implement stamp detection - labels = emptyList(), // TODO: implement label detection - marks = emptyList(), // TODO: implement mark detection - stats = stats, - moves = moves, - ability = ability, - nature = nature, - originalTrainerName = otName, - originalTrainerId = otId, - extractionConfidence = calculateYOLOExtractionConfidence(detections, nickname, level, species) - ) - + // Use the dedicated Pokemon data extractor with async/await pattern + pokemonDataExtractor?.extractPokemonInfo(extractorDetections, mat) } catch (e: Exception) { - PGHLog.e(TAG, "Error extracting Pokemon info from YOLO detections", e) - return null + PGHLog.e(TAG, "Error extracting Pokemon info via PokemonDataExtractor", e) + null } } @@ -1246,6 +1174,7 @@ class ScreenCaptureService : Service() { // TODO: Re-enable when DetectionController is updated // detectionController.clearUICallbacks() mlInferenceEngine?.cleanup() + pokemonDataExtractor?.cleanup() // Release screen capture manager if (::screenCaptureManager.isInitialized) { diff --git a/app/src/main/java/com/quillstudios/pokegoalshelper/data/PokemonDataExtractor.kt b/app/src/main/java/com/quillstudios/pokegoalshelper/data/PokemonDataExtractor.kt new file mode 100644 index 0000000..fcb2492 --- /dev/null +++ b/app/src/main/java/com/quillstudios/pokegoalshelper/data/PokemonDataExtractor.kt @@ -0,0 +1,46 @@ +package com.quillstudios.pokegoalshelper.data + +import android.util.Size +import com.quillstudios.pokegoalshelper.ml.Detection +import com.quillstudios.pokegoalshelper.PokemonInfo +import org.opencv.core.Mat + +/** + * Interface for extracting Pokemon information from detection results. + * + * This component handles the OCR processing and data extraction pipeline that converts + * ML detection results (bounding boxes) into structured Pokemon data through text recognition + * and intelligent parsing. + */ +interface PokemonDataExtractor { + + /** + * Extract Pokemon information from detection results and screen image. + * + * @param detections List of detected UI elements from ML inference + * @param screenMat OpenCV Mat containing the full screen capture + * @return PokemonInfo object with extracted data, or null if extraction failed + */ + suspend fun extractPokemonInfo(detections: List, screenMat: Mat): PokemonInfo? + + /** + * Configure the extractor for specific screen dimensions. + * This allows the extractor to optimize region processing based on screen size. + * + * @param screenSize Screen dimensions for coordinate calculations + */ + fun configureRegions(screenSize: Size) + + /** + * Configure OCR timeout settings. + * + * @param timeoutSeconds Maximum time to wait for OCR operations + */ + fun configureTimeout(timeoutSeconds: Long) + + /** + * Release any resources held by the extractor. + * Should be called when the extractor is no longer needed. + */ + fun cleanup() +} \ No newline at end of file diff --git a/app/src/main/java/com/quillstudios/pokegoalshelper/data/PokemonDataExtractorImpl.kt b/app/src/main/java/com/quillstudios/pokegoalshelper/data/PokemonDataExtractorImpl.kt new file mode 100644 index 0000000..7468ff4 --- /dev/null +++ b/app/src/main/java/com/quillstudios/pokegoalshelper/data/PokemonDataExtractorImpl.kt @@ -0,0 +1,606 @@ +package com.quillstudios.pokegoalshelper.data + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.RectF +import android.util.Size +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.text.TextRecognition +import com.google.mlkit.vision.text.latin.TextRecognizerOptions +import com.quillstudios.pokegoalshelper.ml.Detection +import com.quillstudios.pokegoalshelper.PokemonInfo +import com.quillstudios.pokegoalshelper.PokemonStats +import com.quillstudios.pokegoalshelper.utils.PGHLog +import com.quillstudios.pokegoalshelper.utils.MatUtils.use +import kotlinx.coroutines.* +import org.opencv.android.Utils +import org.opencv.core.* +import org.opencv.imgproc.Imgproc +import java.util.concurrent.Executors +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine + +/** + * Implementation of PokemonDataExtractor that handles OCR processing and data extraction + * from Pokemon detection results using a modern async/await pattern instead of blocking latches. + */ +class PokemonDataExtractorImpl( + private val context: Context +) : PokemonDataExtractor { + + companion object { + private const val TAG = "PokemonDataExtractor" + + // OCR Configuration + private const val DEFAULT_TIMEOUT_SECONDS = 10L + private const val INDIVIDUAL_OCR_TIMEOUT_SECONDS = 5L + private const val BBOX_EXPANSION_FACTOR = 0.05f // 5% expansion for better OCR + + // OCR Image Processing Constants + private const val MIN_OCR_WIDTH = 50 + private const val MIN_OCR_HEIGHT = 50 + private const val GAUSSIAN_BLUR_KERNEL_SIZE = 3.0 + private const val GAUSSIAN_BLUR_SIGMA = 0.5 + private const val CLAHE_CLIP_LIMIT = 1.5 + private const val CLAHE_TILE_SIZE = 8.0 + } + + private var screenSize: Size? = null + private var ocrTimeout: Long = DEFAULT_TIMEOUT_SECONDS + + // Dedicated dispatcher for OCR operations + private val ocrDispatcher = Executors.newFixedThreadPool(4).asCoroutineDispatcher() + + override suspend fun extractPokemonInfo(detections: List, screenMat: Mat): PokemonInfo? { + return withContext(Dispatchers.IO) { + try { + PGHLog.i(TAG, "🎯 Extracting Pokemon info from ${detections.size} detections") + + // Group detections by type for easy lookup + val detectionMap = detections.groupBy { it.className } + + // Extract text-based information using parallel coroutines + val textExtractionResults = extractTextDataAsync(screenMat, detectionMap) + + // Extract icon-based information (synchronous, fast) + val iconData = extractIconData(detectionMap) + + // Combine all extracted data + val pokemonInfo = buildPokemonInfo(textExtractionResults, iconData, detections) + + if (pokemonInfo != null) { + PGHLog.i(TAG, "✅ Successfully extracted Pokemon info") + logExtractionSummary(pokemonInfo) + } else { + PGHLog.w(TAG, "⚠️ Failed to extract essential Pokemon data") + } + + pokemonInfo + + } catch (e: Exception) { + PGHLog.e(TAG, "❌ Error extracting Pokemon info", e) + null + } + } + } + + override fun configureRegions(screenSize: Size) { + this.screenSize = screenSize + PGHLog.d(TAG, "📱 Screen size configured: ${screenSize.width}x${screenSize.height}") + } + + override fun configureTimeout(timeoutSeconds: Long) { + this.ocrTimeout = timeoutSeconds + PGHLog.d(TAG, "⏱️ OCR timeout configured: ${timeoutSeconds}s") + } + + override fun cleanup() { + ocrDispatcher.close() + PGHLog.d(TAG, "🧹 PokemonDataExtractor cleanup completed") + } + + /** + * Extract text-based Pokemon data using parallel OCR operations with coroutines + */ + private suspend fun extractTextDataAsync( + screenMat: Mat, + detectionMap: Map> + ): TextExtractionResults = withContext(ocrDispatcher) { + + // Launch parallel OCR operations + val ocrJobs = listOf( + async { extractTextFromDetection("nickname", screenMat, detectionMap["pokemon_nickname"]?.firstOrNull()) }, + async { extractTextFromDetection("species", screenMat, detectionMap["pokemon_species"]?.firstOrNull()) }, + async { extractTextFromDetection("nature", screenMat, detectionMap["nature_name"]?.firstOrNull()) }, + async { extractTextFromDetection("ability", screenMat, detectionMap["ability_name"]?.firstOrNull()) }, + async { extractTextFromDetection("otName", screenMat, detectionMap["original_trainer_name"]?.firstOrNull()) }, + async { extractTextFromDetection("otId", screenMat, detectionMap["original_trainder_number"]?.firstOrNull()) }, + async { extractLevelFromDetection(screenMat, detectionMap["pokemon_level"]?.maxByOrNull { it.boundingBox.width }) }, + async { extractStatsFromDetections(screenMat, detectionMap) }, + async { extractMovesFromDetections(screenMat, detectionMap) }, + async { extractTypesFromDetections(screenMat, detectionMap) } + ) + + // Wait for all OCR operations with timeout + val results = withTimeoutOrNull(ocrTimeout * 1000) { + ocrJobs.awaitAll() + } + + if (results == null) { + PGHLog.w(TAG, "⏱️ OCR operations timed out after ${ocrTimeout}s") + // Cancel remaining operations + ocrJobs.forEach { it.cancel() } + } + + TextExtractionResults( + nickname = results?.get(0) as? String, + species = results?.get(1) as? String, + nature = results?.get(2) as? String, + ability = results?.get(3) as? String, + otName = results?.get(4) as? String, + otId = results?.get(5) as? String, + level = results?.get(6) as? Int, + stats = results?.get(7) as? PokemonStats, + moves = results?.get(8) as? List ?: emptyList(), + types = results?.get(9) as? List ?: emptyList() + ) + } + + /** + * Extract icon-based Pokemon data (gender, pokeball, shiny, tera, etc.) + */ + private fun extractIconData(detectionMap: Map>): IconExtractionResults { + return IconExtractionResults( + gender = when { + detectionMap["gender_icon_male"]?.isNotEmpty() == true -> "Male" + detectionMap["gender_icon_female"]?.isNotEmpty() == true -> "Female" + else -> null + }, + pokeballType = detectPokeballType(detectionMap), + isShiny = detectionMap["shiny_icon"]?.isNotEmpty() == true, + teraType = detectTeraType(detectionMap), + gameSource = detectGameSource(detectionMap) + ) + } + + /** + * Extract text from a specific detection using OCR with preprocessing + */ + private suspend fun extractTextFromDetection(key: String, screenMat: Mat, detection: Detection?): String? { + if (detection == null) { + PGHLog.d(TAG, "🔍 No detection found for $key") + return null + } + + return try { + // Convert BoundingBox to RectF for expansion + val bbox = detection.boundingBox + val rectF = RectF(bbox.left, bbox.top, bbox.right, bbox.bottom) + + // Create expanded bounding box for better OCR accuracy + val expandedBbox = expandBoundingBox(rectF, screenMat.size()) + + // Extract region of interest + val roi = Mat(screenMat, expandedBbox) + val processedRoi = preprocessImageForOCR(roi) + val bitmap = Bitmap.createBitmap(processedRoi.cols(), processedRoi.rows(), Bitmap.Config.ARGB_8888) + + try { + // Convert to bitmap for ML Kit + Utils.matToBitmap(processedRoi, bitmap) + + // Perform OCR + performOCRWithTimeout(bitmap, key) + } finally { + // Cleanup resources + bitmap.recycle() + processedRoi.release() + roi.release() + }.also { extractedText -> + if (extractedText != null) { + PGHLog.i(TAG, "✅ OCR SUCCESS: $key = '$extractedText'") + } else { + PGHLog.w(TAG, "❌ OCR FAILED: $key - no text found") + } + } + + } catch (e: Exception) { + PGHLog.e(TAG, "❌ Error extracting text for $key", e) + null + } + } + + /** + * Extract Pokemon level with special number-only processing + */ + private suspend fun extractLevelFromDetection(screenMat: Mat, detection: Detection?): Int? { + val levelText = extractTextFromDetection("level", screenMat, detection) + return levelText?.replace("[^0-9]".toRegex(), "")?.toIntOrNull() + } + + /** + * Extract Pokemon stats from multiple stat detections + */ + private suspend fun extractStatsFromDetections( + screenMat: Mat, + detectionMap: Map> + ): PokemonStats? = withContext(ocrDispatcher) { + + val statJobs = listOf( + async { extractTextFromDetection("hp", screenMat, detectionMap["hp_value"]?.firstOrNull())?.toIntOrNull() }, + async { extractTextFromDetection("attack", screenMat, detectionMap["attack_value"]?.firstOrNull())?.toIntOrNull() }, + async { extractTextFromDetection("defense", screenMat, detectionMap["defense_value"]?.firstOrNull())?.toIntOrNull() }, + async { extractTextFromDetection("spAttack", screenMat, detectionMap["sp_atk_value"]?.firstOrNull())?.toIntOrNull() }, + async { extractTextFromDetection("spDefense", screenMat, detectionMap["sp_def_value"]?.firstOrNull())?.toIntOrNull() }, + async { extractTextFromDetection("speed", screenMat, detectionMap["speed_value"]?.firstOrNull())?.toIntOrNull() } + ) + + val stats = statJobs.awaitAll() + val hp = stats[0] + val attack = stats[1] + val defense = stats[2] + val spAttack = stats[3] + val spDefense = stats[4] + val speed = stats[5] + + if (stats.any { it != null }) { + PokemonStats(hp, attack, defense, spAttack, spDefense, speed) + } else null + } + + /** + * Extract Pokemon moves from move detections + */ + private suspend fun extractMovesFromDetections( + screenMat: Mat, + detectionMap: Map> + ): List = withContext(ocrDispatcher) { + + val moveDetections = detectionMap["move_name"] ?: emptyList() + val moveJobs = moveDetections.map { detection -> + async { extractTextFromDetection("move", screenMat, detection) } + } + + moveJobs.awaitAll() + .filterNotNull() + .filter { it.isNotBlank() } + .map { it.trim() } + .take(4) // Pokemon can have max 4 moves + } + + /** + * Extract Pokemon types from type detections + */ + private suspend fun extractTypesFromDetections( + screenMat: Mat, + detectionMap: Map> + ): List = withContext(ocrDispatcher) { + + val types = mutableListOf() + + // Extract type 1 + extractTextFromDetection("type1", screenMat, detectionMap["type_1"]?.firstOrNull())?.let { type1 -> + if (type1.isNotBlank()) types.add(type1.trim()) + } + + // Extract type 2 + extractTextFromDetection("type2", screenMat, detectionMap["type_2"]?.firstOrNull())?.let { type2 -> + if (type2.isNotBlank()) types.add(type2.trim()) + } + + types + } + + /** + * Detect pokeball type from icon detections + */ + private fun detectPokeballType(detectionMap: Map>): String? { + val pokeballTypes = mapOf( + "ball_icon_pokeball" to "Poké Ball", + "ball_icon_greatball" to "Great Ball", + "ball_icon_ultraball" to "Ultra Ball", + "ball_icon_masterball" to "Master Ball", + "ball_icon_heavyball" to "Heavy Ball", + "ball_icon_premierball" to "Premier Ball", + "ball_icon_repeatball" to "Repeat Ball", + "ball_icon_timerball" to "Timer Ball", + "ball_icon_nestball" to "Nest Ball", + "ball_icon_netball" to "Net Ball", + "ball_icon_diveball" to "Dive Ball", + "ball_icon_luxuryball" to "Luxury Ball", + "ball_icon_healball" to "Heal Ball", + "ball_icon_quickball" to "Quick Ball", + "ball_icon_duskball" to "Dusk Ball", + "ball_icon_cherishball" to "Cherish Ball", + "ball_icon_dreamball" to "Dream Ball", + "ball_icon_beastball" to "Beast Ball" + ) + + for ((className, ballName) in pokeballTypes) { + if (detectionMap[className]?.isNotEmpty() == true) { + return ballName + } + } + return null + } + + /** + * Detect tera type from tera icon detections + */ + private fun detectTeraType(detectionMap: Map>): String? { + val teraTypes = mapOf( + "tera_ice" to "Ice", + "tera_fairy" to "Fairy", + "tera_poison" to "Poison", + "tera_ghost" to "Ghost", + "tera_steel" to "Steel", + "tera_grass" to "Grass", + "tera_normal" to "Normal", + "tera_fire" to "Fire", + "tera_electric" to "Electric", + "tera_fighting" to "Fighting", + "tera_ground" to "Ground", + "tera_flying" to "Flying", + "tera_bug" to "Bug", + "tera_rock" to "Rock", + "tera_dark" to "Dark", + "tera_dragon" to "Dragon", + "tera_water" to "Water", + "tera_psychic" to "Psychic" + ) + + for ((className, teraName) in teraTypes) { + if (detectionMap[className]?.isNotEmpty() == true) { + return teraName + } + } + return null + } + + /** + * Detect game source from stamp detections + */ + private fun detectGameSource(detectionMap: Map>): String? { + val gameSources = mapOf( + "last_game_stamp_sh" to "Sword/Shield", + "last_game_stamp_bank" to "Bank", + "last_game_stamp_pla" to "Legends: Arceus", + "last_game_stamp_sc" to "Scarlet/Violet", + "last_game_stamp_vi" to "Violet", + "last_game_stamp_go" to "Pokémon GO", + "origin_icon_vc" to "Virtual Console", + "origin_icon_xyoras" to "XY/ORAS", + "origin_icon_smusum" to "SM/USUM", + "origin_icon_lg" to "Let's Go", + "origin_icon_swsh" to "Sword/Shield", + "origin_icon_go" to "Pokémon GO", + "origin_icon_bdsp" to "BDSP", + "origin_icon_pla" to "Legends: Arceus", + "origin_icon_sv" to "Scarlet/Violet" + ) + + for ((className, sourceName) in gameSources) { + if (detectionMap[className]?.isNotEmpty() == true) { + return sourceName + } + } + return null + } + + /** + * Expand bounding box for better OCR accuracy + */ + private fun expandBoundingBox(bbox: RectF, imageSize: org.opencv.core.Size): org.opencv.core.Rect { + val widthExpansion = (bbox.width() * BBOX_EXPANSION_FACTOR).toInt() + val heightExpansion = (bbox.height() * BBOX_EXPANSION_FACTOR).toInt() + + val expandedBbox = org.opencv.core.Rect( + (bbox.left - widthExpansion).toInt(), + (bbox.top - heightExpansion).toInt(), + (bbox.width() + 2 * widthExpansion).toInt(), + (bbox.height() + 2 * heightExpansion).toInt() + ) + + // Clamp to image boundaries + return org.opencv.core.Rect( + maxOf(0, minOf(expandedBbox.x, imageSize.width.toInt() - 1)), + maxOf(0, minOf(expandedBbox.y, imageSize.height.toInt() - 1)), + maxOf(1, minOf(expandedBbox.width, imageSize.width.toInt() - expandedBbox.x)), + maxOf(1, minOf(expandedBbox.height, imageSize.height.toInt() - expandedBbox.y)) + ) + } + + /** + * Preprocess image region for optimal OCR accuracy + */ + private fun preprocessImageForOCR(roi: Mat): Mat { + return try { + // Scale up small regions + val scaledRoi = if (roi.width() < MIN_OCR_WIDTH || roi.height() < MIN_OCR_HEIGHT) { + val scaleX = maxOf(1.0, MIN_OCR_WIDTH.toDouble() / roi.width()) + val scaleY = maxOf(1.0, MIN_OCR_HEIGHT.toDouble() / roi.height()) + val scale = maxOf(scaleX, scaleY) + + val resized = Mat() + Imgproc.resize(roi, resized, Size(roi.width() * scale, roi.height() * scale)) + resized + } else { + val copy = Mat() + roi.copyTo(copy) + copy + } + + // Convert to grayscale + val gray = Mat() + if (scaledRoi.channels() == 3) { + Imgproc.cvtColor(scaledRoi, gray, Imgproc.COLOR_BGR2GRAY) + } else if (scaledRoi.channels() == 4) { + Imgproc.cvtColor(scaledRoi, gray, Imgproc.COLOR_BGRA2GRAY) + } else { + scaledRoi.copyTo(gray) + } + + // Apply CLAHE for contrast enhancement + val enhanced = Mat() + val clahe = Imgproc.createCLAHE(CLAHE_CLIP_LIMIT, Size(CLAHE_TILE_SIZE, CLAHE_TILE_SIZE)) + clahe.apply(gray, enhanced) + + // Apply slight gaussian blur to reduce noise + val denoised = Mat() + Imgproc.GaussianBlur(enhanced, denoised, Size(GAUSSIAN_BLUR_KERNEL_SIZE, GAUSSIAN_BLUR_KERNEL_SIZE), GAUSSIAN_BLUR_SIGMA) + + // Cleanup intermediate results + scaledRoi.release() + gray.release() + enhanced.release() + + denoised + + } catch (e: Exception) { + PGHLog.e(TAG, "Error preprocessing image for OCR", e) + // Return copy of original if preprocessing fails + val result = Mat() + roi.copyTo(result) + result + } + } + + /** + * Perform OCR with timeout using ML Kit + */ + private suspend fun performOCRWithTimeout(bitmap: Bitmap, purpose: String): String? { + return withTimeoutOrNull(INDIVIDUAL_OCR_TIMEOUT_SECONDS * 1000) { + suspendCoroutine { continuation -> + try { + val image = InputImage.fromBitmap(bitmap, 0) + val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS) + + recognizer.process(image) + .addOnSuccessListener { visionText -> + val result = visionText.text.trim() + continuation.resume(if (result.isBlank()) null else result) + } + .addOnFailureListener { e -> + PGHLog.e(TAG, "OCR failed for $purpose: ${e.message}") + continuation.resume(null) + } + } catch (e: Exception) { + PGHLog.e(TAG, "Error setting up OCR for $purpose", e) + continuation.resume(null) + } + } + } + } + + /** + * Build final PokemonInfo from extracted data + */ + private fun buildPokemonInfo( + textData: TextExtractionResults, + iconData: IconExtractionResults, + detections: List + ): PokemonInfo? { + + // Require at least one essential piece of data + if (textData.nickname.isNullOrBlank() && + textData.species.isNullOrBlank() && + textData.level == null) { + return null + } + + return PokemonInfo( + pokeballType = iconData.pokeballType, + nickname = textData.nickname, + gender = iconData.gender, + level = textData.level, + language = null, // TODO: Implement language detection + gameSource = iconData.gameSource, + isFavorited = false, // TODO: Detect favorite status + nationalDexNumber = null, // TODO: Implement dex number mapping + species = textData.species, + primaryType = textData.types.getOrNull(0), + secondaryType = textData.types.getOrNull(1), + teraType = iconData.teraType, + isShiny = iconData.isShiny, + stamps = emptyList(), // TODO: Extract stamps + labels = emptyList(), // TODO: Extract labels + marks = emptyList(), // TODO: Extract marks + stats = textData.stats, + moves = textData.moves, + ability = textData.ability, + nature = textData.nature, + originalTrainerName = textData.otName, + originalTrainerId = textData.otId, + extractionConfidence = calculateExtractionConfidence(detections, textData, iconData) + ) + } + + /** + * Calculate confidence score for extraction quality + */ + private fun calculateExtractionConfidence( + detections: List, + textData: TextExtractionResults, + iconData: IconExtractionResults + ): Double { + var confidence = 0.0 + + // Base confidence from detection quality + val avgDetectionConfidence = detections.map { it.confidence.toDouble() }.average() + confidence += avgDetectionConfidence * 0.4 + + // Boost for successful text extractions + if (!textData.nickname.isNullOrBlank()) confidence += 0.15 + if (textData.level != null && textData.level > 0) confidence += 0.15 + if (!textData.species.isNullOrBlank()) confidence += 0.15 + if (!textData.nature.isNullOrBlank()) confidence += 0.1 + if (!textData.ability.isNullOrBlank()) confidence += 0.05 + + return confidence.coerceIn(0.0, 1.0) + } + + /** + * Log extraction summary for debugging + */ + private fun logExtractionSummary(pokemonInfo: PokemonInfo) { + PGHLog.i(TAG, "📊 Extraction Summary:") + PGHLog.i(TAG, " Nickname: '${pokemonInfo.nickname ?: "null"}'") + PGHLog.i(TAG, " Level: ${pokemonInfo.level ?: "null"}") + PGHLog.i(TAG, " Species: '${pokemonInfo.species ?: "null"}'") + PGHLog.i(TAG, " Nature: '${pokemonInfo.nature ?: "null"}'") + PGHLog.i(TAG, " Ability: '${pokemonInfo.ability ?: "null"}'") + PGHLog.i(TAG, " Gender: '${pokemonInfo.gender ?: "null"}'") + PGHLog.i(TAG, " Pokeball: '${pokemonInfo.pokeballType ?: "null"}'") + PGHLog.i(TAG, " Types: [${pokemonInfo.primaryType ?: ""}, ${pokemonInfo.secondaryType ?: ""}]") + PGHLog.i(TAG, " Tera: '${pokemonInfo.teraType ?: "null"}'") + PGHLog.i(TAG, " Shiny: ${pokemonInfo.isShiny}") + PGHLog.i(TAG, " Confidence: ${String.format("%.2f", pokemonInfo.extractionConfidence)}") + } + + /** + * Data class for text extraction results + */ + private data class TextExtractionResults( + val nickname: String?, + val species: String?, + val nature: String?, + val ability: String?, + val otName: String?, + val otId: String?, + val level: Int?, + val stats: PokemonStats?, + val moves: List, + val types: List + ) + + /** + * Data class for icon extraction results + */ + private data class IconExtractionResults( + val gender: String?, + val pokeballType: String?, + val isShiny: Boolean, + val teraType: String?, + val gameSource: String? + ) +} \ No newline at end of file