package com.quillstudios.pokegoalshelper import android.app.* import android.content.Context import android.content.Intent import android.os.Binder import android.graphics.Bitmap import android.graphics.ImageFormat import android.graphics.PixelFormat import android.hardware.display.DisplayManager import android.hardware.display.VirtualDisplay import android.media.Image import android.media.ImageReader import android.media.projection.MediaProjection import android.media.projection.MediaProjectionManager import android.os.* import android.util.DisplayMetrics import android.util.Log import com.quillstudios.pokegoalshelper.utils.PGHLog import com.quillstudios.pokegoalshelper.ml.MLErrorType import android.view.WindowManager import android.view.View import android.view.Gravity import android.widget.Button import android.widget.LinearLayout import androidx.core.app.NotificationCompat import com.quillstudios.pokegoalshelper.controllers.DetectionController import com.quillstudios.pokegoalshelper.ui.EnhancedFloatingFAB import com.quillstudios.pokegoalshelper.capture.ScreenCaptureManager import com.quillstudios.pokegoalshelper.capture.ScreenCaptureManagerImpl import com.quillstudios.pokegoalshelper.ml.MLInferenceEngine import com.quillstudios.pokegoalshelper.ml.YOLOInferenceEngine import com.quillstudios.pokegoalshelper.ml.Detection as MLDetection import com.quillstudios.pokegoalshelper.data.PokemonDataExtractor import com.quillstudios.pokegoalshelper.data.PokemonDataExtractorImpl import kotlinx.coroutines.runBlocking import org.opencv.android.Utils import org.opencv.core.* import org.opencv.imgproc.Imgproc import org.opencv.imgcodecs.Imgcodecs import com.quillstudios.pokegoalshelper.utils.MatUtils.use import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.text.TextRecognition import com.google.mlkit.vision.text.latin.TextRecognizerOptions import java.util.concurrent.CountDownLatch import java.io.File import java.util.concurrent.TimeUnit import java.util.concurrent.Executors import java.util.concurrent.ThreadPoolExecutor import java.nio.ByteBuffer data class PokemonInfo( val pokeballType: String?, val nickname: String?, val gender: String?, val level: Int?, val language: String?, val gameSource: String?, val isFavorited: Boolean, val nationalDexNumber: Int?, val species: String?, val primaryType: String?, val secondaryType: String?, val teraType: String?, val isShiny: Boolean, val stamps: List, val labels: List, val marks: List, val stats: PokemonStats?, val moves: List, val ability: String?, val nature: String?, val originalTrainerName: String?, val originalTrainerId: String?, val extractionConfidence: Double ) data class PokemonStats( val hp: Int?, val attack: Int?, val defense: Int?, val spAttack: Int?, val spDefense: Int?, val speed: Int? ) data class ScreenRegion( val x: Int, val y: Int, val width: Int, val height: Int, val purpose: String ) class ScreenCaptureService : Service() { companion object { private const val TAG = "ScreenCaptureService" private const val NOTIFICATION_ID = 1001 private const val CHANNEL_ID = "screen_capture_channel" // Timeout Constants private const val ANALYSIS_STUCK_TIMEOUT_MS = 30000L // 30 seconds private const val OCR_TASK_TIMEOUT_SECONDS = 10L private const val DEFAULT_CAPTURE_INTERVAL_MS = 2000L // 2 seconds private const val OCR_INDIVIDUAL_TIMEOUT_SECONDS = 5L // UI Constants private const val MIN_PIXEL_CHANNELS = 3 const val ACTION_START = "START_SCREEN_CAPTURE" const val ACTION_STOP = "STOP_SCREEN_CAPTURE" const val EXTRA_RESULT_DATA = "result_data" } /** * Binder for external communication (if needed in future) */ inner class LocalBinder : Binder() { fun getService(): ScreenCaptureService = this@ScreenCaptureService } private val binder = LocalBinder() // ML inference engine private var mlInferenceEngine: MLInferenceEngine? = null // Pokemon data extractor private var pokemonDataExtractor: PokemonDataExtractor? = null private lateinit var screenCaptureManager: ScreenCaptureManager private var detectionOverlay: DetectionOverlay? = null // MVC Components private lateinit var detectionController: DetectionController private var enhancedFloatingFAB: EnhancedFloatingFAB? = null private val handler = Handler(Looper.getMainLooper()) private var captureInterval = DEFAULT_CAPTURE_INTERVAL_MS private var autoProcessing = false // Disable automatic processing // Thread pool for OCR processing (4 threads for parallel text extraction) private val ocrExecutor = Executors.newFixedThreadPool(4) // Flag to prevent overlapping analysis cycles private var isAnalyzing = false private var analysisStartTime = 0L private val captureRunnable = object : Runnable { override fun run() { captureScreen() handler.postDelayed(this, captureInterval) } } override fun onCreate() { super.onCreate() createNotificationChannel() // Initialize screen capture manager screenCaptureManager = ScreenCaptureManagerImpl(this, handler) screenCaptureManager.setImageCallback { image -> handleCapturedImage(image) } // Initialize ML inference engine mlInferenceEngine = YOLOInferenceEngine(this) Thread { runBlocking { mlInferenceEngine!!.initialize() .onSuccess { PGHLog.i(TAG, "โœ… ML inference engine initialized for screen capture") } .onError { errorType, exception, message -> PGHLog.e(TAG, "โŒ Failed to initialize ML inference engine: $message (${errorType.name})", exception) } } }.start() // Initialize Pokemon data extractor pokemonDataExtractor = PokemonDataExtractorImpl(this) PGHLog.i(TAG, "โœ… Pokemon data extractor initialized") // Initialize MVC components detectionController = DetectionController(mlInferenceEngine!!) detectionController.setDetectionRequestCallback { triggerManualDetection() } // Initialize enhanced floating FAB enhancedFloatingFAB = EnhancedFloatingFAB( context = this, onDetectionRequested = { triggerDetection() }, onClassFilterRequested = { className -> setClassFilter(className) }, onDebugToggled = { toggleDebugMode() }, onClose = { stopSelf() } ) PGHLog.d(TAG, "โœ… MVC architecture initialized") } override fun onBind(intent: Intent?): IBinder = binder // === Public API for external communication === /** * Trigger manual detection from UI */ fun triggerDetection() { triggerManualDetection() } /** * Set class filter from UI */ fun setClassFilter(className: String?) { detectionController.onClassFilterChanged(className) } /** * Toggle debug mode from UI */ fun toggleDebugMode() { detectionController.onDebugModeToggled() } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { when (intent?.action) { ACTION_START -> { val resultData = intent.getParcelableExtra(EXTRA_RESULT_DATA) if (resultData != null) { startScreenCapture(resultData) } } ACTION_STOP -> { stopScreenCapture() } } return START_STICKY } private fun createNotificationChannel() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { val channel = NotificationChannel( CHANNEL_ID, "Screen Capture Service", NotificationManager.IMPORTANCE_LOW ).apply { description = "Pokemon analysis screen capture" setShowBadge(false) } val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager manager.createNotificationChannel(channel) } } private fun getStatusBarHeight(): Int { var result = 0 val resourceId = resources.getIdentifier("status_bar_height", "dimen", "android") if (resourceId > 0) { result = resources.getDimensionPixelSize(resourceId) } return result } private fun startScreenCapture(resultData: Intent) { PGHLog.d(TAG, "Starting screen capture") try { val notification = NotificationCompat.Builder(this, CHANNEL_ID) .setContentTitle("Pokemon Analysis Active") .setContentText("Analyzing Pokemon Home screens...") .setSmallIcon(R.drawable.ic_launcher_foreground) .setOngoing(true) .addAction( R.drawable.ic_launcher_foreground, "Stop", PendingIntent.getService( this, 0, Intent(this, ScreenCaptureService::class.java).apply { action = ACTION_STOP }, PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE ) ) .build() PGHLog.d(TAG, "Starting foreground service") startForeground(NOTIFICATION_ID, notification) // Use the screen capture manager to start capture if (!screenCaptureManager.startCapture(resultData)) { PGHLog.e(TAG, "Failed to start screen capture via manager") stopSelf() return } PGHLog.d(TAG, "Screen capture setup complete") // Show floating overlay enhancedFloatingFAB?.show() } catch (e: Exception) { PGHLog.e(TAG, "Error starting screen capture", e) stopSelf() } } private fun stopScreenCapture() { PGHLog.d(TAG, "Stopping screen capture") handler.removeCallbacks(captureRunnable) hideDetectionOverlay() enhancedFloatingFAB?.hide() latestImage?.close() latestImage = null // Use the screen capture manager to stop capture screenCaptureManager.stopCapture() stopForeground(true) stopSelf() } /** * Handle captured images from the ScreenCaptureManager */ private fun handleCapturedImage(image: Image) { try { if (autoProcessing) { processImage(image) image.close() } else { // Store the latest image for manual processing latestImage?.close() // Release previous image latestImage = image // Don't close the image yet - it will be closed in triggerManualDetection } } catch (e: Exception) { PGHLog.e(TAG, "Error handling captured image", e) } } private var latestImage: Image? = null private fun captureScreen() { // Trigger image capture by reading from the ImageReader // The onImageAvailableListener will handle the actual processing PGHLog.d(TAG, "Triggering screen capture...") } private fun processImage(image: Image) { var bitmap: Bitmap? = null var croppedBitmap: Bitmap? = null try { // Get screen dimensions from the manager val screenDimensions = screenCaptureManager.getScreenDimensions() if (screenDimensions == null) { PGHLog.e(TAG, "Screen dimensions not available from manager") return } val (screenWidth, screenHeight) = screenDimensions // Configure Pokemon data extractor with screen dimensions pokemonDataExtractor?.configureRegions(android.util.Size(screenWidth, screenHeight)) val planes = image.planes val buffer = planes[0].buffer val pixelStride = planes[0].pixelStride val rowStride = planes[0].rowStride val rowPadding = rowStride - pixelStride * screenWidth PGHLog.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: pixelStride=$pixelStride, rowStride=$rowStride, rowPadding=$rowPadding") PGHLog.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: screenSize=${screenWidth}x${screenHeight}, expected bitmap=${screenWidth + rowPadding / pixelStride}x${screenHeight}") // Create bitmap from image bitmap = Bitmap.createBitmap( screenWidth + rowPadding / pixelStride, screenHeight, Bitmap.Config.ARGB_8888 ) bitmap.copyPixelsFromBuffer(buffer) PGHLog.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: created bitmap=${bitmap.width}x${bitmap.height}") // Convert to cropped bitmap if needed croppedBitmap = if (rowPadding == 0) { PGHLog.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: No padding, using original bitmap") bitmap } else { PGHLog.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: Cropping bitmap from ${bitmap.width}x${bitmap.height} to ${screenWidth}x${screenHeight}") Bitmap.createBitmap(bitmap, 0, 0, screenWidth, screenHeight) } PGHLog.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: final bitmap=${croppedBitmap.width}x${croppedBitmap.height}") // Convert to OpenCV Mat for analysis - with proper resource management Mat().use { mat -> Utils.bitmapToMat(croppedBitmap, mat) // DEBUG: Check color conversion PGHLog.d(TAG, "๐ŸŽจ COLOR DEBUG: Mat type=${mat.type()}, channels=${mat.channels()}") PGHLog.d(TAG, "๐ŸŽจ COLOR DEBUG: OpenCV expects BGR, Android Bitmap is ARGB") // Sample a center pixel to check color values if (mat.rows() > 0 && mat.cols() > 0) { val centerY = mat.rows() / 2 val centerX = mat.cols() / 2 val pixel = mat.get(centerY, centerX) if (pixel != null && pixel.size >= MIN_PIXEL_CHANNELS) { val b = pixel[0].toInt() val g = pixel[1].toInt() val r = pixel[2].toInt() PGHLog.d(TAG, "๐ŸŽจ COLOR DEBUG: Center pixel (${centerX},${centerY}) BGR=($b,$g,$r) -> RGB=(${r},${g},${b})") PGHLog.d(TAG, "๐ŸŽจ COLOR DEBUG: Center pixel hex = #${String.format("%02x%02x%02x", r, g, b)}") } } // Run YOLO analysis analyzePokemonScreen(mat) } } catch (e: Exception) { PGHLog.e(TAG, "Error processing image", e) } finally { // Always clean up bitmaps in finally block to prevent leaks if (croppedBitmap != null && croppedBitmap != bitmap) { croppedBitmap.recycle() } bitmap?.recycle() } } private fun analyzePokemonScreen(mat: Mat) { PGHLog.i(TAG, "๐Ÿ“ฑ ANALYZING SCREEN: ${mat.cols()}x${mat.rows()}") // Check if analysis has been stuck for too long (30 seconds max) val currentTime = System.currentTimeMillis() if (isAnalyzing && (currentTime - analysisStartTime) > ANALYSIS_STUCK_TIMEOUT_MS) { PGHLog.w(TAG, "โš ๏ธ Analysis stuck for >30s, resetting flag") isAnalyzing = false } // Skip if already analyzing if (isAnalyzing) { PGHLog.d(TAG, "โญ๏ธ Skipping analysis - previous cycle still in progress") return } isAnalyzing = true analysisStartTime = currentTime PGHLog.d(TAG, "๐Ÿ”„ Starting new analysis cycle") try { // Run ML inference first val bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888) Utils.matToBitmap(mat, bitmap) val detections: List = runBlocking { mlInferenceEngine?.detect(bitmap) ?.onErrorType(MLErrorType.MEMORY_ERROR) { exception, message -> PGHLog.w(TAG, "โš ๏ธ Memory error during detection, may retry with smaller image: $message") } ?.onErrorType(MLErrorType.TIMEOUT_ERROR) { exception, message -> PGHLog.w(TAG, "โš ๏ธ Detection timed out, analysis may be too complex: $message") } ?.onErrorType(MLErrorType.INVALID_INPUT) { exception, message -> PGHLog.e(TAG, "โŒ Invalid input to ML engine: $message") } ?.getOrDefault(emptyList()) ?: emptyList() } if (detections.isEmpty()) { PGHLog.i(TAG, "๐Ÿ” No Pokemon UI elements detected by ONNX YOLO") isAnalyzing = false return } PGHLog.i(TAG, "๐ŸŽฏ ONNX YOLO detected ${detections.size} UI elements") // Log ALL detections for debugging detections.forEachIndexed { index, detection -> PGHLog.i(TAG, " $index: ${detection.className} (${String.format("%.3f", detection.confidence)}) at [${detection.boundingBox.left}, ${detection.boundingBox.top}, ${detection.boundingBox.width}, ${detection.boundingBox.height}]") } // Show breakdown by type val detectionCounts = detections.groupBy { it.className }.mapValues { it.value.size } PGHLog.i(TAG, "๐Ÿ” Detection counts by type: $detectionCounts") // Check for commonly missing elements val expectedElements = listOf("pokemon_level", "attack_value", "sp_def_value", "shiny_icon", "ball_icon_pokeball", "ball_icon_greatball", "ball_icon_ultraball", "ball_icon_masterball") val missingElements = expectedElements.filter { expected -> detections.none { detection -> detection.className.startsWith(expected.split("_").take(2).joinToString("_")) } } if (missingElements.isNotEmpty()) { PGHLog.w(TAG, "โš ๏ธ Missing expected elements: $missingElements") } // Show detection overlay IMMEDIATELY (no OCR blocking) showYOLODetectionOverlay(detections) PGHLog.i(TAG, "๐Ÿ“บ Overlay displayed with ${detections.size} detections") // Extract Pokemon info using YOLO detections in background extractPokemonInfoFromYOLOAsync(mat, detections) } catch (e: Exception) { PGHLog.e(TAG, "Error analyzing Pokemon screen", e) isAnalyzing = false } } private fun extractPokemonInfoFromYOLOAsync(mat: Mat, detections: List) { // Create a copy of the Mat for background processing val matCopy = Mat() mat.copyTo(matCopy) // Process in background thread with coroutines ocrExecutor.submit { try { val pokemonInfo = runBlocking { extractPokemonInfoFromYOLO(matCopy, detections) } // Post results back to main thread handler.post { try { if (pokemonInfo != null) { PGHLog.i(TAG, "๐Ÿ”ฅ POKEMON DATA EXTRACTED SUCCESSFULLY!") logPokemonInfo(pokemonInfo) // TODO: Send to your API // sendToAPI(pokemonInfo) } else { PGHLog.i(TAG, "โŒ Could not extract complete Pokemon info") } } finally { // Analysis cycle complete, allow next one isAnalyzing = false val duration = System.currentTimeMillis() - analysisStartTime PGHLog.d(TAG, "โœ… Analysis cycle complete after ${duration}ms - ready for next") } } // Clean up matCopy.release() } catch (e: Exception) { PGHLog.e(TAG, "Error in async Pokemon extraction", e) matCopy.release() // Clear flag on error too handler.post { isAnalyzing = false PGHLog.d(TAG, "โŒ Analysis cycle failed - ready for next") } } } } private suspend fun extractPokemonInfoFromYOLO(mat: Mat, detections: List): PokemonInfo? { return try { PGHLog.i(TAG, "๐ŸŽฏ Delegating Pokemon extraction to PokemonDataExtractor") // Convert MLDetection to Detection for the extractor val extractorDetections = detections.map { mlDetection -> com.quillstudios.pokegoalshelper.ml.Detection( className = mlDetection.className, confidence = mlDetection.confidence, boundingBox = mlDetection.boundingBox ) } // Use the dedicated Pokemon data extractor with async/await pattern pokemonDataExtractor?.extractPokemonInfo(extractorDetections, mat) } catch (e: Exception) { PGHLog.e(TAG, "Error extracting Pokemon info via PokemonDataExtractor", e) null } } private fun submitOCRTask(key: String, mat: Mat, detection: MLDetection?, results: MutableMap, latch: CountDownLatch) { ocrExecutor.submit { try { val text = extractTextFromDetection(mat, detection) synchronized(results) { results[key] = text } } catch (e: Exception) { PGHLog.e(TAG, "Error in OCR task for $key", e) synchronized(results) { results[key] = null } } finally { latch.countDown() } } } private fun submitLevelOCRTask(key: String, mat: Mat, detection: MLDetection?, results: MutableMap, latch: CountDownLatch) { ocrExecutor.submit { try { val levelText = extractTextFromDetection(mat, detection) val level = levelText?.replace("[^0-9]".toRegex(), "") synchronized(results) { results[key] = level } } catch (e: Exception) { PGHLog.e(TAG, "Error in level OCR task", e) synchronized(results) { results[key] = null } } finally { latch.countDown() } } } private fun extractTextFromDetection(mat: Mat, detection: MLDetection?): String? { if (detection == null) return null try { // Expand bounding box by 5% for all OCR classes to improve text extraction accuracy val bbox = detection.boundingBox val expansionFactor = 0.05f // 5% expansion val widthExpansion = (bbox.width * expansionFactor).toInt() val heightExpansion = (bbox.height * expansionFactor).toInt() val expandedBbox = Rect( bbox.left.toInt() - widthExpansion, bbox.top.toInt() - heightExpansion, bbox.width.toInt() + (2 * widthExpansion), bbox.height.toInt() + (2 * heightExpansion) ) // Validate and clip bounding box to image boundaries val clippedX = kotlin.math.max(0, kotlin.math.min(expandedBbox.x, mat.cols() - 1)) val clippedY = kotlin.math.max(0, kotlin.math.min(expandedBbox.y, mat.rows() - 1)) val clippedWidth = kotlin.math.max(1, kotlin.math.min(expandedBbox.width, mat.cols() - clippedX)) val clippedHeight = kotlin.math.max(1, kotlin.math.min(expandedBbox.height, mat.rows() - clippedY)) val safeBbox = Rect(clippedX, clippedY, clippedWidth, clippedHeight) // Debug logging for bounding box transformations PGHLog.d(TAG, "๐Ÿ“ Expanded bbox for ${detection.className}: [${bbox.left},${bbox.top},${bbox.width},${bbox.height}] โ†’ [${expandedBbox.x},${expandedBbox.y},${expandedBbox.width},${expandedBbox.height}]") if (safeBbox.x != expandedBbox.x || safeBbox.y != expandedBbox.y || safeBbox.width != expandedBbox.width || safeBbox.height != expandedBbox.height) { PGHLog.w(TAG, "โš ๏ธ Clipped bbox for ${detection.className}: expanded=[${expandedBbox.x},${expandedBbox.y},${expandedBbox.width},${expandedBbox.height}] โ†’ safe=[${safeBbox.x},${safeBbox.y},${safeBbox.width},${safeBbox.height}] (image: ${mat.cols()}x${mat.rows()})") } // Extract region of interest using safe bounding box val roi = Mat(mat, safeBbox) // Preprocess image for better OCR val processedRoi = preprocessImageForOCR(roi) // Convert to bitmap for ML Kit val bitmap = Bitmap.createBitmap(processedRoi.cols(), processedRoi.rows(), Bitmap.Config.ARGB_8888) Utils.matToBitmap(processedRoi, bitmap) // Use ML Kit OCR val extractedText = try { performOCR(bitmap, detection.className) } finally { // Always clean up bitmap after OCR bitmap.recycle() } // Clean up roi.release() processedRoi.release() if (extractedText != null) { PGHLog.i(TAG, "โœ… YOLO SUCCESS: ${detection.className} = '$extractedText' (conf: ${String.format("%.2f", detection.confidence)})") } else { PGHLog.w(TAG, "โŒ YOLO FAILED: ${detection.className} - no text found (conf: ${String.format("%.2f", detection.confidence)})") } return extractedText } catch (e: Exception) { PGHLog.e(TAG, "Error extracting text from YOLO detection ${detection.className}", e) return null } } private fun extractLevelFromDetection(mat: Mat, detection: MLDetection?): Int? { val levelText = extractTextFromDetection(mat, detection) return levelText?.replace("[^0-9]".toRegex(), "")?.toIntOrNull() } private fun extractStatsFromDetections(mat: Mat, detectionMap: Map>): PokemonStats? { val hp = extractTextFromDetection(mat, detectionMap["hp_value"]?.firstOrNull())?.toIntOrNull() val attack = extractTextFromDetection(mat, detectionMap["attack_value"]?.firstOrNull())?.toIntOrNull() val defense = extractTextFromDetection(mat, detectionMap["defense_value"]?.firstOrNull())?.toIntOrNull() val spAttack = extractTextFromDetection(mat, detectionMap["sp_atk_value"]?.firstOrNull())?.toIntOrNull() val spDefense = extractTextFromDetection(mat, detectionMap["sp_def_value"]?.firstOrNull())?.toIntOrNull() val speed = extractTextFromDetection(mat, detectionMap["speed_value"]?.firstOrNull())?.toIntOrNull() return if (hp != null || attack != null || defense != null || spAttack != null || spDefense != null || speed != null) { PokemonStats(hp, attack, defense, spAttack, spDefense, speed) } else null } private fun extractMovesFromDetections(mat: Mat, detectionMap: Map>): List { val moves = mutableListOf() detectionMap["move_name"]?.forEach { detection -> val moveText = extractTextFromDetection(mat, detection) if (!moveText.isNullOrBlank()) { moves.add(moveText.trim()) } } return moves.take(4) // Pokemon can have max 4 moves } private fun detectPokeballTypeFromDetections(detectionMap: Map>): String? { // Check for specific pokeball types detected by YOLO val pokeballTypes = mapOf( "ball_icon_pokeball" to "Pokรฉ Ball", "ball_icon_greatball" to "Great Ball", "ball_icon_ultraball" to "Ultra Ball", "ball_icon_heavyball" to "Heavy Ball", "ball_icon_premierball" to "Premier Ball", "ball_icon_repeatball" to "Repeat Ball", "ball_icon_timerball" to "Timer Ball", "ball_icon_diveball" to "Dive Ball", "ball_icon_quickball" to "Quick Ball", "ball_icon_duskball" to "Dusk Ball", "ball_icon_cherishball" to "Cherish Ball", "ball_icon_originball" to "Origin Ball", "ball_icon_pokeball_hisui" to "Hisuian Pokรฉ Ball", "ball_icon_ultraball_husui" to "Hisuian Ultra Ball" ) for ((className, ballName) in pokeballTypes) { if (detectionMap[className]?.isNotEmpty() == true) { return ballName } } return null } private fun extractTypesFromDetections(mat: Mat, detectionMap: Map>): List { val types = mutableListOf() extractTextFromDetection(mat, detectionMap["type_1"]?.firstOrNull())?.let { type1 -> if (type1.isNotBlank()) types.add(type1.trim()) } extractTextFromDetection(mat, detectionMap["type_2"]?.firstOrNull())?.let { type2 -> if (type2.isNotBlank()) types.add(type2.trim()) } return types } private fun detectTeraTypeFromDetections(detectionMap: Map>): String? { val teraTypes = mapOf( "tera_ice" to "Ice", "tera_fairy" to "Fairy", "tera_poison" to "Poison", "tera_ghost" to "Ghost", "tera_steel" to "Steel", "tera_grass" to "Grass", "tera_normal" to "Normal", "tera_fire" to "Fire", "tera_electric" to "Electric", "tera_ground" to "Ground", "tera_flying" to "Flying", "tera_bug" to "Bug", "tera_dark" to "Dark", "tera_water" to "Water", "tera_psychic" to "Psychic", "tera_dragon" to "Dragon", "tera_fighting" to "Fighting", "tera_rock" to "Rock" ) for ((className, teraName) in teraTypes) { if (detectionMap[className]?.isNotEmpty() == true) { return teraName } } return null } private fun detectGameSourceFromDetections(detectionMap: Map>): String? { val gameSources = mapOf( "last_game_stamp_sh" to "Sword/Shield", "last_game_stamp_bank" to "Bank", "last_game_stamp_pla" to "Legends: Arceus", "last_game_stamp_sc" to "Scarlet/Violet", "last_game_stamp_vi" to "Violet", "last_game_stamp_go" to "Pokรฉmon GO", "origin_icon_vc" to "Virtual Console", "origin_icon_xyoras" to "XY/ORAS", "origin_icon_smusum" to "SM/USUM", "origin_icon_swsh" to "Sword/Shield", "origin_icon_go" to "Pokรฉmon GO", "origin_icon_pla" to "Legends: Arceus", "origin_icon_sv" to "Scarlet/Violet" ) for ((className, sourceName) in gameSources) { if (detectionMap[className]?.isNotEmpty() == true) { return sourceName } } return null } private fun calculateYOLOExtractionConfidence(detections: List, nickname: String?, level: Int?, species: String?): Double { var confidence = 0.0 // Base confidence from YOLO detections val avgDetectionConfidence = detections.map { it.confidence.toDouble() }.average() confidence += avgDetectionConfidence * 0.4 // Boost confidence based on extracted data if (!nickname.isNullOrBlank()) confidence += 0.2 if (level != null && level > 0) confidence += 0.2 if (!species.isNullOrBlank()) confidence += 0.2 return confidence.coerceIn(0.0, 1.0) } private fun showYOLODetectionOverlay(detections: List) { try { PGHLog.i(TAG, "๐ŸŽจ Creating YOLO detection overlay for ${detections.size} detections") if (detectionOverlay == null) { PGHLog.i(TAG, "๐Ÿ†• Creating new DetectionOverlay instance") detectionOverlay = DetectionOverlay(this) } // Convert YOLO detections to screen regions for overlay val statusBarHeight = getStatusBarHeight() val regions = detections.mapIndexed { index, detection -> "${detection.className}_$index" to ScreenRegion( x = detection.boundingBox.left.toInt(), y = detection.boundingBox.top.toInt() - statusBarHeight, // Subtract status bar offset width = detection.boundingBox.width.toInt(), height = detection.boundingBox.height.toInt(), purpose = "${detection.className} (${String.format("%.2f", detection.confidence)})" ) }.toMap() PGHLog.i(TAG, "๐Ÿ“บ Showing YOLO overlay with ${regions.size} regions...") detectionOverlay?.showOverlay(regions) PGHLog.i(TAG, "โœ… YOLO overlay show command sent") } catch (e: Exception) { PGHLog.e(TAG, "โŒ Error showing YOLO detection overlay", e) } } private fun hideDetectionOverlay() { detectionOverlay?.hideOverlay() } private fun performOCR(bitmap: Bitmap, purpose: String): String? { try { PGHLog.d(TAG, "๐Ÿ” Starting OCR for $purpose - bitmap: ${bitmap.width}x${bitmap.height}") // Create InputImage for ML Kit val image = InputImage.fromBitmap(bitmap, 0) val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS) // Use CountDownLatch to make async call synchronous val latch = CountDownLatch(1) var result: String? = null var ocrError: Exception? = null recognizer.process(image) .addOnSuccessListener { visionText -> result = visionText.text.trim() PGHLog.d(TAG, "๐Ÿ” Raw OCR result for $purpose: '${visionText.text}' (blocks: ${visionText.textBlocks.size})") if (result.isNullOrBlank()) { PGHLog.w(TAG, "โš ๏ธ OCR for $purpose: NO TEXT DETECTED - ML Kit found ${visionText.textBlocks.size} text blocks but text is empty") } else { PGHLog.i(TAG, "โœ… OCR SUCCESS for $purpose: '${result}' (${result!!.length} chars)") } latch.countDown() } .addOnFailureListener { e -> ocrError = e PGHLog.e(TAG, "โŒ OCR failed for $purpose: ${e.message}", e) latch.countDown() } // Wait for OCR to complete (max 5 seconds to allow ML Kit to work) val completed = latch.await(OCR_INDIVIDUAL_TIMEOUT_SECONDS, TimeUnit.SECONDS) if (!completed) { PGHLog.e(TAG, "โฑ๏ธ OCR timeout for $purpose after ${OCR_INDIVIDUAL_TIMEOUT_SECONDS} seconds") return null } if (ocrError != null) { PGHLog.e(TAG, "โŒ OCR error for $purpose: ${ocrError!!.message}") return null } // Clean and process the result val cleanedResult = cleanOCRResult(result, purpose) PGHLog.d(TAG, "๐Ÿง™ Cleaned result for $purpose: '${cleanedResult}'") return cleanedResult } catch (e: Exception) { PGHLog.e(TAG, "โŒ Error in OCR for $purpose", e) return null } } private fun cleanOCRResult(rawText: String?, purpose: String): String? { if (rawText.isNullOrBlank()) return null val cleaned = rawText.trim() return when (purpose) { "pokemon_level" -> { // Extract level number (look for "Lv." or just numbers) val levelRegex = """(?:Lv\.?\s*)?([0-9]+)""".toRegex(RegexOption.IGNORE_CASE) levelRegex.find(cleaned)?.groupValues?.get(1) } "pokemon_nickname", "pokemon_species" -> { // Clean up common OCR mistakes for Pokemon names cleaned.replace("[^a-zA-Z0-9 \\-.'โ™‚โ™€]".toRegex(), "") .replace("\\s+".toRegex(), " ") .trim() .takeIf { it.isNotEmpty() } } "nature_name", "ability_name" -> { // Clean up nature/ability names cleaned.replace("[^a-zA-Z ]".toRegex(), "") .replace("\\s+".toRegex(), " ") .trim() .takeIf { it.isNotEmpty() } } "original_trainer_name" -> { // Clean trainer names cleaned.replace("[^a-zA-Z0-9 ]".toRegex(), "") .replace("\\s+".toRegex(), " ") .trim() .takeIf { it.isNotEmpty() } } "original_trainder_number" -> { // Extract ID numbers cleaned.replace("[^0-9]".toRegex(), "") .takeIf { it.isNotEmpty() } } else -> cleaned.takeIf { it.isNotEmpty() } } } private fun preprocessImageForOCR(roi: Mat): Mat { try { // Resize small regions for better OCR (minimum 150x50 pixels) val minWidth = 150 val minHeight = 50 val scaledRoi = if (roi.width() < minWidth || roi.height() < minHeight) { val scaleX = maxOf(1.0, minWidth.toDouble() / roi.width()) val scaleY = maxOf(1.0, minHeight.toDouble() / roi.height()) val scale = maxOf(scaleX, scaleY) val resized = Mat() Imgproc.resize(roi, resized, Size(roi.width() * scale, roi.height() * scale)) PGHLog.d(TAG, "๐Ÿ” Upscaled OCR region from ${roi.width()}x${roi.height()} to ${resized.width()}x${resized.height()}") resized } else { val copy = Mat() roi.copyTo(copy) copy } // Convert to grayscale if needed val gray = Mat() if (scaledRoi.channels() > 1) { Imgproc.cvtColor(scaledRoi, gray, Imgproc.COLOR_RGBA2GRAY) } else { scaledRoi.copyTo(gray) } // Enhance contrast with CLAHE (Contrast Limited Adaptive Histogram Equalization) val clahe = Imgproc.createCLAHE(2.0, Size(8.0, 8.0)) val enhanced = Mat() clahe.apply(gray, enhanced) // Light denoising only if text is very small val denoised = if (scaledRoi.width() < 100 || scaledRoi.height() < 30) { val temp = Mat() Imgproc.GaussianBlur(enhanced, temp, Size(1.0, 1.0), 0.0) temp } else { enhanced.clone() } // Convert back to RGBA for ML Kit val result = Mat() Imgproc.cvtColor(denoised, result, Imgproc.COLOR_GRAY2RGBA) // Clean up intermediate matrices scaledRoi.release() gray.release() enhanced.release() if (denoised !== enhanced) denoised.release() return result } catch (e: Exception) { PGHLog.e(TAG, "Error preprocessing image, using original", e) // Return copy of original if preprocessing fails val result = Mat() roi.copyTo(result) return result } } private fun detectLanguage(nickname: String?, species: String?): String? { // Simple language detection based on character patterns val text = "$nickname $species" return when { text.any { it in '\u3040'..'\u309F' || it in '\u30A0'..'\u30FF' } -> "JP" // Hiragana/Katakana text.any { it in '\u4E00'..'\u9FAF' } -> "ZH" // Chinese characters text.any { it in '\uAC00'..'\uD7AF' } -> "KO" // Korean else -> "EN" // Default to English } } private fun extractDexNumber(species: String?): Int? { // For now, return null - would need a Pokemon species to dex number mapping return null } private fun logPokemonInfo(pokemonInfo: PokemonInfo) { PGHLog.i(TAG, "====== POKEMON INFO EXTRACTED ======") PGHLog.i(TAG, "๐ŸŽพ Pokeball: ${pokemonInfo.pokeballType}") PGHLog.i(TAG, "๐Ÿ“› Nickname: ${pokemonInfo.nickname}") PGHLog.i(TAG, "โšค Gender: ${pokemonInfo.gender}") PGHLog.i(TAG, "๐Ÿ“Š Level: ${pokemonInfo.level}") PGHLog.i(TAG, "๐ŸŒ Language: ${pokemonInfo.language}") PGHLog.i(TAG, "๐ŸŽฎ Game Source: ${pokemonInfo.gameSource}") PGHLog.i(TAG, "โญ Favorited: ${pokemonInfo.isFavorited}") PGHLog.i(TAG, "๐Ÿ”ข Dex #: ${pokemonInfo.nationalDexNumber}") PGHLog.i(TAG, "๐Ÿพ Species: ${pokemonInfo.species}") PGHLog.i(TAG, "๐Ÿท๏ธ Type 1: ${pokemonInfo.primaryType}") PGHLog.i(TAG, "๐Ÿท๏ธ Type 2: ${pokemonInfo.secondaryType}") PGHLog.i(TAG, "๐Ÿ† Stamps: ${pokemonInfo.stamps}") PGHLog.i(TAG, "๐Ÿท๏ธ Labels: ${pokemonInfo.labels}") PGHLog.i(TAG, "โœ… Marks: ${pokemonInfo.marks}") if (pokemonInfo.stats != null) { PGHLog.i(TAG, "๐Ÿ“ˆ Stats: HP:${pokemonInfo.stats.hp} ATK:${pokemonInfo.stats.attack} DEF:${pokemonInfo.stats.defense}") PGHLog.i(TAG, " SP.ATK:${pokemonInfo.stats.spAttack} SP.DEF:${pokemonInfo.stats.spDefense} SPD:${pokemonInfo.stats.speed}") } PGHLog.i(TAG, "โš”๏ธ Moves: ${pokemonInfo.moves}") PGHLog.i(TAG, "๐Ÿ’ช Ability: ${pokemonInfo.ability}") PGHLog.i(TAG, "๐ŸŽญ Nature: ${pokemonInfo.nature}") PGHLog.i(TAG, "๐Ÿ‘ค OT: ${pokemonInfo.originalTrainerName}") PGHLog.i(TAG, "๐Ÿ”ข ID: ${pokemonInfo.originalTrainerId}") PGHLog.i(TAG, "๐ŸŽฏ Confidence: ${String.format("%.2f", pokemonInfo.extractionConfidence)}") PGHLog.i(TAG, "====================================") } private fun convertImageToMat(image: Image): Mat? { var bitmap: Bitmap? = null var croppedBitmap: Bitmap? = null var mat: Mat? = null return try { // Get screen dimensions from the manager val screenDimensions = screenCaptureManager.getScreenDimensions() if (screenDimensions == null) { PGHLog.e(TAG, "Screen dimensions not available from manager") return null } val (screenWidth, screenHeight) = screenDimensions val planes = image.planes val buffer = planes[0].buffer val pixelStride = planes[0].pixelStride val rowStride = planes[0].rowStride val rowPadding = rowStride - pixelStride * screenWidth // Create bitmap from image bitmap = Bitmap.createBitmap( screenWidth + rowPadding / pixelStride, screenHeight, Bitmap.Config.ARGB_8888 ) bitmap.copyPixelsFromBuffer(buffer) // Crop bitmap to remove padding if needed croppedBitmap = if (rowPadding == 0) { bitmap } else { val cropped = Bitmap.createBitmap(bitmap, 0, 0, screenWidth, screenHeight) bitmap.recycle() // Clean up original bitmap = null // Mark as recycled cropped } // Convert bitmap to Mat mat = Mat() Utils.bitmapToMat(croppedBitmap, mat) // Convert from RGBA to BGR (OpenCV format for proper color channel handling) val bgrMat = Mat() Imgproc.cvtColor(mat, bgrMat, Imgproc.COLOR_RGBA2BGR) // Clean up intermediate resources mat.release() croppedBitmap.recycle() bgrMat } catch (e: Exception) { PGHLog.e(TAG, "โŒ Error converting image to Mat", e) // Clean up on error mat?.release() if (croppedBitmap != null && croppedBitmap != bitmap) { croppedBitmap.recycle() } bitmap?.recycle() null } } private fun triggerManualDetection() { PGHLog.d(TAG, "๐Ÿ” Manual detection triggered via MVC!") latestImage?.let { image -> try { // Convert image to Mat for processing val mat = convertImageToMat(image) if (mat != null) { // Convert Mat to Bitmap and run inference val bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888) Utils.matToBitmap(mat, bitmap) val detections: List = runBlocking { mlInferenceEngine?.detect(bitmap)?.getOrDefault(emptyList()) ?: emptyList() } // Show detection overlay with results if (detections.isNotEmpty()) { showYOLODetectionOverlay(detections) // Extract Pokemon info using YOLO detections with OCR extractPokemonInfoFromYOLOAsync(mat, detections) } mat.release() } else { PGHLog.e(TAG, "โŒ Failed to convert image to Mat") } // Close the image after processing to free the buffer image.close() latestImage = null } catch (e: Exception) { PGHLog.e(TAG, "โŒ Error in manual detection", e) } } ?: run { PGHLog.w(TAG, "โš ๏ธ No image available for detection") } } override fun onDestroy() { super.onDestroy() hideDetectionOverlay() enhancedFloatingFAB?.hide() // TODO: Re-enable when DetectionController is updated // detectionController.clearUICallbacks() mlInferenceEngine?.cleanup() pokemonDataExtractor?.cleanup() // Release screen capture manager if (::screenCaptureManager.isInitialized) { screenCaptureManager.release() } // Proper executor shutdown with timeout ocrExecutor.shutdown() try { if (!ocrExecutor.awaitTermination(OCR_INDIVIDUAL_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { PGHLog.w(TAG, "OCR executor did not terminate gracefully, forcing shutdown") ocrExecutor.shutdownNow() } } catch (e: InterruptedException) { PGHLog.w(TAG, "Interrupted while waiting for OCR executor termination") ocrExecutor.shutdownNow() Thread.currentThread().interrupt() } stopScreenCapture() } }