package com.quillstudios.pokegoalshelper import android.app.* import android.content.Context import android.content.Intent import android.os.Binder import android.graphics.Bitmap import android.graphics.ImageFormat import android.graphics.PixelFormat import android.hardware.display.DisplayManager import android.hardware.display.VirtualDisplay import android.media.Image import android.media.ImageReader import android.media.projection.MediaProjection import android.media.projection.MediaProjectionManager import android.os.* import android.util.DisplayMetrics import android.util.Log import android.view.WindowManager import android.view.View import android.view.Gravity import android.widget.Button import android.widget.LinearLayout import androidx.core.app.NotificationCompat import com.quillstudios.pokegoalshelper.controllers.DetectionController import com.quillstudios.pokegoalshelper.ui.EnhancedFloatingFAB import org.opencv.android.Utils import org.opencv.core.* import org.opencv.imgproc.Imgproc import org.opencv.imgcodecs.Imgcodecs import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.text.TextRecognition import com.google.mlkit.vision.text.latin.TextRecognizerOptions import java.util.concurrent.CountDownLatch import java.io.File import java.util.concurrent.TimeUnit import java.util.concurrent.Executors import java.util.concurrent.ThreadPoolExecutor import java.nio.ByteBuffer data class PokemonInfo( val pokeballType: String?, val nickname: String?, val gender: String?, val level: Int?, val language: String?, val gameSource: String?, val isFavorited: Boolean, val nationalDexNumber: Int?, val species: String?, val primaryType: String?, val secondaryType: String?, val stamps: List, val labels: List, val marks: List, val stats: PokemonStats?, val moves: List, val ability: String?, val nature: String?, val originalTrainerName: String?, val originalTrainerId: String?, val extractionConfidence: Double ) data class PokemonStats( val hp: Int?, val attack: Int?, val defense: Int?, val spAttack: Int?, val spDefense: Int?, val speed: Int? ) data class ScreenRegion( val x: Int, val y: Int, val width: Int, val height: Int, val purpose: String ) class ScreenCaptureService : Service() { companion object { private const val TAG = "ScreenCaptureService" private const val NOTIFICATION_ID = 1001 private const val CHANNEL_ID = "screen_capture_channel" const val ACTION_START = "START_SCREEN_CAPTURE" const val ACTION_STOP = "STOP_SCREEN_CAPTURE" const val EXTRA_RESULT_DATA = "result_data" } /** * Binder for communication with FloatingUIActivity */ inner class LocalBinder : Binder() { fun getService(): ScreenCaptureService = this@ScreenCaptureService } private val binder = LocalBinder() // ONNX YOLO detector instance private var yoloDetector: YOLOOnnxDetector? = null private var mediaProjection: MediaProjection? = null private var virtualDisplay: VirtualDisplay? = null private var imageReader: ImageReader? = null private var mediaProjectionManager: MediaProjectionManager? = null private var screenWidth = 0 private var screenHeight = 0 private var screenDensity = 0 private var detectionOverlay: DetectionOverlay? = null // MVC Components private lateinit var detectionController: DetectionController private var enhancedFloatingFAB: EnhancedFloatingFAB? = null private val handler = Handler(Looper.getMainLooper()) private var captureInterval = 2000L // Capture every 2 seconds private var autoProcessing = false // Disable automatic processing // Thread pool for OCR processing (4 threads for parallel text extraction) private val ocrExecutor = Executors.newFixedThreadPool(4) // Flag to prevent overlapping analysis cycles private var isAnalyzing = false private var analysisStartTime = 0L private val captureRunnable = object : Runnable { override fun run() { captureScreen() handler.postDelayed(this, captureInterval) } } private val mediaProjectionCallback = object : MediaProjection.Callback() { override fun onStop() { Log.d(TAG, "MediaProjection stopped") stopScreenCapture() } override fun onCapturedContentResize(width: Int, height: Int) { Log.d(TAG, "Screen size changed: ${width}x${height}") } override fun onCapturedContentVisibilityChanged(isVisible: Boolean) { Log.d(TAG, "Content visibility changed: $isVisible") } } override fun onCreate() { super.onCreate() createNotificationChannel() getScreenMetrics() mediaProjectionManager = getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager // Initialize ONNX YOLO detector yoloDetector = YOLOOnnxDetector(this) if (!yoloDetector!!.initialize()) { Log.e(TAG, "โŒ Failed to initialize ONNX YOLO detector") } else { Log.i(TAG, "โœ… ONNX YOLO detector initialized for screen capture") } // Initialize MVC components detectionController = DetectionController(yoloDetector!!) detectionController.setDetectionRequestCallback { triggerManualDetection() } // Initialize enhanced floating FAB enhancedFloatingFAB = EnhancedFloatingFAB( context = this, onDetectionRequested = { triggerDetection() }, onClassFilterRequested = { className -> setClassFilter(className) }, onDebugToggled = { toggleDebugMode() }, onClose = { stopSelf() } ) Log.d(TAG, "โœ… MVC architecture initialized") } override fun onBind(intent: Intent?): IBinder = binder // === Public API for FloatingUIActivity === /** * Trigger manual detection from UI */ fun triggerDetection() { triggerManualDetection() } /** * Set class filter from UI */ fun setClassFilter(className: String?) { detectionController.onClassFilterChanged(className) } /** * Toggle debug mode from UI */ fun toggleDebugMode() { detectionController.onDebugModeToggled() } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { when (intent?.action) { ACTION_START -> { val resultData = intent.getParcelableExtra(EXTRA_RESULT_DATA) if (resultData != null) { startScreenCapture(resultData) } } ACTION_STOP -> { stopScreenCapture() } } return START_STICKY } private fun createNotificationChannel() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { val channel = NotificationChannel( CHANNEL_ID, "Screen Capture Service", NotificationManager.IMPORTANCE_LOW ).apply { description = "Pokemon analysis screen capture" setShowBadge(false) } val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager manager.createNotificationChannel(channel) } } private fun getScreenMetrics() { val displayMetrics = DisplayMetrics() val windowManager = getSystemService(Context.WINDOW_SERVICE) as android.view.WindowManager windowManager.defaultDisplay.getMetrics(displayMetrics) screenWidth = displayMetrics.widthPixels screenHeight = displayMetrics.heightPixels screenDensity = displayMetrics.densityDpi Log.d(TAG, "Screen metrics: ${screenWidth}x${screenHeight}, density: $screenDensity") // Get status bar height for coordinate adjustment val statusBarHeight = getStatusBarHeight() Log.d(TAG, "Status bar height: ${statusBarHeight}px") } private fun getStatusBarHeight(): Int { var result = 0 val resourceId = resources.getIdentifier("status_bar_height", "dimen", "android") if (resourceId > 0) { result = resources.getDimensionPixelSize(resourceId) } return result } private fun startScreenCapture(resultData: Intent) { Log.d(TAG, "Starting screen capture") try { val notification = NotificationCompat.Builder(this, CHANNEL_ID) .setContentTitle("Pokemon Analysis Active") .setContentText("Analyzing Pokemon Home screens...") .setSmallIcon(R.drawable.ic_launcher_foreground) .setOngoing(true) .addAction( R.drawable.ic_launcher_foreground, "Stop", PendingIntent.getService( this, 0, Intent(this, ScreenCaptureService::class.java).apply { action = ACTION_STOP }, PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE ) ) .build() Log.d(TAG, "Starting foreground service") startForeground(NOTIFICATION_ID, notification) Log.d(TAG, "Getting MediaProjection") mediaProjection = mediaProjectionManager?.getMediaProjection(Activity.RESULT_OK, resultData) if (mediaProjection == null) { Log.e(TAG, "Failed to get MediaProjection") stopSelf() return } Log.d(TAG, "Registering MediaProjection callback") mediaProjection?.registerCallback(mediaProjectionCallback, handler) Log.d(TAG, "Creating ImageReader: ${screenWidth}x${screenHeight}") imageReader = ImageReader.newInstance(screenWidth, screenHeight, PixelFormat.RGBA_8888, 3) // Increased buffer count imageReader?.setOnImageAvailableListener(onImageAvailableListener, handler) Log.d(TAG, "Creating VirtualDisplay") virtualDisplay = mediaProjection?.createVirtualDisplay( "ScreenCapture", screenWidth, screenHeight, screenDensity, DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, imageReader?.surface, null, null ) if (virtualDisplay == null) { Log.e(TAG, "Failed to create VirtualDisplay") stopSelf() return } Log.d(TAG, "Screen capture setup complete") // Show floating overlay enhancedFloatingFAB?.show() } catch (e: Exception) { Log.e(TAG, "Error starting screen capture", e) stopSelf() } } private fun stopScreenCapture() { Log.d(TAG, "Stopping screen capture") handler.removeCallbacks(captureRunnable) hideDetectionOverlay() enhancedFloatingFAB?.hide() latestImage?.close() latestImage = null virtualDisplay?.release() imageReader?.close() mediaProjection?.unregisterCallback(mediaProjectionCallback) mediaProjection?.stop() virtualDisplay = null imageReader = null mediaProjection = null stopForeground(true) stopSelf() } private val onImageAvailableListener = ImageReader.OnImageAvailableListener { reader -> try { val image = reader.acquireLatestImage() if (image != null) { if (autoProcessing) { processImage(image) image.close() } else { // Store the latest image for manual processing latestImage?.close() // Release previous image latestImage = image // Don't close the image yet - it will be closed in triggerManualDetection } } } catch (e: Exception) { Log.e(TAG, "Error in onImageAvailableListener", e) } } private var latestImage: Image? = null private fun captureScreen() { // Trigger image capture by reading from the ImageReader // The onImageAvailableListener will handle the actual processing Log.d(TAG, "Triggering screen capture...") } private fun processImage(image: Image) { try { val planes = image.planes val buffer = planes[0].buffer val pixelStride = planes[0].pixelStride val rowStride = planes[0].rowStride val rowPadding = rowStride - pixelStride * screenWidth Log.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: pixelStride=$pixelStride, rowStride=$rowStride, rowPadding=$rowPadding") Log.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: screenSize=${screenWidth}x${screenHeight}, expected bitmap=${screenWidth + rowPadding / pixelStride}x${screenHeight}") // Create bitmap from image val bitmap = Bitmap.createBitmap( screenWidth + rowPadding / pixelStride, screenHeight, Bitmap.Config.ARGB_8888 ) bitmap.copyPixelsFromBuffer(buffer) Log.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: created bitmap=${bitmap.width}x${bitmap.height}") // Convert to cropped bitmap if needed val croppedBitmap = if (rowPadding == 0) { Log.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: No padding, using original bitmap") bitmap } else { Log.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: Cropping bitmap from ${bitmap.width}x${bitmap.height} to ${screenWidth}x${screenHeight}") Bitmap.createBitmap(bitmap, 0, 0, screenWidth, screenHeight) } Log.d(TAG, "๐Ÿ–ผ๏ธ CAPTURE DEBUG: final bitmap=${croppedBitmap.width}x${croppedBitmap.height}") // Convert to OpenCV Mat for analysis val mat = Mat() Utils.bitmapToMat(croppedBitmap, mat) // DEBUG: Check color conversion Log.d(TAG, "๐ŸŽจ COLOR DEBUG: Mat type=${mat.type()}, channels=${mat.channels()}") Log.d(TAG, "๐ŸŽจ COLOR DEBUG: OpenCV expects BGR, Android Bitmap is ARGB") // Sample a center pixel to check color values if (mat.rows() > 0 && mat.cols() > 0) { val centerY = mat.rows() / 2 val centerX = mat.cols() / 2 val pixel = mat.get(centerY, centerX) if (pixel != null && pixel.size >= 3) { val b = pixel[0].toInt() val g = pixel[1].toInt() val r = pixel[2].toInt() Log.d(TAG, "๐ŸŽจ COLOR DEBUG: Center pixel (${centerX},${centerY}) BGR=($b,$g,$r) -> RGB=(${r},${g},${b})") Log.d(TAG, "๐ŸŽจ COLOR DEBUG: Center pixel hex = #${String.format("%02x%02x%02x", r, g, b)}") } } // Run YOLO analysis analyzePokemonScreen(mat) // Clean up if (croppedBitmap != bitmap) { croppedBitmap.recycle() } bitmap.recycle() } catch (e: Exception) { Log.e(TAG, "Error processing image", e) } } private fun analyzePokemonScreen(mat: Mat) { Log.i(TAG, "๐Ÿ“ฑ ANALYZING SCREEN: ${mat.cols()}x${mat.rows()}") // Check if analysis has been stuck for too long (30 seconds max) val currentTime = System.currentTimeMillis() if (isAnalyzing && (currentTime - analysisStartTime) > 30000) { Log.w(TAG, "โš ๏ธ Analysis stuck for >30s, resetting flag") isAnalyzing = false } // Skip if already analyzing if (isAnalyzing) { Log.d(TAG, "โญ๏ธ Skipping analysis - previous cycle still in progress") return } isAnalyzing = true analysisStartTime = currentTime Log.d(TAG, "๐Ÿ”„ Starting new analysis cycle") try { // Run YOLO detection first val detections = yoloDetector?.detect(mat) ?: emptyList() if (detections.isEmpty()) { Log.i(TAG, "๐Ÿ” No Pokemon UI elements detected by ONNX YOLO") isAnalyzing = false return } Log.i(TAG, "๐ŸŽฏ ONNX YOLO detected ${detections.size} UI elements") // Log ALL detections for debugging detections.forEachIndexed { index, detection -> Log.i(TAG, " $index: ${detection.className} (${String.format("%.3f", detection.confidence)}) at [${detection.boundingBox.x}, ${detection.boundingBox.y}, ${detection.boundingBox.width}, ${detection.boundingBox.height}]") } // Show breakdown by type val detectionCounts = detections.groupBy { it.className }.mapValues { it.value.size } Log.i(TAG, "๐Ÿ” Detection counts by type: $detectionCounts") // Check for commonly missing elements val expectedElements = listOf("pokemon_level", "attack_value", "sp_def_value", "shiny_icon", "ball_icon_pokeball", "ball_icon_greatball", "ball_icon_ultraball", "ball_icon_masterball") val missingElements = expectedElements.filter { expected -> detections.none { it.className.startsWith(expected.split("_").take(2).joinToString("_")) } } if (missingElements.isNotEmpty()) { Log.w(TAG, "โš ๏ธ Missing expected elements: $missingElements") } // Show detection overlay IMMEDIATELY (no OCR blocking) showYOLODetectionOverlay(detections) Log.i(TAG, "๐Ÿ“บ Overlay displayed with ${detections.size} detections") // Extract Pokemon info using YOLO detections in background extractPokemonInfoFromYOLOAsync(mat, detections) } catch (e: Exception) { Log.e(TAG, "Error analyzing Pokemon screen", e) isAnalyzing = false } } private fun extractPokemonInfoFromYOLOAsync(mat: Mat, detections: List) { // Create a copy of the Mat for background processing val matCopy = Mat() mat.copyTo(matCopy) // Process in background thread ocrExecutor.submit { try { val pokemonInfo = extractPokemonInfoFromYOLO(matCopy, detections) // Post results back to main thread handler.post { try { if (pokemonInfo != null) { Log.i(TAG, "๐Ÿ”ฅ POKEMON DATA EXTRACTED SUCCESSFULLY!") logPokemonInfo(pokemonInfo) // TODO: Send to your API // sendToAPI(pokemonInfo) } else { Log.i(TAG, "โŒ Could not extract complete Pokemon info") } } finally { // Analysis cycle complete, allow next one isAnalyzing = false val duration = System.currentTimeMillis() - analysisStartTime Log.d(TAG, "โœ… Analysis cycle complete after ${duration}ms - ready for next") } } // Clean up matCopy.release() } catch (e: Exception) { Log.e(TAG, "Error in async Pokemon extraction", e) matCopy.release() // Clear flag on error too handler.post { isAnalyzing = false Log.d(TAG, "โŒ Analysis cycle failed - ready for next") } } } } private fun extractPokemonInfoFromYOLO(mat: Mat, detections: List): PokemonInfo? { try { Log.i(TAG, "๐ŸŽฏ Extracting Pokemon info from ${detections.size} YOLO detections") // Group detections by type val detectionMap = detections.groupBy { it.className } // Extract key information using YOLO bounding boxes (async OCR calls) val ocrResults = mutableMapOf() val latch = CountDownLatch(7) // Wait for 7 OCR operations // Submit all OCR tasks in parallel submitOCRTask("nickname", mat, detectionMap["pokemon_nickname"]?.firstOrNull(), ocrResults, latch) submitOCRTask("species", mat, detectionMap["pokemon_species"]?.firstOrNull(), ocrResults, latch) submitOCRTask("nature", mat, detectionMap["nature_name"]?.firstOrNull(), ocrResults, latch) submitOCRTask("ability", mat, detectionMap["ability_name"]?.firstOrNull(), ocrResults, latch) submitOCRTask("otName", mat, detectionMap["original_trainer_name"]?.firstOrNull(), ocrResults, latch) submitOCRTask("otId", mat, detectionMap["original_trainder_number"]?.firstOrNull(), ocrResults, latch) // For level, prioritize wider bounding boxes (more likely to contain full level text) val levelDetection = detectionMap["pokemon_level"]?.maxByOrNull { it.boundingBox.width } submitLevelOCRTask("level", mat, levelDetection, ocrResults, latch) // Wait for all OCR tasks to complete (max 10 seconds total) val completed = latch.await(10, TimeUnit.SECONDS) if (!completed) { Log.w(TAG, "โฑ๏ธ Some OCR tasks timed out after 10 seconds") } // Extract results val nickname = ocrResults["nickname"] val level = ocrResults["level"]?.toIntOrNull() val species = ocrResults["species"] val nature = ocrResults["nature"] val ability = ocrResults["ability"] val otName = ocrResults["otName"] val otId = ocrResults["otId"] // Extract stats (multiple detections) val stats = extractStatsFromDetections(mat, detectionMap) // Extract moves (multiple detections) val moves = extractMovesFromDetections(mat, detectionMap) // Detect gender val gender = when { detectionMap["gender_icon_male"]?.isNotEmpty() == true -> "Male" detectionMap["gender_icon_female"]?.isNotEmpty() == true -> "Female" else -> null } // Detect pokeball type val pokeballType = detectPokeballTypeFromDetections(detectionMap) // Detect types val types = extractTypesFromDetections(mat, detectionMap) // Detect shiny status val isShiny = detectionMap["shiny_icon"]?.isNotEmpty() == true // Detect tera type val teraType = detectTeraTypeFromDetections(detectionMap) Log.i(TAG, "๐Ÿ“Š YOLO extraction summary:") Log.i(TAG, " Nickname: '${nickname ?: "null"}'") Log.i(TAG, " Level: ${level ?: "null"}") Log.i(TAG, " Species: '${species ?: "null"}'") Log.i(TAG, " Nature: '${nature ?: "null"}'") Log.i(TAG, " Ability: '${ability ?: "null"}'") Log.i(TAG, " Gender: '${gender ?: "null"}'") Log.i(TAG, " Pokeball: '${pokeballType ?: "null"}'") Log.i(TAG, " Types: ${types}") Log.i(TAG, " Tera: '${teraType ?: "null"}'") Log.i(TAG, " Shiny: $isShiny") if (nickname.isNullOrBlank() && species.isNullOrBlank() && level == null) { Log.w(TAG, "โš ๏ธ No essential Pokemon data found with YOLO detection") return null } return PokemonInfo( pokeballType = pokeballType, nickname = nickname, gender = gender, level = level, language = detectLanguage(nickname, species), gameSource = detectGameSourceFromDetections(detectionMap), isFavorited = false, // TODO: implement favorite detection nationalDexNumber = extractDexNumber(species), species = species, primaryType = types.getOrNull(0), secondaryType = types.getOrNull(1), stamps = emptyList(), // TODO: implement stamp detection labels = emptyList(), // TODO: implement label detection marks = emptyList(), // TODO: implement mark detection stats = stats, moves = moves, ability = ability, nature = nature, originalTrainerName = otName, originalTrainerId = otId, extractionConfidence = calculateYOLOExtractionConfidence(detections, nickname, level, species) ) } catch (e: Exception) { Log.e(TAG, "Error extracting Pokemon info from YOLO detections", e) return null } } private fun submitOCRTask(key: String, mat: Mat, detection: Detection?, results: MutableMap, latch: CountDownLatch) { ocrExecutor.submit { try { val text = extractTextFromDetection(mat, detection) synchronized(results) { results[key] = text } } catch (e: Exception) { Log.e(TAG, "Error in OCR task for $key", e) synchronized(results) { results[key] = null } } finally { latch.countDown() } } } private fun submitLevelOCRTask(key: String, mat: Mat, detection: Detection?, results: MutableMap, latch: CountDownLatch) { ocrExecutor.submit { try { val levelText = extractTextFromDetection(mat, detection) val level = levelText?.replace("[^0-9]".toRegex(), "") synchronized(results) { results[key] = level } } catch (e: Exception) { Log.e(TAG, "Error in level OCR task", e) synchronized(results) { results[key] = null } } finally { latch.countDown() } } } private fun extractTextFromDetection(mat: Mat, detection: Detection?): String? { if (detection == null) return null try { // Expand bounding box by 5% for all OCR classes to improve text extraction accuracy val bbox = detection.boundingBox val expansionFactor = 0.05f // 5% expansion val widthExpansion = (bbox.width * expansionFactor).toInt() val heightExpansion = (bbox.height * expansionFactor).toInt() val expandedBbox = Rect( bbox.x - widthExpansion, bbox.y - heightExpansion, bbox.width + (2 * widthExpansion), bbox.height + (2 * heightExpansion) ) // Validate and clip bounding box to image boundaries val clippedX = kotlin.math.max(0, kotlin.math.min(expandedBbox.x, mat.cols() - 1)) val clippedY = kotlin.math.max(0, kotlin.math.min(expandedBbox.y, mat.rows() - 1)) val clippedWidth = kotlin.math.max(1, kotlin.math.min(expandedBbox.width, mat.cols() - clippedX)) val clippedHeight = kotlin.math.max(1, kotlin.math.min(expandedBbox.height, mat.rows() - clippedY)) val safeBbox = Rect(clippedX, clippedY, clippedWidth, clippedHeight) // Debug logging for bounding box transformations if (expandedBbox != bbox) { Log.d(TAG, "๐Ÿ“ Expanded bbox for ${detection.className}: [${bbox.x},${bbox.y},${bbox.width},${bbox.height}] โ†’ [${expandedBbox.x},${expandedBbox.y},${expandedBbox.width},${expandedBbox.height}]") } if (safeBbox.x != expandedBbox.x || safeBbox.y != expandedBbox.y || safeBbox.width != expandedBbox.width || safeBbox.height != expandedBbox.height) { Log.w(TAG, "โš ๏ธ Clipped bbox for ${detection.className}: expanded=[${expandedBbox.x},${expandedBbox.y},${expandedBbox.width},${expandedBbox.height}] โ†’ safe=[${safeBbox.x},${safeBbox.y},${safeBbox.width},${safeBbox.height}] (image: ${mat.cols()}x${mat.rows()})") } // Extract region of interest using safe bounding box val roi = Mat(mat, safeBbox) // Preprocess image for better OCR val processedRoi = preprocessImageForOCR(roi) // Convert to bitmap for ML Kit val bitmap = Bitmap.createBitmap(processedRoi.cols(), processedRoi.rows(), Bitmap.Config.ARGB_8888) Utils.matToBitmap(processedRoi, bitmap) // Use ML Kit OCR val extractedText = performOCR(bitmap, detection.className) // Clean up roi.release() processedRoi.release() bitmap.recycle() if (extractedText != null) { Log.i(TAG, "โœ… YOLO SUCCESS: ${detection.className} = '$extractedText' (conf: ${String.format("%.2f", detection.confidence)})") } else { Log.w(TAG, "โŒ YOLO FAILED: ${detection.className} - no text found (conf: ${String.format("%.2f", detection.confidence)})") } return extractedText } catch (e: Exception) { Log.e(TAG, "Error extracting text from YOLO detection ${detection.className}", e) return null } } private fun extractLevelFromDetection(mat: Mat, detection: Detection?): Int? { val levelText = extractTextFromDetection(mat, detection) return levelText?.replace("[^0-9]".toRegex(), "")?.toIntOrNull() } private fun extractStatsFromDetections(mat: Mat, detectionMap: Map>): PokemonStats? { val hp = extractTextFromDetection(mat, detectionMap["hp_value"]?.firstOrNull())?.toIntOrNull() val attack = extractTextFromDetection(mat, detectionMap["attack_value"]?.firstOrNull())?.toIntOrNull() val defense = extractTextFromDetection(mat, detectionMap["defense_value"]?.firstOrNull())?.toIntOrNull() val spAttack = extractTextFromDetection(mat, detectionMap["sp_atk_value"]?.firstOrNull())?.toIntOrNull() val spDefense = extractTextFromDetection(mat, detectionMap["sp_def_value"]?.firstOrNull())?.toIntOrNull() val speed = extractTextFromDetection(mat, detectionMap["speed_value"]?.firstOrNull())?.toIntOrNull() return if (hp != null || attack != null || defense != null || spAttack != null || spDefense != null || speed != null) { PokemonStats(hp, attack, defense, spAttack, spDefense, speed) } else null } private fun extractMovesFromDetections(mat: Mat, detectionMap: Map>): List { val moves = mutableListOf() detectionMap["move_name"]?.forEach { detection -> val moveText = extractTextFromDetection(mat, detection) if (!moveText.isNullOrBlank()) { moves.add(moveText.trim()) } } return moves.take(4) // Pokemon can have max 4 moves } private fun detectPokeballTypeFromDetections(detectionMap: Map>): String? { // Check for specific pokeball types detected by YOLO val pokeballTypes = mapOf( "ball_icon_pokeball" to "Pokรฉ Ball", "ball_icon_greatball" to "Great Ball", "ball_icon_ultraball" to "Ultra Ball", "ball_icon_heavyball" to "Heavy Ball", "ball_icon_premierball" to "Premier Ball", "ball_icon_repeatball" to "Repeat Ball", "ball_icon_timerball" to "Timer Ball", "ball_icon_diveball" to "Dive Ball", "ball_icon_quickball" to "Quick Ball", "ball_icon_duskball" to "Dusk Ball", "ball_icon_cherishball" to "Cherish Ball", "ball_icon_originball" to "Origin Ball", "ball_icon_pokeball_hisui" to "Hisuian Pokรฉ Ball", "ball_icon_ultraball_husui" to "Hisuian Ultra Ball" ) for ((className, ballName) in pokeballTypes) { if (detectionMap[className]?.isNotEmpty() == true) { return ballName } } return null } private fun extractTypesFromDetections(mat: Mat, detectionMap: Map>): List { val types = mutableListOf() extractTextFromDetection(mat, detectionMap["type_1"]?.firstOrNull())?.let { type1 -> if (type1.isNotBlank()) types.add(type1.trim()) } extractTextFromDetection(mat, detectionMap["type_2"]?.firstOrNull())?.let { type2 -> if (type2.isNotBlank()) types.add(type2.trim()) } return types } private fun detectTeraTypeFromDetections(detectionMap: Map>): String? { val teraTypes = mapOf( "tera_ice" to "Ice", "tera_fairy" to "Fairy", "tera_poison" to "Poison", "tera_ghost" to "Ghost", "tera_steel" to "Steel", "tera_grass" to "Grass", "tera_normal" to "Normal", "tera_fire" to "Fire", "tera_electric" to "Electric", "tera_ground" to "Ground", "tera_flying" to "Flying", "tera_bug" to "Bug", "tera_dark" to "Dark", "tera_water" to "Water", "tera_psychic" to "Psychic", "tera_dragon" to "Dragon", "tera_fighting" to "Fighting", "tera_rock" to "Rock" ) for ((className, teraName) in teraTypes) { if (detectionMap[className]?.isNotEmpty() == true) { return teraName } } return null } private fun detectGameSourceFromDetections(detectionMap: Map>): String? { val gameSources = mapOf( "last_game_stamp_sh" to "Sword/Shield", "last_game_stamp_bank" to "Bank", "last_game_stamp_pla" to "Legends: Arceus", "last_game_stamp_sc" to "Scarlet/Violet", "last_game_stamp_vi" to "Violet", "last_game_stamp_go" to "Pokรฉmon GO", "origin_icon_vc" to "Virtual Console", "origin_icon_xyoras" to "XY/ORAS", "origin_icon_smusum" to "SM/USUM", "origin_icon_swsh" to "Sword/Shield", "origin_icon_go" to "Pokรฉmon GO", "origin_icon_pla" to "Legends: Arceus", "origin_icon_sv" to "Scarlet/Violet" ) for ((className, sourceName) in gameSources) { if (detectionMap[className]?.isNotEmpty() == true) { return sourceName } } return null } private fun calculateYOLOExtractionConfidence(detections: List, nickname: String?, level: Int?, species: String?): Double { var confidence = 0.0 // Base confidence from YOLO detections val avgDetectionConfidence = detections.map { it.confidence.toDouble() }.average() confidence += avgDetectionConfidence * 0.4 // Boost confidence based on extracted data if (!nickname.isNullOrBlank()) confidence += 0.2 if (level != null && level > 0) confidence += 0.2 if (!species.isNullOrBlank()) confidence += 0.2 return confidence.coerceIn(0.0, 1.0) } private fun showYOLODetectionOverlay(detections: List) { try { Log.i(TAG, "๐ŸŽจ Creating YOLO detection overlay for ${detections.size} detections") if (detectionOverlay == null) { Log.i(TAG, "๐Ÿ†• Creating new DetectionOverlay instance") detectionOverlay = DetectionOverlay(this) } // Convert YOLO detections to screen regions for overlay val statusBarHeight = getStatusBarHeight() val regions = detections.mapIndexed { index, detection -> "${detection.className}_$index" to ScreenRegion( x = detection.boundingBox.x, y = detection.boundingBox.y - statusBarHeight, // Subtract status bar offset width = detection.boundingBox.width, height = detection.boundingBox.height, purpose = "${detection.className} (${String.format("%.2f", detection.confidence)})" ) }.toMap() Log.i(TAG, "๐Ÿ“บ Showing YOLO overlay with ${regions.size} regions...") detectionOverlay?.showOverlay(regions) Log.i(TAG, "โœ… YOLO overlay show command sent") } catch (e: Exception) { Log.e(TAG, "โŒ Error showing YOLO detection overlay", e) } } private fun hideDetectionOverlay() { detectionOverlay?.hideOverlay() } private fun performOCR(bitmap: Bitmap, purpose: String): String? { try { Log.d(TAG, "๐Ÿ” Starting OCR for $purpose - bitmap: ${bitmap.width}x${bitmap.height}") // Create InputImage for ML Kit val image = InputImage.fromBitmap(bitmap, 0) val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS) // Use CountDownLatch to make async call synchronous val latch = CountDownLatch(1) var result: String? = null var ocrError: Exception? = null recognizer.process(image) .addOnSuccessListener { visionText -> result = visionText.text.trim() Log.d(TAG, "๐Ÿ” Raw OCR result for $purpose: '${visionText.text}' (blocks: ${visionText.textBlocks.size})") if (result.isNullOrBlank()) { Log.w(TAG, "โš ๏ธ OCR for $purpose: NO TEXT DETECTED - ML Kit found ${visionText.textBlocks.size} text blocks but text is empty") } else { Log.i(TAG, "โœ… OCR SUCCESS for $purpose: '${result}' (${result!!.length} chars)") } latch.countDown() } .addOnFailureListener { e -> ocrError = e Log.e(TAG, "โŒ OCR failed for $purpose: ${e.message}", e) latch.countDown() } // Wait for OCR to complete (max 5 seconds to allow ML Kit to work) val completed = latch.await(5, TimeUnit.SECONDS) if (!completed) { Log.e(TAG, "โฑ๏ธ OCR timeout for $purpose after 5 seconds") return null } if (ocrError != null) { Log.e(TAG, "โŒ OCR error for $purpose: ${ocrError!!.message}") return null } // Clean and process the result val cleanedResult = cleanOCRResult(result, purpose) Log.d(TAG, "๐Ÿง™ Cleaned result for $purpose: '${cleanedResult}'") return cleanedResult } catch (e: Exception) { Log.e(TAG, "โŒ Error in OCR for $purpose", e) return null } } private fun cleanOCRResult(rawText: String?, purpose: String): String? { if (rawText.isNullOrBlank()) return null val cleaned = rawText.trim() return when (purpose) { "pokemon_level" -> { // Extract level number (look for "Lv." or just numbers) val levelRegex = """(?:Lv\.?\s*)?([0-9]+)""".toRegex(RegexOption.IGNORE_CASE) levelRegex.find(cleaned)?.groupValues?.get(1) } "pokemon_nickname", "pokemon_species" -> { // Clean up common OCR mistakes for Pokemon names cleaned.replace("[^a-zA-Z0-9 \\-.'โ™‚โ™€]".toRegex(), "") .replace("\\s+".toRegex(), " ") .trim() .takeIf { it.isNotEmpty() } } "nature_name", "ability_name" -> { // Clean up nature/ability names cleaned.replace("[^a-zA-Z ]".toRegex(), "") .replace("\\s+".toRegex(), " ") .trim() .takeIf { it.isNotEmpty() } } "original_trainer_name" -> { // Clean trainer names cleaned.replace("[^a-zA-Z0-9 ]".toRegex(), "") .replace("\\s+".toRegex(), " ") .trim() .takeIf { it.isNotEmpty() } } "original_trainder_number" -> { // Extract ID numbers cleaned.replace("[^0-9]".toRegex(), "") .takeIf { it.isNotEmpty() } } else -> cleaned.takeIf { it.isNotEmpty() } } } private fun preprocessImageForOCR(roi: Mat): Mat { try { // Resize small regions for better OCR (minimum 150x50 pixels) val minWidth = 150 val minHeight = 50 val scaledRoi = if (roi.width() < minWidth || roi.height() < minHeight) { val scaleX = maxOf(1.0, minWidth.toDouble() / roi.width()) val scaleY = maxOf(1.0, minHeight.toDouble() / roi.height()) val scale = maxOf(scaleX, scaleY) val resized = Mat() Imgproc.resize(roi, resized, Size(roi.width() * scale, roi.height() * scale)) Log.d(TAG, "๐Ÿ” Upscaled OCR region from ${roi.width()}x${roi.height()} to ${resized.width()}x${resized.height()}") resized } else { val copy = Mat() roi.copyTo(copy) copy } // Convert to grayscale if needed val gray = Mat() if (scaledRoi.channels() > 1) { Imgproc.cvtColor(scaledRoi, gray, Imgproc.COLOR_RGBA2GRAY) } else { scaledRoi.copyTo(gray) } // Enhance contrast with CLAHE (Contrast Limited Adaptive Histogram Equalization) val clahe = Imgproc.createCLAHE(2.0, Size(8.0, 8.0)) val enhanced = Mat() clahe.apply(gray, enhanced) // Light denoising only if text is very small val denoised = if (scaledRoi.width() < 100 || scaledRoi.height() < 30) { val temp = Mat() Imgproc.GaussianBlur(enhanced, temp, Size(1.0, 1.0), 0.0) temp } else { enhanced.clone() } // Convert back to RGBA for ML Kit val result = Mat() Imgproc.cvtColor(denoised, result, Imgproc.COLOR_GRAY2RGBA) // Clean up intermediate matrices scaledRoi.release() gray.release() enhanced.release() if (denoised !== enhanced) denoised.release() return result } catch (e: Exception) { Log.e(TAG, "Error preprocessing image, using original", e) // Return copy of original if preprocessing fails val result = Mat() roi.copyTo(result) return result } } private fun detectLanguage(nickname: String?, species: String?): String? { // Simple language detection based on character patterns val text = "$nickname $species" return when { text.any { it in '\u3040'..'\u309F' || it in '\u30A0'..'\u30FF' } -> "JP" // Hiragana/Katakana text.any { it in '\u4E00'..'\u9FAF' } -> "ZH" // Chinese characters text.any { it in '\uAC00'..'\uD7AF' } -> "KO" // Korean else -> "EN" // Default to English } } private fun extractDexNumber(species: String?): Int? { // For now, return null - would need a Pokemon species to dex number mapping return null } private fun logPokemonInfo(pokemonInfo: PokemonInfo) { Log.i(TAG, "====== POKEMON INFO EXTRACTED ======") Log.i(TAG, "๐ŸŽพ Pokeball: ${pokemonInfo.pokeballType}") Log.i(TAG, "๐Ÿ“› Nickname: ${pokemonInfo.nickname}") Log.i(TAG, "โšค Gender: ${pokemonInfo.gender}") Log.i(TAG, "๐Ÿ“Š Level: ${pokemonInfo.level}") Log.i(TAG, "๐ŸŒ Language: ${pokemonInfo.language}") Log.i(TAG, "๐ŸŽฎ Game Source: ${pokemonInfo.gameSource}") Log.i(TAG, "โญ Favorited: ${pokemonInfo.isFavorited}") Log.i(TAG, "๐Ÿ”ข Dex #: ${pokemonInfo.nationalDexNumber}") Log.i(TAG, "๐Ÿพ Species: ${pokemonInfo.species}") Log.i(TAG, "๐Ÿท๏ธ Type 1: ${pokemonInfo.primaryType}") Log.i(TAG, "๐Ÿท๏ธ Type 2: ${pokemonInfo.secondaryType}") Log.i(TAG, "๐Ÿ† Stamps: ${pokemonInfo.stamps}") Log.i(TAG, "๐Ÿท๏ธ Labels: ${pokemonInfo.labels}") Log.i(TAG, "โœ… Marks: ${pokemonInfo.marks}") if (pokemonInfo.stats != null) { Log.i(TAG, "๐Ÿ“ˆ Stats: HP:${pokemonInfo.stats.hp} ATK:${pokemonInfo.stats.attack} DEF:${pokemonInfo.stats.defense}") Log.i(TAG, " SP.ATK:${pokemonInfo.stats.spAttack} SP.DEF:${pokemonInfo.stats.spDefense} SPD:${pokemonInfo.stats.speed}") } Log.i(TAG, "โš”๏ธ Moves: ${pokemonInfo.moves}") Log.i(TAG, "๐Ÿ’ช Ability: ${pokemonInfo.ability}") Log.i(TAG, "๐ŸŽญ Nature: ${pokemonInfo.nature}") Log.i(TAG, "๐Ÿ‘ค OT: ${pokemonInfo.originalTrainerName}") Log.i(TAG, "๐Ÿ”ข ID: ${pokemonInfo.originalTrainerId}") Log.i(TAG, "๐ŸŽฏ Confidence: ${String.format("%.2f", pokemonInfo.extractionConfidence)}") Log.i(TAG, "====================================") } private fun convertImageToMat(image: Image): Mat? { return try { val planes = image.planes val buffer = planes[0].buffer val pixelStride = planes[0].pixelStride val rowStride = planes[0].rowStride val rowPadding = rowStride - pixelStride * screenWidth // Create bitmap from image val bitmap = Bitmap.createBitmap( screenWidth + rowPadding / pixelStride, screenHeight, Bitmap.Config.ARGB_8888 ) bitmap.copyPixelsFromBuffer(buffer) // Crop bitmap to remove padding if needed val croppedBitmap = if (rowPadding == 0) { bitmap } else { val cropped = Bitmap.createBitmap(bitmap, 0, 0, screenWidth, screenHeight) bitmap.recycle() // Clean up original cropped } // Convert bitmap to Mat val mat = Mat() Utils.bitmapToMat(croppedBitmap, mat) // Convert from RGBA to BGR (OpenCV format for proper color channel handling) val bgrMat = Mat() Imgproc.cvtColor(mat, bgrMat, Imgproc.COLOR_RGBA2BGR) // Clean up mat.release() croppedBitmap.recycle() bgrMat } catch (e: Exception) { Log.e(TAG, "โŒ Error converting image to Mat", e) null } } private fun triggerManualDetection() { Log.d(TAG, "๐Ÿ” Manual detection triggered via MVC!") latestImage?.let { image -> try { // Convert image to Mat for processing val mat = convertImageToMat(image) if (mat != null) { // Use controller to process detection (this will notify UI via callbacks) val detections = detectionController.processDetection(mat) // Show detection overlay with results if (detections.isNotEmpty()) { showYOLODetectionOverlay(detections) // Extract Pokemon info using YOLO detections with OCR extractPokemonInfoFromYOLOAsync(mat, detections) } mat.release() } else { Log.e(TAG, "โŒ Failed to convert image to Mat") } // Close the image after processing to free the buffer image.close() latestImage = null } catch (e: Exception) { Log.e(TAG, "โŒ Error in manual detection", e) } } ?: run { Log.w(TAG, "โš ๏ธ No image available for detection") } } override fun onDestroy() { super.onDestroy() hideDetectionOverlay() enhancedFloatingFAB?.hide() detectionController.clearUICallbacks() yoloDetector?.release() ocrExecutor.shutdown() stopScreenCapture() } }