You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1199 lines
48 KiB
1199 lines
48 KiB
package com.quillstudios.pokegoalshelper
|
|
|
|
import android.app.*
|
|
import android.content.Context
|
|
import android.content.Intent
|
|
import android.os.Binder
|
|
import android.graphics.Bitmap
|
|
import android.graphics.ImageFormat
|
|
import android.graphics.PixelFormat
|
|
import android.hardware.display.DisplayManager
|
|
import android.hardware.display.VirtualDisplay
|
|
import android.media.Image
|
|
import android.media.ImageReader
|
|
import android.media.projection.MediaProjection
|
|
import android.media.projection.MediaProjectionManager
|
|
import android.os.*
|
|
import android.util.DisplayMetrics
|
|
import android.util.Log
|
|
import com.quillstudios.pokegoalshelper.utils.PGHLog
|
|
import com.quillstudios.pokegoalshelper.ml.MLErrorType
|
|
import android.view.WindowManager
|
|
import android.view.View
|
|
import android.view.Gravity
|
|
import android.widget.Button
|
|
import android.widget.LinearLayout
|
|
import androidx.core.app.NotificationCompat
|
|
import com.quillstudios.pokegoalshelper.controllers.DetectionController
|
|
import com.quillstudios.pokegoalshelper.ui.EnhancedFloatingFAB
|
|
import com.quillstudios.pokegoalshelper.capture.ScreenCaptureManager
|
|
import com.quillstudios.pokegoalshelper.capture.ScreenCaptureManagerImpl
|
|
import com.quillstudios.pokegoalshelper.ml.MLInferenceEngine
|
|
import com.quillstudios.pokegoalshelper.ml.YOLOInferenceEngine
|
|
import com.quillstudios.pokegoalshelper.ml.Detection as MLDetection
|
|
import com.quillstudios.pokegoalshelper.data.PokemonDataExtractor
|
|
import com.quillstudios.pokegoalshelper.data.PokemonDataExtractorImpl
|
|
import kotlinx.coroutines.runBlocking
|
|
import org.opencv.android.Utils
|
|
import org.opencv.core.*
|
|
import org.opencv.imgproc.Imgproc
|
|
import org.opencv.imgcodecs.Imgcodecs
|
|
import com.quillstudios.pokegoalshelper.utils.MatUtils.use
|
|
import com.google.mlkit.vision.common.InputImage
|
|
import com.google.mlkit.vision.text.TextRecognition
|
|
import com.google.mlkit.vision.text.latin.TextRecognizerOptions
|
|
import java.util.concurrent.CountDownLatch
|
|
import java.io.File
|
|
import java.util.concurrent.TimeUnit
|
|
import java.util.concurrent.Executors
|
|
import java.util.concurrent.ThreadPoolExecutor
|
|
import java.nio.ByteBuffer
|
|
|
|
data class PokemonInfo(
|
|
val pokeballType: String?,
|
|
val nickname: String?,
|
|
val gender: String?,
|
|
val level: Int?,
|
|
val language: String?,
|
|
val gameSource: String?,
|
|
val isFavorited: Boolean,
|
|
val nationalDexNumber: Int?,
|
|
val species: String?,
|
|
val primaryType: String?,
|
|
val secondaryType: String?,
|
|
val teraType: String?,
|
|
val isShiny: Boolean,
|
|
val stamps: List<String>,
|
|
val labels: List<String>,
|
|
val marks: List<String>,
|
|
val stats: PokemonStats?,
|
|
val moves: List<String>,
|
|
val ability: String?,
|
|
val nature: String?,
|
|
val originalTrainerName: String?,
|
|
val originalTrainerId: String?,
|
|
val extractionConfidence: Double
|
|
)
|
|
|
|
data class PokemonStats(
|
|
val hp: Int?,
|
|
val attack: Int?,
|
|
val defense: Int?,
|
|
val spAttack: Int?,
|
|
val spDefense: Int?,
|
|
val speed: Int?
|
|
)
|
|
|
|
data class ScreenRegion(
|
|
val x: Int,
|
|
val y: Int,
|
|
val width: Int,
|
|
val height: Int,
|
|
val purpose: String
|
|
)
|
|
|
|
class ScreenCaptureService : Service() {
|
|
|
|
companion object {
|
|
private const val TAG = "ScreenCaptureService"
|
|
private const val NOTIFICATION_ID = 1001
|
|
private const val CHANNEL_ID = "screen_capture_channel"
|
|
|
|
// Timeout Constants
|
|
private const val ANALYSIS_STUCK_TIMEOUT_MS = 30000L // 30 seconds
|
|
private const val OCR_TASK_TIMEOUT_SECONDS = 10L
|
|
private const val DEFAULT_CAPTURE_INTERVAL_MS = 2000L // 2 seconds
|
|
private const val OCR_INDIVIDUAL_TIMEOUT_SECONDS = 5L
|
|
|
|
// UI Constants
|
|
private const val MIN_PIXEL_CHANNELS = 3
|
|
|
|
const val ACTION_START = "START_SCREEN_CAPTURE"
|
|
const val ACTION_STOP = "STOP_SCREEN_CAPTURE"
|
|
const val EXTRA_RESULT_DATA = "result_data"
|
|
}
|
|
|
|
/**
|
|
* Binder for external communication (if needed in future)
|
|
*/
|
|
inner class LocalBinder : Binder() {
|
|
fun getService(): ScreenCaptureService = this@ScreenCaptureService
|
|
}
|
|
|
|
private val binder = LocalBinder()
|
|
|
|
// ML inference engine
|
|
private var mlInferenceEngine: MLInferenceEngine? = null
|
|
|
|
// Pokemon data extractor
|
|
private var pokemonDataExtractor: PokemonDataExtractor? = null
|
|
|
|
private lateinit var screenCaptureManager: ScreenCaptureManager
|
|
private var detectionOverlay: DetectionOverlay? = null
|
|
|
|
// MVC Components
|
|
private lateinit var detectionController: DetectionController
|
|
private var enhancedFloatingFAB: EnhancedFloatingFAB? = null
|
|
|
|
private val handler = Handler(Looper.getMainLooper())
|
|
private var captureInterval = DEFAULT_CAPTURE_INTERVAL_MS
|
|
private var autoProcessing = false // Disable automatic processing
|
|
|
|
// Thread pool for OCR processing (4 threads for parallel text extraction)
|
|
private val ocrExecutor = Executors.newFixedThreadPool(4)
|
|
|
|
// Flag to prevent overlapping analysis cycles
|
|
private var isAnalyzing = false
|
|
private var analysisStartTime = 0L
|
|
|
|
private val captureRunnable = object : Runnable {
|
|
override fun run() {
|
|
captureScreen()
|
|
handler.postDelayed(this, captureInterval)
|
|
}
|
|
}
|
|
|
|
|
|
override fun onCreate() {
|
|
super.onCreate()
|
|
createNotificationChannel()
|
|
|
|
// Initialize screen capture manager
|
|
screenCaptureManager = ScreenCaptureManagerImpl(this, handler)
|
|
screenCaptureManager.setImageCallback { image -> handleCapturedImage(image) }
|
|
|
|
// Initialize ML inference engine
|
|
mlInferenceEngine = YOLOInferenceEngine(this)
|
|
Thread {
|
|
runBlocking {
|
|
mlInferenceEngine!!.initialize()
|
|
.onSuccess {
|
|
PGHLog.i(TAG, "✅ ML inference engine initialized for screen capture")
|
|
}
|
|
.onError { errorType, exception, message ->
|
|
PGHLog.e(TAG, "❌ Failed to initialize ML inference engine: $message (${errorType.name})", exception)
|
|
}
|
|
}
|
|
}.start()
|
|
|
|
// Initialize Pokemon data extractor
|
|
pokemonDataExtractor = PokemonDataExtractorImpl(this)
|
|
PGHLog.i(TAG, "✅ Pokemon data extractor initialized")
|
|
|
|
// Initialize MVC components
|
|
detectionController = DetectionController(mlInferenceEngine!!)
|
|
detectionController.setDetectionRequestCallback { triggerManualDetection() }
|
|
|
|
// Initialize enhanced floating FAB
|
|
enhancedFloatingFAB = EnhancedFloatingFAB(
|
|
context = this,
|
|
onDetectionRequested = { triggerDetection() },
|
|
onClassFilterRequested = { className -> setClassFilter(className) },
|
|
onDebugToggled = { toggleDebugMode() },
|
|
onClose = { stopSelf() }
|
|
)
|
|
|
|
PGHLog.d(TAG, "✅ MVC architecture initialized")
|
|
}
|
|
|
|
override fun onBind(intent: Intent?): IBinder = binder
|
|
|
|
// === Public API for external communication ===
|
|
|
|
/**
|
|
* Trigger manual detection from UI
|
|
*/
|
|
fun triggerDetection() {
|
|
triggerManualDetection()
|
|
}
|
|
|
|
/**
|
|
* Set class filter from UI
|
|
*/
|
|
fun setClassFilter(className: String?) {
|
|
detectionController.onClassFilterChanged(className)
|
|
}
|
|
|
|
/**
|
|
* Toggle debug mode from UI
|
|
*/
|
|
fun toggleDebugMode() {
|
|
detectionController.onDebugModeToggled()
|
|
}
|
|
|
|
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
|
when (intent?.action) {
|
|
ACTION_START -> {
|
|
val resultData = intent.getParcelableExtra<Intent>(EXTRA_RESULT_DATA)
|
|
if (resultData != null) {
|
|
startScreenCapture(resultData)
|
|
}
|
|
}
|
|
ACTION_STOP -> {
|
|
stopScreenCapture()
|
|
}
|
|
}
|
|
return START_STICKY
|
|
}
|
|
|
|
private fun createNotificationChannel() {
|
|
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
|
val channel = NotificationChannel(
|
|
CHANNEL_ID,
|
|
"Screen Capture Service",
|
|
NotificationManager.IMPORTANCE_LOW
|
|
).apply {
|
|
description = "Pokemon analysis screen capture"
|
|
setShowBadge(false)
|
|
}
|
|
|
|
val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
|
|
manager.createNotificationChannel(channel)
|
|
}
|
|
}
|
|
|
|
private fun getStatusBarHeight(): Int {
|
|
var result = 0
|
|
val resourceId = resources.getIdentifier("status_bar_height", "dimen", "android")
|
|
if (resourceId > 0) {
|
|
result = resources.getDimensionPixelSize(resourceId)
|
|
}
|
|
return result
|
|
}
|
|
|
|
private fun startScreenCapture(resultData: Intent) {
|
|
PGHLog.d(TAG, "Starting screen capture")
|
|
|
|
try {
|
|
val notification = NotificationCompat.Builder(this, CHANNEL_ID)
|
|
.setContentTitle("Pokemon Analysis Active")
|
|
.setContentText("Analyzing Pokemon Home screens...")
|
|
.setSmallIcon(R.drawable.ic_launcher_foreground)
|
|
.setOngoing(true)
|
|
.addAction(
|
|
R.drawable.ic_launcher_foreground,
|
|
"Stop",
|
|
PendingIntent.getService(
|
|
this,
|
|
0,
|
|
Intent(this, ScreenCaptureService::class.java).apply {
|
|
action = ACTION_STOP
|
|
},
|
|
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
|
|
)
|
|
)
|
|
.build()
|
|
|
|
PGHLog.d(TAG, "Starting foreground service")
|
|
startForeground(NOTIFICATION_ID, notification)
|
|
|
|
// Use the screen capture manager to start capture
|
|
if (!screenCaptureManager.startCapture(resultData)) {
|
|
PGHLog.e(TAG, "Failed to start screen capture via manager")
|
|
stopSelf()
|
|
return
|
|
}
|
|
|
|
PGHLog.d(TAG, "Screen capture setup complete")
|
|
// Show floating overlay
|
|
enhancedFloatingFAB?.show()
|
|
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "Error starting screen capture", e)
|
|
stopSelf()
|
|
}
|
|
}
|
|
|
|
private fun stopScreenCapture() {
|
|
PGHLog.d(TAG, "Stopping screen capture")
|
|
|
|
handler.removeCallbacks(captureRunnable)
|
|
hideDetectionOverlay()
|
|
enhancedFloatingFAB?.hide()
|
|
latestImage?.close()
|
|
latestImage = null
|
|
|
|
// Use the screen capture manager to stop capture
|
|
screenCaptureManager.stopCapture()
|
|
|
|
stopForeground(true)
|
|
stopSelf()
|
|
}
|
|
|
|
/**
|
|
* Handle captured images from the ScreenCaptureManager
|
|
*/
|
|
private fun handleCapturedImage(image: Image) {
|
|
try {
|
|
if (autoProcessing) {
|
|
processImage(image)
|
|
image.close()
|
|
} else {
|
|
// Store the latest image for manual processing
|
|
latestImage?.close() // Release previous image
|
|
latestImage = image
|
|
// Don't close the image yet - it will be closed in triggerManualDetection
|
|
}
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "Error handling captured image", e)
|
|
}
|
|
}
|
|
|
|
private var latestImage: Image? = null
|
|
|
|
private fun captureScreen() {
|
|
// Trigger image capture by reading from the ImageReader
|
|
// The onImageAvailableListener will handle the actual processing
|
|
PGHLog.d(TAG, "Triggering screen capture...")
|
|
}
|
|
|
|
private fun processImage(image: Image) {
|
|
var bitmap: Bitmap? = null
|
|
var croppedBitmap: Bitmap? = null
|
|
|
|
try {
|
|
// Get screen dimensions from the manager
|
|
val screenDimensions = screenCaptureManager.getScreenDimensions()
|
|
if (screenDimensions == null) {
|
|
PGHLog.e(TAG, "Screen dimensions not available from manager")
|
|
return
|
|
}
|
|
val (screenWidth, screenHeight) = screenDimensions
|
|
|
|
// Configure Pokemon data extractor with screen dimensions
|
|
pokemonDataExtractor?.configureRegions(android.util.Size(screenWidth, screenHeight))
|
|
|
|
val planes = image.planes
|
|
val buffer = planes[0].buffer
|
|
val pixelStride = planes[0].pixelStride
|
|
val rowStride = planes[0].rowStride
|
|
val rowPadding = rowStride - pixelStride * screenWidth
|
|
|
|
PGHLog.d(TAG, "🖼️ CAPTURE DEBUG: pixelStride=$pixelStride, rowStride=$rowStride, rowPadding=$rowPadding")
|
|
PGHLog.d(TAG, "🖼️ CAPTURE DEBUG: screenSize=${screenWidth}x${screenHeight}, expected bitmap=${screenWidth + rowPadding / pixelStride}x${screenHeight}")
|
|
|
|
// Create bitmap from image
|
|
bitmap = Bitmap.createBitmap(
|
|
screenWidth + rowPadding / pixelStride,
|
|
screenHeight,
|
|
Bitmap.Config.ARGB_8888
|
|
)
|
|
bitmap.copyPixelsFromBuffer(buffer)
|
|
|
|
PGHLog.d(TAG, "🖼️ CAPTURE DEBUG: created bitmap=${bitmap.width}x${bitmap.height}")
|
|
|
|
// Convert to cropped bitmap if needed
|
|
croppedBitmap = if (rowPadding == 0) {
|
|
PGHLog.d(TAG, "🖼️ CAPTURE DEBUG: No padding, using original bitmap")
|
|
bitmap
|
|
} else {
|
|
PGHLog.d(TAG, "🖼️ CAPTURE DEBUG: Cropping bitmap from ${bitmap.width}x${bitmap.height} to ${screenWidth}x${screenHeight}")
|
|
Bitmap.createBitmap(bitmap, 0, 0, screenWidth, screenHeight)
|
|
}
|
|
|
|
PGHLog.d(TAG, "🖼️ CAPTURE DEBUG: final bitmap=${croppedBitmap.width}x${croppedBitmap.height}")
|
|
|
|
// Convert to OpenCV Mat for analysis - with proper resource management
|
|
Mat().use { mat ->
|
|
Utils.bitmapToMat(croppedBitmap, mat)
|
|
|
|
// DEBUG: Check color conversion
|
|
PGHLog.d(TAG, "🎨 COLOR DEBUG: Mat type=${mat.type()}, channels=${mat.channels()}")
|
|
PGHLog.d(TAG, "🎨 COLOR DEBUG: OpenCV expects BGR, Android Bitmap is ARGB")
|
|
|
|
// Sample a center pixel to check color values
|
|
if (mat.rows() > 0 && mat.cols() > 0) {
|
|
val centerY = mat.rows() / 2
|
|
val centerX = mat.cols() / 2
|
|
val pixel = mat.get(centerY, centerX)
|
|
if (pixel != null && pixel.size >= MIN_PIXEL_CHANNELS) {
|
|
val b = pixel[0].toInt()
|
|
val g = pixel[1].toInt()
|
|
val r = pixel[2].toInt()
|
|
PGHLog.d(TAG, "🎨 COLOR DEBUG: Center pixel (${centerX},${centerY}) BGR=($b,$g,$r) -> RGB=(${r},${g},${b})")
|
|
PGHLog.d(TAG, "🎨 COLOR DEBUG: Center pixel hex = #${String.format("%02x%02x%02x", r, g, b)}")
|
|
}
|
|
}
|
|
|
|
// Run YOLO analysis
|
|
analyzePokemonScreen(mat)
|
|
}
|
|
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "Error processing image", e)
|
|
} finally {
|
|
// Always clean up bitmaps in finally block to prevent leaks
|
|
if (croppedBitmap != null && croppedBitmap != bitmap) {
|
|
croppedBitmap.recycle()
|
|
}
|
|
bitmap?.recycle()
|
|
}
|
|
}
|
|
|
|
private fun analyzePokemonScreen(mat: Mat) {
|
|
PGHLog.i(TAG, "📱 ANALYZING SCREEN: ${mat.cols()}x${mat.rows()}")
|
|
|
|
// Check if analysis has been stuck for too long (30 seconds max)
|
|
val currentTime = System.currentTimeMillis()
|
|
if (isAnalyzing && (currentTime - analysisStartTime) > ANALYSIS_STUCK_TIMEOUT_MS) {
|
|
PGHLog.w(TAG, "⚠️ Analysis stuck for >30s, resetting flag")
|
|
isAnalyzing = false
|
|
}
|
|
|
|
// Skip if already analyzing
|
|
if (isAnalyzing) {
|
|
PGHLog.d(TAG, "⏭️ Skipping analysis - previous cycle still in progress")
|
|
return
|
|
}
|
|
|
|
isAnalyzing = true
|
|
analysisStartTime = currentTime
|
|
PGHLog.d(TAG, "🔄 Starting new analysis cycle")
|
|
|
|
try {
|
|
// Run ML inference first
|
|
val bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888)
|
|
Utils.matToBitmap(mat, bitmap)
|
|
|
|
val detections: List<MLDetection> = runBlocking {
|
|
mlInferenceEngine?.detect(bitmap)
|
|
?.onErrorType(MLErrorType.MEMORY_ERROR) { exception, message ->
|
|
PGHLog.w(TAG, "⚠️ Memory error during detection, may retry with smaller image: $message")
|
|
}
|
|
?.onErrorType(MLErrorType.TIMEOUT_ERROR) { exception, message ->
|
|
PGHLog.w(TAG, "⚠️ Detection timed out, analysis may be too complex: $message")
|
|
}
|
|
?.onErrorType(MLErrorType.INVALID_INPUT) { exception, message ->
|
|
PGHLog.e(TAG, "❌ Invalid input to ML engine: $message")
|
|
}
|
|
?.getOrDefault(emptyList()) ?: emptyList()
|
|
}
|
|
|
|
if (detections.isEmpty()) {
|
|
PGHLog.i(TAG, "🔍 No Pokemon UI elements detected by ONNX YOLO")
|
|
isAnalyzing = false
|
|
return
|
|
}
|
|
|
|
PGHLog.i(TAG, "🎯 ONNX YOLO detected ${detections.size} UI elements")
|
|
|
|
// Log ALL detections for debugging
|
|
detections.forEachIndexed { index, detection ->
|
|
PGHLog.i(TAG, " $index: ${detection.className} (${String.format("%.3f", detection.confidence)}) at [${detection.boundingBox.left}, ${detection.boundingBox.top}, ${detection.boundingBox.width}, ${detection.boundingBox.height}]")
|
|
}
|
|
|
|
// Show breakdown by type
|
|
val detectionCounts = detections.groupBy { it.className }.mapValues { it.value.size }
|
|
PGHLog.i(TAG, "🔍 Detection counts by type: $detectionCounts")
|
|
|
|
// Check for commonly missing elements
|
|
val expectedElements = listOf("pokemon_level", "attack_value", "sp_def_value", "shiny_icon",
|
|
"ball_icon_pokeball", "ball_icon_greatball", "ball_icon_ultraball", "ball_icon_masterball")
|
|
val missingElements = expectedElements.filter { expected ->
|
|
detections.none { detection -> detection.className.startsWith(expected.split("_").take(2).joinToString("_")) }
|
|
}
|
|
if (missingElements.isNotEmpty()) {
|
|
PGHLog.w(TAG, "⚠️ Missing expected elements: $missingElements")
|
|
}
|
|
|
|
// Show detection overlay IMMEDIATELY (no OCR blocking)
|
|
showYOLODetectionOverlay(detections)
|
|
PGHLog.i(TAG, "📺 Overlay displayed with ${detections.size} detections")
|
|
|
|
// Extract Pokemon info using YOLO detections in background
|
|
extractPokemonInfoFromYOLOAsync(mat, detections)
|
|
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "Error analyzing Pokemon screen", e)
|
|
isAnalyzing = false
|
|
}
|
|
}
|
|
|
|
private fun extractPokemonInfoFromYOLOAsync(mat: Mat, detections: List<MLDetection>) {
|
|
// Create a copy of the Mat for background processing
|
|
val matCopy = Mat()
|
|
mat.copyTo(matCopy)
|
|
|
|
// Process in background thread with coroutines
|
|
ocrExecutor.submit {
|
|
try {
|
|
val pokemonInfo = runBlocking {
|
|
extractPokemonInfoFromYOLO(matCopy, detections)
|
|
}
|
|
|
|
// Post results back to main thread
|
|
handler.post {
|
|
try {
|
|
if (pokemonInfo != null) {
|
|
PGHLog.i(TAG, "🔥 POKEMON DATA EXTRACTED SUCCESSFULLY!")
|
|
logPokemonInfo(pokemonInfo)
|
|
// TODO: Send to your API
|
|
// sendToAPI(pokemonInfo)
|
|
} else {
|
|
PGHLog.i(TAG, "❌ Could not extract complete Pokemon info")
|
|
}
|
|
} finally {
|
|
// Analysis cycle complete, allow next one
|
|
isAnalyzing = false
|
|
val duration = System.currentTimeMillis() - analysisStartTime
|
|
PGHLog.d(TAG, "✅ Analysis cycle complete after ${duration}ms - ready for next")
|
|
}
|
|
}
|
|
|
|
// Clean up
|
|
matCopy.release()
|
|
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "Error in async Pokemon extraction", e)
|
|
matCopy.release()
|
|
|
|
// Clear flag on error too
|
|
handler.post {
|
|
isAnalyzing = false
|
|
PGHLog.d(TAG, "❌ Analysis cycle failed - ready for next")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private suspend fun extractPokemonInfoFromYOLO(mat: Mat, detections: List<MLDetection>): PokemonInfo? {
|
|
return try {
|
|
PGHLog.i(TAG, "🎯 Delegating Pokemon extraction to PokemonDataExtractor")
|
|
|
|
// Convert MLDetection to Detection for the extractor
|
|
val extractorDetections = detections.map { mlDetection ->
|
|
com.quillstudios.pokegoalshelper.ml.Detection(
|
|
className = mlDetection.className,
|
|
confidence = mlDetection.confidence,
|
|
boundingBox = mlDetection.boundingBox
|
|
)
|
|
}
|
|
|
|
// Use the dedicated Pokemon data extractor with async/await pattern
|
|
pokemonDataExtractor?.extractPokemonInfo(extractorDetections, mat)
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "Error extracting Pokemon info via PokemonDataExtractor", e)
|
|
null
|
|
}
|
|
}
|
|
|
|
private fun submitOCRTask(key: String, mat: Mat, detection: MLDetection?, results: MutableMap<String, String?>, latch: CountDownLatch) {
|
|
ocrExecutor.submit {
|
|
try {
|
|
val text = extractTextFromDetection(mat, detection)
|
|
synchronized(results) {
|
|
results[key] = text
|
|
}
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "Error in OCR task for $key", e)
|
|
synchronized(results) {
|
|
results[key] = null
|
|
}
|
|
} finally {
|
|
latch.countDown()
|
|
}
|
|
}
|
|
}
|
|
|
|
private fun submitLevelOCRTask(key: String, mat: Mat, detection: MLDetection?, results: MutableMap<String, String?>, latch: CountDownLatch) {
|
|
ocrExecutor.submit {
|
|
try {
|
|
val levelText = extractTextFromDetection(mat, detection)
|
|
val level = levelText?.replace("[^0-9]".toRegex(), "")
|
|
synchronized(results) {
|
|
results[key] = level
|
|
}
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "Error in level OCR task", e)
|
|
synchronized(results) {
|
|
results[key] = null
|
|
}
|
|
} finally {
|
|
latch.countDown()
|
|
}
|
|
}
|
|
}
|
|
|
|
private fun extractTextFromDetection(mat: Mat, detection: MLDetection?): String? {
|
|
if (detection == null) return null
|
|
|
|
try {
|
|
// Expand bounding box by 5% for all OCR classes to improve text extraction accuracy
|
|
val bbox = detection.boundingBox
|
|
val expansionFactor = 0.05f // 5% expansion
|
|
val widthExpansion = (bbox.width * expansionFactor).toInt()
|
|
val heightExpansion = (bbox.height * expansionFactor).toInt()
|
|
|
|
val expandedBbox = Rect(
|
|
bbox.left.toInt() - widthExpansion,
|
|
bbox.top.toInt() - heightExpansion,
|
|
bbox.width.toInt() + (2 * widthExpansion),
|
|
bbox.height.toInt() + (2 * heightExpansion)
|
|
)
|
|
|
|
// Validate and clip bounding box to image boundaries
|
|
val clippedX = kotlin.math.max(0, kotlin.math.min(expandedBbox.x, mat.cols() - 1))
|
|
val clippedY = kotlin.math.max(0, kotlin.math.min(expandedBbox.y, mat.rows() - 1))
|
|
val clippedWidth = kotlin.math.max(1, kotlin.math.min(expandedBbox.width, mat.cols() - clippedX))
|
|
val clippedHeight = kotlin.math.max(1, kotlin.math.min(expandedBbox.height, mat.rows() - clippedY))
|
|
|
|
val safeBbox = Rect(clippedX, clippedY, clippedWidth, clippedHeight)
|
|
|
|
// Debug logging for bounding box transformations
|
|
PGHLog.d(TAG, "📏 Expanded bbox for ${detection.className}: [${bbox.left},${bbox.top},${bbox.width},${bbox.height}] → [${expandedBbox.x},${expandedBbox.y},${expandedBbox.width},${expandedBbox.height}]")
|
|
if (safeBbox.x != expandedBbox.x || safeBbox.y != expandedBbox.y || safeBbox.width != expandedBbox.width || safeBbox.height != expandedBbox.height) {
|
|
PGHLog.w(TAG, "⚠️ Clipped bbox for ${detection.className}: expanded=[${expandedBbox.x},${expandedBbox.y},${expandedBbox.width},${expandedBbox.height}] → safe=[${safeBbox.x},${safeBbox.y},${safeBbox.width},${safeBbox.height}] (image: ${mat.cols()}x${mat.rows()})")
|
|
}
|
|
|
|
// Extract region of interest using safe bounding box
|
|
val roi = Mat(mat, safeBbox)
|
|
|
|
// Preprocess image for better OCR
|
|
val processedRoi = preprocessImageForOCR(roi)
|
|
|
|
// Convert to bitmap for ML Kit
|
|
val bitmap = Bitmap.createBitmap(processedRoi.cols(), processedRoi.rows(), Bitmap.Config.ARGB_8888)
|
|
Utils.matToBitmap(processedRoi, bitmap)
|
|
|
|
// Use ML Kit OCR
|
|
val extractedText = try {
|
|
performOCR(bitmap, detection.className)
|
|
} finally {
|
|
// Always clean up bitmap after OCR
|
|
bitmap.recycle()
|
|
}
|
|
|
|
// Clean up
|
|
roi.release()
|
|
processedRoi.release()
|
|
|
|
if (extractedText != null) {
|
|
PGHLog.i(TAG, "✅ YOLO SUCCESS: ${detection.className} = '$extractedText' (conf: ${String.format("%.2f", detection.confidence)})")
|
|
} else {
|
|
PGHLog.w(TAG, "❌ YOLO FAILED: ${detection.className} - no text found (conf: ${String.format("%.2f", detection.confidence)})")
|
|
}
|
|
return extractedText
|
|
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "Error extracting text from YOLO detection ${detection.className}", e)
|
|
return null
|
|
}
|
|
}
|
|
|
|
private fun extractLevelFromDetection(mat: Mat, detection: MLDetection?): Int? {
|
|
val levelText = extractTextFromDetection(mat, detection)
|
|
return levelText?.replace("[^0-9]".toRegex(), "")?.toIntOrNull()
|
|
}
|
|
|
|
private fun extractStatsFromDetections(mat: Mat, detectionMap: Map<String, List<MLDetection>>): PokemonStats? {
|
|
val hp = extractTextFromDetection(mat, detectionMap["hp_value"]?.firstOrNull())?.toIntOrNull()
|
|
val attack = extractTextFromDetection(mat, detectionMap["attack_value"]?.firstOrNull())?.toIntOrNull()
|
|
val defense = extractTextFromDetection(mat, detectionMap["defense_value"]?.firstOrNull())?.toIntOrNull()
|
|
val spAttack = extractTextFromDetection(mat, detectionMap["sp_atk_value"]?.firstOrNull())?.toIntOrNull()
|
|
val spDefense = extractTextFromDetection(mat, detectionMap["sp_def_value"]?.firstOrNull())?.toIntOrNull()
|
|
val speed = extractTextFromDetection(mat, detectionMap["speed_value"]?.firstOrNull())?.toIntOrNull()
|
|
|
|
return if (hp != null || attack != null || defense != null || spAttack != null || spDefense != null || speed != null) {
|
|
PokemonStats(hp, attack, defense, spAttack, spDefense, speed)
|
|
} else null
|
|
}
|
|
|
|
private fun extractMovesFromDetections(mat: Mat, detectionMap: Map<String, List<MLDetection>>): List<String> {
|
|
val moves = mutableListOf<String>()
|
|
detectionMap["move_name"]?.forEach { detection ->
|
|
val moveText = extractTextFromDetection(mat, detection)
|
|
if (!moveText.isNullOrBlank()) {
|
|
moves.add(moveText.trim())
|
|
}
|
|
}
|
|
return moves.take(4) // Pokemon can have max 4 moves
|
|
}
|
|
|
|
private fun detectPokeballTypeFromDetections(detectionMap: Map<String, List<MLDetection>>): String? {
|
|
// Check for specific pokeball types detected by YOLO
|
|
val pokeballTypes = mapOf(
|
|
"ball_icon_pokeball" to "Poké Ball",
|
|
"ball_icon_greatball" to "Great Ball",
|
|
"ball_icon_ultraball" to "Ultra Ball",
|
|
"ball_icon_heavyball" to "Heavy Ball",
|
|
"ball_icon_premierball" to "Premier Ball",
|
|
"ball_icon_repeatball" to "Repeat Ball",
|
|
"ball_icon_timerball" to "Timer Ball",
|
|
"ball_icon_diveball" to "Dive Ball",
|
|
"ball_icon_quickball" to "Quick Ball",
|
|
"ball_icon_duskball" to "Dusk Ball",
|
|
"ball_icon_cherishball" to "Cherish Ball",
|
|
"ball_icon_originball" to "Origin Ball",
|
|
"ball_icon_pokeball_hisui" to "Hisuian Poké Ball",
|
|
"ball_icon_ultraball_husui" to "Hisuian Ultra Ball"
|
|
)
|
|
|
|
for ((className, ballName) in pokeballTypes) {
|
|
if (detectionMap[className]?.isNotEmpty() == true) {
|
|
return ballName
|
|
}
|
|
}
|
|
|
|
return null
|
|
}
|
|
|
|
private fun extractTypesFromDetections(mat: Mat, detectionMap: Map<String, List<MLDetection>>): List<String> {
|
|
val types = mutableListOf<String>()
|
|
|
|
extractTextFromDetection(mat, detectionMap["type_1"]?.firstOrNull())?.let { type1 ->
|
|
if (type1.isNotBlank()) types.add(type1.trim())
|
|
}
|
|
|
|
extractTextFromDetection(mat, detectionMap["type_2"]?.firstOrNull())?.let { type2 ->
|
|
if (type2.isNotBlank()) types.add(type2.trim())
|
|
}
|
|
|
|
return types
|
|
}
|
|
|
|
private fun detectTeraTypeFromDetections(detectionMap: Map<String, List<MLDetection>>): String? {
|
|
val teraTypes = mapOf(
|
|
"tera_ice" to "Ice",
|
|
"tera_fairy" to "Fairy",
|
|
"tera_poison" to "Poison",
|
|
"tera_ghost" to "Ghost",
|
|
"tera_steel" to "Steel",
|
|
"tera_grass" to "Grass",
|
|
"tera_normal" to "Normal",
|
|
"tera_fire" to "Fire",
|
|
"tera_electric" to "Electric",
|
|
"tera_ground" to "Ground",
|
|
"tera_flying" to "Flying",
|
|
"tera_bug" to "Bug",
|
|
"tera_dark" to "Dark",
|
|
"tera_water" to "Water",
|
|
"tera_psychic" to "Psychic",
|
|
"tera_dragon" to "Dragon",
|
|
"tera_fighting" to "Fighting",
|
|
"tera_rock" to "Rock"
|
|
)
|
|
|
|
for ((className, teraName) in teraTypes) {
|
|
if (detectionMap[className]?.isNotEmpty() == true) {
|
|
return teraName
|
|
}
|
|
}
|
|
|
|
return null
|
|
}
|
|
|
|
private fun detectGameSourceFromDetections(detectionMap: Map<String, List<MLDetection>>): String? {
|
|
val gameSources = mapOf(
|
|
"last_game_stamp_sh" to "Sword/Shield",
|
|
"last_game_stamp_bank" to "Bank",
|
|
"last_game_stamp_pla" to "Legends: Arceus",
|
|
"last_game_stamp_sc" to "Scarlet/Violet",
|
|
"last_game_stamp_vi" to "Violet",
|
|
"last_game_stamp_go" to "Pokémon GO",
|
|
"origin_icon_vc" to "Virtual Console",
|
|
"origin_icon_xyoras" to "XY/ORAS",
|
|
"origin_icon_smusum" to "SM/USUM",
|
|
"origin_icon_swsh" to "Sword/Shield",
|
|
"origin_icon_go" to "Pokémon GO",
|
|
"origin_icon_pla" to "Legends: Arceus",
|
|
"origin_icon_sv" to "Scarlet/Violet"
|
|
)
|
|
|
|
for ((className, sourceName) in gameSources) {
|
|
if (detectionMap[className]?.isNotEmpty() == true) {
|
|
return sourceName
|
|
}
|
|
}
|
|
|
|
return null
|
|
}
|
|
|
|
private fun calculateYOLOExtractionConfidence(detections: List<MLDetection>, nickname: String?, level: Int?, species: String?): Double {
|
|
var confidence = 0.0
|
|
|
|
// Base confidence from YOLO detections
|
|
val avgDetectionConfidence = detections.map { it.confidence.toDouble() }.average()
|
|
confidence += avgDetectionConfidence * 0.4
|
|
|
|
// Boost confidence based on extracted data
|
|
if (!nickname.isNullOrBlank()) confidence += 0.2
|
|
if (level != null && level > 0) confidence += 0.2
|
|
if (!species.isNullOrBlank()) confidence += 0.2
|
|
|
|
return confidence.coerceIn(0.0, 1.0)
|
|
}
|
|
|
|
private fun showYOLODetectionOverlay(detections: List<MLDetection>) {
|
|
try {
|
|
PGHLog.i(TAG, "🎨 Creating YOLO detection overlay for ${detections.size} detections")
|
|
|
|
if (detectionOverlay == null) {
|
|
PGHLog.i(TAG, "🆕 Creating new DetectionOverlay instance")
|
|
detectionOverlay = DetectionOverlay(this)
|
|
}
|
|
|
|
// Convert YOLO detections to screen regions for overlay
|
|
val statusBarHeight = getStatusBarHeight()
|
|
val regions = detections.mapIndexed { index, detection ->
|
|
"${detection.className}_$index" to ScreenRegion(
|
|
x = detection.boundingBox.left.toInt(),
|
|
y = detection.boundingBox.top.toInt() - statusBarHeight, // Subtract status bar offset
|
|
width = detection.boundingBox.width.toInt(),
|
|
height = detection.boundingBox.height.toInt(),
|
|
purpose = "${detection.className} (${String.format("%.2f", detection.confidence)})"
|
|
)
|
|
}.toMap()
|
|
|
|
PGHLog.i(TAG, "📺 Showing YOLO overlay with ${regions.size} regions...")
|
|
detectionOverlay?.showOverlay(regions)
|
|
PGHLog.i(TAG, "✅ YOLO overlay show command sent")
|
|
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "❌ Error showing YOLO detection overlay", e)
|
|
}
|
|
}
|
|
|
|
private fun hideDetectionOverlay() {
|
|
detectionOverlay?.hideOverlay()
|
|
}
|
|
|
|
private fun performOCR(bitmap: Bitmap, purpose: String): String? {
|
|
try {
|
|
PGHLog.d(TAG, "🔍 Starting OCR for $purpose - bitmap: ${bitmap.width}x${bitmap.height}")
|
|
|
|
// Create InputImage for ML Kit
|
|
val image = InputImage.fromBitmap(bitmap, 0)
|
|
val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)
|
|
|
|
// Use CountDownLatch to make async call synchronous
|
|
val latch = CountDownLatch(1)
|
|
var result: String? = null
|
|
var ocrError: Exception? = null
|
|
|
|
recognizer.process(image)
|
|
.addOnSuccessListener { visionText ->
|
|
result = visionText.text.trim()
|
|
PGHLog.d(TAG, "🔍 Raw OCR result for $purpose: '${visionText.text}' (blocks: ${visionText.textBlocks.size})")
|
|
|
|
if (result.isNullOrBlank()) {
|
|
PGHLog.w(TAG, "⚠️ OCR for $purpose: NO TEXT DETECTED - ML Kit found ${visionText.textBlocks.size} text blocks but text is empty")
|
|
} else {
|
|
PGHLog.i(TAG, "✅ OCR SUCCESS for $purpose: '${result}' (${result!!.length} chars)")
|
|
}
|
|
latch.countDown()
|
|
}
|
|
.addOnFailureListener { e ->
|
|
ocrError = e
|
|
PGHLog.e(TAG, "❌ OCR failed for $purpose: ${e.message}", e)
|
|
latch.countDown()
|
|
}
|
|
|
|
// Wait for OCR to complete (max 5 seconds to allow ML Kit to work)
|
|
val completed = latch.await(OCR_INDIVIDUAL_TIMEOUT_SECONDS, TimeUnit.SECONDS)
|
|
|
|
if (!completed) {
|
|
PGHLog.e(TAG, "⏱️ OCR timeout for $purpose after ${OCR_INDIVIDUAL_TIMEOUT_SECONDS} seconds")
|
|
return null
|
|
}
|
|
|
|
if (ocrError != null) {
|
|
PGHLog.e(TAG, "❌ OCR error for $purpose: ${ocrError!!.message}")
|
|
return null
|
|
}
|
|
|
|
// Clean and process the result
|
|
val cleanedResult = cleanOCRResult(result, purpose)
|
|
PGHLog.d(TAG, "🧙 Cleaned result for $purpose: '${cleanedResult}'")
|
|
return cleanedResult
|
|
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "❌ Error in OCR for $purpose", e)
|
|
return null
|
|
}
|
|
}
|
|
|
|
private fun cleanOCRResult(rawText: String?, purpose: String): String? {
|
|
if (rawText.isNullOrBlank()) return null
|
|
|
|
val cleaned = rawText.trim()
|
|
|
|
return when (purpose) {
|
|
"pokemon_level" -> {
|
|
// Extract level number (look for "Lv." or just numbers)
|
|
val levelRegex = """(?:Lv\.?\s*)?([0-9]+)""".toRegex(RegexOption.IGNORE_CASE)
|
|
levelRegex.find(cleaned)?.groupValues?.get(1)
|
|
}
|
|
"pokemon_nickname", "pokemon_species" -> {
|
|
// Clean up common OCR mistakes for Pokemon names
|
|
cleaned.replace("[^a-zA-Z0-9 \\-.'♂♀]".toRegex(), "")
|
|
.replace("\\s+".toRegex(), " ")
|
|
.trim()
|
|
.takeIf { it.isNotEmpty() }
|
|
}
|
|
"nature_name", "ability_name" -> {
|
|
// Clean up nature/ability names
|
|
cleaned.replace("[^a-zA-Z ]".toRegex(), "")
|
|
.replace("\\s+".toRegex(), " ")
|
|
.trim()
|
|
.takeIf { it.isNotEmpty() }
|
|
}
|
|
"original_trainer_name" -> {
|
|
// Clean trainer names
|
|
cleaned.replace("[^a-zA-Z0-9 ]".toRegex(), "")
|
|
.replace("\\s+".toRegex(), " ")
|
|
.trim()
|
|
.takeIf { it.isNotEmpty() }
|
|
}
|
|
"original_trainder_number" -> {
|
|
// Extract ID numbers
|
|
cleaned.replace("[^0-9]".toRegex(), "")
|
|
.takeIf { it.isNotEmpty() }
|
|
}
|
|
else -> cleaned.takeIf { it.isNotEmpty() }
|
|
}
|
|
}
|
|
|
|
private fun preprocessImageForOCR(roi: Mat): Mat {
|
|
try {
|
|
// Resize small regions for better OCR (minimum 150x50 pixels)
|
|
val minWidth = 150
|
|
val minHeight = 50
|
|
val scaledRoi = if (roi.width() < minWidth || roi.height() < minHeight) {
|
|
val scaleX = maxOf(1.0, minWidth.toDouble() / roi.width())
|
|
val scaleY = maxOf(1.0, minHeight.toDouble() / roi.height())
|
|
val scale = maxOf(scaleX, scaleY)
|
|
|
|
val resized = Mat()
|
|
Imgproc.resize(roi, resized, Size(roi.width() * scale, roi.height() * scale))
|
|
PGHLog.d(TAG, "🔍 Upscaled OCR region from ${roi.width()}x${roi.height()} to ${resized.width()}x${resized.height()}")
|
|
resized
|
|
} else {
|
|
val copy = Mat()
|
|
roi.copyTo(copy)
|
|
copy
|
|
}
|
|
|
|
// Convert to grayscale if needed
|
|
val gray = Mat()
|
|
if (scaledRoi.channels() > 1) {
|
|
Imgproc.cvtColor(scaledRoi, gray, Imgproc.COLOR_RGBA2GRAY)
|
|
} else {
|
|
scaledRoi.copyTo(gray)
|
|
}
|
|
|
|
// Enhance contrast with CLAHE (Contrast Limited Adaptive Histogram Equalization)
|
|
val clahe = Imgproc.createCLAHE(2.0, Size(8.0, 8.0))
|
|
val enhanced = Mat()
|
|
clahe.apply(gray, enhanced)
|
|
|
|
// Light denoising only if text is very small
|
|
val denoised = if (scaledRoi.width() < 100 || scaledRoi.height() < 30) {
|
|
val temp = Mat()
|
|
Imgproc.GaussianBlur(enhanced, temp, Size(1.0, 1.0), 0.0)
|
|
temp
|
|
} else {
|
|
enhanced.clone()
|
|
}
|
|
|
|
// Convert back to RGBA for ML Kit
|
|
val result = Mat()
|
|
Imgproc.cvtColor(denoised, result, Imgproc.COLOR_GRAY2RGBA)
|
|
|
|
// Clean up intermediate matrices
|
|
scaledRoi.release()
|
|
gray.release()
|
|
enhanced.release()
|
|
if (denoised !== enhanced) denoised.release()
|
|
|
|
return result
|
|
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "Error preprocessing image, using original", e)
|
|
// Return copy of original if preprocessing fails
|
|
val result = Mat()
|
|
roi.copyTo(result)
|
|
return result
|
|
}
|
|
}
|
|
|
|
private fun detectLanguage(nickname: String?, species: String?): String? {
|
|
// Simple language detection based on character patterns
|
|
val text = "$nickname $species"
|
|
return when {
|
|
text.any { it in '\u3040'..'\u309F' || it in '\u30A0'..'\u30FF' } -> "JP" // Hiragana/Katakana
|
|
text.any { it in '\u4E00'..'\u9FAF' } -> "ZH" // Chinese characters
|
|
text.any { it in '\uAC00'..'\uD7AF' } -> "KO" // Korean
|
|
else -> "EN" // Default to English
|
|
}
|
|
}
|
|
|
|
private fun extractDexNumber(species: String?): Int? {
|
|
// For now, return null - would need a Pokemon species to dex number mapping
|
|
return null
|
|
}
|
|
|
|
private fun logPokemonInfo(pokemonInfo: PokemonInfo) {
|
|
PGHLog.i(TAG, "====== POKEMON INFO EXTRACTED ======")
|
|
PGHLog.i(TAG, "🎾 Pokeball: ${pokemonInfo.pokeballType}")
|
|
PGHLog.i(TAG, "📛 Nickname: ${pokemonInfo.nickname}")
|
|
PGHLog.i(TAG, "⚤ Gender: ${pokemonInfo.gender}")
|
|
PGHLog.i(TAG, "📊 Level: ${pokemonInfo.level}")
|
|
PGHLog.i(TAG, "🌍 Language: ${pokemonInfo.language}")
|
|
PGHLog.i(TAG, "🎮 Game Source: ${pokemonInfo.gameSource}")
|
|
PGHLog.i(TAG, "⭐ Favorited: ${pokemonInfo.isFavorited}")
|
|
PGHLog.i(TAG, "🔢 Dex #: ${pokemonInfo.nationalDexNumber}")
|
|
PGHLog.i(TAG, "🐾 Species: ${pokemonInfo.species}")
|
|
PGHLog.i(TAG, "🏷️ Type 1: ${pokemonInfo.primaryType}")
|
|
PGHLog.i(TAG, "🏷️ Type 2: ${pokemonInfo.secondaryType}")
|
|
PGHLog.i(TAG, "🏆 Stamps: ${pokemonInfo.stamps}")
|
|
PGHLog.i(TAG, "🏷️ Labels: ${pokemonInfo.labels}")
|
|
PGHLog.i(TAG, "✅ Marks: ${pokemonInfo.marks}")
|
|
if (pokemonInfo.stats != null) {
|
|
PGHLog.i(TAG, "📈 Stats: HP:${pokemonInfo.stats.hp} ATK:${pokemonInfo.stats.attack} DEF:${pokemonInfo.stats.defense}")
|
|
PGHLog.i(TAG, " SP.ATK:${pokemonInfo.stats.spAttack} SP.DEF:${pokemonInfo.stats.spDefense} SPD:${pokemonInfo.stats.speed}")
|
|
}
|
|
PGHLog.i(TAG, "⚔️ Moves: ${pokemonInfo.moves}")
|
|
PGHLog.i(TAG, "💪 Ability: ${pokemonInfo.ability}")
|
|
PGHLog.i(TAG, "🎭 Nature: ${pokemonInfo.nature}")
|
|
PGHLog.i(TAG, "👤 OT: ${pokemonInfo.originalTrainerName}")
|
|
PGHLog.i(TAG, "🔢 ID: ${pokemonInfo.originalTrainerId}")
|
|
PGHLog.i(TAG, "🎯 Confidence: ${String.format("%.2f", pokemonInfo.extractionConfidence)}")
|
|
PGHLog.i(TAG, "====================================")
|
|
}
|
|
|
|
private fun convertImageToMat(image: Image): Mat? {
|
|
var bitmap: Bitmap? = null
|
|
var croppedBitmap: Bitmap? = null
|
|
var mat: Mat? = null
|
|
|
|
return try {
|
|
// Get screen dimensions from the manager
|
|
val screenDimensions = screenCaptureManager.getScreenDimensions()
|
|
if (screenDimensions == null) {
|
|
PGHLog.e(TAG, "Screen dimensions not available from manager")
|
|
return null
|
|
}
|
|
val (screenWidth, screenHeight) = screenDimensions
|
|
|
|
val planes = image.planes
|
|
val buffer = planes[0].buffer
|
|
val pixelStride = planes[0].pixelStride
|
|
val rowStride = planes[0].rowStride
|
|
val rowPadding = rowStride - pixelStride * screenWidth
|
|
|
|
// Create bitmap from image
|
|
bitmap = Bitmap.createBitmap(
|
|
screenWidth + rowPadding / pixelStride,
|
|
screenHeight,
|
|
Bitmap.Config.ARGB_8888
|
|
)
|
|
bitmap.copyPixelsFromBuffer(buffer)
|
|
|
|
// Crop bitmap to remove padding if needed
|
|
croppedBitmap = if (rowPadding == 0) {
|
|
bitmap
|
|
} else {
|
|
val cropped = Bitmap.createBitmap(bitmap, 0, 0, screenWidth, screenHeight)
|
|
bitmap.recycle() // Clean up original
|
|
bitmap = null // Mark as recycled
|
|
cropped
|
|
}
|
|
|
|
// Convert bitmap to Mat
|
|
mat = Mat()
|
|
Utils.bitmapToMat(croppedBitmap, mat)
|
|
|
|
// Convert from RGBA to BGR (OpenCV format for proper color channel handling)
|
|
val bgrMat = Mat()
|
|
Imgproc.cvtColor(mat, bgrMat, Imgproc.COLOR_RGBA2BGR)
|
|
|
|
// Clean up intermediate resources
|
|
mat.release()
|
|
croppedBitmap.recycle()
|
|
|
|
bgrMat
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "❌ Error converting image to Mat", e)
|
|
// Clean up on error
|
|
mat?.release()
|
|
if (croppedBitmap != null && croppedBitmap != bitmap) {
|
|
croppedBitmap.recycle()
|
|
}
|
|
bitmap?.recycle()
|
|
null
|
|
}
|
|
}
|
|
|
|
private fun triggerManualDetection() {
|
|
PGHLog.d(TAG, "🔍 Manual detection triggered via MVC!")
|
|
|
|
latestImage?.let { image ->
|
|
try {
|
|
// Convert image to Mat for processing
|
|
val mat = convertImageToMat(image)
|
|
|
|
if (mat != null) {
|
|
// Convert Mat to Bitmap and run inference
|
|
val bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888)
|
|
Utils.matToBitmap(mat, bitmap)
|
|
|
|
val detections: List<MLDetection> = runBlocking {
|
|
mlInferenceEngine?.detect(bitmap)?.getOrDefault(emptyList()) ?: emptyList()
|
|
}
|
|
|
|
// Show detection overlay with results
|
|
if (detections.isNotEmpty()) {
|
|
showYOLODetectionOverlay(detections)
|
|
|
|
// Extract Pokemon info using YOLO detections with OCR
|
|
extractPokemonInfoFromYOLOAsync(mat, detections)
|
|
}
|
|
|
|
mat.release()
|
|
} else {
|
|
PGHLog.e(TAG, "❌ Failed to convert image to Mat")
|
|
}
|
|
|
|
// Close the image after processing to free the buffer
|
|
image.close()
|
|
latestImage = null
|
|
|
|
} catch (e: Exception) {
|
|
PGHLog.e(TAG, "❌ Error in manual detection", e)
|
|
}
|
|
} ?: run {
|
|
PGHLog.w(TAG, "⚠️ No image available for detection")
|
|
}
|
|
}
|
|
|
|
|
|
override fun onDestroy() {
|
|
super.onDestroy()
|
|
hideDetectionOverlay()
|
|
enhancedFloatingFAB?.hide()
|
|
// TODO: Re-enable when DetectionController is updated
|
|
// detectionController.clearUICallbacks()
|
|
mlInferenceEngine?.cleanup()
|
|
pokemonDataExtractor?.cleanup()
|
|
|
|
// Release screen capture manager
|
|
if (::screenCaptureManager.isInitialized) {
|
|
screenCaptureManager.release()
|
|
}
|
|
|
|
// Proper executor shutdown with timeout
|
|
ocrExecutor.shutdown()
|
|
try {
|
|
if (!ocrExecutor.awaitTermination(OCR_INDIVIDUAL_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
|
|
PGHLog.w(TAG, "OCR executor did not terminate gracefully, forcing shutdown")
|
|
ocrExecutor.shutdownNow()
|
|
}
|
|
} catch (e: InterruptedException) {
|
|
PGHLog.w(TAG, "Interrupted while waiting for OCR executor termination")
|
|
ocrExecutor.shutdownNow()
|
|
Thread.currentThread().interrupt()
|
|
}
|
|
stopScreenCapture()
|
|
}
|
|
|
|
}
|