|
|
|
|
package com.quillstudios.pokegoalshelper
|
|
|
|
|
|
|
|
|
|
import android.app.*
|
|
|
|
|
import android.content.Context
|
|
|
|
|
import android.content.Intent
|
|
|
|
|
import android.os.Binder
|
|
|
|
|
import android.graphics.Bitmap
|
|
|
|
|
import android.graphics.ImageFormat
|
|
|
|
|
import android.graphics.PixelFormat
|
|
|
|
|
import android.hardware.display.DisplayManager
|
|
|
|
|
import android.hardware.display.VirtualDisplay
|
|
|
|
|
import android.media.Image
|
|
|
|
|
import android.media.ImageReader
|
|
|
|
|
import android.media.projection.MediaProjection
|
|
|
|
|
import android.media.projection.MediaProjectionManager
|
|
|
|
|
import android.os.*
|
|
|
|
|
import android.util.DisplayMetrics
|
|
|
|
|
import android.util.Log
|
|
|
|
|
import android.view.WindowManager
|
|
|
|
|
import android.view.View
|
|
|
|
|
import android.view.Gravity
|
|
|
|
|
import android.widget.Button
|
|
|
|
|
import android.widget.LinearLayout
|
|
|
|
|
import androidx.core.app.NotificationCompat
|
|
|
|
|
import com.quillstudios.pokegoalshelper.controllers.DetectionController
|
|
|
|
|
import com.quillstudios.pokegoalshelper.ui.EnhancedFloatingFAB
|
|
|
|
|
import org.opencv.android.Utils
|
|
|
|
|
import org.opencv.core.*
|
|
|
|
|
import org.opencv.imgproc.Imgproc
|
|
|
|
|
import org.opencv.imgcodecs.Imgcodecs
|
|
|
|
|
import com.google.mlkit.vision.common.InputImage
|
|
|
|
|
import com.google.mlkit.vision.text.TextRecognition
|
|
|
|
|
import com.google.mlkit.vision.text.latin.TextRecognizerOptions
|
|
|
|
|
import java.util.concurrent.CountDownLatch
|
|
|
|
|
import java.io.File
|
|
|
|
|
import java.util.concurrent.TimeUnit
|
|
|
|
|
import java.util.concurrent.Executors
|
|
|
|
|
import java.util.concurrent.ThreadPoolExecutor
|
|
|
|
|
import java.nio.ByteBuffer
|
|
|
|
|
|
|
|
|
|
data class PokemonInfo(
|
|
|
|
|
val pokeballType: String?,
|
|
|
|
|
val nickname: String?,
|
|
|
|
|
val gender: String?,
|
|
|
|
|
val level: Int?,
|
|
|
|
|
val language: String?,
|
|
|
|
|
val gameSource: String?,
|
|
|
|
|
val isFavorited: Boolean,
|
|
|
|
|
val nationalDexNumber: Int?,
|
|
|
|
|
val species: String?,
|
|
|
|
|
val primaryType: String?,
|
|
|
|
|
val secondaryType: String?,
|
|
|
|
|
val stamps: List<String>,
|
|
|
|
|
val labels: List<String>,
|
|
|
|
|
val marks: List<String>,
|
|
|
|
|
val stats: PokemonStats?,
|
|
|
|
|
val moves: List<String>,
|
|
|
|
|
val ability: String?,
|
|
|
|
|
val nature: String?,
|
|
|
|
|
val originalTrainerName: String?,
|
|
|
|
|
val originalTrainerId: String?,
|
|
|
|
|
val extractionConfidence: Double
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
data class PokemonStats(
|
|
|
|
|
val hp: Int?,
|
|
|
|
|
val attack: Int?,
|
|
|
|
|
val defense: Int?,
|
|
|
|
|
val spAttack: Int?,
|
|
|
|
|
val spDefense: Int?,
|
|
|
|
|
val speed: Int?
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
data class ScreenRegion(
|
|
|
|
|
val x: Int,
|
|
|
|
|
val y: Int,
|
|
|
|
|
val width: Int,
|
|
|
|
|
val height: Int,
|
|
|
|
|
val purpose: String
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
class ScreenCaptureService : Service() {
|
|
|
|
|
|
|
|
|
|
companion object {
|
|
|
|
|
private const val TAG = "ScreenCaptureService"
|
|
|
|
|
private const val NOTIFICATION_ID = 1001
|
|
|
|
|
private const val CHANNEL_ID = "screen_capture_channel"
|
|
|
|
|
|
|
|
|
|
const val ACTION_START = "START_SCREEN_CAPTURE"
|
|
|
|
|
const val ACTION_STOP = "STOP_SCREEN_CAPTURE"
|
|
|
|
|
const val EXTRA_RESULT_DATA = "result_data"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Binder for communication with FloatingUIActivity
|
|
|
|
|
*/
|
|
|
|
|
inner class LocalBinder : Binder() {
|
|
|
|
|
fun getService(): ScreenCaptureService = this@ScreenCaptureService
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private val binder = LocalBinder()
|
|
|
|
|
|
|
|
|
|
// ONNX YOLO detector instance
|
|
|
|
|
private var yoloDetector: YOLOOnnxDetector? = null
|
|
|
|
|
|
|
|
|
|
private var mediaProjection: MediaProjection? = null
|
|
|
|
|
private var virtualDisplay: VirtualDisplay? = null
|
|
|
|
|
private var imageReader: ImageReader? = null
|
|
|
|
|
private var mediaProjectionManager: MediaProjectionManager? = null
|
|
|
|
|
private var screenWidth = 0
|
|
|
|
|
private var screenHeight = 0
|
|
|
|
|
private var screenDensity = 0
|
|
|
|
|
private var detectionOverlay: DetectionOverlay? = null
|
|
|
|
|
|
|
|
|
|
// MVC Components
|
|
|
|
|
private lateinit var detectionController: DetectionController
|
|
|
|
|
private var enhancedFloatingFAB: EnhancedFloatingFAB? = null
|
|
|
|
|
|
|
|
|
|
private val handler = Handler(Looper.getMainLooper())
|
|
|
|
|
private var captureInterval = 2000L // Capture every 2 seconds
|
|
|
|
|
private var autoProcessing = false // Disable automatic processing
|
|
|
|
|
|
|
|
|
|
// Thread pool for OCR processing (4 threads for parallel text extraction)
|
|
|
|
|
private val ocrExecutor = Executors.newFixedThreadPool(4)
|
|
|
|
|
|
|
|
|
|
// Flag to prevent overlapping analysis cycles
|
|
|
|
|
private var isAnalyzing = false
|
|
|
|
|
private var analysisStartTime = 0L
|
|
|
|
|
|
|
|
|
|
private val captureRunnable = object : Runnable {
|
|
|
|
|
override fun run() {
|
|
|
|
|
captureScreen()
|
|
|
|
|
handler.postDelayed(this, captureInterval)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private val mediaProjectionCallback = object : MediaProjection.Callback() {
|
|
|
|
|
override fun onStop() {
|
|
|
|
|
Log.d(TAG, "MediaProjection stopped")
|
|
|
|
|
stopScreenCapture()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
override fun onCapturedContentResize(width: Int, height: Int) {
|
|
|
|
|
Log.d(TAG, "Screen size changed: ${width}x${height}")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
override fun onCapturedContentVisibilityChanged(isVisible: Boolean) {
|
|
|
|
|
Log.d(TAG, "Content visibility changed: $isVisible")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
override fun onCreate() {
|
|
|
|
|
super.onCreate()
|
|
|
|
|
createNotificationChannel()
|
|
|
|
|
getScreenMetrics()
|
|
|
|
|
mediaProjectionManager = getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
|
|
|
|
|
|
|
|
|
// Initialize ONNX YOLO detector
|
|
|
|
|
yoloDetector = YOLOOnnxDetector(this)
|
|
|
|
|
if (!yoloDetector!!.initialize()) {
|
|
|
|
|
Log.e(TAG, "❌ Failed to initialize ONNX YOLO detector")
|
|
|
|
|
} else {
|
|
|
|
|
Log.i(TAG, "✅ ONNX YOLO detector initialized for screen capture")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Initialize MVC components
|
|
|
|
|
detectionController = DetectionController(yoloDetector!!)
|
|
|
|
|
detectionController.setDetectionRequestCallback { triggerManualDetection() }
|
|
|
|
|
|
|
|
|
|
// Initialize enhanced floating FAB
|
|
|
|
|
enhancedFloatingFAB = EnhancedFloatingFAB(
|
|
|
|
|
context = this,
|
|
|
|
|
onDetectionRequested = { triggerDetection() },
|
|
|
|
|
onClassFilterRequested = { className -> setClassFilter(className) },
|
|
|
|
|
onDebugToggled = { toggleDebugMode() },
|
|
|
|
|
onClose = { stopSelf() }
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
Log.d(TAG, "✅ MVC architecture initialized")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
override fun onBind(intent: Intent?): IBinder = binder
|
|
|
|
|
|
|
|
|
|
// === Public API for FloatingUIActivity ===
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Trigger manual detection from UI
|
|
|
|
|
*/
|
|
|
|
|
fun triggerDetection() {
|
|
|
|
|
triggerManualDetection()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Set class filter from UI
|
|
|
|
|
*/
|
|
|
|
|
fun setClassFilter(className: String?) {
|
|
|
|
|
detectionController.onClassFilterChanged(className)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Toggle debug mode from UI
|
|
|
|
|
*/
|
|
|
|
|
fun toggleDebugMode() {
|
|
|
|
|
detectionController.onDebugModeToggled()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
|
|
|
|
when (intent?.action) {
|
|
|
|
|
ACTION_START -> {
|
|
|
|
|
val resultData = intent.getParcelableExtra<Intent>(EXTRA_RESULT_DATA)
|
|
|
|
|
if (resultData != null) {
|
|
|
|
|
startScreenCapture(resultData)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
ACTION_STOP -> {
|
|
|
|
|
stopScreenCapture()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return START_STICKY
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun createNotificationChannel() {
|
|
|
|
|
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
|
|
|
|
val channel = NotificationChannel(
|
|
|
|
|
CHANNEL_ID,
|
|
|
|
|
"Screen Capture Service",
|
|
|
|
|
NotificationManager.IMPORTANCE_LOW
|
|
|
|
|
).apply {
|
|
|
|
|
description = "Pokemon analysis screen capture"
|
|
|
|
|
setShowBadge(false)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
|
|
|
|
|
manager.createNotificationChannel(channel)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun getScreenMetrics() {
|
|
|
|
|
val displayMetrics = DisplayMetrics()
|
|
|
|
|
val windowManager = getSystemService(Context.WINDOW_SERVICE) as android.view.WindowManager
|
|
|
|
|
windowManager.defaultDisplay.getMetrics(displayMetrics)
|
|
|
|
|
|
|
|
|
|
screenWidth = displayMetrics.widthPixels
|
|
|
|
|
screenHeight = displayMetrics.heightPixels
|
|
|
|
|
screenDensity = displayMetrics.densityDpi
|
|
|
|
|
|
|
|
|
|
Log.d(TAG, "Screen metrics: ${screenWidth}x${screenHeight}, density: $screenDensity")
|
|
|
|
|
|
|
|
|
|
// Get status bar height for coordinate adjustment
|
|
|
|
|
val statusBarHeight = getStatusBarHeight()
|
|
|
|
|
Log.d(TAG, "Status bar height: ${statusBarHeight}px")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun getStatusBarHeight(): Int {
|
|
|
|
|
var result = 0
|
|
|
|
|
val resourceId = resources.getIdentifier("status_bar_height", "dimen", "android")
|
|
|
|
|
if (resourceId > 0) {
|
|
|
|
|
result = resources.getDimensionPixelSize(resourceId)
|
|
|
|
|
}
|
|
|
|
|
return result
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun startScreenCapture(resultData: Intent) {
|
|
|
|
|
Log.d(TAG, "Starting screen capture")
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
val notification = NotificationCompat.Builder(this, CHANNEL_ID)
|
|
|
|
|
.setContentTitle("Pokemon Analysis Active")
|
|
|
|
|
.setContentText("Analyzing Pokemon Home screens...")
|
|
|
|
|
.setSmallIcon(R.drawable.ic_launcher_foreground)
|
|
|
|
|
.setOngoing(true)
|
|
|
|
|
.addAction(
|
|
|
|
|
R.drawable.ic_launcher_foreground,
|
|
|
|
|
"Stop",
|
|
|
|
|
PendingIntent.getService(
|
|
|
|
|
this,
|
|
|
|
|
0,
|
|
|
|
|
Intent(this, ScreenCaptureService::class.java).apply {
|
|
|
|
|
action = ACTION_STOP
|
|
|
|
|
},
|
|
|
|
|
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
.build()
|
|
|
|
|
|
|
|
|
|
Log.d(TAG, "Starting foreground service")
|
|
|
|
|
startForeground(NOTIFICATION_ID, notification)
|
|
|
|
|
|
|
|
|
|
Log.d(TAG, "Getting MediaProjection")
|
|
|
|
|
mediaProjection = mediaProjectionManager?.getMediaProjection(Activity.RESULT_OK, resultData)
|
|
|
|
|
|
|
|
|
|
if (mediaProjection == null) {
|
|
|
|
|
Log.e(TAG, "Failed to get MediaProjection")
|
|
|
|
|
stopSelf()
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Log.d(TAG, "Registering MediaProjection callback")
|
|
|
|
|
mediaProjection?.registerCallback(mediaProjectionCallback, handler)
|
|
|
|
|
|
|
|
|
|
Log.d(TAG, "Creating ImageReader: ${screenWidth}x${screenHeight}")
|
|
|
|
|
imageReader = ImageReader.newInstance(screenWidth, screenHeight, PixelFormat.RGBA_8888, 3) // Increased buffer count
|
|
|
|
|
imageReader?.setOnImageAvailableListener(onImageAvailableListener, handler)
|
|
|
|
|
|
|
|
|
|
Log.d(TAG, "Creating VirtualDisplay")
|
|
|
|
|
virtualDisplay = mediaProjection?.createVirtualDisplay(
|
|
|
|
|
"ScreenCapture",
|
|
|
|
|
screenWidth,
|
|
|
|
|
screenHeight,
|
|
|
|
|
screenDensity,
|
|
|
|
|
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
|
|
|
|
|
imageReader?.surface,
|
|
|
|
|
null,
|
|
|
|
|
null
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if (virtualDisplay == null) {
|
|
|
|
|
Log.e(TAG, "Failed to create VirtualDisplay")
|
|
|
|
|
stopSelf()
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Log.d(TAG, "Screen capture setup complete")
|
|
|
|
|
// Show floating overlay
|
|
|
|
|
enhancedFloatingFAB?.show()
|
|
|
|
|
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "Error starting screen capture", e)
|
|
|
|
|
stopSelf()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun stopScreenCapture() {
|
|
|
|
|
Log.d(TAG, "Stopping screen capture")
|
|
|
|
|
|
|
|
|
|
handler.removeCallbacks(captureRunnable)
|
|
|
|
|
hideDetectionOverlay()
|
|
|
|
|
enhancedFloatingFAB?.hide()
|
|
|
|
|
latestImage?.close()
|
|
|
|
|
latestImage = null
|
|
|
|
|
virtualDisplay?.release()
|
|
|
|
|
imageReader?.close()
|
|
|
|
|
mediaProjection?.unregisterCallback(mediaProjectionCallback)
|
|
|
|
|
mediaProjection?.stop()
|
|
|
|
|
|
|
|
|
|
virtualDisplay = null
|
|
|
|
|
imageReader = null
|
|
|
|
|
mediaProjection = null
|
|
|
|
|
|
|
|
|
|
stopForeground(true)
|
|
|
|
|
stopSelf()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private val onImageAvailableListener = ImageReader.OnImageAvailableListener { reader ->
|
|
|
|
|
try {
|
|
|
|
|
val image = reader.acquireLatestImage()
|
|
|
|
|
if (image != null) {
|
|
|
|
|
if (autoProcessing) {
|
|
|
|
|
processImage(image)
|
|
|
|
|
image.close()
|
|
|
|
|
} else {
|
|
|
|
|
// Store the latest image for manual processing
|
|
|
|
|
latestImage?.close() // Release previous image
|
|
|
|
|
latestImage = image
|
|
|
|
|
// Don't close the image yet - it will be closed in triggerManualDetection
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "Error in onImageAvailableListener", e)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private var latestImage: Image? = null
|
|
|
|
|
|
|
|
|
|
private fun captureScreen() {
|
|
|
|
|
// Trigger image capture by reading from the ImageReader
|
|
|
|
|
// The onImageAvailableListener will handle the actual processing
|
|
|
|
|
Log.d(TAG, "Triggering screen capture...")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun processImage(image: Image) {
|
|
|
|
|
try {
|
|
|
|
|
val planes = image.planes
|
|
|
|
|
val buffer = planes[0].buffer
|
|
|
|
|
val pixelStride = planes[0].pixelStride
|
|
|
|
|
val rowStride = planes[0].rowStride
|
|
|
|
|
val rowPadding = rowStride - pixelStride * screenWidth
|
|
|
|
|
|
|
|
|
|
Log.d(TAG, "🖼️ CAPTURE DEBUG: pixelStride=$pixelStride, rowStride=$rowStride, rowPadding=$rowPadding")
|
|
|
|
|
Log.d(TAG, "🖼️ CAPTURE DEBUG: screenSize=${screenWidth}x${screenHeight}, expected bitmap=${screenWidth + rowPadding / pixelStride}x${screenHeight}")
|
|
|
|
|
|
|
|
|
|
// Create bitmap from image
|
|
|
|
|
val bitmap = Bitmap.createBitmap(
|
|
|
|
|
screenWidth + rowPadding / pixelStride,
|
|
|
|
|
screenHeight,
|
|
|
|
|
Bitmap.Config.ARGB_8888
|
|
|
|
|
)
|
|
|
|
|
bitmap.copyPixelsFromBuffer(buffer)
|
|
|
|
|
|
|
|
|
|
Log.d(TAG, "🖼️ CAPTURE DEBUG: created bitmap=${bitmap.width}x${bitmap.height}")
|
|
|
|
|
|
|
|
|
|
// Convert to cropped bitmap if needed
|
|
|
|
|
val croppedBitmap = if (rowPadding == 0) {
|
|
|
|
|
Log.d(TAG, "🖼️ CAPTURE DEBUG: No padding, using original bitmap")
|
|
|
|
|
bitmap
|
|
|
|
|
} else {
|
|
|
|
|
Log.d(TAG, "🖼️ CAPTURE DEBUG: Cropping bitmap from ${bitmap.width}x${bitmap.height} to ${screenWidth}x${screenHeight}")
|
|
|
|
|
Bitmap.createBitmap(bitmap, 0, 0, screenWidth, screenHeight)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Log.d(TAG, "🖼️ CAPTURE DEBUG: final bitmap=${croppedBitmap.width}x${croppedBitmap.height}")
|
|
|
|
|
|
|
|
|
|
// Convert to OpenCV Mat for analysis
|
|
|
|
|
val mat = Mat()
|
|
|
|
|
Utils.bitmapToMat(croppedBitmap, mat)
|
|
|
|
|
|
|
|
|
|
// DEBUG: Check color conversion
|
|
|
|
|
Log.d(TAG, "🎨 COLOR DEBUG: Mat type=${mat.type()}, channels=${mat.channels()}")
|
|
|
|
|
Log.d(TAG, "🎨 COLOR DEBUG: OpenCV expects BGR, Android Bitmap is ARGB")
|
|
|
|
|
|
|
|
|
|
// Sample a center pixel to check color values
|
|
|
|
|
if (mat.rows() > 0 && mat.cols() > 0) {
|
|
|
|
|
val centerY = mat.rows() / 2
|
|
|
|
|
val centerX = mat.cols() / 2
|
|
|
|
|
val pixel = mat.get(centerY, centerX)
|
|
|
|
|
if (pixel != null && pixel.size >= 3) {
|
|
|
|
|
val b = pixel[0].toInt()
|
|
|
|
|
val g = pixel[1].toInt()
|
|
|
|
|
val r = pixel[2].toInt()
|
|
|
|
|
Log.d(TAG, "🎨 COLOR DEBUG: Center pixel (${centerX},${centerY}) BGR=($b,$g,$r) -> RGB=(${r},${g},${b})")
|
|
|
|
|
Log.d(TAG, "🎨 COLOR DEBUG: Center pixel hex = #${String.format("%02x%02x%02x", r, g, b)}")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Run YOLO analysis
|
|
|
|
|
analyzePokemonScreen(mat)
|
|
|
|
|
|
|
|
|
|
// Clean up
|
|
|
|
|
if (croppedBitmap != bitmap) {
|
|
|
|
|
croppedBitmap.recycle()
|
|
|
|
|
}
|
|
|
|
|
bitmap.recycle()
|
|
|
|
|
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "Error processing image", e)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun analyzePokemonScreen(mat: Mat) {
|
|
|
|
|
Log.i(TAG, "📱 ANALYZING SCREEN: ${mat.cols()}x${mat.rows()}")
|
|
|
|
|
|
|
|
|
|
// Check if analysis has been stuck for too long (30 seconds max)
|
|
|
|
|
val currentTime = System.currentTimeMillis()
|
|
|
|
|
if (isAnalyzing && (currentTime - analysisStartTime) > 30000) {
|
|
|
|
|
Log.w(TAG, "⚠️ Analysis stuck for >30s, resetting flag")
|
|
|
|
|
isAnalyzing = false
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Skip if already analyzing
|
|
|
|
|
if (isAnalyzing) {
|
|
|
|
|
Log.d(TAG, "⏭️ Skipping analysis - previous cycle still in progress")
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
isAnalyzing = true
|
|
|
|
|
analysisStartTime = currentTime
|
|
|
|
|
Log.d(TAG, "🔄 Starting new analysis cycle")
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
// Run YOLO detection first
|
|
|
|
|
val detections = yoloDetector?.detect(mat) ?: emptyList()
|
|
|
|
|
|
|
|
|
|
if (detections.isEmpty()) {
|
|
|
|
|
Log.i(TAG, "🔍 No Pokemon UI elements detected by ONNX YOLO")
|
|
|
|
|
isAnalyzing = false
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Log.i(TAG, "🎯 ONNX YOLO detected ${detections.size} UI elements")
|
|
|
|
|
|
|
|
|
|
// Log ALL detections for debugging
|
|
|
|
|
detections.forEachIndexed { index, detection ->
|
|
|
|
|
Log.i(TAG, " $index: ${detection.className} (${String.format("%.3f", detection.confidence)}) at [${detection.boundingBox.x}, ${detection.boundingBox.y}, ${detection.boundingBox.width}, ${detection.boundingBox.height}]")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Show breakdown by type
|
|
|
|
|
val detectionCounts = detections.groupBy { it.className }.mapValues { it.value.size }
|
|
|
|
|
Log.i(TAG, "🔍 Detection counts by type: $detectionCounts")
|
|
|
|
|
|
|
|
|
|
// Check for commonly missing elements
|
|
|
|
|
val expectedElements = listOf("pokemon_level", "attack_value", "sp_def_value", "shiny_icon",
|
|
|
|
|
"ball_icon_pokeball", "ball_icon_greatball", "ball_icon_ultraball", "ball_icon_masterball")
|
|
|
|
|
val missingElements = expectedElements.filter { expected ->
|
|
|
|
|
detections.none { it.className.startsWith(expected.split("_").take(2).joinToString("_")) }
|
|
|
|
|
}
|
|
|
|
|
if (missingElements.isNotEmpty()) {
|
|
|
|
|
Log.w(TAG, "⚠️ Missing expected elements: $missingElements")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Show detection overlay IMMEDIATELY (no OCR blocking)
|
|
|
|
|
showYOLODetectionOverlay(detections)
|
|
|
|
|
Log.i(TAG, "📺 Overlay displayed with ${detections.size} detections")
|
|
|
|
|
|
|
|
|
|
// Extract Pokemon info using YOLO detections in background
|
|
|
|
|
extractPokemonInfoFromYOLOAsync(mat, detections)
|
|
|
|
|
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "Error analyzing Pokemon screen", e)
|
|
|
|
|
isAnalyzing = false
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun extractPokemonInfoFromYOLOAsync(mat: Mat, detections: List<Detection>) {
|
|
|
|
|
// Create a copy of the Mat for background processing
|
|
|
|
|
val matCopy = Mat()
|
|
|
|
|
mat.copyTo(matCopy)
|
|
|
|
|
|
|
|
|
|
// Process in background thread
|
|
|
|
|
ocrExecutor.submit {
|
|
|
|
|
try {
|
|
|
|
|
val pokemonInfo = extractPokemonInfoFromYOLO(matCopy, detections)
|
|
|
|
|
|
|
|
|
|
// Post results back to main thread
|
|
|
|
|
handler.post {
|
|
|
|
|
try {
|
|
|
|
|
if (pokemonInfo != null) {
|
|
|
|
|
Log.i(TAG, "🔥 POKEMON DATA EXTRACTED SUCCESSFULLY!")
|
|
|
|
|
logPokemonInfo(pokemonInfo)
|
|
|
|
|
// TODO: Send to your API
|
|
|
|
|
// sendToAPI(pokemonInfo)
|
|
|
|
|
} else {
|
|
|
|
|
Log.i(TAG, "❌ Could not extract complete Pokemon info")
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
// Analysis cycle complete, allow next one
|
|
|
|
|
isAnalyzing = false
|
|
|
|
|
val duration = System.currentTimeMillis() - analysisStartTime
|
|
|
|
|
Log.d(TAG, "✅ Analysis cycle complete after ${duration}ms - ready for next")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Clean up
|
|
|
|
|
matCopy.release()
|
|
|
|
|
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "Error in async Pokemon extraction", e)
|
|
|
|
|
matCopy.release()
|
|
|
|
|
|
|
|
|
|
// Clear flag on error too
|
|
|
|
|
handler.post {
|
|
|
|
|
isAnalyzing = false
|
|
|
|
|
Log.d(TAG, "❌ Analysis cycle failed - ready for next")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun extractPokemonInfoFromYOLO(mat: Mat, detections: List<Detection>): PokemonInfo? {
|
|
|
|
|
try {
|
|
|
|
|
Log.i(TAG, "🎯 Extracting Pokemon info from ${detections.size} YOLO detections")
|
|
|
|
|
|
|
|
|
|
// Group detections by type
|
|
|
|
|
val detectionMap = detections.groupBy { it.className }
|
|
|
|
|
|
|
|
|
|
// Extract key information using YOLO bounding boxes (async OCR calls)
|
|
|
|
|
val ocrResults = mutableMapOf<String, String?>()
|
|
|
|
|
val latch = CountDownLatch(7) // Wait for 7 OCR operations
|
|
|
|
|
|
|
|
|
|
// Submit all OCR tasks in parallel
|
|
|
|
|
submitOCRTask("nickname", mat, detectionMap["pokemon_nickname"]?.firstOrNull(), ocrResults, latch)
|
|
|
|
|
submitOCRTask("species", mat, detectionMap["pokemon_species"]?.firstOrNull(), ocrResults, latch)
|
|
|
|
|
submitOCRTask("nature", mat, detectionMap["nature_name"]?.firstOrNull(), ocrResults, latch)
|
|
|
|
|
submitOCRTask("ability", mat, detectionMap["ability_name"]?.firstOrNull(), ocrResults, latch)
|
|
|
|
|
submitOCRTask("otName", mat, detectionMap["original_trainer_name"]?.firstOrNull(), ocrResults, latch)
|
|
|
|
|
submitOCRTask("otId", mat, detectionMap["original_trainder_number"]?.firstOrNull(), ocrResults, latch)
|
|
|
|
|
// For level, prioritize wider bounding boxes (more likely to contain full level text)
|
|
|
|
|
val levelDetection = detectionMap["pokemon_level"]?.maxByOrNull { it.boundingBox.width }
|
|
|
|
|
submitLevelOCRTask("level", mat, levelDetection, ocrResults, latch)
|
|
|
|
|
|
|
|
|
|
// Wait for all OCR tasks to complete (max 10 seconds total)
|
|
|
|
|
val completed = latch.await(10, TimeUnit.SECONDS)
|
|
|
|
|
if (!completed) {
|
|
|
|
|
Log.w(TAG, "⏱️ Some OCR tasks timed out after 10 seconds")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Extract results
|
|
|
|
|
val nickname = ocrResults["nickname"]
|
|
|
|
|
val level = ocrResults["level"]?.toIntOrNull()
|
|
|
|
|
val species = ocrResults["species"]
|
|
|
|
|
val nature = ocrResults["nature"]
|
|
|
|
|
val ability = ocrResults["ability"]
|
|
|
|
|
val otName = ocrResults["otName"]
|
|
|
|
|
val otId = ocrResults["otId"]
|
|
|
|
|
|
|
|
|
|
// Extract stats (multiple detections)
|
|
|
|
|
val stats = extractStatsFromDetections(mat, detectionMap)
|
|
|
|
|
|
|
|
|
|
// Extract moves (multiple detections)
|
|
|
|
|
val moves = extractMovesFromDetections(mat, detectionMap)
|
|
|
|
|
|
|
|
|
|
// Detect gender
|
|
|
|
|
val gender = when {
|
|
|
|
|
detectionMap["gender_icon_male"]?.isNotEmpty() == true -> "Male"
|
|
|
|
|
detectionMap["gender_icon_female"]?.isNotEmpty() == true -> "Female"
|
|
|
|
|
else -> null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Detect pokeball type
|
|
|
|
|
val pokeballType = detectPokeballTypeFromDetections(detectionMap)
|
|
|
|
|
|
|
|
|
|
// Detect types
|
|
|
|
|
val types = extractTypesFromDetections(mat, detectionMap)
|
|
|
|
|
|
|
|
|
|
// Detect shiny status
|
|
|
|
|
val isShiny = detectionMap["shiny_icon"]?.isNotEmpty() == true
|
|
|
|
|
|
|
|
|
|
// Detect tera type
|
|
|
|
|
val teraType = detectTeraTypeFromDetections(detectionMap)
|
|
|
|
|
|
|
|
|
|
Log.i(TAG, "📊 YOLO extraction summary:")
|
|
|
|
|
Log.i(TAG, " Nickname: '${nickname ?: "null"}'")
|
|
|
|
|
Log.i(TAG, " Level: ${level ?: "null"}")
|
|
|
|
|
Log.i(TAG, " Species: '${species ?: "null"}'")
|
|
|
|
|
Log.i(TAG, " Nature: '${nature ?: "null"}'")
|
|
|
|
|
Log.i(TAG, " Ability: '${ability ?: "null"}'")
|
|
|
|
|
Log.i(TAG, " Gender: '${gender ?: "null"}'")
|
|
|
|
|
Log.i(TAG, " Pokeball: '${pokeballType ?: "null"}'")
|
|
|
|
|
Log.i(TAG, " Types: ${types}")
|
|
|
|
|
Log.i(TAG, " Tera: '${teraType ?: "null"}'")
|
|
|
|
|
Log.i(TAG, " Shiny: $isShiny")
|
|
|
|
|
|
|
|
|
|
if (nickname.isNullOrBlank() && species.isNullOrBlank() && level == null) {
|
|
|
|
|
Log.w(TAG, "⚠️ No essential Pokemon data found with YOLO detection")
|
|
|
|
|
return null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return PokemonInfo(
|
|
|
|
|
pokeballType = pokeballType,
|
|
|
|
|
nickname = nickname,
|
|
|
|
|
gender = gender,
|
|
|
|
|
level = level,
|
|
|
|
|
language = detectLanguage(nickname, species),
|
|
|
|
|
gameSource = detectGameSourceFromDetections(detectionMap),
|
|
|
|
|
isFavorited = false, // TODO: implement favorite detection
|
|
|
|
|
nationalDexNumber = extractDexNumber(species),
|
|
|
|
|
species = species,
|
|
|
|
|
primaryType = types.getOrNull(0),
|
|
|
|
|
secondaryType = types.getOrNull(1),
|
|
|
|
|
stamps = emptyList(), // TODO: implement stamp detection
|
|
|
|
|
labels = emptyList(), // TODO: implement label detection
|
|
|
|
|
marks = emptyList(), // TODO: implement mark detection
|
|
|
|
|
stats = stats,
|
|
|
|
|
moves = moves,
|
|
|
|
|
ability = ability,
|
|
|
|
|
nature = nature,
|
|
|
|
|
originalTrainerName = otName,
|
|
|
|
|
originalTrainerId = otId,
|
|
|
|
|
extractionConfidence = calculateYOLOExtractionConfidence(detections, nickname, level, species)
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "Error extracting Pokemon info from YOLO detections", e)
|
|
|
|
|
return null
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun submitOCRTask(key: String, mat: Mat, detection: Detection?, results: MutableMap<String, String?>, latch: CountDownLatch) {
|
|
|
|
|
ocrExecutor.submit {
|
|
|
|
|
try {
|
|
|
|
|
val text = extractTextFromDetection(mat, detection)
|
|
|
|
|
synchronized(results) {
|
|
|
|
|
results[key] = text
|
|
|
|
|
}
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "Error in OCR task for $key", e)
|
|
|
|
|
synchronized(results) {
|
|
|
|
|
results[key] = null
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
latch.countDown()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun submitLevelOCRTask(key: String, mat: Mat, detection: Detection?, results: MutableMap<String, String?>, latch: CountDownLatch) {
|
|
|
|
|
ocrExecutor.submit {
|
|
|
|
|
try {
|
|
|
|
|
val levelText = extractTextFromDetection(mat, detection)
|
|
|
|
|
val level = levelText?.replace("[^0-9]".toRegex(), "")
|
|
|
|
|
synchronized(results) {
|
|
|
|
|
results[key] = level
|
|
|
|
|
}
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "Error in level OCR task", e)
|
|
|
|
|
synchronized(results) {
|
|
|
|
|
results[key] = null
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
latch.countDown()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun extractTextFromDetection(mat: Mat, detection: Detection?): String? {
|
|
|
|
|
if (detection == null) return null
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
// Expand bounding box by 5% for all OCR classes to improve text extraction accuracy
|
|
|
|
|
val bbox = detection.boundingBox
|
|
|
|
|
val expansionFactor = 0.05f // 5% expansion
|
|
|
|
|
val widthExpansion = (bbox.width * expansionFactor).toInt()
|
|
|
|
|
val heightExpansion = (bbox.height * expansionFactor).toInt()
|
|
|
|
|
|
|
|
|
|
val expandedBbox = Rect(
|
|
|
|
|
bbox.x - widthExpansion,
|
|
|
|
|
bbox.y - heightExpansion,
|
|
|
|
|
bbox.width + (2 * widthExpansion),
|
|
|
|
|
bbox.height + (2 * heightExpansion)
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// Validate and clip bounding box to image boundaries
|
|
|
|
|
val clippedX = kotlin.math.max(0, kotlin.math.min(expandedBbox.x, mat.cols() - 1))
|
|
|
|
|
val clippedY = kotlin.math.max(0, kotlin.math.min(expandedBbox.y, mat.rows() - 1))
|
|
|
|
|
val clippedWidth = kotlin.math.max(1, kotlin.math.min(expandedBbox.width, mat.cols() - clippedX))
|
|
|
|
|
val clippedHeight = kotlin.math.max(1, kotlin.math.min(expandedBbox.height, mat.rows() - clippedY))
|
|
|
|
|
|
|
|
|
|
val safeBbox = Rect(clippedX, clippedY, clippedWidth, clippedHeight)
|
|
|
|
|
|
|
|
|
|
// Debug logging for bounding box transformations
|
|
|
|
|
if (expandedBbox != bbox) {
|
|
|
|
|
Log.d(TAG, "📏 Expanded bbox for ${detection.className}: [${bbox.x},${bbox.y},${bbox.width},${bbox.height}] → [${expandedBbox.x},${expandedBbox.y},${expandedBbox.width},${expandedBbox.height}]")
|
|
|
|
|
}
|
|
|
|
|
if (safeBbox.x != expandedBbox.x || safeBbox.y != expandedBbox.y || safeBbox.width != expandedBbox.width || safeBbox.height != expandedBbox.height) {
|
|
|
|
|
Log.w(TAG, "⚠️ Clipped bbox for ${detection.className}: expanded=[${expandedBbox.x},${expandedBbox.y},${expandedBbox.width},${expandedBbox.height}] → safe=[${safeBbox.x},${safeBbox.y},${safeBbox.width},${safeBbox.height}] (image: ${mat.cols()}x${mat.rows()})")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Extract region of interest using safe bounding box
|
|
|
|
|
val roi = Mat(mat, safeBbox)
|
|
|
|
|
|
|
|
|
|
// Preprocess image for better OCR
|
|
|
|
|
val processedRoi = preprocessImageForOCR(roi)
|
|
|
|
|
|
|
|
|
|
// Convert to bitmap for ML Kit
|
|
|
|
|
val bitmap = Bitmap.createBitmap(processedRoi.cols(), processedRoi.rows(), Bitmap.Config.ARGB_8888)
|
|
|
|
|
Utils.matToBitmap(processedRoi, bitmap)
|
|
|
|
|
|
|
|
|
|
// Use ML Kit OCR
|
|
|
|
|
val extractedText = performOCR(bitmap, detection.className)
|
|
|
|
|
|
|
|
|
|
// Clean up
|
|
|
|
|
roi.release()
|
|
|
|
|
processedRoi.release()
|
|
|
|
|
bitmap.recycle()
|
|
|
|
|
|
|
|
|
|
if (extractedText != null) {
|
|
|
|
|
Log.i(TAG, "✅ YOLO SUCCESS: ${detection.className} = '$extractedText' (conf: ${String.format("%.2f", detection.confidence)})")
|
|
|
|
|
} else {
|
|
|
|
|
Log.w(TAG, "❌ YOLO FAILED: ${detection.className} - no text found (conf: ${String.format("%.2f", detection.confidence)})")
|
|
|
|
|
}
|
|
|
|
|
return extractedText
|
|
|
|
|
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "Error extracting text from YOLO detection ${detection.className}", e)
|
|
|
|
|
return null
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun extractLevelFromDetection(mat: Mat, detection: Detection?): Int? {
|
|
|
|
|
val levelText = extractTextFromDetection(mat, detection)
|
|
|
|
|
return levelText?.replace("[^0-9]".toRegex(), "")?.toIntOrNull()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun extractStatsFromDetections(mat: Mat, detectionMap: Map<String, List<Detection>>): PokemonStats? {
|
|
|
|
|
val hp = extractTextFromDetection(mat, detectionMap["hp_value"]?.firstOrNull())?.toIntOrNull()
|
|
|
|
|
val attack = extractTextFromDetection(mat, detectionMap["attack_value"]?.firstOrNull())?.toIntOrNull()
|
|
|
|
|
val defense = extractTextFromDetection(mat, detectionMap["defense_value"]?.firstOrNull())?.toIntOrNull()
|
|
|
|
|
val spAttack = extractTextFromDetection(mat, detectionMap["sp_atk_value"]?.firstOrNull())?.toIntOrNull()
|
|
|
|
|
val spDefense = extractTextFromDetection(mat, detectionMap["sp_def_value"]?.firstOrNull())?.toIntOrNull()
|
|
|
|
|
val speed = extractTextFromDetection(mat, detectionMap["speed_value"]?.firstOrNull())?.toIntOrNull()
|
|
|
|
|
|
|
|
|
|
return if (hp != null || attack != null || defense != null || spAttack != null || spDefense != null || speed != null) {
|
|
|
|
|
PokemonStats(hp, attack, defense, spAttack, spDefense, speed)
|
|
|
|
|
} else null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun extractMovesFromDetections(mat: Mat, detectionMap: Map<String, List<Detection>>): List<String> {
|
|
|
|
|
val moves = mutableListOf<String>()
|
|
|
|
|
detectionMap["move_name"]?.forEach { detection ->
|
|
|
|
|
val moveText = extractTextFromDetection(mat, detection)
|
|
|
|
|
if (!moveText.isNullOrBlank()) {
|
|
|
|
|
moves.add(moveText.trim())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return moves.take(4) // Pokemon can have max 4 moves
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun detectPokeballTypeFromDetections(detectionMap: Map<String, List<Detection>>): String? {
|
|
|
|
|
// Check for specific pokeball types detected by YOLO
|
|
|
|
|
val pokeballTypes = mapOf(
|
|
|
|
|
"ball_icon_pokeball" to "Poké Ball",
|
|
|
|
|
"ball_icon_greatball" to "Great Ball",
|
|
|
|
|
"ball_icon_ultraball" to "Ultra Ball",
|
|
|
|
|
"ball_icon_heavyball" to "Heavy Ball",
|
|
|
|
|
"ball_icon_premierball" to "Premier Ball",
|
|
|
|
|
"ball_icon_repeatball" to "Repeat Ball",
|
|
|
|
|
"ball_icon_timerball" to "Timer Ball",
|
|
|
|
|
"ball_icon_diveball" to "Dive Ball",
|
|
|
|
|
"ball_icon_quickball" to "Quick Ball",
|
|
|
|
|
"ball_icon_duskball" to "Dusk Ball",
|
|
|
|
|
"ball_icon_cherishball" to "Cherish Ball",
|
|
|
|
|
"ball_icon_originball" to "Origin Ball",
|
|
|
|
|
"ball_icon_pokeball_hisui" to "Hisuian Poké Ball",
|
|
|
|
|
"ball_icon_ultraball_husui" to "Hisuian Ultra Ball"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
for ((className, ballName) in pokeballTypes) {
|
|
|
|
|
if (detectionMap[className]?.isNotEmpty() == true) {
|
|
|
|
|
return ballName
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun extractTypesFromDetections(mat: Mat, detectionMap: Map<String, List<Detection>>): List<String> {
|
|
|
|
|
val types = mutableListOf<String>()
|
|
|
|
|
|
|
|
|
|
extractTextFromDetection(mat, detectionMap["type_1"]?.firstOrNull())?.let { type1 ->
|
|
|
|
|
if (type1.isNotBlank()) types.add(type1.trim())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
extractTextFromDetection(mat, detectionMap["type_2"]?.firstOrNull())?.let { type2 ->
|
|
|
|
|
if (type2.isNotBlank()) types.add(type2.trim())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return types
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun detectTeraTypeFromDetections(detectionMap: Map<String, List<Detection>>): String? {
|
|
|
|
|
val teraTypes = mapOf(
|
|
|
|
|
"tera_ice" to "Ice",
|
|
|
|
|
"tera_fairy" to "Fairy",
|
|
|
|
|
"tera_poison" to "Poison",
|
|
|
|
|
"tera_ghost" to "Ghost",
|
|
|
|
|
"tera_steel" to "Steel",
|
|
|
|
|
"tera_grass" to "Grass",
|
|
|
|
|
"tera_normal" to "Normal",
|
|
|
|
|
"tera_fire" to "Fire",
|
|
|
|
|
"tera_electric" to "Electric",
|
|
|
|
|
"tera_ground" to "Ground",
|
|
|
|
|
"tera_flying" to "Flying",
|
|
|
|
|
"tera_bug" to "Bug",
|
|
|
|
|
"tera_dark" to "Dark",
|
|
|
|
|
"tera_water" to "Water",
|
|
|
|
|
"tera_psychic" to "Psychic",
|
|
|
|
|
"tera_dragon" to "Dragon",
|
|
|
|
|
"tera_fighting" to "Fighting",
|
|
|
|
|
"tera_rock" to "Rock"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
for ((className, teraName) in teraTypes) {
|
|
|
|
|
if (detectionMap[className]?.isNotEmpty() == true) {
|
|
|
|
|
return teraName
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun detectGameSourceFromDetections(detectionMap: Map<String, List<Detection>>): String? {
|
|
|
|
|
val gameSources = mapOf(
|
|
|
|
|
"last_game_stamp_sh" to "Sword/Shield",
|
|
|
|
|
"last_game_stamp_bank" to "Bank",
|
|
|
|
|
"last_game_stamp_pla" to "Legends: Arceus",
|
|
|
|
|
"last_game_stamp_sc" to "Scarlet/Violet",
|
|
|
|
|
"last_game_stamp_vi" to "Violet",
|
|
|
|
|
"last_game_stamp_go" to "Pokémon GO",
|
|
|
|
|
"origin_icon_vc" to "Virtual Console",
|
|
|
|
|
"origin_icon_xyoras" to "XY/ORAS",
|
|
|
|
|
"origin_icon_smusum" to "SM/USUM",
|
|
|
|
|
"origin_icon_swsh" to "Sword/Shield",
|
|
|
|
|
"origin_icon_go" to "Pokémon GO",
|
|
|
|
|
"origin_icon_pla" to "Legends: Arceus",
|
|
|
|
|
"origin_icon_sv" to "Scarlet/Violet"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
for ((className, sourceName) in gameSources) {
|
|
|
|
|
if (detectionMap[className]?.isNotEmpty() == true) {
|
|
|
|
|
return sourceName
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun calculateYOLOExtractionConfidence(detections: List<Detection>, nickname: String?, level: Int?, species: String?): Double {
|
|
|
|
|
var confidence = 0.0
|
|
|
|
|
|
|
|
|
|
// Base confidence from YOLO detections
|
|
|
|
|
val avgDetectionConfidence = detections.map { it.confidence.toDouble() }.average()
|
|
|
|
|
confidence += avgDetectionConfidence * 0.4
|
|
|
|
|
|
|
|
|
|
// Boost confidence based on extracted data
|
|
|
|
|
if (!nickname.isNullOrBlank()) confidence += 0.2
|
|
|
|
|
if (level != null && level > 0) confidence += 0.2
|
|
|
|
|
if (!species.isNullOrBlank()) confidence += 0.2
|
|
|
|
|
|
|
|
|
|
return confidence.coerceIn(0.0, 1.0)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun showYOLODetectionOverlay(detections: List<Detection>) {
|
|
|
|
|
try {
|
|
|
|
|
Log.i(TAG, "🎨 Creating YOLO detection overlay for ${detections.size} detections")
|
|
|
|
|
|
|
|
|
|
if (detectionOverlay == null) {
|
|
|
|
|
Log.i(TAG, "🆕 Creating new DetectionOverlay instance")
|
|
|
|
|
detectionOverlay = DetectionOverlay(this)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Convert YOLO detections to screen regions for overlay
|
|
|
|
|
val statusBarHeight = getStatusBarHeight()
|
|
|
|
|
val regions = detections.mapIndexed { index, detection ->
|
|
|
|
|
"${detection.className}_$index" to ScreenRegion(
|
|
|
|
|
x = detection.boundingBox.x,
|
|
|
|
|
y = detection.boundingBox.y - statusBarHeight, // Subtract status bar offset
|
|
|
|
|
width = detection.boundingBox.width,
|
|
|
|
|
height = detection.boundingBox.height,
|
|
|
|
|
purpose = "${detection.className} (${String.format("%.2f", detection.confidence)})"
|
|
|
|
|
)
|
|
|
|
|
}.toMap()
|
|
|
|
|
|
|
|
|
|
Log.i(TAG, "📺 Showing YOLO overlay with ${regions.size} regions...")
|
|
|
|
|
detectionOverlay?.showOverlay(regions)
|
|
|
|
|
Log.i(TAG, "✅ YOLO overlay show command sent")
|
|
|
|
|
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "❌ Error showing YOLO detection overlay", e)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun hideDetectionOverlay() {
|
|
|
|
|
detectionOverlay?.hideOverlay()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun performOCR(bitmap: Bitmap, purpose: String): String? {
|
|
|
|
|
try {
|
|
|
|
|
Log.d(TAG, "🔍 Starting OCR for $purpose - bitmap: ${bitmap.width}x${bitmap.height}")
|
|
|
|
|
|
|
|
|
|
// Create InputImage for ML Kit
|
|
|
|
|
val image = InputImage.fromBitmap(bitmap, 0)
|
|
|
|
|
val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)
|
|
|
|
|
|
|
|
|
|
// Use CountDownLatch to make async call synchronous
|
|
|
|
|
val latch = CountDownLatch(1)
|
|
|
|
|
var result: String? = null
|
|
|
|
|
var ocrError: Exception? = null
|
|
|
|
|
|
|
|
|
|
recognizer.process(image)
|
|
|
|
|
.addOnSuccessListener { visionText ->
|
|
|
|
|
result = visionText.text.trim()
|
|
|
|
|
Log.d(TAG, "🔍 Raw OCR result for $purpose: '${visionText.text}' (blocks: ${visionText.textBlocks.size})")
|
|
|
|
|
|
|
|
|
|
if (result.isNullOrBlank()) {
|
|
|
|
|
Log.w(TAG, "⚠️ OCR for $purpose: NO TEXT DETECTED - ML Kit found ${visionText.textBlocks.size} text blocks but text is empty")
|
|
|
|
|
} else {
|
|
|
|
|
Log.i(TAG, "✅ OCR SUCCESS for $purpose: '${result}' (${result!!.length} chars)")
|
|
|
|
|
}
|
|
|
|
|
latch.countDown()
|
|
|
|
|
}
|
|
|
|
|
.addOnFailureListener { e ->
|
|
|
|
|
ocrError = e
|
|
|
|
|
Log.e(TAG, "❌ OCR failed for $purpose: ${e.message}", e)
|
|
|
|
|
latch.countDown()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Wait for OCR to complete (max 5 seconds to allow ML Kit to work)
|
|
|
|
|
val completed = latch.await(5, TimeUnit.SECONDS)
|
|
|
|
|
|
|
|
|
|
if (!completed) {
|
|
|
|
|
Log.e(TAG, "⏱️ OCR timeout for $purpose after 5 seconds")
|
|
|
|
|
return null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (ocrError != null) {
|
|
|
|
|
Log.e(TAG, "❌ OCR error for $purpose: ${ocrError!!.message}")
|
|
|
|
|
return null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Clean and process the result
|
|
|
|
|
val cleanedResult = cleanOCRResult(result, purpose)
|
|
|
|
|
Log.d(TAG, "🧙 Cleaned result for $purpose: '${cleanedResult}'")
|
|
|
|
|
return cleanedResult
|
|
|
|
|
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "❌ Error in OCR for $purpose", e)
|
|
|
|
|
return null
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun cleanOCRResult(rawText: String?, purpose: String): String? {
|
|
|
|
|
if (rawText.isNullOrBlank()) return null
|
|
|
|
|
|
|
|
|
|
val cleaned = rawText.trim()
|
|
|
|
|
|
|
|
|
|
return when (purpose) {
|
|
|
|
|
"pokemon_level" -> {
|
|
|
|
|
// Extract level number (look for "Lv." or just numbers)
|
|
|
|
|
val levelRegex = """(?:Lv\.?\s*)?([0-9]+)""".toRegex(RegexOption.IGNORE_CASE)
|
|
|
|
|
levelRegex.find(cleaned)?.groupValues?.get(1)
|
|
|
|
|
}
|
|
|
|
|
"pokemon_nickname", "pokemon_species" -> {
|
|
|
|
|
// Clean up common OCR mistakes for Pokemon names
|
|
|
|
|
cleaned.replace("[^a-zA-Z0-9 \\-.'♂♀]".toRegex(), "")
|
|
|
|
|
.replace("\\s+".toRegex(), " ")
|
|
|
|
|
.trim()
|
|
|
|
|
.takeIf { it.isNotEmpty() }
|
|
|
|
|
}
|
|
|
|
|
"nature_name", "ability_name" -> {
|
|
|
|
|
// Clean up nature/ability names
|
|
|
|
|
cleaned.replace("[^a-zA-Z ]".toRegex(), "")
|
|
|
|
|
.replace("\\s+".toRegex(), " ")
|
|
|
|
|
.trim()
|
|
|
|
|
.takeIf { it.isNotEmpty() }
|
|
|
|
|
}
|
|
|
|
|
"original_trainer_name" -> {
|
|
|
|
|
// Clean trainer names
|
|
|
|
|
cleaned.replace("[^a-zA-Z0-9 ]".toRegex(), "")
|
|
|
|
|
.replace("\\s+".toRegex(), " ")
|
|
|
|
|
.trim()
|
|
|
|
|
.takeIf { it.isNotEmpty() }
|
|
|
|
|
}
|
|
|
|
|
"original_trainder_number" -> {
|
|
|
|
|
// Extract ID numbers
|
|
|
|
|
cleaned.replace("[^0-9]".toRegex(), "")
|
|
|
|
|
.takeIf { it.isNotEmpty() }
|
|
|
|
|
}
|
|
|
|
|
else -> cleaned.takeIf { it.isNotEmpty() }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun preprocessImageForOCR(roi: Mat): Mat {
|
|
|
|
|
try {
|
|
|
|
|
// Resize small regions for better OCR (minimum 150x50 pixels)
|
|
|
|
|
val minWidth = 150
|
|
|
|
|
val minHeight = 50
|
|
|
|
|
val scaledRoi = if (roi.width() < minWidth || roi.height() < minHeight) {
|
|
|
|
|
val scaleX = maxOf(1.0, minWidth.toDouble() / roi.width())
|
|
|
|
|
val scaleY = maxOf(1.0, minHeight.toDouble() / roi.height())
|
|
|
|
|
val scale = maxOf(scaleX, scaleY)
|
|
|
|
|
|
|
|
|
|
val resized = Mat()
|
|
|
|
|
Imgproc.resize(roi, resized, Size(roi.width() * scale, roi.height() * scale))
|
|
|
|
|
Log.d(TAG, "🔍 Upscaled OCR region from ${roi.width()}x${roi.height()} to ${resized.width()}x${resized.height()}")
|
|
|
|
|
resized
|
|
|
|
|
} else {
|
|
|
|
|
val copy = Mat()
|
|
|
|
|
roi.copyTo(copy)
|
|
|
|
|
copy
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Convert to grayscale if needed
|
|
|
|
|
val gray = Mat()
|
|
|
|
|
if (scaledRoi.channels() > 1) {
|
|
|
|
|
Imgproc.cvtColor(scaledRoi, gray, Imgproc.COLOR_RGBA2GRAY)
|
|
|
|
|
} else {
|
|
|
|
|
scaledRoi.copyTo(gray)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Enhance contrast with CLAHE (Contrast Limited Adaptive Histogram Equalization)
|
|
|
|
|
val clahe = Imgproc.createCLAHE(2.0, Size(8.0, 8.0))
|
|
|
|
|
val enhanced = Mat()
|
|
|
|
|
clahe.apply(gray, enhanced)
|
|
|
|
|
|
|
|
|
|
// Light denoising only if text is very small
|
|
|
|
|
val denoised = if (scaledRoi.width() < 100 || scaledRoi.height() < 30) {
|
|
|
|
|
val temp = Mat()
|
|
|
|
|
Imgproc.GaussianBlur(enhanced, temp, Size(1.0, 1.0), 0.0)
|
|
|
|
|
temp
|
|
|
|
|
} else {
|
|
|
|
|
enhanced.clone()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Convert back to RGBA for ML Kit
|
|
|
|
|
val result = Mat()
|
|
|
|
|
Imgproc.cvtColor(denoised, result, Imgproc.COLOR_GRAY2RGBA)
|
|
|
|
|
|
|
|
|
|
// Clean up intermediate matrices
|
|
|
|
|
scaledRoi.release()
|
|
|
|
|
gray.release()
|
|
|
|
|
enhanced.release()
|
|
|
|
|
if (denoised !== enhanced) denoised.release()
|
|
|
|
|
|
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "Error preprocessing image, using original", e)
|
|
|
|
|
// Return copy of original if preprocessing fails
|
|
|
|
|
val result = Mat()
|
|
|
|
|
roi.copyTo(result)
|
|
|
|
|
return result
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun detectLanguage(nickname: String?, species: String?): String? {
|
|
|
|
|
// Simple language detection based on character patterns
|
|
|
|
|
val text = "$nickname $species"
|
|
|
|
|
return when {
|
|
|
|
|
text.any { it in '\u3040'..'\u309F' || it in '\u30A0'..'\u30FF' } -> "JP" // Hiragana/Katakana
|
|
|
|
|
text.any { it in '\u4E00'..'\u9FAF' } -> "ZH" // Chinese characters
|
|
|
|
|
text.any { it in '\uAC00'..'\uD7AF' } -> "KO" // Korean
|
|
|
|
|
else -> "EN" // Default to English
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun extractDexNumber(species: String?): Int? {
|
|
|
|
|
// For now, return null - would need a Pokemon species to dex number mapping
|
|
|
|
|
return null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun logPokemonInfo(pokemonInfo: PokemonInfo) {
|
|
|
|
|
Log.i(TAG, "====== POKEMON INFO EXTRACTED ======")
|
|
|
|
|
Log.i(TAG, "🎾 Pokeball: ${pokemonInfo.pokeballType}")
|
|
|
|
|
Log.i(TAG, "📛 Nickname: ${pokemonInfo.nickname}")
|
|
|
|
|
Log.i(TAG, "⚤ Gender: ${pokemonInfo.gender}")
|
|
|
|
|
Log.i(TAG, "📊 Level: ${pokemonInfo.level}")
|
|
|
|
|
Log.i(TAG, "🌍 Language: ${pokemonInfo.language}")
|
|
|
|
|
Log.i(TAG, "🎮 Game Source: ${pokemonInfo.gameSource}")
|
|
|
|
|
Log.i(TAG, "⭐ Favorited: ${pokemonInfo.isFavorited}")
|
|
|
|
|
Log.i(TAG, "🔢 Dex #: ${pokemonInfo.nationalDexNumber}")
|
|
|
|
|
Log.i(TAG, "🐾 Species: ${pokemonInfo.species}")
|
|
|
|
|
Log.i(TAG, "🏷️ Type 1: ${pokemonInfo.primaryType}")
|
|
|
|
|
Log.i(TAG, "🏷️ Type 2: ${pokemonInfo.secondaryType}")
|
|
|
|
|
Log.i(TAG, "🏆 Stamps: ${pokemonInfo.stamps}")
|
|
|
|
|
Log.i(TAG, "🏷️ Labels: ${pokemonInfo.labels}")
|
|
|
|
|
Log.i(TAG, "✅ Marks: ${pokemonInfo.marks}")
|
|
|
|
|
if (pokemonInfo.stats != null) {
|
|
|
|
|
Log.i(TAG, "📈 Stats: HP:${pokemonInfo.stats.hp} ATK:${pokemonInfo.stats.attack} DEF:${pokemonInfo.stats.defense}")
|
|
|
|
|
Log.i(TAG, " SP.ATK:${pokemonInfo.stats.spAttack} SP.DEF:${pokemonInfo.stats.spDefense} SPD:${pokemonInfo.stats.speed}")
|
|
|
|
|
}
|
|
|
|
|
Log.i(TAG, "⚔️ Moves: ${pokemonInfo.moves}")
|
|
|
|
|
Log.i(TAG, "💪 Ability: ${pokemonInfo.ability}")
|
|
|
|
|
Log.i(TAG, "🎭 Nature: ${pokemonInfo.nature}")
|
|
|
|
|
Log.i(TAG, "👤 OT: ${pokemonInfo.originalTrainerName}")
|
|
|
|
|
Log.i(TAG, "🔢 ID: ${pokemonInfo.originalTrainerId}")
|
|
|
|
|
Log.i(TAG, "🎯 Confidence: ${String.format("%.2f", pokemonInfo.extractionConfidence)}")
|
|
|
|
|
Log.i(TAG, "====================================")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun convertImageToMat(image: Image): Mat? {
|
|
|
|
|
return try {
|
|
|
|
|
val planes = image.planes
|
|
|
|
|
val buffer = planes[0].buffer
|
|
|
|
|
val pixelStride = planes[0].pixelStride
|
|
|
|
|
val rowStride = planes[0].rowStride
|
|
|
|
|
val rowPadding = rowStride - pixelStride * screenWidth
|
|
|
|
|
|
|
|
|
|
// Create bitmap from image
|
|
|
|
|
val bitmap = Bitmap.createBitmap(
|
|
|
|
|
screenWidth + rowPadding / pixelStride,
|
|
|
|
|
screenHeight,
|
|
|
|
|
Bitmap.Config.ARGB_8888
|
|
|
|
|
)
|
|
|
|
|
bitmap.copyPixelsFromBuffer(buffer)
|
|
|
|
|
|
|
|
|
|
// Crop bitmap to remove padding if needed
|
|
|
|
|
val croppedBitmap = if (rowPadding == 0) {
|
|
|
|
|
bitmap
|
|
|
|
|
} else {
|
|
|
|
|
val cropped = Bitmap.createBitmap(bitmap, 0, 0, screenWidth, screenHeight)
|
|
|
|
|
bitmap.recycle() // Clean up original
|
|
|
|
|
cropped
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Convert bitmap to Mat
|
|
|
|
|
val mat = Mat()
|
|
|
|
|
Utils.bitmapToMat(croppedBitmap, mat)
|
|
|
|
|
|
|
|
|
|
// Convert from RGBA to BGR (OpenCV format for proper color channel handling)
|
|
|
|
|
val bgrMat = Mat()
|
|
|
|
|
Imgproc.cvtColor(mat, bgrMat, Imgproc.COLOR_RGBA2BGR)
|
|
|
|
|
|
|
|
|
|
// Clean up
|
|
|
|
|
mat.release()
|
|
|
|
|
croppedBitmap.recycle()
|
|
|
|
|
|
|
|
|
|
bgrMat
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "❌ Error converting image to Mat", e)
|
|
|
|
|
null
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun triggerManualDetection() {
|
|
|
|
|
Log.d(TAG, "🔍 Manual detection triggered via MVC!")
|
|
|
|
|
|
|
|
|
|
latestImage?.let { image ->
|
|
|
|
|
try {
|
|
|
|
|
// Convert image to Mat for processing
|
|
|
|
|
val mat = convertImageToMat(image)
|
|
|
|
|
|
|
|
|
|
if (mat != null) {
|
|
|
|
|
// Use controller to process detection (this will notify UI via callbacks)
|
|
|
|
|
val detections = detectionController.processDetection(mat)
|
|
|
|
|
|
|
|
|
|
// Show detection overlay with results
|
|
|
|
|
if (detections.isNotEmpty()) {
|
|
|
|
|
showYOLODetectionOverlay(detections)
|
|
|
|
|
|
|
|
|
|
// Extract Pokemon info using YOLO detections with OCR
|
|
|
|
|
extractPokemonInfoFromYOLOAsync(mat, detections)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
mat.release()
|
|
|
|
|
} else {
|
|
|
|
|
Log.e(TAG, "❌ Failed to convert image to Mat")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Close the image after processing to free the buffer
|
|
|
|
|
image.close()
|
|
|
|
|
latestImage = null
|
|
|
|
|
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
Log.e(TAG, "❌ Error in manual detection", e)
|
|
|
|
|
}
|
|
|
|
|
} ?: run {
|
|
|
|
|
Log.w(TAG, "⚠️ No image available for detection")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
override fun onDestroy() {
|
|
|
|
|
super.onDestroy()
|
|
|
|
|
hideDetectionOverlay()
|
|
|
|
|
enhancedFloatingFAB?.hide()
|
|
|
|
|
detectionController.clearUICallbacks()
|
|
|
|
|
yoloDetector?.release()
|
|
|
|
|
ocrExecutor.shutdown()
|
|
|
|
|
stopScreenCapture()
|
|
|
|
|
}
|
|
|
|
|
}
|