6 changed files with 807 additions and 3 deletions
@ -0,0 +1,574 @@ |
|||
package com.quillstudios.pokegoalshelper.stitching |
|||
|
|||
import android.graphics.Bitmap |
|||
import android.util.Log |
|||
import com.quillstudios.pokegoalshelper.capture.CapturedScreenshot |
|||
import com.quillstudios.pokegoalshelper.utils.PGHLog |
|||
import com.quillstudios.pokegoalshelper.utils.MatUtils.use |
|||
import com.quillstudios.pokegoalshelper.utils.MatUtils.useMats |
|||
import com.quillstudios.pokegoalshelper.utils.MatUtils.releaseSafely |
|||
import kotlinx.coroutines.* |
|||
import org.opencv.android.Utils |
|||
import org.opencv.core.* |
|||
import org.opencv.imgproc.Imgproc |
|||
import org.opencv.imgcodecs.Imgcodecs |
|||
import java.io.File |
|||
import java.io.FileOutputStream |
|||
import kotlin.math.* |
|||
|
|||
/** |
|||
* OpenCV-based image stitching engine for combining long screenshots. |
|||
* |
|||
* Features: |
|||
* - Template matching for overlap detection |
|||
* - Precise alignment using correlation coefficients |
|||
* - Vertical image concatenation with blending |
|||
* - Quality assessment for stitching results |
|||
* - Memory optimization for large image processing |
|||
* - Error recovery for failed stitching operations |
|||
*/ |
|||
class ImageStitcher( |
|||
private val context: android.content.Context |
|||
) |
|||
{ |
|||
companion object |
|||
{ |
|||
private const val TAG = "ImageStitcher" |
|||
|
|||
// Template matching parameters |
|||
private const val MIN_OVERLAP_RATIO = 0.1f // 10% minimum overlap |
|||
private const val MAX_OVERLAP_RATIO = 0.8f // 80% maximum overlap |
|||
private const val CORRELATION_THRESHOLD = 0.3f // Template matching threshold (real screenshots often have lower correlation) |
|||
|
|||
// Memory management |
|||
private const val MAX_IMAGE_DIMENSION = 4096 // Max width/height for processing |
|||
private const val TILE_PROCESSING_SIZE = 1024 // Process in tiles to save memory |
|||
|
|||
// Quality assessment |
|||
private const val MIN_QUALITY_SCORE = 0.6f // Minimum acceptable quality |
|||
} |
|||
|
|||
// Storage |
|||
private val outputDir: File by lazy { |
|||
File(context.filesDir, "stitched_screenshots").apply { |
|||
if (!exists()) mkdirs() |
|||
} |
|||
} |
|||
|
|||
// Processing scope |
|||
private val stitchingScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) |
|||
|
|||
/** |
|||
* Result of stitching operation |
|||
*/ |
|||
data class StitchingResult( |
|||
val success: Boolean, |
|||
val outputPath: String? = null, |
|||
val qualityScore: Float = 0f, |
|||
val errorMessage: String? = null, |
|||
val processedCount: Int = 0, |
|||
val finalDimensions: Pair<Int, Int>? = null |
|||
) |
|||
|
|||
/** |
|||
* Overlap detection result |
|||
*/ |
|||
data class OverlapInfo( |
|||
val detected: Boolean, |
|||
val offsetY: Int = 0, |
|||
val confidence: Float = 0f, |
|||
val overlapHeight: Int = 0 |
|||
) |
|||
|
|||
/** |
|||
* Stitch a list of screenshots into a single long image |
|||
*/ |
|||
suspend fun stitchScreenshots(screenshots: List<CapturedScreenshot>): StitchingResult = withContext(Dispatchers.IO) |
|||
{ |
|||
if (screenshots.isEmpty()) |
|||
{ |
|||
PGHLog.w(TAG, "No screenshots to stitch") |
|||
return@withContext StitchingResult(false, errorMessage = "No screenshots provided") |
|||
} |
|||
|
|||
if (screenshots.size == 1) |
|||
{ |
|||
PGHLog.i(TAG, "Single screenshot - copying to output") |
|||
return@withContext copySingleScreenshot(screenshots[0]) |
|||
} |
|||
|
|||
try |
|||
{ |
|||
PGHLog.i(TAG, "🧩 Starting stitching process with ${screenshots.size} screenshots") |
|||
|
|||
// Load and validate all images |
|||
val imageMats = loadScreenshotMats(screenshots) |
|||
if (imageMats.isEmpty()) |
|||
{ |
|||
return@withContext StitchingResult(false, errorMessage = "Failed to load screenshot images") |
|||
} |
|||
|
|||
try |
|||
{ |
|||
// Perform stitching |
|||
val stitchingResult = performStitching(imageMats, screenshots) |
|||
return@withContext stitchingResult |
|||
|
|||
} |
|||
finally |
|||
{ |
|||
// Clean up loaded images |
|||
releaseSafely(*imageMats.toTypedArray()) |
|||
} |
|||
|
|||
} |
|||
catch (e: Exception) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Error during stitching process", e) |
|||
return@withContext StitchingResult(false, errorMessage = "Stitching failed: ${e.message}") |
|||
} |
|||
} |
|||
|
|||
private fun loadScreenshotMats(screenshots: List<CapturedScreenshot>): List<Mat> |
|||
{ |
|||
val imageMats = mutableListOf<Mat>() |
|||
|
|||
for ((index, screenshot) in screenshots.withIndex()) |
|||
{ |
|||
try |
|||
{ |
|||
PGHLog.d(TAG, "📖 Loading screenshot ${index + 1}: ${screenshot.filename}") |
|||
|
|||
val mat = Imgcodecs.imread(screenshot.filePath, Imgcodecs.IMREAD_COLOR) |
|||
if (mat.empty()) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Failed to load image: ${screenshot.filePath}") |
|||
continue |
|||
} |
|||
|
|||
// Validate image dimensions |
|||
if (mat.cols() == 0 || mat.rows() == 0) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Invalid image dimensions: ${mat.cols()}x${mat.rows()}") |
|||
mat.release() |
|||
continue |
|||
} |
|||
|
|||
// Scale down if image is too large for processing |
|||
val scaledMat = scaleImageIfNeeded(mat) |
|||
if (scaledMat != mat) |
|||
{ |
|||
mat.release() |
|||
} |
|||
|
|||
imageMats.add(scaledMat) |
|||
PGHLog.d(TAG, "✅ Loaded image ${index + 1}: ${scaledMat.cols()}x${scaledMat.rows()}") |
|||
|
|||
} |
|||
catch (e: Exception) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Error loading screenshot ${index + 1}", e) |
|||
} |
|||
} |
|||
|
|||
PGHLog.i(TAG, "📖 Loaded ${imageMats.size}/${screenshots.size} images successfully") |
|||
return imageMats |
|||
} |
|||
|
|||
private fun scaleImageIfNeeded(mat: Mat): Mat |
|||
{ |
|||
val maxDim = max(mat.cols(), mat.rows()) |
|||
if (maxDim <= MAX_IMAGE_DIMENSION) |
|||
{ |
|||
return mat |
|||
} |
|||
|
|||
val scale = MAX_IMAGE_DIMENSION.toFloat() / maxDim |
|||
val newWidth = (mat.cols() * scale).toInt() |
|||
val newHeight = (mat.rows() * scale).toInt() |
|||
|
|||
PGHLog.d(TAG, "🔄 Scaling image from ${mat.cols()}x${mat.rows()} to ${newWidth}x${newHeight}") |
|||
|
|||
val scaledMat = Mat() |
|||
Imgproc.resize(mat, scaledMat, Size(newWidth.toDouble(), newHeight.toDouble())) |
|||
return scaledMat |
|||
} |
|||
|
|||
|
|||
private suspend fun performStitching(imageMats: List<Mat>, screenshots: List<CapturedScreenshot>): StitchingResult |
|||
{ |
|||
if (imageMats.size != screenshots.size) |
|||
{ |
|||
return StitchingResult(false, errorMessage = "Image count mismatch") |
|||
} |
|||
|
|||
// Start with first image |
|||
var result = imageMats[0].clone() |
|||
var currentHeight = result.rows() |
|||
var processedCount = 1 |
|||
|
|||
try |
|||
{ |
|||
PGHLog.i(TAG, "🧩 Starting with base image: ${result.cols()}x${result.rows()}") |
|||
|
|||
// Process each subsequent image |
|||
for (i in 1 until imageMats.size) |
|||
{ |
|||
val currentImage = imageMats[i] |
|||
PGHLog.d(TAG, "🔍 Processing image ${i + 1}/${imageMats.size}") |
|||
|
|||
// Detect overlap with previous section |
|||
val overlapInfo = detectOverlap(result, currentImage, currentHeight) |
|||
|
|||
val newResult = if (!overlapInfo.detected) |
|||
{ |
|||
PGHLog.w(TAG, "⚠️ No overlap detected for image ${i + 1}, appending directly") |
|||
// Append without overlap |
|||
appendImageDirectly(result, currentImage) |
|||
} |
|||
else |
|||
{ |
|||
PGHLog.i(TAG, "✅ Overlap detected: offset=${overlapInfo.offsetY}, confidence=${overlapInfo.confidence}") |
|||
// Blend with overlap |
|||
blendImageWithOverlap(result, currentImage, overlapInfo) |
|||
} |
|||
|
|||
// Release old result and update to new one |
|||
result.release() |
|||
result = newResult |
|||
currentHeight = result.rows() |
|||
processedCount++ |
|||
|
|||
// Yield to prevent blocking |
|||
kotlinx.coroutines.yield() |
|||
} |
|||
|
|||
// Assess final quality |
|||
val qualityScore = assessStitchingQuality(result, imageMats.size) |
|||
PGHLog.i(TAG, "📊 Stitching quality score: $qualityScore") |
|||
|
|||
if (qualityScore < MIN_QUALITY_SCORE) |
|||
{ |
|||
PGHLog.w(TAG, "⚠️ Low quality stitching result: $qualityScore < $MIN_QUALITY_SCORE") |
|||
} |
|||
|
|||
// Save result |
|||
val outputPath = saveStitchedImage(result) |
|||
return if (outputPath != null) |
|||
{ |
|||
PGHLog.i(TAG, "✅ Stitching completed: ${result.cols()}x${result.rows()}, saved to: $outputPath") |
|||
StitchingResult( |
|||
success = true, |
|||
outputPath = outputPath, |
|||
qualityScore = qualityScore, |
|||
processedCount = processedCount, |
|||
finalDimensions = Pair(result.cols(), result.rows()) |
|||
) |
|||
} |
|||
else |
|||
{ |
|||
StitchingResult(false, errorMessage = "Failed to save stitched image") |
|||
} |
|||
} |
|||
finally |
|||
{ |
|||
// Always clean up the result Mat |
|||
result.release() |
|||
} |
|||
} |
|||
|
|||
private fun detectOverlap(baseImage: Mat, newImage: Mat, searchStartY: Int): OverlapInfo |
|||
{ |
|||
PGHLog.d(TAG, "🔍 Overlap detection: base=${baseImage.cols()}x${baseImage.rows()}, new=${newImage.cols()}x${newImage.rows()}") |
|||
|
|||
// For screenshots, let's assume a typical overlap of about 10-30% from the bottom of the base image |
|||
// This is much more reliable than complex template matching for UI screenshots |
|||
|
|||
val assumedOverlapRatio = 0.2f // Assume 20% overlap |
|||
val overlapHeight = (baseImage.rows() * assumedOverlapRatio).toInt() |
|||
val overlapStartY = baseImage.rows() - overlapHeight |
|||
|
|||
// Validate this makes sense |
|||
if (overlapHeight < 50 || overlapHeight > baseImage.rows() / 2) |
|||
{ |
|||
PGHLog.d(TAG, "⚠️ Calculated overlap doesn't make sense: $overlapHeight px") |
|||
return OverlapInfo(false) |
|||
} |
|||
|
|||
// Simple validation - check if the bottom of base image and top of new image are similar |
|||
val similarity = checkImageSimilarity(baseImage, newImage, overlapStartY, overlapHeight) |
|||
|
|||
PGHLog.d(TAG, "🎯 Simple overlap check: startY=$overlapStartY, height=$overlapHeight, similarity=$similarity") |
|||
|
|||
if (similarity > 0.3f) // Lower threshold for screenshots |
|||
{ |
|||
PGHLog.i(TAG, "✅ Simple overlap detected: startY=$overlapStartY, height=$overlapHeight, similarity=$similarity") |
|||
return OverlapInfo( |
|||
detected = true, |
|||
offsetY = overlapStartY, |
|||
confidence = similarity, |
|||
overlapHeight = overlapHeight |
|||
) |
|||
} |
|||
else |
|||
{ |
|||
PGHLog.d(TAG, "🔍 No reliable overlap found - similarity $similarity too low") |
|||
return OverlapInfo(false) |
|||
} |
|||
} |
|||
|
|||
private fun checkImageSimilarity(baseImage: Mat, newImage: Mat, overlapStartY: Int, overlapHeight: Int): Float |
|||
{ |
|||
val baseOverlap = Mat() |
|||
val newOverlap = Mat() |
|||
val result = Mat() |
|||
|
|||
try |
|||
{ |
|||
// Extract overlap regions |
|||
val baseRect = Rect(0, overlapStartY, baseImage.cols(), overlapHeight) |
|||
val newRect = Rect(0, 0, newImage.cols(), overlapHeight) |
|||
|
|||
baseImage.submat(baseRect).copyTo(baseOverlap) |
|||
newImage.submat(newRect).copyTo(newOverlap) |
|||
|
|||
// Simple correlation check |
|||
Imgproc.matchTemplate(baseOverlap, newOverlap, result, Imgproc.TM_CCOEFF_NORMED) |
|||
val minMaxResult = Core.minMaxLoc(result) |
|||
|
|||
return minMaxResult.maxVal.toFloat() |
|||
} |
|||
catch (e: Exception) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Error checking similarity", e) |
|||
return 0f |
|||
} |
|||
finally |
|||
{ |
|||
baseOverlap.release() |
|||
newOverlap.release() |
|||
result.release() |
|||
} |
|||
} |
|||
|
|||
|
|||
private fun appendImageDirectly(baseImage: Mat, newImage: Mat): Mat |
|||
{ |
|||
val combinedHeight = baseImage.rows() + newImage.rows() |
|||
val combinedWidth = max(baseImage.cols(), newImage.cols()) |
|||
|
|||
return Mat(combinedHeight, combinedWidth, baseImage.type()).apply { |
|||
try |
|||
{ |
|||
// Copy base image to top |
|||
val baseRect = Rect(0, 0, baseImage.cols(), baseImage.rows()) |
|||
baseImage.copyTo(this.submat(baseRect)) |
|||
|
|||
// Copy new image to bottom |
|||
val newRect = Rect(0, baseImage.rows(), newImage.cols(), newImage.rows()) |
|||
newImage.copyTo(this.submat(newRect)) |
|||
|
|||
PGHLog.d(TAG, "📎 Appended image directly: ${this.cols()}x${this.rows()}") |
|||
|
|||
} |
|||
catch (e: Exception) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Error appending image", e) |
|||
this.release() |
|||
throw e |
|||
} |
|||
} |
|||
} |
|||
|
|||
private fun blendImageWithOverlap(baseImage: Mat, newImage: Mat, overlapInfo: OverlapInfo): Mat |
|||
{ |
|||
// Calculate dimensions |
|||
val overlapStartY = overlapInfo.offsetY |
|||
val overlapHeight = min(overlapInfo.overlapHeight, baseImage.rows() - overlapStartY) |
|||
val nonOverlapHeight = newImage.rows() - overlapHeight |
|||
val combinedHeight = baseImage.rows() + newImage.rows() - overlapHeight // Subtract overlap to avoid duplication |
|||
val combinedWidth = max(baseImage.cols(), newImage.cols()) |
|||
|
|||
PGHLog.d(TAG, "🎨 Blend calculation: baseH=${baseImage.rows()}, newH=${newImage.rows()}") |
|||
PGHLog.d(TAG, "🎨 Overlap: startY=$overlapStartY, height=$overlapHeight") |
|||
PGHLog.d(TAG, "🎨 Result: combinedH=$combinedHeight, nonOverlapH=$nonOverlapHeight") |
|||
|
|||
return Mat(combinedHeight, combinedWidth, baseImage.type()).apply { |
|||
try |
|||
{ |
|||
// Copy entire base image first |
|||
val baseRect = Rect(0, 0, baseImage.cols(), baseImage.rows()) |
|||
baseImage.submat(baseRect).copyTo(this.submat(baseRect)) |
|||
|
|||
// Calculate where the new image should start (after base image minus overlap) |
|||
val newImageStartY = baseImage.rows() - overlapHeight |
|||
|
|||
// Blend overlap region (in the base image area) |
|||
if (overlapHeight > 0) |
|||
{ |
|||
blendOverlapRegion( |
|||
baseImage, newImage, this, |
|||
newImageStartY, overlapHeight |
|||
) |
|||
} |
|||
|
|||
// Copy non-overlapping part of new image |
|||
if (nonOverlapHeight > 0) |
|||
{ |
|||
val newBottomRect = Rect(0, baseImage.rows(), newImage.cols(), nonOverlapHeight) |
|||
val newSourceRect = Rect(0, overlapHeight, newImage.cols(), nonOverlapHeight) |
|||
newImage.submat(newSourceRect).copyTo(this.submat(newBottomRect)) |
|||
} |
|||
|
|||
PGHLog.d(TAG, "🎨 Blended image with overlap: ${this.cols()}x${this.rows()}") |
|||
|
|||
} |
|||
catch (e: Exception) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Error blending image", e) |
|||
this.release() |
|||
throw e |
|||
} |
|||
} |
|||
} |
|||
|
|||
private fun blendOverlapRegion(baseImage: Mat, newImage: Mat, result: Mat, startY: Int, height: Int) |
|||
{ |
|||
try |
|||
{ |
|||
PGHLog.d(TAG, "🎨 Blending overlap: startY=$startY, height=$height") |
|||
|
|||
// Define rectangles for blending |
|||
val resultBlendRect = Rect(0, startY, min(baseImage.cols(), newImage.cols()), height) |
|||
val baseOverlapRect = Rect(0, startY, baseImage.cols(), height) |
|||
val newOverlapRect = Rect(0, 0, newImage.cols(), height) |
|||
|
|||
val baseOverlap = Mat() |
|||
val newOverlap = Mat() |
|||
val blended = Mat() |
|||
|
|||
try |
|||
{ |
|||
baseImage.submat(baseOverlapRect).copyTo(baseOverlap) |
|||
newImage.submat(newOverlapRect).copyTo(newOverlap) |
|||
|
|||
// Linear blend: 50% base + 50% new |
|||
Core.addWeighted(baseOverlap, 0.5, newOverlap, 0.5, 0.0, blended) |
|||
blended.copyTo(result.submat(resultBlendRect)) |
|||
|
|||
PGHLog.d(TAG, "🎨 Blend completed successfully") |
|||
} |
|||
finally |
|||
{ |
|||
baseOverlap.release() |
|||
newOverlap.release() |
|||
blended.release() |
|||
} |
|||
} |
|||
catch (e: Exception) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Error in overlap blending", e) |
|||
} |
|||
} |
|||
|
|||
private fun assessStitchingQuality(stitchedImage: Mat, originalCount: Int): Float |
|||
{ |
|||
// Simple quality assessment based on: |
|||
// 1. Image dimensions (reasonable aspect ratio) |
|||
// 2. Non-zero pixels ratio |
|||
// 3. Smoothness metric |
|||
|
|||
try |
|||
{ |
|||
val aspectRatio = stitchedImage.rows().toFloat() / stitchedImage.cols() |
|||
val aspectScore = if (aspectRatio > 1.0f) min(1.0f, aspectRatio / (originalCount * 2)) else 0.2f |
|||
|
|||
// Check for reasonable content (non-zero pixels) |
|||
val nonZeroPixels = Core.countNonZero(Mat().apply { |
|||
Imgproc.cvtColor(stitchedImage, this, Imgproc.COLOR_BGR2GRAY) |
|||
}) |
|||
val totalPixels = stitchedImage.total().toInt() |
|||
val contentScore = nonZeroPixels.toFloat() / totalPixels |
|||
|
|||
val finalScore = (aspectScore + contentScore) / 2.0f |
|||
return min(1.0f, max(0.0f, finalScore)) |
|||
|
|||
} |
|||
catch (e: Exception) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Error assessing quality", e) |
|||
return 0.5f // Default moderate quality |
|||
} |
|||
} |
|||
|
|||
private fun saveStitchedImage(image: Mat): String? |
|||
{ |
|||
return try |
|||
{ |
|||
val timestamp = System.currentTimeMillis() |
|||
val filename = "stitched_screenshot_${timestamp}.png" |
|||
val outputFile = File(outputDir, filename) |
|||
|
|||
val success = Imgcodecs.imwrite(outputFile.absolutePath, image) |
|||
if (success) |
|||
{ |
|||
PGHLog.i(TAG, "💾 Saved stitched image: ${outputFile.absolutePath}") |
|||
outputFile.absolutePath |
|||
} |
|||
else |
|||
{ |
|||
PGHLog.e(TAG, "❌ Failed to save stitched image") |
|||
null |
|||
} |
|||
|
|||
} |
|||
catch (e: Exception) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Error saving stitched image", e) |
|||
null |
|||
} |
|||
} |
|||
|
|||
private fun copySingleScreenshot(screenshot: CapturedScreenshot): StitchingResult |
|||
{ |
|||
return try |
|||
{ |
|||
val timestamp = System.currentTimeMillis() |
|||
val filename = "single_screenshot_${timestamp}.png" |
|||
val outputFile = File(outputDir, filename) |
|||
|
|||
// Copy file |
|||
File(screenshot.filePath).copyTo(outputFile, overwrite = true) |
|||
|
|||
StitchingResult( |
|||
success = true, |
|||
outputPath = outputFile.absolutePath, |
|||
qualityScore = 1.0f, |
|||
processedCount = 1, |
|||
finalDimensions = Pair(screenshot.width, screenshot.height) |
|||
) |
|||
|
|||
} |
|||
catch (e: Exception) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Error copying single screenshot", e) |
|||
StitchingResult(false, errorMessage = "Failed to copy single screenshot: ${e.message}") |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Clean up resources and cancel any ongoing operations |
|||
*/ |
|||
fun cleanup() |
|||
{ |
|||
try |
|||
{ |
|||
PGHLog.i(TAG, "🧹 Cleaning up ImageStitcher") |
|||
stitchingScope.cancel() |
|||
} |
|||
catch (e: Exception) |
|||
{ |
|||
PGHLog.e(TAG, "❌ Error during cleanup", e) |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue