#!/usr/bin/env python3 """ Compare .pt model predictions with ONNX model outputs on the same static test image Provides detailed debug output to identify differences in preprocessing and inference """ import cv2 import numpy as np from ultralytics import YOLO import onnxruntime as ort import torch import os from pathlib import Path # Force CPU-only execution to avoid CUDA compatibility issues os.environ['CUDA_VISIBLE_DEVICES'] = '' torch.cuda.is_available = lambda: False def letterbox(im, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32): """Letterbox preprocessing - exact copy of YOLO preprocessing""" shape = im.shape[:2] # current shape [height, width] if isinstance(new_shape, int): new_shape = (new_shape, new_shape) # Scale ratio (new / old) r = min(new_shape[0] / shape[0], new_shape[1] / shape[1]) if not scaleup: # only scale down, do not scale up (for better val mAP) r = min(r, 1.0) # Compute padding ratio = r, r # width, height ratios new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r)) dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding if auto: # minimum rectangle dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding elif scaleFill: # stretch dw, dh = 0.0, 0.0 new_unpad = (new_shape[1], new_shape[0]) ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios dw /= 2 # divide padding into 2 sides dh /= 2 if shape[::-1] != new_unpad: # resize im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR) top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1)) left, right = int(round(dw - 0.1)), int(round(dw + 0.1)) im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border return im, ratio, (dw, dh) def preprocess_image(image_path, target_size=(640, 640)): """Preprocess image for ONNX model - matches Android preprocessing""" print(f"šŸ“ø Loading image: {image_path}") # Load image img = cv2.imread(str(image_path)) if img is None: raise ValueError(f"Could not load image: {image_path}") print(f" Original size: {img.shape}") # Apply letterbox (same as YOLO preprocessing) img_processed, ratio, pad = letterbox(img, target_size) print(f" Letterboxed size: {img_processed.shape}") print(f" Scale ratio: {ratio}") print(f" Padding (dw, dh): {pad}") # Convert BGR to RGB img_rgb = cv2.cvtColor(img_processed, cv2.COLOR_BGR2RGB) # Normalize to [0, 1] and convert to CHW format img_normalized = img_rgb.astype(np.float32) / 255.0 img_chw = np.transpose(img_normalized, (2, 0, 1)) img_batch = np.expand_dims(img_chw, axis=0) print(f" Final tensor shape: {img_batch.shape}") print(f" Value range: [{img_batch.min():.3f}, {img_batch.max():.3f}]") return img_batch, img, ratio, pad def run_pt_model(model_path, image_path): """Run .pt model prediction with full debug output""" print("\nšŸ”„ Running .pt model prediction:") print(f" Model: {model_path}") # Load model model = YOLO(model_path) # Run prediction with verbose output results = model.predict( source=str(image_path), conf=0.01, # Very low confidence to catch everything iou=0.5, max_det=1000, verbose=True, save=False ) result = results[0] print(f" Found {len(result.boxes)} detections") # Extract raw data if len(result.boxes) > 0: boxes = result.boxes.xyxy.cpu().numpy() # x1, y1, x2, y2 confidences = result.boxes.conf.cpu().numpy() classes = result.boxes.cls.cpu().numpy().astype(int) print(f"\nšŸ“Š .pt Model Results Summary:") print(f" Total detections: {len(boxes)}") # Group by class class_counts = {} for cls_id in classes: class_counts[cls_id] = class_counts.get(cls_id, 0) + 1 print(f" Classes found: {sorted(class_counts.keys())}") # Focus on shiny icon (class 50) shiny_detections = [(i, conf) for i, (cls_id, conf) in enumerate(zip(classes, confidences)) if cls_id == 50] if shiny_detections: print(f"\n✨ SHINY ICON DETECTIONS (Class 50):") for i, conf in shiny_detections: box = boxes[i] print(f" Detection {i}: conf={conf:.6f}, box=[{box[0]:.1f},{box[1]:.1f},{box[2]:.1f},{box[3]:.1f}]") else: print(f"\nāŒ NO SHINY ICON DETECTIONS (Class 50)") # Show all detections with confidence > 0.1 high_conf_detections = [(i, cls_id, conf) for i, (cls_id, conf) in enumerate(zip(classes, confidences)) if conf > 0.1] if high_conf_detections: print(f"\nšŸŽÆ High confidence detections (>0.1):") for i, cls_id, conf in high_conf_detections[:10]: # Show top 10 box = boxes[i] print(f" Class {cls_id}: conf={conf:.6f}, box=[{box[0]:.1f},{box[1]:.1f},{box[2]:.1f},{box[3]:.1f}]") return boxes, confidences, classes else: print(f"\nāŒ NO DETECTIONS FOUND") return None, None, None def run_onnx_model(model_path, preprocessed_img): """Run ONNX model inference with full debug output""" print(f"\nšŸ”§ Running ONNX model inference:") print(f" Model: {model_path}") # Load ONNX model session = ort.InferenceSession(str(model_path)) # Get model info input_name = session.get_inputs()[0].name output_names = [output.name for output in session.get_outputs()] print(f" Input name: {input_name}") print(f" Output names: {output_names}") print(f" Input shape: {preprocessed_img.shape}") # Run inference outputs = session.run(output_names, {input_name: preprocessed_img}) print(f" Number of outputs: {len(outputs)}") for i, output in enumerate(outputs): print(f" Output {i} shape: {output.shape}") # Process main output (should be detections) detections = outputs[0] # Usually the first output contains detections if len(detections.shape) == 3: batch_size, num_detections, num_values = detections.shape print(f" Detections tensor: [{batch_size}, {num_detections}, {num_values}]") # Extract detections from batch detection_data = detections[0] # Remove batch dimension if num_values == 6: # NMS format: [x, y, w, h, conf, class] print(f" Format: NMS output (x, y, w, h, conf, class)") # Count valid detections (non-zero confidence) valid_mask = detection_data[:, 4] > 0.000001 # conf > 0 valid_detections = detection_data[valid_mask] print(f" Valid detections: {len(valid_detections)} / {num_detections}") if len(valid_detections) > 0: confidences = valid_detections[:, 4] classes = valid_detections[:, 5].astype(int) # Group by class class_counts = {} for cls_id in classes: class_counts[cls_id] = class_counts.get(cls_id, 0) + 1 print(f" Classes found: {sorted(class_counts.keys())}") # Focus on shiny icon (class 50) shiny_mask = classes == 50 shiny_detections = valid_detections[shiny_mask] if len(shiny_detections) > 0: print(f"\n✨ SHINY ICON DETECTIONS (Class 50): {len(shiny_detections)}") for i, det in enumerate(shiny_detections): print(f" Detection {i}: conf={det[4]:.6f}, box=[{det[0]:.1f},{det[1]:.1f},{det[2]:.1f},{det[3]:.1f}]") else: print(f"\nāŒ NO SHINY ICON DETECTIONS (Class 50)") # Show high confidence detections high_conf_mask = confidences > 0.1 high_conf_detections = valid_detections[high_conf_mask] if len(high_conf_detections) > 0: print(f"\nšŸŽÆ High confidence detections (>0.1): {len(high_conf_detections)}") for i, det in enumerate(high_conf_detections[:10]): # Show top 10 print(f" Class {int(det[5])}: conf={det[4]:.6f}, box=[{det[0]:.1f},{det[1]:.1f},{det[2]:.1f},{det[3]:.1f}]") return valid_detections elif num_values > 80: # Raw format: [x, y, w, h, obj, class0, class1, ...] print(f" Format: Raw output ({num_values-5} classes)") # This would need more complex processing for raw outputs print(f" āš ļø Raw format detected - would need objectness * class confidence processing") return None else: print(f" āš ļø Unexpected output shape: {detections.shape}") return None def compare_models(pt_model_path, onnx_model_path, test_image_path): """Compare .pt and ONNX model outputs on the same image""" print("="*80) print("šŸ” MODEL COMPARISON DEBUG SESSION") print("="*80) # Check if files exist for path, name in [(pt_model_path, ".pt model"), (onnx_model_path, "ONNX model"), (test_image_path, "test image")]: if not Path(path).exists(): print(f"āŒ {name} not found: {path}") return # Preprocess image for ONNX try: preprocessed_img, original_img, ratio, pad = preprocess_image(test_image_path) except Exception as e: print(f"āŒ Failed to preprocess image: {e}") return # Run .pt model try: pt_boxes, pt_confidences, pt_classes = run_pt_model(pt_model_path, test_image_path) except Exception as e: print(f"āŒ Failed to run .pt model: {e}") pt_boxes, pt_confidences, pt_classes = None, None, None # Run ONNX model try: onnx_detections = run_onnx_model(onnx_model_path, preprocessed_img) except Exception as e: print(f"āŒ Failed to run ONNX model: {e}") onnx_detections = None # Compare results print("\n" + "="*80) print("šŸ“Š COMPARISON SUMMARY") print("="*80) # Count shiny detections pt_shiny_count = 0 onnx_shiny_count = 0 if pt_classes is not None: pt_shiny_count = np.sum(pt_classes == 50) if onnx_detections is not None and len(onnx_detections) > 0: if onnx_detections.shape[1] == 6: # NMS format onnx_classes = onnx_detections[:, 5].astype(int) onnx_shiny_count = np.sum(onnx_classes == 50) print(f"šŸ”„ .pt Model Results:") print(f" Total detections: {len(pt_boxes) if pt_boxes is not None else 0}") print(f" Shiny icons (class 50): {pt_shiny_count}") print(f"\nšŸ”§ ONNX Model Results:") print(f" Total detections: {len(onnx_detections) if onnx_detections is not None else 0}") print(f" Shiny icons (class 50): {onnx_shiny_count}") if pt_shiny_count > 0 and onnx_shiny_count == 0: print(f"\n🚨 ISSUE CONFIRMED: .pt model finds {pt_shiny_count} shiny icons, ONNX finds 0") print(f" This confirms the preprocessing/inference discrepancy") elif pt_shiny_count == onnx_shiny_count and pt_shiny_count > 0: print(f"\nāœ… Both models find {pt_shiny_count} shiny icons - issue may be elsewhere") print("\n" + "="*80) if __name__ == "__main__": # Test with available models and image pt_model = "raw_models/best.pt" # Test multiple ONNX variants onnx_models = [ "app/src/main/assets/best.onnx", "raw_models/exports/best_no_nms.onnx", "raw_models/exports/best_nms_relaxed.onnx", "raw_models/exports/best_nms_very_relaxed.onnx" ] # You'll need to provide a test image with known shiny icon test_image = "test_images/shiny_test.jpg" # Replace with actual test image path print("šŸ” Looking for test images...") # Try to find a suitable test image test_image_candidates = [ "test_images/shiny_test.jpg", "test_images/test.jpg", "screenshots/shiny.jpg", "screenshots/test.png" ] test_image_found = None for candidate in test_image_candidates: if Path(candidate).exists(): test_image_found = candidate print(f" Found test image: {candidate}") break if not test_image_found: print("āŒ No test image found. Please provide a test image with shiny icon at one of these paths:") for candidate in test_image_candidates: print(f" {candidate}") print("\nYou can capture a screenshot with shiny icon and save it as test_images/shiny_test.jpg") exit(1) # Run comparison for each ONNX model for onnx_model in onnx_models: if Path(onnx_model).exists(): print(f"\nšŸ”„ Testing ONNX model: {onnx_model}") compare_models(pt_model, onnx_model, test_image_found) print("\n" + "="*120 + "\n") else: print(f"āš ļø ONNX model not found: {onnx_model}")