feat: recaptcha system in FastAPI

.dockerignore

@@ -24,6 +24,9 @@ models/*
 !models/captcha_svm_model.pkl
 !models/label_encoder.pkl
 !models/scaler.pkl
+!models/efficientnet_v2_best.pth
+!models/v2_classes.txt
+!models/cnn_best_model.pth
 
 !app/
 !scripts/

.gitignore

@@ -44,6 +44,9 @@ models/*
 !models/captcha_svm_model.pkl
 !models/label_encoder.pkl
 !models/scaler.pkl
+!models/efficientnet_v2_best.pth
+!models/v2_classes.txt
+!models/cnn_best_model.pth
 
 # Project Specific
 app/uploads/*

app/routers/captcha_v2.py

@@ -3,6 +3,7 @@
 from fastapi import APIRouter, Request
 from fastapi.responses import JSONResponse
 from ..services.dataset_service import dataset_cache
+from ..services.captcha_v2_service import captcha_v2_service
 
 router = APIRouter(prefix="/api/v2")
 
@@ -32,18 +33,42 @@ async def get_challenge():
         "grid": grid_with_images
     }
 
+# ============================
+# Model Integration
+# ============================
+from ..services.captcha_v2_service import captcha_v2_service
+
+@router.on_event("startup")
+async def load_v2_model():
+    captcha_v2_service.load_model()
+
 @router.post("/solve")
 async def solve_v2(request: Request):
-    """Simulate AI solving the 3x3 grid."""
+    """Real AI solving the 3x3 grid using EfficientNet."""
     data = await request.json()
-    grid_ids = data.get("grid_ids", []) # IDs of images currently in the grid
+    grid_ids = data.get("grid_ids", []) 
     target_name = data.get("target")
 
-    if target_name not in dataset_cache.v2_data:
-        return {"correct_indices": []}
-
-    target_files = set(dataset_cache.v2_data[target_name])
-    correct_indices = [idx for idx, img_id in enumerate(grid_ids) if img_id in target_files]
+    correct_indices = []
+
+    # Reverse lookup map (ID -> Path)
+    id_to_path = {}
+    for cat, ids in dataset_cache.v2_data.items():
+        for img_id in ids:
+            id_to_path[img_id] = dataset_cache.v2_root / cat / img_id
+
+    for idx, img_id in enumerate(grid_ids):
+        if img_id not in id_to_path:
+            continue
+
+        img_path = id_to_path[img_id]
+        predicted_class = captcha_v2_service.predict(img_path)
+
+        # Determine match
+        # Note: predicted_class is singular (e.g. 'bus'), target_name is 'bus'.
+        # Exact match.
+        if predicted_class and predicted_class.lower() == target_name.lower():
+            correct_indices.append(idx)
 
     return {"correct_indices": correct_indices}
 

app/services/__init__.py

@@ -0,0 +1,2 @@
+from .dataset_service import dataset_cache
+from .captcha_v2_service import captcha_v2_service

app/services/captcha_v2_service.py

@@ -0,0 +1,115 @@
+import torch
+import torch.nn as nn
+from torchvision import models, transforms
+from PIL import Image
+from pathlib import Path
+import json
+
+# Constants
+DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+if torch.backends.mps.is_available():
+    DEVICE = torch.device('mps')
+
+# Paths are relative to project root usually, but let's make them robust
+BASE_DIR = Path(__file__).parent.parent.parent
+MODEL_PATH = BASE_DIR / "models" / "cnn_best_model.pth"
+METADATA_PATH = BASE_DIR / "models" / "model_metadata_v2.json"
+CLASSES_PATH = BASE_DIR / "models" / "v2_classes.txt"
+
+class CaptchaV2Service:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(CaptchaV2Service, cls).__new__(cls)
+            cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if self._initialized: return
+
+        self.model = None
+        self.classes = []
+        self.transform = transforms.Compose([
+            transforms.Resize((224, 224)),
+            transforms.ToTensor(),
+            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+        ])
+
+        self.load_model()
+        self._initialized = True
+
+    def load_model(self):
+        try:
+            # Load Classes from metadata
+            if METADATA_PATH.exists():
+                with open(METADATA_PATH, "r") as f:
+                    meta = json.load(f)
+                    self.classes = meta.get("classes", [])
+                    print(f"✅ [V2 Service] Loaded {len(self.classes)} classes from metadata")
+
+            if not self.classes:
+                # Fallback classes (11 classes as detected in model weights)
+                self.classes = [
+                    "Bicycle", "Bridge", "Bus", "Car", "Chimney", 
+                    "Crosswalk", "Hydrant", "Motorcycle", "Other", "Palm", "Stair"
+                ]
+                print(f"⚠️ [V2 Service] Using fallback 11 classes")
+
+            # Load Model: cnn_best_model.pth is EfficientNet-B1
+            self.model = models.efficientnet_b1(weights=None)
+
+            # Match the nested classifier structure found in the state_dict (classifier.1.1)
+            num_ftrs = self.model.classifier[1].in_features
+            self.model.classifier[1] = nn.Sequential(
+                nn.Identity(), 
+                nn.Linear(num_ftrs, len(self.classes))
+            )
+
+            if MODEL_PATH.exists():
+                state_dict = torch.load(MODEL_PATH, map_location=DEVICE)
+                # Handle 'model.' prefix if present
+                new_state_dict = {}
+                for k, v in state_dict.items():
+                    if k.startswith("model."):
+                        new_state_dict[k[6:]] = v
+                    else:
+                        new_state_dict[k] = v
+
+                self.model.load_state_dict(new_state_dict)
+                self.model.to(DEVICE)
+                self.model.eval()
+                print(f"✅ [V2 Service] EfficientNet-B1 Model loaded from {MODEL_PATH}")
+            else:
+                print(f"⚠️ [V2 Service] Model file not found at {MODEL_PATH}")
+                self.model = None
+
+        except Exception as e:
+            print(f"❌ [V2 Service] Failed to load model: {e}")
+            self.model = None
+
+    def predict(self, img_path: Path):
+        """
+        Predicts the class of the image at img_path.
+        Returns the class name (str) or None if prediction fails.
+        """
+        if not self.model: 
+            # Try reloading if missing (e.g. usage before startup)
+            self.load_model()
+            if not self.model: return None
+
+        try:
+            img = Image.open(img_path).convert('RGB')
+            img_tensor = self.transform(img).unsqueeze(0).to(DEVICE)
+
+            with torch.no_grad():
+                outputs = self.model(img_tensor)
+                _, pred_idx = torch.max(outputs, 1)
+
+            return self.classes[pred_idx.item()]
+        except Exception as e:
+            print(f"❌ [V2 Service] Prediction error for {img_path}: {e}")
+            return None
+
+# Singleton export
+captcha_v2_service = CaptchaV2Service()

app/services/metadata_service.py

@@ -26,7 +26,29 @@ def load_metadata():
             with open(v2_path, "r") as f:
                 data = json.load(f)
                 STATE["v2_metadata"].update(data)
-                print(f"✅ V2 Metadata loaded from {v2_path.name}")
+
+                # Parse Stats for Dashboard
+                if "fold_results" in data:
+                    best_acc = 0.0
+                    avg_loss = 0.0
+                    total_folds = len(data["fold_results"])
+
+                    for fold in data["fold_results"]:
+                        if fold["best_val_acc"] > best_acc:
+                            best_acc = fold["best_val_acc"]
+
+                        # Take last validation loss as proxy
+                        if "history" in fold and "val_loss" in fold["history"]:
+                             avg_loss += fold["history"]["val_loss"][-1]
+
+                    if total_folds > 0:
+                        avg_loss /= total_folds
+
+                    STATE["v2_metadata"]["accuracy"] = f"{best_acc*100:.2f}%"
+                    STATE["v2_metadata"]["loss"] = f"{avg_loss:.4f}"
+                    STATE["v2_metadata"]["type"] = "EfficientNet-B0 (K-Fold)"
+
+                print(f"✅ V2 Metadata loaded from {v2_path.name} (Acc: {STATE['v2_metadata']['accuracy']})")
         except Exception as e:
             print(f"❌ Failed to load V2 metadata: {e}")
 

models/model_metadata_v2.json

@@ -1,12 +1,25 @@
 {
-    "accuracy": "12.34%",
-    "char_accuracy": "12.34%",
-    "precision": "0.5",
-    "recall": "0.5",
-    "f1_score": "0.5",
-    "loss_value": "0.1",
-    "type": "CNN",
-    "loss": "CrossEntropy",
-    "features": "Image Classification",
-    "preprocessing": "Resize (28x28), Normalization"
-}
+    "accuracy": "72.98%",
+    "char_accuracy": "72.98%",
+    "loss": "1.1722",
+    "loss_value": "1.1722",
+    "precision": "0.67",
+    "recall": "0.73",
+    "f1_score": "0.70",
+    "type": "EfficientNet-B0 (Re-eval)",
+    "timestamp": "2026-01-28T01:00:10",
+    "classes": [
+        "Bicycle",
+        "Bridge",
+        "Bus",
+        "Car",
+        "Chimney",
+        "Crosswalk",
+        "Hydrant",
+        "Motorcycle",
+        "Other",
+        "Palm",
+        "Stair"
+    ],
+    "dataset_path": "/Users/parkyoungdu/Downloads/samples_v2/images"
+}

models/v2_classes.txt

@@ -0,0 +1,12 @@
+Bicycle
+Bridge
+Bus
+Car
+Chimney
+Crosswalk
+Hydrant
+Motorcycle
+Other
+Palm
+Stair
+Traffic Light

requirements.txt

@@ -17,3 +17,7 @@ websockets
 # ML Production (CPU optimized)
 torch --index-url https://download.pytorch.org/whl/cpu
 torchinfo
+torchvision
+mlflow
+opencv-python
+accelerate