fix: workers stuck, images not processing after restart, dashboard stats frozen

backend/database.py

@@ -75,6 +75,15 @@ async def init_db(db_path: str) -> aiosqlite.Connection:
     """Open (or create) the SQLite database and ensure tables exist."""
     db = await aiosqlite.connect(db_path)
     db.row_factory = aiosqlite.Row
+
+    # WAL mode: readers never block writers and vice versa — critical for
+    # concurrent dashboard polling + background worker writes.
+    await db.execute("PRAGMA journal_mode=WAL")
+    await db.execute("PRAGMA synchronous=NORMAL")   # safe with WAL, much faster
+    await db.execute("PRAGMA cache_size=-32000")    # 32 MB page cache
+    await db.execute("PRAGMA temp_store=MEMORY")
+    await db.commit()
+
     await db.executescript(_SCHEMA)
     await db.commit()
 
@@ -186,6 +195,13 @@ async def get_image(db: aiosqlite.Connection, image_id: int) -> dict | None:
     return _row_to_dict(row)
 
 
+async def get_image_path(db: aiosqlite.Connection, image_id: int) -> str | None:
+    """Fetch only the file_path for an image — fast path for thumbnail serving."""
+    cursor = await db.execute("SELECT file_path FROM images WHERE id = ?", (image_id,))
+    row = await cursor.fetchone()
+    return row[0] if row else None
+
+
 async def get_image_by_hash(db: aiosqlite.Connection, file_hash: str) -> dict | None:
     """Find an image by its file hash (for dedup/skip detection)."""
     cursor = await db.execute(

backend/main.py

@@ -79,6 +79,17 @@ async def lifespan(app: FastAPI):
     app.state.worker = worker
     await worker.start()
 
+    # Reset any images that were mid-flight when the app last stopped, then
+    # re-enqueue all pending images so they aren't stuck after a restart.
+    await db.execute("UPDATE images SET status = 'pending' WHERE status = 'processing'")
+    await db.commit()
+    cursor = await db.execute("SELECT id FROM images WHERE status = 'pending'")
+    pending_rows = await cursor.fetchall()
+    pending_ids = [row[0] for row in pending_rows]
+    if pending_ids:
+        await worker.enqueue(pending_ids)
+        logger.info("Re-enqueued %d pending image(s) from previous session", len(pending_ids))
+
     # File watcher
     watcher = FileWatcher(db=db, settings=settings, worker=worker)
     app.state.watcher = watcher

backend/routes/images.py

@@ -21,6 +21,7 @@
     delete_all_history,
     delete_reverted_history,
     get_image,
+    get_image_path,
     get_rename_history,
     insert_rename_history,
     list_images,
@@ -167,12 +168,14 @@ async def api_get_image(request: Request, image_id: int):
 async def api_get_thumbnail(request: Request, image_id: int):
     db = request.app.state.db
     settings = request.app.state.settings
-    image = await get_image(db, image_id)
-    if not image:
+
+    # Only fetch the file_path — no need to load the full image row
+    file_path = await get_image_path(db, image_id)
+    if not file_path:
         raise HTTPException(404, "Image not found")
 
     photos_dir = Path(settings.photos_dir)
-    source_path = _safe_path(photos_dir, image["file_path"])
+    source_path = _safe_path(photos_dir, file_path)
     if not source_path.exists():
         raise HTTPException(404, "Source image not found")
 
@@ -184,7 +187,11 @@ async def api_get_thumbnail(request: Request, image_id: int):
     if not thumb:
         raise HTTPException(500, "Failed to generate thumbnail")
 
-    return FileResponse(thumb, media_type="image/jpeg")
+    return FileResponse(
+        thumb,
+        media_type="image/jpeg",
+        headers={"Cache-Control": "max-age=86400, immutable"},
+    )
 
 
 @router.get("/images/{image_id}/file")
@@ -282,6 +289,20 @@ async def api_process_batch(request: Request, body: BatchProcessRequest):
     return {"status": "enqueued", "count": count}
 
 
+@router.post("/images/retry-all-errors")
+async def api_retry_all_errors(request: Request):
+    """Reset all error images to pending and enqueue them for reprocessing."""
+    db = request.app.state.db
+    await db.execute("UPDATE images SET status = 'pending', error_message = NULL WHERE status = 'error'")
+    await db.commit()
+    cursor = await db.execute("SELECT id FROM images WHERE status = 'pending'")
+    rows = await cursor.fetchall()
+    ids = [row[0] for row in rows]
+    worker = request.app.state.worker
+    count = await worker.enqueue(ids)
+    return {"status": "enqueued", "count": count}
+
+
 @router.post("/images/download-batch")
 async def api_download_batch(request: Request, body: BatchDownloadRequest):
     """Generate a zip file containing selected images and stream it."""

backend/routes/scan.py

@@ -42,6 +42,23 @@ async def _do_scan():
     return {"status": "started", "message": "Scan started" + (" with context" if context else "")}
 
 
+@router.post("/scan/resume")
+async def resume_processing(request: Request):
+    """Clear the stop flag and re-enqueue any pending images."""
+    worker = request.app.state.worker
+    db = request.app.state.db
+
+    worker.clear_stop()
+
+    cursor = await db.execute("SELECT id FROM images WHERE status = 'pending'")
+    rows = await cursor.fetchall()
+    pending_ids = [row[0] for row in rows]
+    if pending_ids:
+        await worker.enqueue(pending_ids)
+
+    return {"resumed": True, "enqueued": len(pending_ids)}
+
+
 @router.post("/scan/stop")
 async def stop_processing(request: Request):
     """Stop all active processing gracefully."""

backend/routes/settings.py

@@ -91,6 +91,8 @@ async def _apply_settings_update(request: Request, body: SettingsUpdate):
 
     # Update worker, watcher, scheduler, and workspace references
     request.app.state.worker.settings = new_settings
+    if "concurrent_workers" in updates:
+        await request.app.state.worker.resize(new_settings.concurrent_workers)
     request.app.state.watcher.settings = new_settings
     scheduler = getattr(request.app.state, "scheduler", None)
     if scheduler:

backend/thumbnails.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import logging
 from pathlib import Path
 
@@ -10,6 +11,23 @@
 logger = logging.getLogger(__name__)
 
 
+def _generate_thumbnail_sync(
+    source_path: Path,
+    thumb_path: Path,
+    max_size: int,
+    quality: int,
+) -> Path | None:
+    """CPU/IO-bound thumbnail work — runs in a thread pool, not the event loop."""
+    try:
+        img = open_image(source_path)
+        img.thumbnail((max_size, max_size), Image.LANCZOS)
+        img.save(thumb_path, "JPEG", quality=quality)
+        return thumb_path
+    except Exception:
+        logger.warning("Failed to generate thumbnail for %s", source_path, exc_info=True)
+        return None
+
+
 async def get_or_create_thumbnail(
     image_id: int,
     source_path: Path,
@@ -30,19 +48,13 @@ async def get_or_create_thumbnail(
     if thumb_path.exists():
         return thumb_path
 
-    try:
-        img = open_image(source_path)
-
-        # Thumbnail preserves aspect ratio, fits within max_size x max_size
-        img.thumbnail((max_size, max_size), Image.LANCZOS)
-        img.save(thumb_path, "JPEG", quality=quality)
-
+    # Run PIL work in a thread so it doesn't block the async event loop
+    result = await asyncio.to_thread(
+        _generate_thumbnail_sync, source_path, thumb_path, max_size, quality
+    )
+    if result:
         logger.debug("Generated thumbnail for image %d at %s", image_id, thumb_path)
-        return thumb_path
-
-    except Exception:
-        logger.warning("Failed to generate thumbnail for %s", source_path, exc_info=True)
-        return None
+    return result
 
 
 async def delete_thumbnail(image_id: int, data_dir: Path) -> None:

backend/watcher.py

@@ -217,7 +217,7 @@ async def _scan(self) -> tuple[int, int, list[int]]:
         photos_dir = Path(self.settings.photos_dir)
         if not photos_dir.exists():
             logger.warning("Photos directory does not exist: %s", photos_dir)
-            return 0, 0
+            return 0, 0, []
 
         new_ids: list[int] = []
         skipped = 0

backend/worker.py

@@ -123,6 +123,24 @@ async def stop(self) -> None:
         self._tasks.clear()
         logger.info("Workers stopped")
 
+    async def resize(self, new_count: int) -> None:
+        """Stop all workers and restart with new_count — called when the setting changes."""
+        if new_count == len(self._tasks):
+            return
+        self._running = False
+        self._resume_event.set()
+        for _ in self._tasks:
+            await self._queue.put(-1)
+        for task in self._tasks:
+            task.cancel()
+        await asyncio.gather(*self._tasks, return_exceptions=True)
+        self._tasks.clear()
+        self._running = True
+        for i in range(new_count):
+            task = asyncio.create_task(self._worker_loop(i))
+            self._tasks.append(task)
+        logger.info("Worker pool resized to %d", new_count)
+
     async def enqueue(self, image_ids: list[int]) -> int:
         """Add image IDs to the processing queue. Returns count enqueued."""
         self._stop_requested = False  # Clear any previous stop

frontend/static/js/app.js

@@ -161,6 +161,9 @@ function updateDashboard() {
                 _setText('review-count', proposed);
             }
 
+            // Worker data needed below
+            const w = data.worker || {};
+
             // Toggle idle/active progress sections
             const p = data.progress || {};
             var isActive = (stats.pending || 0) + (stats.processing || 0) > 0;
@@ -169,22 +172,25 @@ function updateDashboard() {
             if (progressActive) progressActive.style.display = isActive ? '' : 'none';
             if (progressIdle) progressIdle.style.display = isActive ? 'none' : '';
 
-            // Stop button visibility
+            // Stop / Resume button
             var stopBtn = document.getElementById('stop-processing-btn');
             if (stopBtn) {
                 var stopRequested = w.stop_requested || false;
                 if (!isActive) {
                     stopBtn.style.display = 'none';
                     stopBtn.disabled = false;
                     stopBtn.textContent = 'Stop Processing';
+                    stopBtn.onclick = stopProcessing;
                 } else if (stopRequested) {
                     stopBtn.style.display = '';
-                    stopBtn.disabled = true;
-                    stopBtn.textContent = 'Stopping...';
+                    stopBtn.disabled = false;
+                    stopBtn.textContent = 'Resume Processing';
+                    stopBtn.onclick = resumeProcessing;
                 } else {
                     stopBtn.style.display = '';
                     stopBtn.disabled = false;
                     stopBtn.textContent = 'Stop Processing';
+                    stopBtn.onclick = stopProcessing;
                 }
             }
 
@@ -224,7 +230,6 @@ function updateDashboard() {
             updateScheduleStatus(data);
 
             // Update worker status
-            const w = data.worker || {};
             var workerLabel = w.running ? 'Active' : 'Stopped';
             if (w.paused) workerLabel = 'Paused (scheduled)';
             _setText('worker-running', workerLabel);
@@ -600,7 +605,18 @@ document.addEventListener('DOMContentLoaded', function() {
     // Dashboard page: start live polling + init upload/workspace
     if (document.getElementById('stat-total')) {
         updateDashboard();
-        setInterval(updateDashboard, 5000);
+        // Poll fast (5s) while active, slow (15s) when idle to reduce DB load
+        var _dashInterval = null;
+        function _scheduleDashPoll(active) {
+            if (_dashInterval) clearInterval(_dashInterval);
+            _dashInterval = setInterval(function() {
+                var wasActive = (document.getElementById('progress-active') || {}).style &&
+                    document.getElementById('progress-active').style.display !== 'none';
+                updateDashboard();
+                _scheduleDashPoll(wasActive);
+            }, active ? 5000 : 15000);
+        }
+        _scheduleDashPoll(false);
 
         // Library upload drop zone
         initUploadDropZone('upload-drop-zone', 'upload-file-input', function(files) {
@@ -2647,6 +2663,26 @@ function stopProcessing() {
         .catch(function() { showToast('Failed to stop processing', 'error'); });
 }
 
+function resumeProcessing() {
+    var btn = document.getElementById('stop-processing-btn');
+    if (btn) { btn.textContent = 'Resuming...'; btn.disabled = true; }
+    fetch('/api/scan/resume', { method: 'POST' })
+        .then(function(r) { if (!r.ok) throw new Error(r.status); return r.json(); })
+        .then(function() { showToast('Processing resumed'); updateDashboard(); })
+        .catch(function() { showToast('Failed to resume processing', 'error'); });
+}
+
+function retryAllErrors() {
+    var prefix = (typeof API_PREFIX !== 'undefined' ? API_PREFIX : '/api');
+    fetch(prefix + '/images/retry-all-errors', { method: 'POST' })
+        .then(function(r) { if (!r.ok) throw new Error(r.status); return r.json(); })
+        .then(function(data) {
+            showToast('Retrying ' + (data.count || 0) + ' error image(s)');
+            setTimeout(function() { window.location.reload(); }, 600);
+        })
+        .catch(function() { showToast('Failed to retry errors', 'error'); });
+}
+
 // ── Settings: Processing Modes ──────────────────────────────
 
 function toggleCatalogueMode() {

frontend/templates/queue.html

@@ -89,6 +89,7 @@ <h1>{% if workspace_mode %}Workspace {% endif %}All Images</h1>
         {% if workspace_mode %}<button class="btn btn-sm btn-error" onclick="wsBulkDelete()">Delete Selected</button>{% endif %}
         {% elif status_filter == 'error' %}
         <button class="btn btn-sm btn-primary" onclick="bulkAction('process')">Retry Selected</button>
+        <button class="btn btn-sm btn-primary" onclick="retryAllErrors()">Retry All Errors</button>
         <button class="btn btn-sm" onclick="bulkQueueAdd()">Add to Queue</button>
         {% if workspace_mode %}<button class="btn btn-sm btn-error" onclick="wsBulkDelete()">Delete Selected</button>{% else %}<button class="btn btn-sm btn-error" onclick="bulkTrash()">Delete Selected</button>{% endif %}
         {% elif status_filter == 'quality_issues' %}