Feature: Release v1.2.0

.gitignore

@@ -4,6 +4,7 @@ settings.json
 
 # SQLite cache
 modaq_upload_cache.db
+s3_cache.db
 
 # Log files
 /logs/

CHANGELOG.md

@@ -0,0 +1,14 @@
+# Changelog
+
+## v1.2.0
+
+- S3 file downloads: download files directly from the S3 browser.
+- Large upload to existing folder: target an existing S3 prefix when running a large folder sync, not just newly created folders.
+- Application log syncing: app logs and global frontend/backend errors are captured and uploaded to S3 for diagnostics.
+- Per-user event logs: unique event logs are generated for each user.
+- Faster S3 browsing: more efficient S3 listing strategy plus frontend improvements for the file browser.
+- OS-standard log directories: logs now use platform-standard locations.
+
+## v1.1.0
+
+- Initial tracked release with in-app updater (git pull + pip install) from Settings.

app/__init__.py

@@ -3,15 +3,30 @@
 import atexit
 import os
 import threading
+import time
+import traceback
 
-from flask import Flask
+from flask import Flask, g, jsonify, request
+from flask.wrappers import Response
+from werkzeug.exceptions import HTTPException
 
 from app.config import get_package_version, get_settings
 
 # Background cleanup thread control
 _cleanup_thread: threading.Thread | None = None
 _cleanup_stop_event = threading.Event()
 
+# Background log-sync thread control
+_log_sync_thread: threading.Thread | None = None
+_log_sync_stop_event = threading.Event()
+# Set to wake the sync worker immediately (e.g. right after an error is logged).
+_log_sync_trigger_event = threading.Event()
+# How often to sync logs to S3 when nothing has triggered an early sync (seconds).
+_LOG_SYNC_INTERVAL = 300
+# Request paths to exclude from automatic per-request logging (noisy / would
+# create feedback loops with the logging pipeline itself).
+_REQUEST_LOG_SKIP_PREFIXES = ("/api/logs", "/api/upload/progress")
+
 
 def _sse_cleanup_worker() -> None:
     """Background worker that periodically cleans up stale SSE queues."""
@@ -52,6 +67,157 @@ def _stop_sse_cleanup() -> None:
         _cleanup_thread.join(timeout=2.0)
 
 
+def _run_log_sync() -> None:
+    """Sync log files to S3 once, if a bucket is configured.
+
+    Failures are logged as WARNING (never ERROR) so that a persistently failing
+    sync cannot re-trigger itself through the error hook and hot-loop.
+    """
+    from app.services import s3_service
+    from app.services.log_service import get_log_service
+
+    settings = get_settings()
+    log = get_log_service()
+
+    if not settings.s3_bucket:
+        return  # Nothing to sync to yet; stay quiet.
+
+    try:
+        client = s3_service.create_s3_client(settings.aws_profile, settings.aws_region)
+        log.sync_logs_to_s3(client, settings.s3_bucket)
+    except Exception as e:
+        log.warning(
+            "sync",
+            "log_sync_failed",
+            f"Automatic log sync to S3 failed: {e}",
+            {"error": str(e)},
+        )
+
+
+def _log_sync_worker() -> None:
+    """Background worker: sync logs to S3 periodically and on demand.
+
+    Wakes every ``_LOG_SYNC_INTERVAL`` seconds, or immediately whenever
+    ``_log_sync_trigger_event`` is set (e.g. just after an error is logged).
+    """
+    while not _log_sync_stop_event.is_set():
+        # Wait for either the interval to elapse or an explicit trigger.
+        _log_sync_trigger_event.wait(timeout=_LOG_SYNC_INTERVAL)
+        if _log_sync_stop_event.is_set():
+            break
+        _log_sync_trigger_event.clear()
+        _run_log_sync()
+
+
+def _start_log_sync() -> None:
+    """Start the background log-sync thread and register the error trigger."""
+    global _log_sync_thread
+
+    from app.services.log_service import get_log_service
+
+    # Logging an ERROR wakes the sync worker for a near-immediate upload.
+    get_log_service().set_error_callback(_log_sync_trigger_event.set)
+
+    if _log_sync_thread is None:
+        _log_sync_thread = threading.Thread(target=_log_sync_worker, daemon=True, name="LogSync")
+        _log_sync_thread.start()
+
+
+def _stop_log_sync() -> None:
+    """Stop the background log-sync thread (flushing one final sync)."""
+    _log_sync_stop_event.set()
+    _log_sync_trigger_event.set()  # Wake it so it can observe the stop flag.
+    if _log_sync_thread:
+        _log_sync_thread.join(timeout=2.0)
+
+
+def _register_logging_hooks(app: Flask) -> None:
+    """Wire up automatic per-request logging and a global error handler.
+
+    - Every ``/api/*`` request is logged (method, path, status, duration) so user
+      actions are captured without per-route boilerplate.
+    - Any uncaught exception is logged at ERROR level with a traceback, which also
+      triggers a near-immediate S3 sync via the log service's error hook.
+    """
+    from app.services.log_service import get_log_service
+
+    def _should_log(path: str) -> bool:
+        return path.startswith("/api/") and not path.startswith(_REQUEST_LOG_SKIP_PREFIXES)
+
+    @app.before_request
+    def _stamp_request_start() -> None:
+        g._request_start = time.monotonic()
+
+    @app.after_request
+    def _log_request(response: Response) -> Response:
+        if _should_log(request.path):
+            start = getattr(g, "_request_start", None)
+            duration_ms = round((time.monotonic() - start) * 1000, 1) if start else None
+            level = "WARNING" if response.status_code >= 400 else "INFO"
+            get_log_service().log(
+                level,
+                "request",
+                "http_request",
+                f"{request.method} {request.path} -> {response.status_code}",
+                {
+                    "method": request.method,
+                    "path": request.path,
+                    "status": response.status_code,
+                    "duration_ms": duration_ms,
+                    "remote_addr": request.remote_addr,
+                },
+            )
+        return response
+
+    @app.errorhandler(Exception)
+    def _handle_uncaught(error: Exception) -> Response | tuple[Response, int] | HTTPException:
+        # Let normal HTTP errors (404, 405, explicit aborts, ...) keep their
+        # default handling; only true crashes are logged as ERROR.
+        if isinstance(error, HTTPException):
+            return error
+        get_log_service().error(
+            "error",
+            "unhandled_exception",
+            f"Unhandled {type(error).__name__} on {request.method} {request.path}: {error}",
+            {
+                "exception_type": type(error).__name__,
+                "method": request.method,
+                "path": request.path,
+                "traceback": traceback.format_exc(),
+            },
+        )
+        return jsonify({"error": "Internal server error"}), 500
+
+
+def _start_s3_warmup() -> None:
+    """Pre-resolve AWS credentials and open an S3 connection in the background.
+
+    The first S3 call in a fresh process pays a one-time cost — credential
+    resolution (which for SSO/assume-role can be slow) plus TLS connection setup.
+    Without warming, that cost lands on the user's first file-browser request,
+    making the initial listing feel slow even though warm calls are ~tens of ms.
+    This does the cold work up front, off the request path, and swallows all
+    errors (no bucket/credentials configured yet is fine).
+    """
+
+    def _warm() -> None:
+        try:
+            settings = get_settings()
+            bucket = settings.s3_bucket
+            if not bucket:
+                return
+            from app.services import s3_service
+
+            client = s3_service.create_s3_client(settings.aws_profile, settings.aws_region)
+            # Cheap call that forces credential resolution + a live connection.
+            client.head_bucket(Bucket=bucket)
+        except Exception:
+            # Warmup is best-effort; the real request will surface any error.
+            pass
+
+    threading.Thread(target=_warm, name="s3-warmup", daemon=True).start()
+
+
 def create_app() -> Flask:
     """Create and configure the Flask application."""
     app = Flask(__name__)
@@ -86,11 +252,21 @@ def inject_display_name() -> dict[str, str]:
     app.register_blueprint(delete_bp, url_prefix="/api/delete")
     app.register_blueprint(large_folder_upload_bp, url_prefix="/api/large-folder-upload")
 
+    # Automatic request/error logging hooks
+    _register_logging_hooks(app)
+
+    # Warm the S3 client/credentials so the first file-browser request isn't cold
+    _start_s3_warmup()
+
     # Start background SSE cleanup thread
     _start_sse_cleanup()
 
+    # Start background log-sync thread (periodic + error-triggered S3 upload)
+    _start_log_sync()
+
     # Register cleanup on shutdown
     atexit.register(_stop_sse_cleanup)
+    atexit.register(_stop_log_sync)
 
     # Log application startup
     from app.services.log_service import get_log_service

app/config.py

@@ -3,9 +3,13 @@
 import functools
 import json
 import os
+import platform
+import re
 import subprocess
 import sys
+import tempfile
 import tomllib
+import uuid
 from pathlib import Path
 from typing import Any
 
@@ -14,6 +18,55 @@
 # Base directory for the project
 BASE_DIR = Path(__file__).resolve().parent.parent
 
+# Folder name used under the OS-specific application-data location for logs.
+APP_DIR_NAME = "modaq_upload"
+
+
+def get_default_log_directory(app_name: str = APP_DIR_NAME) -> Path:
+    """Return the OS-appropriate directory for application logs.
+
+    Follows each platform's conventions for per-user application data so logs
+    live in a writable, persistent, standard location instead of inside the
+    repository (which breaks when the app is installed read-only or moved):
+
+    - Windows: ``%LOCALAPPDATA%\\<app_name>\\logs``
+      (e.g. ``C:\\Users\\you\\AppData\\Local\\modaq_upload\\logs``)
+    - macOS:   ``~/Library/Logs/<app_name>``
+    - Linux/other POSIX: ``$XDG_STATE_HOME/<app_name>/logs``
+      (defaults to ``~/.local/state/<app_name>/logs`` per the XDG Base Directory
+      spec, which places logs under "state" data)
+    """
+    home = Path.home()
+
+    if sys.platform == "win32":
+        base = os.environ.get("LOCALAPPDATA")
+        root = Path(base) if base else home / "AppData" / "Local"
+        return root / app_name / "logs"
+
+    if sys.platform == "darwin":
+        return home / "Library" / "Logs" / app_name
+
+    # Linux and other POSIX systems: XDG state directory.
+    base = os.environ.get("XDG_STATE_HOME")
+    root = Path(base) if base else home / ".local" / "state"
+    return root / app_name / "logs"
+
+
+def _is_under_temp_dir(path: Path) -> bool:
+    """True if ``path`` lives inside the OS temp directory.
+
+    A configured log directory under the system temp dir is almost always a
+    stale value leaked from a test run (pytest's ``tempfile.mkdtemp``), not a
+    real log location — production logs must never be written there.
+    """
+    try:
+        tmp = Path(tempfile.gettempdir()).resolve()
+        resolved = path.resolve()
+        return resolved == tmp or tmp in resolved.parents
+    except OSError:
+        return False
+
+
 # Load environment variables from .env file
 ENV_FILE = BASE_DIR / ".env"
 load_dotenv(ENV_FILE)
@@ -54,6 +107,44 @@ def get_package_name() -> str:
     return _read_pyproject_field("name", "modaq-uploader")
 
 
+def _sanitize_partition_value(value: str) -> str:
+    """Make a string safe to use as a hive partition value (a path segment).
+
+    Replaces path-hostile characters (slashes, ``=``, and whitespace) with ``-`` so
+    the value can't break out of its partition directory or confuse hive parsing.
+    Falls back to ``"unknown"`` when the result is empty.
+    """
+    cleaned = re.sub(r"[/\\=\s]+", "-", value.strip()).strip("-")
+    return cleaned or "unknown"
+
+
+def get_os_name() -> str:
+    """Return a partition-safe OS name (e.g. ``Darwin``/``Windows``/``Linux``)."""
+    return _sanitize_partition_value(platform.system())
+
+
+def get_os_version() -> str:
+    """Return a partition-safe OS version (e.g. ``25.4.0``).
+
+    Uses ``platform.release()`` for cross-platform uniformity (on macOS this is the
+    Darwin kernel version; ``platform.mac_ver()[0]`` would give the product version).
+    """
+    return _sanitize_partition_value(platform.release())
+
+
+def get_session_partitions() -> list[str]:
+    """Return hive partition segments identifying this install's log session.
+
+    Combines OS, OS version, and a persistent per-install ID so logs from multiple
+    machines syncing to one S3 bucket land at distinct, queryable paths.
+    """
+    return [
+        f"os={get_os_name()}",
+        f"os_version={get_os_version()}",
+        f"session={get_settings().install_id}",
+    ]
+
+
 class Settings:
     """Manages application settings stored in JSON format."""
 
@@ -84,7 +175,7 @@ def _load_settings(self) -> None:
             "s3_bucket": "",
             "default_upload_folder": "",
             "display_name": "MODAQ Uploader",
-            "log_directory": "logs",
+            "log_directory": str(get_default_log_directory()),
             "file_categories": [
                 {
                     "name": "data",
@@ -149,6 +240,17 @@ def _load_settings(self) -> None:
                 defaults[key] = value
                 provenance[key] = {"source": "env", "env_var": env_var}
 
+        # Repair an empty or temp-dir log_directory (e.g. a path leaked from a
+        # test run that got persisted to settings.json). Fall back to the
+        # OS-standard location so logs never silently write to a temp dir.
+        raw_log_dir = str(defaults.get("log_directory") or "").strip()
+        resolved_log_dir = (
+            Path(raw_log_dir) if Path(raw_log_dir).is_absolute() else BASE_DIR / raw_log_dir
+        )
+        if not raw_log_dir or _is_under_temp_dir(resolved_log_dir):
+            defaults["log_directory"] = str(get_default_log_directory())
+            provenance["log_directory"] = {"source": "builtin"}
+
         self._settings = defaults
         self._provenance = provenance
 
@@ -212,6 +314,20 @@ def display_name(self) -> str:
         """Get the display name for the application."""
         return str(self._settings["display_name"])
 
+    @property
+    def install_id(self) -> str:
+        """Get a stable per-install identifier, generating and persisting it once.
+
+        Stored in the gitignored settings.json so it survives restarts and stays
+        unique per machine. Used to give each install's logs a distinct S3 path.
+        """
+        existing = self._settings.get("install_id")
+        if existing:
+            return str(existing)
+        new_id = uuid.uuid4().hex[:12]
+        self.set("install_id", new_id)
+        return new_id
+
     @property
     def log_directory(self) -> Path:
         """Get the log directory path (resolved to absolute path relative to BASE_DIR)."""