Feature: Add S3 Downloads

.gitignore

@@ -4,6 +4,7 @@ settings.json
 
 # SQLite cache
 modaq_upload_cache.db
+s3_cache.db
 
 # Log files
 /logs/

app/__init__.py

@@ -189,6 +189,35 @@ def _handle_uncaught(error: Exception) -> Response | tuple[Response, int] | HTTP
         return jsonify({"error": "Internal server error"}), 500
 
 
+def _start_s3_warmup() -> None:
+    """Pre-resolve AWS credentials and open an S3 connection in the background.
+
+    The first S3 call in a fresh process pays a one-time cost — credential
+    resolution (which for SSO/assume-role can be slow) plus TLS connection setup.
+    Without warming, that cost lands on the user's first file-browser request,
+    making the initial listing feel slow even though warm calls are ~tens of ms.
+    This does the cold work up front, off the request path, and swallows all
+    errors (no bucket/credentials configured yet is fine).
+    """
+
+    def _warm() -> None:
+        try:
+            settings = get_settings()
+            bucket = settings.s3_bucket
+            if not bucket:
+                return
+            from app.services import s3_service
+
+            client = s3_service.create_s3_client(settings.aws_profile, settings.aws_region)
+            # Cheap call that forces credential resolution + a live connection.
+            client.head_bucket(Bucket=bucket)
+        except Exception:
+            # Warmup is best-effort; the real request will surface any error.
+            pass
+
+    threading.Thread(target=_warm, name="s3-warmup", daemon=True).start()
+
+
 def create_app() -> Flask:
     """Create and configure the Flask application."""
     app = Flask(__name__)
@@ -226,6 +255,9 @@ def inject_display_name() -> dict[str, str]:
     # Automatic request/error logging hooks
     _register_logging_hooks(app)
 
+    # Warm the S3 client/credentials so the first file-browser request isn't cold
+    _start_s3_warmup()
+
     # Start background SSE cleanup thread
     _start_sse_cleanup()
 

app/config.py

@@ -3,9 +3,13 @@
 import functools
 import json
 import os
+import platform
+import re
 import subprocess
 import sys
+import tempfile
 import tomllib
+import uuid
 from pathlib import Path
 from typing import Any
 
@@ -14,6 +18,55 @@
 # Base directory for the project
 BASE_DIR = Path(__file__).resolve().parent.parent
 
+# Folder name used under the OS-specific application-data location for logs.
+APP_DIR_NAME = "modaq_upload"
+
+
+def get_default_log_directory(app_name: str = APP_DIR_NAME) -> Path:
+    """Return the OS-appropriate directory for application logs.
+
+    Follows each platform's conventions for per-user application data so logs
+    live in a writable, persistent, standard location instead of inside the
+    repository (which breaks when the app is installed read-only or moved):
+
+    - Windows: ``%LOCALAPPDATA%\\<app_name>\\logs``
+      (e.g. ``C:\\Users\\you\\AppData\\Local\\modaq_upload\\logs``)
+    - macOS:   ``~/Library/Logs/<app_name>``
+    - Linux/other POSIX: ``$XDG_STATE_HOME/<app_name>/logs``
+      (defaults to ``~/.local/state/<app_name>/logs`` per the XDG Base Directory
+      spec, which places logs under "state" data)
+    """
+    home = Path.home()
+
+    if sys.platform == "win32":
+        base = os.environ.get("LOCALAPPDATA")
+        root = Path(base) if base else home / "AppData" / "Local"
+        return root / app_name / "logs"
+
+    if sys.platform == "darwin":
+        return home / "Library" / "Logs" / app_name
+
+    # Linux and other POSIX systems: XDG state directory.
+    base = os.environ.get("XDG_STATE_HOME")
+    root = Path(base) if base else home / ".local" / "state"
+    return root / app_name / "logs"
+
+
+def _is_under_temp_dir(path: Path) -> bool:
+    """True if ``path`` lives inside the OS temp directory.
+
+    A configured log directory under the system temp dir is almost always a
+    stale value leaked from a test run (pytest's ``tempfile.mkdtemp``), not a
+    real log location — production logs must never be written there.
+    """
+    try:
+        tmp = Path(tempfile.gettempdir()).resolve()
+        resolved = path.resolve()
+        return resolved == tmp or tmp in resolved.parents
+    except OSError:
+        return False
+
+
 # Load environment variables from .env file
 ENV_FILE = BASE_DIR / ".env"
 load_dotenv(ENV_FILE)
@@ -54,6 +107,44 @@ def get_package_name() -> str:
     return _read_pyproject_field("name", "modaq-uploader")
 
 
+def _sanitize_partition_value(value: str) -> str:
+    """Make a string safe to use as a hive partition value (a path segment).
+
+    Replaces path-hostile characters (slashes, ``=``, and whitespace) with ``-`` so
+    the value can't break out of its partition directory or confuse hive parsing.
+    Falls back to ``"unknown"`` when the result is empty.
+    """
+    cleaned = re.sub(r"[/\\=\s]+", "-", value.strip()).strip("-")
+    return cleaned or "unknown"
+
+
+def get_os_name() -> str:
+    """Return a partition-safe OS name (e.g. ``Darwin``/``Windows``/``Linux``)."""
+    return _sanitize_partition_value(platform.system())
+
+
+def get_os_version() -> str:
+    """Return a partition-safe OS version (e.g. ``25.4.0``).
+
+    Uses ``platform.release()`` for cross-platform uniformity (on macOS this is the
+    Darwin kernel version; ``platform.mac_ver()[0]`` would give the product version).
+    """
+    return _sanitize_partition_value(platform.release())
+
+
+def get_session_partitions() -> list[str]:
+    """Return hive partition segments identifying this install's log session.
+
+    Combines OS, OS version, and a persistent per-install ID so logs from multiple
+    machines syncing to one S3 bucket land at distinct, queryable paths.
+    """
+    return [
+        f"os={get_os_name()}",
+        f"os_version={get_os_version()}",
+        f"session={get_settings().install_id}",
+    ]
+
+
 class Settings:
     """Manages application settings stored in JSON format."""
 
@@ -84,7 +175,7 @@ def _load_settings(self) -> None:
             "s3_bucket": "",
             "default_upload_folder": "",
             "display_name": "MODAQ Uploader",
-            "log_directory": "logs",
+            "log_directory": str(get_default_log_directory()),
             "file_categories": [
                 {
                     "name": "data",
@@ -149,6 +240,17 @@ def _load_settings(self) -> None:
                 defaults[key] = value
                 provenance[key] = {"source": "env", "env_var": env_var}
 
+        # Repair an empty or temp-dir log_directory (e.g. a path leaked from a
+        # test run that got persisted to settings.json). Fall back to the
+        # OS-standard location so logs never silently write to a temp dir.
+        raw_log_dir = str(defaults.get("log_directory") or "").strip()
+        resolved_log_dir = (
+            Path(raw_log_dir) if Path(raw_log_dir).is_absolute() else BASE_DIR / raw_log_dir
+        )
+        if not raw_log_dir or _is_under_temp_dir(resolved_log_dir):
+            defaults["log_directory"] = str(get_default_log_directory())
+            provenance["log_directory"] = {"source": "builtin"}
+
         self._settings = defaults
         self._provenance = provenance
 
@@ -212,6 +314,20 @@ def display_name(self) -> str:
         """Get the display name for the application."""
         return str(self._settings["display_name"])
 
+    @property
+    def install_id(self) -> str:
+        """Get a stable per-install identifier, generating and persisting it once.
+
+        Stored in the gitignored settings.json so it survives restarts and stays
+        unique per machine. Used to give each install's logs a distinct S3 path.
+        """
+        existing = self._settings.get("install_id")
+        if existing:
+            return str(existing)
+        new_id = uuid.uuid4().hex[:12]
+        self.set("install_id", new_id)
+        return new_id
+
     @property
     def log_directory(self) -> Path:
         """Get the log directory path (resolved to absolute path relative to BASE_DIR)."""

app/routes/files.py

@@ -29,12 +29,15 @@ def list_files() -> tuple[Response, int]:
     Query parameters:
         prefix: S3 prefix to filter by (default: "")
         delimiter: Delimiter for grouping (default: "/")
+        token: Continuation token from a previous response's next_token, to fetch
+            the next page of a folder with more results than fit in one page
 
     Returns:
         JSON response with folders and files
     """
     prefix = request.args.get("prefix", "")
     delimiter = request.args.get("delimiter", "/")
+    token = request.args.get("token") or None
 
     try:
         client = s3_service.create_s3_client(
@@ -47,6 +50,7 @@ def list_files() -> tuple[Response, int]:
             g.settings.s3_bucket,
             prefix=prefix,
             delimiter=delimiter,
+            continuation_token=token,
         )
 
         if not result["success"]:
@@ -103,6 +107,77 @@ def get_file_info() -> tuple[Response, int]:
         return jsonify({"error": str(e)}), 500
 
 
+@files_bp.route("/stats", methods=["GET"])
+def get_prefix_stats() -> tuple[Response, int]:
+    """Get subfolder and file counts for a folder (one level).
+
+    Query parameters:
+        prefix: S3 prefix to summarize (default: "" = bucket root)
+        delimiter: Delimiter for grouping (default: "/")
+
+    Returns:
+        JSON response with folder_count and file_count
+    """
+    prefix = request.args.get("prefix", "")
+    delimiter = request.args.get("delimiter", "/")
+
+    try:
+        client = s3_service.create_s3_client(
+            g.settings.aws_profile,
+            g.settings.aws_region,
+        )
+
+        result = s3_service.get_prefix_counts(
+            client,
+            g.settings.s3_bucket,
+            prefix=prefix,
+            delimiter=delimiter,
+        )
+
+        if not result["success"]:
+            return jsonify({"error": result["error"]}), 500
+
+        return jsonify(result), 200
+
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
+@files_bp.route("/download", methods=["GET"])
+def download_file() -> tuple[Response, int]:
+    """Generate a presigned URL for downloading an S3 object.
+
+    Query parameters:
+        key: S3 object key
+
+    Returns:
+        JSON response with a presigned download URL
+    """
+    key = request.args.get("key", "")
+    if not key:
+        return jsonify({"error": "Object key required"}), 400
+
+    try:
+        client = s3_service.create_s3_client(
+            g.settings.aws_profile,
+            g.settings.aws_region,
+        )
+
+        result = s3_service.generate_presigned_download_url(
+            client,
+            g.settings.s3_bucket,
+            key,
+        )
+
+        if not result["success"]:
+            return jsonify({"error": result["error"]}), 404
+
+        return jsonify(result), 200
+
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
 @files_bp.route("/browse", methods=["GET"])
 def browse_local() -> tuple[Response, int]:
     """Browse local filesystem for folder selection.

app/routes/settings.py

@@ -63,6 +63,10 @@ def update_settings() -> tuple[Response, int]:
 
     settings.update(filtered_data)
 
+    # Changing the profile/region invalidates any cached S3 client.
+    if "aws_profile" in filtered_data or "aws_region" in filtered_data:
+        s3_service.reset_s3_client_cache()
+
     log = get_log_service()
     log.info(
         "settings",

app/routes/upload.py

@@ -79,10 +79,7 @@ def progress_callback(job: UploadJob) -> None:
             # Large jobs read per-file results from /api/upload/results (SQLite-backed)
             # rather than receiving a 10k-row payload here. Small jobs keep the
             # legacy behavior — frontend merges the full file array directly.
-            if (
-                large_job_threshold is not None
-                and len(job.files) >= large_job_threshold
-            ):
+            if large_job_threshold is not None and len(job.files) >= large_job_threshold:
                 payload = job.to_progress_dict()
                 payload["terminal"] = True
                 sse.send_event(job.job_id, payload)
@@ -96,9 +93,7 @@ def progress_callback(job: UploadJob) -> None:
 
 def _large_job_threshold() -> int:
     """Read the live setting for the large-job cutoff."""
-    return int(
-        get_settings().batch_processing.get("large_job_threshold", 1000)
-    )
+    return int(get_settings().batch_processing.get("large_job_threshold", 1000))
 
 
 @upload_bp.route("/analyze", methods=["POST"])