diff --git a/.gitignore b/.gitignore index 0716996..4029838 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ settings.json # SQLite cache modaq_upload_cache.db +s3_cache.db # Log files /logs/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a985bc6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,14 @@ +# Changelog + +## v1.2.0 + +- S3 file downloads: download files directly from the S3 browser. +- Large upload to existing folder: target an existing S3 prefix when running a large folder sync, not just newly created folders. +- Application log syncing: app logs and global frontend/backend errors are captured and uploaded to S3 for diagnostics. +- Per-user event logs: unique event logs are generated for each user. +- Faster S3 browsing: more efficient S3 listing strategy plus frontend improvements for the file browser. +- OS-standard log directories: logs now use platform-standard locations. + +## v1.1.0 + +- Initial tracked release with in-app updater (git pull + pip install) from Settings. diff --git a/app/__init__.py b/app/__init__.py index 3a6a505..83d360b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -3,8 +3,12 @@ import atexit import os import threading +import time +import traceback -from flask import Flask +from flask import Flask, g, jsonify, request +from flask.wrappers import Response +from werkzeug.exceptions import HTTPException from app.config import get_package_version, get_settings @@ -12,6 +16,17 @@ _cleanup_thread: threading.Thread | None = None _cleanup_stop_event = threading.Event() +# Background log-sync thread control +_log_sync_thread: threading.Thread | None = None +_log_sync_stop_event = threading.Event() +# Set to wake the sync worker immediately (e.g. right after an error is logged). +_log_sync_trigger_event = threading.Event() +# How often to sync logs to S3 when nothing has triggered an early sync (seconds). +_LOG_SYNC_INTERVAL = 300 +# Request paths to exclude from automatic per-request logging (noisy / would +# create feedback loops with the logging pipeline itself). +_REQUEST_LOG_SKIP_PREFIXES = ("/api/logs", "/api/upload/progress") + def _sse_cleanup_worker() -> None: """Background worker that periodically cleans up stale SSE queues.""" @@ -52,6 +67,157 @@ def _stop_sse_cleanup() -> None: _cleanup_thread.join(timeout=2.0) +def _run_log_sync() -> None: + """Sync log files to S3 once, if a bucket is configured. + + Failures are logged as WARNING (never ERROR) so that a persistently failing + sync cannot re-trigger itself through the error hook and hot-loop. + """ + from app.services import s3_service + from app.services.log_service import get_log_service + + settings = get_settings() + log = get_log_service() + + if not settings.s3_bucket: + return # Nothing to sync to yet; stay quiet. + + try: + client = s3_service.create_s3_client(settings.aws_profile, settings.aws_region) + log.sync_logs_to_s3(client, settings.s3_bucket) + except Exception as e: + log.warning( + "sync", + "log_sync_failed", + f"Automatic log sync to S3 failed: {e}", + {"error": str(e)}, + ) + + +def _log_sync_worker() -> None: + """Background worker: sync logs to S3 periodically and on demand. + + Wakes every ``_LOG_SYNC_INTERVAL`` seconds, or immediately whenever + ``_log_sync_trigger_event`` is set (e.g. just after an error is logged). + """ + while not _log_sync_stop_event.is_set(): + # Wait for either the interval to elapse or an explicit trigger. + _log_sync_trigger_event.wait(timeout=_LOG_SYNC_INTERVAL) + if _log_sync_stop_event.is_set(): + break + _log_sync_trigger_event.clear() + _run_log_sync() + + +def _start_log_sync() -> None: + """Start the background log-sync thread and register the error trigger.""" + global _log_sync_thread + + from app.services.log_service import get_log_service + + # Logging an ERROR wakes the sync worker for a near-immediate upload. + get_log_service().set_error_callback(_log_sync_trigger_event.set) + + if _log_sync_thread is None: + _log_sync_thread = threading.Thread(target=_log_sync_worker, daemon=True, name="LogSync") + _log_sync_thread.start() + + +def _stop_log_sync() -> None: + """Stop the background log-sync thread (flushing one final sync).""" + _log_sync_stop_event.set() + _log_sync_trigger_event.set() # Wake it so it can observe the stop flag. + if _log_sync_thread: + _log_sync_thread.join(timeout=2.0) + + +def _register_logging_hooks(app: Flask) -> None: + """Wire up automatic per-request logging and a global error handler. + + - Every ``/api/*`` request is logged (method, path, status, duration) so user + actions are captured without per-route boilerplate. + - Any uncaught exception is logged at ERROR level with a traceback, which also + triggers a near-immediate S3 sync via the log service's error hook. + """ + from app.services.log_service import get_log_service + + def _should_log(path: str) -> bool: + return path.startswith("/api/") and not path.startswith(_REQUEST_LOG_SKIP_PREFIXES) + + @app.before_request + def _stamp_request_start() -> None: + g._request_start = time.monotonic() + + @app.after_request + def _log_request(response: Response) -> Response: + if _should_log(request.path): + start = getattr(g, "_request_start", None) + duration_ms = round((time.monotonic() - start) * 1000, 1) if start else None + level = "WARNING" if response.status_code >= 400 else "INFO" + get_log_service().log( + level, + "request", + "http_request", + f"{request.method} {request.path} -> {response.status_code}", + { + "method": request.method, + "path": request.path, + "status": response.status_code, + "duration_ms": duration_ms, + "remote_addr": request.remote_addr, + }, + ) + return response + + @app.errorhandler(Exception) + def _handle_uncaught(error: Exception) -> Response | tuple[Response, int] | HTTPException: + # Let normal HTTP errors (404, 405, explicit aborts, ...) keep their + # default handling; only true crashes are logged as ERROR. + if isinstance(error, HTTPException): + return error + get_log_service().error( + "error", + "unhandled_exception", + f"Unhandled {type(error).__name__} on {request.method} {request.path}: {error}", + { + "exception_type": type(error).__name__, + "method": request.method, + "path": request.path, + "traceback": traceback.format_exc(), + }, + ) + return jsonify({"error": "Internal server error"}), 500 + + +def _start_s3_warmup() -> None: + """Pre-resolve AWS credentials and open an S3 connection in the background. + + The first S3 call in a fresh process pays a one-time cost — credential + resolution (which for SSO/assume-role can be slow) plus TLS connection setup. + Without warming, that cost lands on the user's first file-browser request, + making the initial listing feel slow even though warm calls are ~tens of ms. + This does the cold work up front, off the request path, and swallows all + errors (no bucket/credentials configured yet is fine). + """ + + def _warm() -> None: + try: + settings = get_settings() + bucket = settings.s3_bucket + if not bucket: + return + from app.services import s3_service + + client = s3_service.create_s3_client(settings.aws_profile, settings.aws_region) + # Cheap call that forces credential resolution + a live connection. + client.head_bucket(Bucket=bucket) + except Exception: + # Warmup is best-effort; the real request will surface any error. + pass + + threading.Thread(target=_warm, name="s3-warmup", daemon=True).start() + + def create_app() -> Flask: """Create and configure the Flask application.""" app = Flask(__name__) @@ -86,11 +252,21 @@ def inject_display_name() -> dict[str, str]: app.register_blueprint(delete_bp, url_prefix="/api/delete") app.register_blueprint(large_folder_upload_bp, url_prefix="/api/large-folder-upload") + # Automatic request/error logging hooks + _register_logging_hooks(app) + + # Warm the S3 client/credentials so the first file-browser request isn't cold + _start_s3_warmup() + # Start background SSE cleanup thread _start_sse_cleanup() + # Start background log-sync thread (periodic + error-triggered S3 upload) + _start_log_sync() + # Register cleanup on shutdown atexit.register(_stop_sse_cleanup) + atexit.register(_stop_log_sync) # Log application startup from app.services.log_service import get_log_service diff --git a/app/config.py b/app/config.py index f4699d2..6440a2b 100644 --- a/app/config.py +++ b/app/config.py @@ -3,9 +3,13 @@ import functools import json import os +import platform +import re import subprocess import sys +import tempfile import tomllib +import uuid from pathlib import Path from typing import Any @@ -14,6 +18,55 @@ # Base directory for the project BASE_DIR = Path(__file__).resolve().parent.parent +# Folder name used under the OS-specific application-data location for logs. +APP_DIR_NAME = "modaq_upload" + + +def get_default_log_directory(app_name: str = APP_DIR_NAME) -> Path: + """Return the OS-appropriate directory for application logs. + + Follows each platform's conventions for per-user application data so logs + live in a writable, persistent, standard location instead of inside the + repository (which breaks when the app is installed read-only or moved): + + - Windows: ``%LOCALAPPDATA%\\\\logs`` + (e.g. ``C:\\Users\\you\\AppData\\Local\\modaq_upload\\logs``) + - macOS: ``~/Library/Logs/`` + - Linux/other POSIX: ``$XDG_STATE_HOME//logs`` + (defaults to ``~/.local/state//logs`` per the XDG Base Directory + spec, which places logs under "state" data) + """ + home = Path.home() + + if sys.platform == "win32": + base = os.environ.get("LOCALAPPDATA") + root = Path(base) if base else home / "AppData" / "Local" + return root / app_name / "logs" + + if sys.platform == "darwin": + return home / "Library" / "Logs" / app_name + + # Linux and other POSIX systems: XDG state directory. + base = os.environ.get("XDG_STATE_HOME") + root = Path(base) if base else home / ".local" / "state" + return root / app_name / "logs" + + +def _is_under_temp_dir(path: Path) -> bool: + """True if ``path`` lives inside the OS temp directory. + + A configured log directory under the system temp dir is almost always a + stale value leaked from a test run (pytest's ``tempfile.mkdtemp``), not a + real log location — production logs must never be written there. + """ + try: + tmp = Path(tempfile.gettempdir()).resolve() + resolved = path.resolve() + return resolved == tmp or tmp in resolved.parents + except OSError: + return False + + # Load environment variables from .env file ENV_FILE = BASE_DIR / ".env" load_dotenv(ENV_FILE) @@ -54,6 +107,44 @@ def get_package_name() -> str: return _read_pyproject_field("name", "modaq-uploader") +def _sanitize_partition_value(value: str) -> str: + """Make a string safe to use as a hive partition value (a path segment). + + Replaces path-hostile characters (slashes, ``=``, and whitespace) with ``-`` so + the value can't break out of its partition directory or confuse hive parsing. + Falls back to ``"unknown"`` when the result is empty. + """ + cleaned = re.sub(r"[/\\=\s]+", "-", value.strip()).strip("-") + return cleaned or "unknown" + + +def get_os_name() -> str: + """Return a partition-safe OS name (e.g. ``Darwin``/``Windows``/``Linux``).""" + return _sanitize_partition_value(platform.system()) + + +def get_os_version() -> str: + """Return a partition-safe OS version (e.g. ``25.4.0``). + + Uses ``platform.release()`` for cross-platform uniformity (on macOS this is the + Darwin kernel version; ``platform.mac_ver()[0]`` would give the product version). + """ + return _sanitize_partition_value(platform.release()) + + +def get_session_partitions() -> list[str]: + """Return hive partition segments identifying this install's log session. + + Combines OS, OS version, and a persistent per-install ID so logs from multiple + machines syncing to one S3 bucket land at distinct, queryable paths. + """ + return [ + f"os={get_os_name()}", + f"os_version={get_os_version()}", + f"session={get_settings().install_id}", + ] + + class Settings: """Manages application settings stored in JSON format.""" @@ -84,7 +175,7 @@ def _load_settings(self) -> None: "s3_bucket": "", "default_upload_folder": "", "display_name": "MODAQ Uploader", - "log_directory": "logs", + "log_directory": str(get_default_log_directory()), "file_categories": [ { "name": "data", @@ -149,6 +240,17 @@ def _load_settings(self) -> None: defaults[key] = value provenance[key] = {"source": "env", "env_var": env_var} + # Repair an empty or temp-dir log_directory (e.g. a path leaked from a + # test run that got persisted to settings.json). Fall back to the + # OS-standard location so logs never silently write to a temp dir. + raw_log_dir = str(defaults.get("log_directory") or "").strip() + resolved_log_dir = ( + Path(raw_log_dir) if Path(raw_log_dir).is_absolute() else BASE_DIR / raw_log_dir + ) + if not raw_log_dir or _is_under_temp_dir(resolved_log_dir): + defaults["log_directory"] = str(get_default_log_directory()) + provenance["log_directory"] = {"source": "builtin"} + self._settings = defaults self._provenance = provenance @@ -212,6 +314,20 @@ def display_name(self) -> str: """Get the display name for the application.""" return str(self._settings["display_name"]) + @property + def install_id(self) -> str: + """Get a stable per-install identifier, generating and persisting it once. + + Stored in the gitignored settings.json so it survives restarts and stays + unique per machine. Used to give each install's logs a distinct S3 path. + """ + existing = self._settings.get("install_id") + if existing: + return str(existing) + new_id = uuid.uuid4().hex[:12] + self.set("install_id", new_id) + return new_id + @property def log_directory(self) -> Path: """Get the log directory path (resolved to absolute path relative to BASE_DIR).""" diff --git a/app/routes/files.py b/app/routes/files.py index 4621bcd..f565154 100644 --- a/app/routes/files.py +++ b/app/routes/files.py @@ -29,12 +29,15 @@ def list_files() -> tuple[Response, int]: Query parameters: prefix: S3 prefix to filter by (default: "") delimiter: Delimiter for grouping (default: "/") + token: Continuation token from a previous response's next_token, to fetch + the next page of a folder with more results than fit in one page Returns: JSON response with folders and files """ prefix = request.args.get("prefix", "") delimiter = request.args.get("delimiter", "/") + token = request.args.get("token") or None try: client = s3_service.create_s3_client( @@ -47,6 +50,7 @@ def list_files() -> tuple[Response, int]: g.settings.s3_bucket, prefix=prefix, delimiter=delimiter, + continuation_token=token, ) if not result["success"]: @@ -103,6 +107,77 @@ def get_file_info() -> tuple[Response, int]: return jsonify({"error": str(e)}), 500 +@files_bp.route("/stats", methods=["GET"]) +def get_prefix_stats() -> tuple[Response, int]: + """Get subfolder and file counts for a folder (one level). + + Query parameters: + prefix: S3 prefix to summarize (default: "" = bucket root) + delimiter: Delimiter for grouping (default: "/") + + Returns: + JSON response with folder_count and file_count + """ + prefix = request.args.get("prefix", "") + delimiter = request.args.get("delimiter", "/") + + try: + client = s3_service.create_s3_client( + g.settings.aws_profile, + g.settings.aws_region, + ) + + result = s3_service.get_prefix_counts( + client, + g.settings.s3_bucket, + prefix=prefix, + delimiter=delimiter, + ) + + if not result["success"]: + return jsonify({"error": result["error"]}), 500 + + return jsonify(result), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + + +@files_bp.route("/download", methods=["GET"]) +def download_file() -> tuple[Response, int]: + """Generate a presigned URL for downloading an S3 object. + + Query parameters: + key: S3 object key + + Returns: + JSON response with a presigned download URL + """ + key = request.args.get("key", "") + if not key: + return jsonify({"error": "Object key required"}), 400 + + try: + client = s3_service.create_s3_client( + g.settings.aws_profile, + g.settings.aws_region, + ) + + result = s3_service.generate_presigned_download_url( + client, + g.settings.s3_bucket, + key, + ) + + if not result["success"]: + return jsonify({"error": result["error"]}), 404 + + return jsonify(result), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + + @files_bp.route("/browse", methods=["GET"]) def browse_local() -> tuple[Response, int]: """Browse local filesystem for folder selection. diff --git a/app/routes/large_folder_upload.py b/app/routes/large_folder_upload.py index 4199453..ea09378 100644 --- a/app/routes/large_folder_upload.py +++ b/app/routes/large_folder_upload.py @@ -44,6 +44,25 @@ class SyncJob: created_at: float = field(default_factory=time.time) +def _resolve_folder_path(folder: Path) -> Path: + """Resolve a folder to an absolute, symlink-followed real path. + + A relative path is anchored to the configured ``default_upload_folder`` (the user's + data root), never the process cwd — the launcher runs us with cwd=PROJECT_DIR, so + resolving against it would point at the project, not the drive. The base is forced + absolute (falling back to home) so the result is always absolute. ``resolve()`` also + follows the symlinks that external and mounted drives are reached through, so + ``aws s3 sync`` targets the real mount. + """ + folder = folder.expanduser() + if not folder.is_absolute(): + base = Path(get_settings().default_upload_folder).expanduser() + if not base.is_absolute(): + base = Path.home() + folder = base / folder + return folder.resolve() + + def _register(job: SyncJob) -> None: with _jobs_lock: _jobs[job.job_id] = job @@ -316,13 +335,18 @@ def start_sync() -> tuple[Response, int]: return jsonify({"error": "JSON body required"}), 400 data: dict[str, Any] = request.get_json() or {} - folder_path: str = data.get("folder_path", "").strip() + folder_path_raw: str = data.get("folder_path", "").strip() s3_prefix: str = data.get("s3_prefix", "").strip() - if not folder_path: + if not folder_path_raw: return jsonify({"error": "folder_path is required"}), 400 - if not os.path.isdir(folder_path): - return jsonify({"error": f"folder_path does not exist: {folder_path}"}), 400 + folder = _resolve_folder_path(Path(folder_path_raw)) + if not folder.is_dir(): + return jsonify({"error": f"folder_path does not exist: {folder}"}), 400 + + # str at the serialization boundary: the subprocess command, the SyncJob record, + # and the JSON logs all store the path as text. + folder_path = str(folder) if not s3_prefix: return jsonify({"error": "s3_prefix is required"}), 400 diff --git a/app/routes/logs.py b/app/routes/logs.py index 394783f..8d2c2d1 100644 --- a/app/routes/logs.py +++ b/app/routes/logs.py @@ -91,6 +91,39 @@ def get_upload_stats() -> tuple[Response, int]: return jsonify(stats), 200 +@logs_bp.route("/client", methods=["POST"]) +def log_client_event() -> tuple[Response, int]: + """Record a client-side (browser) event, typically a JS error. + + Frontend global error hooks POST here so browser failures land in the same + JSONL pipeline (and S3 sync) as backend logs. + + JSON body: + level: INFO/WARNING/ERROR (default ERROR) + event: machine-readable event name (default client_error) + message: human-readable message + metadata: optional dict of extra context (url, stack, userAgent, ...) + """ + data = request.get_json(silent=True) or {} + + level = str(data.get("level", "ERROR")).upper() + if level not in ("INFO", "WARNING", "ERROR"): + level = "ERROR" + event = str(data.get("event", "client_error")) + message = str(data.get("message", "")) or "Client-side event" + + metadata = data.get("metadata") + if not isinstance(metadata, dict): + metadata = {} + # Always capture where it came from so logs are attributable. + metadata.setdefault("user_agent", request.headers.get("User-Agent", "")) + metadata.setdefault("remote_addr", request.remote_addr) + + log = get_log_service() + log.log(level, "client", event, message, metadata) + return jsonify({"success": True}), 200 + + @logs_bp.route("/sync", methods=["POST"]) def sync_logs() -> tuple[Response, int]: """Trigger S3 sync of log files. diff --git a/app/routes/settings.py b/app/routes/settings.py index 541cfe3..2716629 100644 --- a/app/routes/settings.py +++ b/app/routes/settings.py @@ -63,6 +63,10 @@ def update_settings() -> tuple[Response, int]: settings.update(filtered_data) + # Changing the profile/region invalidates any cached S3 client. + if "aws_profile" in filtered_data or "aws_region" in filtered_data: + s3_service.reset_s3_client_cache() + log = get_log_service() log.info( "settings", diff --git a/app/routes/upload.py b/app/routes/upload.py index 419c916..7c3f1ab 100644 --- a/app/routes/upload.py +++ b/app/routes/upload.py @@ -79,10 +79,7 @@ def progress_callback(job: UploadJob) -> None: # Large jobs read per-file results from /api/upload/results (SQLite-backed) # rather than receiving a 10k-row payload here. Small jobs keep the # legacy behavior — frontend merges the full file array directly. - if ( - large_job_threshold is not None - and len(job.files) >= large_job_threshold - ): + if large_job_threshold is not None and len(job.files) >= large_job_threshold: payload = job.to_progress_dict() payload["terminal"] = True sse.send_event(job.job_id, payload) @@ -96,9 +93,7 @@ def progress_callback(job: UploadJob) -> None: def _large_job_threshold() -> int: """Read the live setting for the large-job cutoff.""" - return int( - get_settings().batch_processing.get("large_job_threshold", 1000) - ) + return int(get_settings().batch_processing.get("large_job_threshold", 1000)) @upload_bp.route("/analyze", methods=["POST"]) diff --git a/app/services/log_service.py b/app/services/log_service.py index 854be2c..84dff19 100644 --- a/app/services/log_service.py +++ b/app/services/log_service.py @@ -1,7 +1,10 @@ """JSONL logging service for application events. -Writes one JSON object per line to hive-partitioned daily .jsonl files. +Writes one JSON object per line to hive-partitioned daily .jsonl files. Files are +partitioned by date (year/month/day) and then by session (os/os_version/session) so +multiple installs can sync to one S3 bucket without overwriting each other. DuckDB-compatible: SELECT * FROM read_json_auto('logs/json/**/events.jsonl', hive_partitioning=true) +exposes year, month, day, os, os_version, and session as columns. """ import csv @@ -9,11 +12,12 @@ import json import re import threading +from collections.abc import Callable from datetime import UTC, datetime from pathlib import Path from typing import Any -from app.config import get_settings +from app.config import get_session_partitions, get_settings class LogService: @@ -22,6 +26,16 @@ class LogService: def __init__(self) -> None: """Initialize the log service.""" self._write_lock = threading.Lock() + self._error_callback: Callable[[], None] | None = None + + def set_error_callback(self, callback: Callable[[], None] | None) -> None: + """Register a callback fired (best-effort) after every ERROR-level log. + + Used to trigger an immediate S3 log sync when something goes wrong. + The callback must be cheap and non-blocking (e.g. set a threading.Event); + it runs on the thread that logged the error. + """ + self._error_callback = callback def _get_log_dir(self) -> Path: """Get the configured log directory, creating it if needed.""" @@ -33,17 +47,24 @@ def _get_log_dir(self) -> Path: def _get_hive_dir(self, subdir: str, dt: datetime) -> Path: """Build a hive-partitioned directory path and create it. + The path is partitioned by date and then by this install's session + (OS, OS version, install ID) so logs from multiple machines syncing to one + S3 bucket land at distinct, queryable paths instead of overwriting each other. + Args: subdir: Top-level subdirectory ('json' or 'csv') dt: Datetime to partition by Returns: - Path like logs/json/year=2026/month=02/day=08/ + Path like + logs/json/year=2026/month=02/day=08/os=Darwin/os_version=25.4.0/session=ab12cd34/ """ log_dir = self._get_log_dir() hive_dir = ( log_dir / subdir / f"year={dt.year:04d}" / f"month={dt.month:02d}" / f"day={dt.day:02d}" ) + for segment in get_session_partitions(): + hive_dir = hive_dir / segment hive_dir.mkdir(parents=True, exist_ok=True) return hive_dir @@ -102,6 +123,14 @@ def log( with open(log_file, "a", encoding="utf-8") as f: f.write(line + "\n") + # Fire the error hook outside the write lock so a slow/failing callback + # can never block logging. Never let it raise back into the caller. + if entry["level"] == "ERROR" and self._error_callback is not None: + try: + self._error_callback() + except Exception: + pass + def info( self, category: str, @@ -310,14 +339,12 @@ def read_log_entries( dt = datetime.strptime(date, "%Y-%m-%d") except ValueError: return {"entries": [], "total": 0, "offset": offset, "limit": limit} - hive_path = ( - json_dir - / f"year={dt.year:04d}" - / f"month={dt.month:02d}" - / f"day={dt.day:02d}" - / "events.jsonl" + day_dir = ( + json_dir / f"year={dt.year:04d}" / f"month={dt.month:02d}" / f"day={dt.day:02d}" ) - files = [hive_path] if hive_path.exists() else [] + # events.jsonl now lives under session sub-dirs (os=/os_version=/session=), + # and a synced-down bucket may hold several sessions for one day. + files = sorted(day_dir.rglob("events.jsonl")) if day_dir.exists() else [] else: if json_dir.exists(): files = sorted(json_dir.rglob("events.jsonl"), reverse=True) @@ -568,7 +595,7 @@ def sync_logs_to_s3( self, s3_client: Any, bucket: str, - prefix: str = "logs/", + prefix: str = "app_logs/", ) -> dict[str, Any]: """Upload new/changed log files (JSONL + CSV) to S3. diff --git a/app/services/s3_service.py b/app/services/s3_service.py index f1afa60..68d50ec 100644 --- a/app/services/s3_service.py +++ b/app/services/s3_service.py @@ -9,6 +9,7 @@ from boto3.s3.transfer import TransferConfig from botocore.exceptions import ClientError, NoCredentialsError from mypy_boto3_s3 import S3Client +from mypy_boto3_s3.type_defs import PaginatorConfigTypeDef class UploadCancelledError(Exception): @@ -62,8 +63,20 @@ def get_available_profiles() -> list[str]: return sorted(profiles) +# Cache S3 clients by (profile, region). Building a boto3 Session resolves +# credentials (which for SSO/assume-role can mean a network round-trip), so +# rebuilding one per request added latency to every list/stats/download call. +# boto3 low-level clients are thread-safe for making calls, so a shared instance +# is safe across the dev server's request threads. +_CLIENT_CACHE: dict[tuple[str, str], S3Client] = {} + + def create_s3_client(profile: str, region: str = "us-west-2") -> S3Client: - """Create an S3 client using the specified AWS profile. + """Create (or reuse) an S3 client for the given AWS profile and region. + + Clients are cached by (profile, region) so repeated calls — e.g. the parallel + list/stats requests on the file browser — don't each rebuild a session and + re-resolve credentials. Args: profile: AWS profile name from ~/.aws/credentials or ~/.aws/config @@ -75,11 +88,26 @@ def create_s3_client(profile: str, region: str = "us-west-2") -> S3Client: Raises: NoCredentialsError: If credentials are not found """ + cache_key = (profile, region) + cached = _CLIENT_CACHE.get(cache_key) + if cached is not None: + return cached + session = boto3.Session(profile_name=profile, region_name=region) client: S3Client = session.client("s3") + _CLIENT_CACHE[cache_key] = client return client +def reset_s3_client_cache() -> None: + """Clear the cached S3 clients. + + Call after changing AWS settings so the next request rebuilds the client with + the new profile/region/credentials. + """ + _CLIENT_CACHE.clear() + + def check_file_exists(client: S3Client, bucket: str, key: str) -> bool: """Check if a file exists in S3. @@ -186,40 +214,54 @@ def list_bucket_objects( prefix: str = "", delimiter: str = "/", max_keys: int = 1000, + continuation_token: str | None = None, ) -> dict[str, Any]: - """List objects in an S3 bucket with prefix filtering. + """List objects in an S3 bucket with prefix filtering, one page at a time. + + Returns at most ``max_keys`` items (folders + files combined). When more + results exist, ``next_token`` is a non-null string the caller passes back as + ``continuation_token`` to fetch the next page. This avoids silently truncating + folders that contain more than ``max_keys`` objects. Args: client: S3 client bucket: S3 bucket name prefix: Object key prefix for filtering delimiter: Delimiter for grouping (default: /) - max_keys: Maximum number of keys to return + max_keys: Maximum number of items to return per page + continuation_token: Opaque token from a previous call's ``next_token`` to + resume listing where the last page left off Returns: - Dictionary containing folders and files + Dictionary containing folders, files, and ``next_token`` (None when done) """ folders: list[dict[str, str]] = [] files: list[dict[str, Any]] = [] try: + pagination_config: PaginatorConfigTypeDef = {"MaxItems": max_keys} + if continuation_token: + pagination_config["StartingToken"] = continuation_token + paginator = client.get_paginator("list_objects_v2") page_iterator = paginator.paginate( Bucket=bucket, Prefix=prefix, Delimiter=delimiter, - PaginationConfig={"MaxItems": max_keys}, + PaginationConfig=pagination_config, ) for page in page_iterator: - # Get common prefixes (folders) - for prefix_info in page.get("CommonPrefixes", []): + # Get common prefixes (folders). Note: when resuming from a + # StartingToken these keys may be present but explicitly None, so + # `or []` guards against iterating over None. + for prefix_info in page.get("CommonPrefixes") or []: folder_prefix = prefix_info.get("Prefix", "") folder_name = folder_prefix.rstrip("/").split("/")[-1] folders.append({"name": folder_name, "prefix": folder_prefix}) # Get objects (files) - for obj in page.get("Contents", []): + for obj in page.get("Contents") or []: key = obj.get("Key", "") # Skip the prefix itself if it's listed if key == prefix: @@ -237,12 +279,16 @@ def list_bucket_objects( } ) + # resume_token is None once all results have been returned. + next_token = page_iterator.resume_token + return { "success": True, "bucket": bucket, "prefix": prefix, "folders": folders, "files": files, + "next_token": next_token, "error": None, } except ClientError as e: @@ -252,6 +298,85 @@ def list_bucket_objects( "prefix": prefix, "folders": [], "files": [], + "next_token": None, + "error": str(e), + } + + +# Max entries (subfolders + files) get_prefix_counts looks at. The count is a +# nice-to-have, so it costs exactly ONE list_objects_v2 call (a single page) and +# never enumerates a large level. S3 counts both keys and common prefixes toward +# MaxKeys, so this bounds work whether the level is files, folders, or a mix. +# Capped at S3's 1,000-key page maximum. +STATS_MAX_ITEMS = 1_000 + + +def get_prefix_counts( + client: S3Client, + bucket: str, + prefix: str = "", + delimiter: str = "/", + max_items: int = STATS_MAX_ITEMS, +) -> dict[str, Any]: + """Count subfolders and files directly under a prefix (one level only). + + Uses ``delimiter`` so S3 returns just the immediate level — this never recurses + into subfolders. It makes a SINGLE ``list_objects_v2`` request (one page of up + to ``max_items`` entries) and reads ``IsTruncated`` to know whether there are + more. So it always costs one S3 call no matter how large the level is; when the + level overflows the page, ``capped`` is True and the counts are lower bounds. + + Args: + client: S3 client + bucket: S3 bucket name + prefix: Object key prefix for the folder to summarize + delimiter: Delimiter for grouping (default: /) + max_items: Page size — entries (folders + files) to look at, clamped to + S3's 1,000 maximum; default ``STATS_MAX_ITEMS`` + + Returns: + Dictionary with ``folder_count``, ``file_count``, and ``capped`` for this + level. When ``capped`` is True the true totals are higher than reported. + """ + folder_prefixes: set[str] = set() + file_count = 0 + + try: + max_keys = max(1, min(max_items, 1000)) + response = client.list_objects_v2( + Bucket=bucket, + Prefix=prefix, + Delimiter=delimiter, + MaxKeys=max_keys, + ) + + for prefix_info in response.get("CommonPrefixes") or []: + folder_prefixes.add(prefix_info.get("Prefix", "")) + for obj in response.get("Contents") or []: + key = obj.get("Key", "") + # Skip the prefix placeholder object and any directory markers. + if key == prefix or not key.split("/")[-1]: + continue + file_count += 1 + + # IsTruncated means the level has more entries than fit in this one page. + capped = bool(response.get("IsTruncated", False)) + + return { + "success": True, + "prefix": prefix, + "folder_count": len(folder_prefixes), + "file_count": file_count, + "capped": capped, + "error": None, + } + except ClientError as e: + return { + "success": False, + "prefix": prefix, + "folder_count": 0, + "file_count": 0, + "capped": False, "error": str(e), } @@ -303,6 +428,59 @@ def get_s3_client_from_settings() -> tuple[S3Client, str]: return client, settings.s3_bucket +def generate_presigned_download_url( + client: S3Client, + bucket: str, + key: str, + expires_in: int = 3600, +) -> dict[str, Any]: + """Generate a presigned URL for downloading an S3 object. + + The URL forces a browser download (Content-Disposition: attachment) with the + object's original filename. Presigned URLs let the browser fetch directly from + S3 rather than proxying large files through Flask. + + Args: + client: S3 client + bucket: S3 bucket name + key: S3 object key + expires_in: URL validity in seconds (default: 3600 = 1 hour) + + Returns: + Dictionary with the presigned URL or an error + """ + filename = key.split("/")[-1] + try: + # Confirm the object exists so a missing key returns 404 rather than a + # presigned URL that later fails when the user clicks it. + client.head_object(Bucket=bucket, Key=key) + + url = client.generate_presigned_url( + "get_object", + Params={ + "Bucket": bucket, + "Key": key, + "ResponseContentDisposition": f'attachment; filename="{filename}"', + }, + ExpiresIn=expires_in, + ) + return { + "success": True, + "url": url, + "key": key, + "filename": filename, + "error": None, + } + except ClientError as e: + error_code = e.response["Error"]["Code"] + error_msg = f"File '{filename}' not found" if error_code == "404" else str(e) + return { + "success": False, + "key": key, + "error": error_msg, + } + + def get_object_metadata(client: S3Client, bucket: str, key: str) -> dict[str, Any]: """Get metadata for an S3 object. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 98300b4..60e861d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "frontend", - "version": "1.1.0", + "version": "1.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "frontend", - "version": "1.1.0", + "version": "1.2.0", "dependencies": { "@tanstack/react-table": "^8.21.3", "@tanstack/react-virtual": "^3.13.18", diff --git a/frontend/package.json b/frontend/package.json index b3f7ed1..a1bbb56 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "1.1.0", + "version": "1.2.0", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index f2a2fcd..6818fc5 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -17,9 +17,13 @@ async function handleResponse(res: Response): Promise { return res.json() as Promise; } -export async function apiGet(url: string, params?: Record): Promise { +export async function apiGet( + url: string, + params?: Record, + signal?: AbortSignal, +): Promise { const qs = params ? `?${new URLSearchParams(params)}` : ''; - const res = await fetch(`${url}${qs}`); + const res = await fetch(`${url}${qs}`, { signal }); return handleResponse(res); } diff --git a/frontend/src/components/common/S3FolderPicker.tsx b/frontend/src/components/common/S3FolderPicker.tsx new file mode 100644 index 0000000..eb606a2 --- /dev/null +++ b/frontend/src/components/common/S3FolderPicker.tsx @@ -0,0 +1,210 @@ +/** + * S3FolderPicker — browse the configured S3 bucket and choose a destination folder. + * + * Renders inline (not a modal). At any level the user can either: + * - sync into the folder they've navigated into, or + * - create a new subfolder at the current location. + * + * Calls `onSelect(prefix)` with the resolved key prefix (no leading slash, no + * trailing slash) whenever a valid destination is chosen. The bucket root is not + * a valid "folder" destination. + */ + +import { useCallback, useEffect, useMemo, useState } from 'react'; + +import { apiGet } from '../../api/client.ts'; +import type { S3ListResponse } from '../../types/api.ts'; +import { CloudIcon, FolderIcon, RefreshIcon } from '../../utils/icons.tsx'; +import Breadcrumb from './Breadcrumb.tsx'; +import Spinner from './Spinner.tsx'; +import { type DestMode, resolveDestination } from './s3FolderPath.ts'; + +interface S3FolderPickerProps { + bucket: string; + /** Called with the resolved prefix (no surrounding slashes) when a destination is chosen. */ + onSelect: (prefix: string) => void; + /** + * Optional suggested top-level folder name. When provided, a shortcut button + * sets the destination to a new folder of this name at the bucket root. + */ + suggestedName?: string; +} + +export default function S3FolderPicker({ bucket, onSelect, suggestedName }: S3FolderPickerProps) { + const [prefix, setPrefix] = useState(''); + const [folders, setFolders] = useState<{ name: string; prefix: string }[]>([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const [mode, setMode] = useState('new'); + const [newName, setNewName] = useState(''); + + const fetchFolders = useCallback(async (currentPrefix: string) => { + setLoading(true); + setError(null); + try { + const params: Record = { delimiter: '/' }; + if (currentPrefix) params.prefix = currentPrefix; + const data = await apiGet('/api/files/list', params); + if (!data.success) { + setError(data.error ?? 'Failed to list folders'); + return; + } + setFolders(data.folders); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to list folders'); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + void fetchFolders(prefix); + }, [prefix, fetchFolders]); + + // Breadcrumb: bucket root + each path segment + const breadcrumbItems = useMemo(() => { + const items: { label: string; onClick?: () => void }[] = [ + { label: bucket || 'bucket', onClick: prefix ? () => setPrefix('') : undefined }, + ]; + if (prefix) { + const parts = prefix.replace(/\/$/, '').split('/'); + for (let i = 0; i < parts.length; i++) { + const partPrefix = `${parts.slice(0, i + 1).join('/')}/`; + const isLast = i === parts.length - 1; + items.push({ label: parts[i], onClick: isLast ? undefined : () => setPrefix(partPrefix) }); + } + } + return items; + }, [bucket, prefix]); + + const atRoot = prefix === ''; + const currentFolderLabel = prefix.replace(/\/$/, '') || '(bucket root)'; + const destination = resolveDestination(prefix, mode, newName); + + // "Sync into this folder" is meaningless at the bucket root — force "new" there. + useEffect(() => { + if (atRoot && mode === 'existing') setMode('new'); + }, [atRoot, mode]); + + // Push the resolved destination up to the parent whenever it changes. + useEffect(() => { + onSelect(destination); + }, [destination, onSelect]); + + return ( +
+ {/* Header: bucket + breadcrumb + refresh */} +
+ +
+ +
+ {suggestedName && ( + + )} + +
+ + {/* Folder list */} +
+ {loading ? ( +
+ +
+ ) : error ? ( +
{error}
+ ) : folders.length === 0 ? ( +
No subfolders here.
+ ) : ( +
    + {folders.map((folder) => ( +
  • + +
  • + ))} +
+ )} +
+ + {/* Destination control */} +
+ {/* Sync into current folder */} + + + {/* Create new subfolder here */} + + + {/* Live destination preview */} +

+ Files land at{' '} + + s3://{bucket || ''}/{destination || '…'}/ + +

+
+
+ ); +} diff --git a/frontend/src/components/common/s3FolderPath.ts b/frontend/src/components/common/s3FolderPath.ts new file mode 100644 index 0000000..080242d --- /dev/null +++ b/frontend/src/components/common/s3FolderPath.ts @@ -0,0 +1,26 @@ +/** Pure helpers for resolving an S3 destination prefix from S3FolderPicker state. */ + +export type DestMode = 'existing' | 'new'; + +/** Trim, drop surrounding slashes, and collapse internal whitespace runs. */ +export function sanitizeFolderName(name: string): string { + return name + .trim() + .replace(/^\/+|\/+$/g, '') + .replace(/\s+/g, ' '); +} + +/** + * Resolve the destination prefix from the current browse location and form state. + * Returns '' when the destination is not a valid folder (e.g. bucket root, or an + * empty new-subfolder name). + */ +export function resolveDestination(prefix: string, mode: DestMode, newName: string): string { + const base = prefix.replace(/\/$/, ''); + if (mode === 'existing') { + return base; // '' at bucket root → invalid + } + const clean = sanitizeFolderName(newName); + if (!clean) return ''; + return base ? `${base}/${clean}` : clean; +} diff --git a/frontend/src/components/files/FileList.tsx b/frontend/src/components/files/FileList.tsx index 54842c1..52aad89 100644 --- a/frontend/src/components/files/FileList.tsx +++ b/frontend/src/components/files/FileList.tsx @@ -1,5 +1,9 @@ -import type { S3File, S3Folder } from '../../types/api.ts'; -import { FileIcon, FolderIcon } from '../../utils/icons.tsx'; +import { useVirtualizer } from '@tanstack/react-virtual'; +import { useMemo, useRef, useState } from 'react'; +import { apiGet } from '../../api/client.ts'; +import { useAppStore } from '../../stores/appStore.ts'; +import type { S3DownloadResponse, S3File, S3Folder } from '../../types/api.ts'; +import { DownloadIcon, FileIcon, FolderIcon, SpinnerIcon } from '../../utils/icons.tsx'; function formatBytes(bytes: number): string { if (bytes === 0) return '0 B'; @@ -23,9 +27,70 @@ interface FileListProps { folders: S3Folder[]; files: S3File[]; onNavigate: (prefix: string) => void; + hasMore?: boolean; + loadingMore?: boolean; + onLoadMore?: () => void; } -export default function FileList({ folders, files, onNavigate }: FileListProps) { +// Folders render first, then files, in a single virtualized list. +type Row = { type: 'folder'; folder: S3Folder } | { type: 'file'; file: S3File }; + +export default function FileList({ + folders, + files, + onNavigate, + hasMore = false, + loadingMore = false, + onLoadMore, +}: FileListProps) { + const addNotification = useAppStore((s) => s.addNotification); + const [downloadingKey, setDownloadingKey] = useState(null); + const parentRef = useRef(null); + + const rows = useMemo( + () => [ + ...folders.map((folder): Row => ({ type: 'folder', folder })), + ...files.map((file): Row => ({ type: 'file', file })), + ], + [folders, files], + ); + + // Virtualize the rows so folders with thousands of files render only the + // handful currently on screen rather than mounting every row. Rows are a fixed + // single-line height, so no per-row measurement is needed. + const virtualizer = useVirtualizer({ + count: rows.length, + getScrollElement: () => parentRef.current, + estimateSize: () => 49, + overscan: 12, + }); + + const handleDownload = async (file: S3File) => { + setDownloadingKey(file.key); + try { + const data = await apiGet('/api/files/download', { key: file.key }); + if (!data.success) { + addNotification('error', data.error ?? `Failed to download ${file.name}`); + return; + } + // Navigating to the presigned URL triggers the browser download + // (the URL sets Content-Disposition: attachment). + const link = document.createElement('a'); + link.href = data.url; + link.download = file.name; + document.body.appendChild(link); + link.click(); + link.remove(); + } catch (err) { + addNotification( + 'error', + err instanceof Error ? err.message : `Failed to download ${file.name}`, + ); + } finally { + setDownloadingKey(null); + } + }; + if (folders.length === 0 && files.length === 0) { return (
@@ -36,36 +101,86 @@ export default function FileList({ folders, files, onNavigate }: FileListProps) } return ( -
- {folders.map((folder) => ( - - ))} + <> +
+
+ {virtualizer.getVirtualItems().map((vItem) => { + const row = rows[vItem.index]; + return ( +
+ {row.type === 'folder' ? ( + + ) : ( +
+ +
+

+ {row.file.name} +

+
+ + {formatBytes(row.file.size)} + + + {formatDate(row.file.last_modified)} + + +
+ )} +
+ ); + })} +
+
- {files.map((file) => ( -
- -
-

- {file.name} -

-
- {formatBytes(file.size)} - - {formatDate(file.last_modified)} - + {hasMore && ( +
+
- ))} -
+ )} + ); } diff --git a/frontend/src/components/files/S3Browser.tsx b/frontend/src/components/files/S3Browser.tsx index a295dc9..ac2a091 100644 --- a/frontend/src/components/files/S3Browser.tsx +++ b/frontend/src/components/files/S3Browser.tsx @@ -1,9 +1,9 @@ -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { apiGet } from '../../api/client.ts'; -import type { S3File, S3Folder, S3ListResponse } from '../../types/api.ts'; +import type { S3File, S3Folder, S3ListResponse, S3StatsResponse } from '../../types/api.ts'; +import { reportClientEvent } from '../../utils/errorReporter.ts'; import { CloudIcon } from '../../utils/icons.tsx'; import Breadcrumb from '../common/Breadcrumb.tsx'; -import Spinner from '../common/Spinner.tsx'; import FileList from './FileList.tsx'; interface S3BrowserProps { @@ -16,7 +16,13 @@ export default function S3Browser({ bucketName, region }: S3BrowserProps) { const [folders, setFolders] = useState([]); const [files, setFiles] = useState([]); const [loading, setLoading] = useState(true); + const [loadingMore, setLoadingMore] = useState(false); + const [nextToken, setNextToken] = useState(null); const [error, setError] = useState(null); + const [stats, setStats] = useState(null); + // Aborts the in-flight stats request when we navigate away before it returns, + // so stale counts can't land late and overwrite the current folder's note. + const statsAbortRef = useRef(null); // Build breadcrumb items from the current prefix const breadcrumbItems = (() => { @@ -45,16 +51,36 @@ export default function S3Browser({ bucketName, region }: S3BrowserProps) { const fetchObjects = useCallback(async (currentPrefix: string) => { setLoading(true); setError(null); + // Clear the previous folder's rows immediately. Otherwise the parent's + // contents linger under the new breadcrumb until this fetch returns, which + // reads as a stale/"stuck" list showing the wrong folder. + setFolders([]); + setFiles([]); + setNextToken(null); + const startedAt = performance.now(); try { const params: Record = { delimiter: '/' }; if (currentPrefix) params.prefix = currentPrefix; const data = await apiGet('/api/files/list', params); + // Browser-perceived round-trip (network + server). Compare against the + // backend's own duration_ms log to separate S3/network time from handling. + const durationMs = Math.round(performance.now() - startedAt); + reportClientEvent('s3_list_timing', `files/list ${durationMs}ms`, { + endpoint: '/api/files/list', + prefix: currentPrefix || '(root)', + duration_ms: durationMs, + ok: data.success, + folder_count: data.folders?.length ?? 0, + file_count: data.files?.length ?? 0, + has_more: data.next_token != null, + }); if (!data.success) { setError(data.error ?? 'Failed to list objects'); return; } setFolders(data.folders); setFiles(data.files); + setNextToken(data.next_token ?? null); } catch (err) { setError(err instanceof Error ? err.message : 'Failed to fetch files'); } finally { @@ -62,14 +88,111 @@ export default function S3Browser({ bucketName, region }: S3BrowserProps) { } }, []); + // Fetch the next page and append, deduping folders that span page boundaries. + const loadMore = useCallback(async () => { + if (!nextToken || loadingMore) return; + setLoadingMore(true); + try { + const params: Record = { delimiter: '/', token: nextToken }; + if (prefix) params.prefix = prefix; + const data = await apiGet('/api/files/list', params); + if (!data.success) { + setError(data.error ?? 'Failed to load more objects'); + return; + } + setFolders((prev) => { + const seen = new Set(prev.map((f) => f.prefix)); + return [...prev, ...data.folders.filter((f) => !seen.has(f.prefix))]; + }); + setFiles((prev) => { + const seen = new Set(prev.map((f) => f.key)); + return [...prev, ...data.files.filter((f) => !seen.has(f.key))]; + }); + setNextToken(data.next_token ?? null); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load more files'); + } finally { + setLoadingMore(false); + } + }, [nextToken, loadingMore, prefix]); + + // Accurate counts for the current level. Fetched separately because the file + // list is paginated (one page at a time), so its loaded length isn't the total. + const fetchStats = useCallback(async (currentPrefix: string) => { + // Cancel any stats request still running for a previously-viewed folder. + statsAbortRef.current?.abort(); + const controller = new AbortController(); + statsAbortRef.current = controller; + + setStats(null); + const startedAt = performance.now(); + try { + const params: Record = { delimiter: '/' }; + if (currentPrefix) params.prefix = currentPrefix; + const data = await apiGet('/api/files/stats', params, controller.signal); + const durationMs = Math.round(performance.now() - startedAt); + reportClientEvent('s3_stats_timing', `files/stats ${durationMs}ms`, { + endpoint: '/api/files/stats', + prefix: currentPrefix || '(root)', + duration_ms: durationMs, + ok: data.success, + folder_count: data.folder_count, + file_count: data.file_count, + capped: data.capped ?? false, + }); + const valid = + data.success && + typeof data.folder_count === 'number' && + typeof data.file_count === 'number'; + setStats(valid ? data : null); + } catch { + // Aborted (navigated away) or failed — counts are non-critical, so don't + // surface an error. An abort leaves the newer request to set the state. + const durationMs = Math.round(performance.now() - startedAt); + reportClientEvent( + 's3_stats_timing', + controller.signal.aborted + ? `files/stats aborted after ${durationMs}ms` + : `files/stats failed after ${durationMs}ms`, + { + endpoint: '/api/files/stats', + prefix: currentPrefix || '(root)', + duration_ms: durationMs, + aborted: controller.signal.aborted, + }, + ); + if (!controller.signal.aborted) setStats(null); + } + }, []); + useEffect(() => { void fetchObjects(prefix); - }, [prefix, fetchObjects]); + void fetchStats(prefix); + }, [prefix, fetchObjects, fetchStats]); const navigateToPrefix = useCallback((newPrefix: string) => { setPrefix(newPrefix); }, []); + // A plain-language note summarizing the current folder's contents. It appears + // quietly once counting finishes; while counting we render nothing rather than + // a "Counting items…" placeholder, so the note never looks like it's holding + // up the file list (which loads independently and is what actually matters). + const statsNote = (() => { + if (!stats) return null; + // Capped: counting stopped early, so both counts are lower bounds — report a + // single "over N items" rather than implying an exact per-type breakdown. + if (stats.capped) { + const total = stats.folder_count + stats.file_count; + return `Large folder — over ${total.toLocaleString()} items.`; + } + const plural = (n: number, word: string) => + `${n.toLocaleString()} ${word}${n === 1 ? '' : 's'}`; + const folders = plural(stats.folder_count, 'subfolder'); + const files = plural(stats.file_count, 'file'); + return `This folder contains ${folders} and ${files}.`; + })(); + return (
{/* Header: Bucket info + breadcrumb */} @@ -82,15 +205,24 @@ export default function S3Browser({ bucketName, region }: S3BrowserProps) {
+ {statsNote &&

{statsNote}

}
{/* Content */} -
- {loading ? ( -
- +
+ {/* Thin indeterminate bar across the top while a fetch is in flight. + The list stays visible underneath rather than being replaced. */} + {(loading || loadingMore) && ( +
+
- ) : error ? ( + )} + + {error ? (

{error}

+ ) : loading && folders.length === 0 && files.length === 0 ? ( + // First load with nothing to show yet — leave the panel empty (the bar + // signals activity) rather than flashing the "no files" empty state. +
) : ( - + )}
diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index 8ec974e..d796907 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -3,6 +3,10 @@ import { createRoot } from 'react-dom/client'; import { BrowserRouter } from 'react-router-dom'; import App from './App.tsx'; import './index.css'; +import { installGlobalErrorReporting } from './utils/errorReporter.ts'; + +// Capture uncaught browser errors / promise rejections and ship them to S3. +installGlobalErrorReporting(); // biome-ignore lint/style/noNonNullAssertion: root element is guaranteed to exist in index.html createRoot(document.getElementById('root')!).render( diff --git a/frontend/src/pages/LargeFolderUploadPage.tsx b/frontend/src/pages/LargeFolderUploadPage.tsx index b1a11df..e5ec8c8 100644 --- a/frontend/src/pages/LargeFolderUploadPage.tsx +++ b/frontend/src/pages/LargeFolderUploadPage.tsx @@ -8,13 +8,15 @@ */ import { useVirtualizer } from '@tanstack/react-virtual'; -import { useCallback, useEffect, useRef, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useNavigate } from 'react-router-dom'; import { cancelLargeFolderSync, startLargeFolderSync } from '../api/largeFolderUpload.ts'; import FolderPicker from '../components/common/FolderPicker.tsx'; +import S3FolderPicker from '../components/common/S3FolderPicker.tsx'; import { useAppStore } from '../stores/appStore.ts'; import { + defaultPrefix, type SyncProgress, type SyncStatus, useLargeFolderUploadStore, @@ -284,6 +286,9 @@ function NamePrefixStep() { const settings = useAppStore((s) => s.settings); const notifications = useAppStore((s) => s.addNotification); + // Stable per-mount timestamp folder name for the "Use timestamp default" shortcut. + const suggestedName = useMemo(() => defaultPrefix(), []); + const handleBack = useCallback(() => { // Return to folder picker — clear folderPath so SetupPhase shows picker again setFolderPath(''); @@ -319,35 +324,20 @@ function NamePrefixStep() {

- Name your cloud destination folder + Choose your cloud destination folder

- The default name is fine. It's a timestamp so each - upload gets its own folder in NLR Cloud Storage and nothing gets overwritten. You can - change it if you're continuing a previous upload or want a friendlier name. + Browse NLR Cloud Storage and pick an existing folder to add to it, or create a new + subfolder. Already-uploaded files are skipped automatically — safe to re-run.
-
- - setS3Prefix(e.target.value)} - className="w-full border border-gray-300 rounded-md px-3 py-2 text-sm font-mono focus:outline-none focus:ring-2 focus:ring-nlr-blue" - placeholder="user_upload_2025-01-01T12-00-00" - // biome-ignore lint/a11y/noAutofocus: focus is intentional — user is here to name the folder - autoFocus - /> -

- Files land at{' '} - s3://<bucket>/{s3Prefix.trim() || '…'}/ -

-
+