diff --git a/.gitignore b/.gitignore index 0716996..4029838 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ settings.json # SQLite cache modaq_upload_cache.db +s3_cache.db # Log files /logs/ diff --git a/app/__init__.py b/app/__init__.py index 3c018a0..83d360b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -189,6 +189,35 @@ def _handle_uncaught(error: Exception) -> Response | tuple[Response, int] | HTTP return jsonify({"error": "Internal server error"}), 500 +def _start_s3_warmup() -> None: + """Pre-resolve AWS credentials and open an S3 connection in the background. + + The first S3 call in a fresh process pays a one-time cost — credential + resolution (which for SSO/assume-role can be slow) plus TLS connection setup. + Without warming, that cost lands on the user's first file-browser request, + making the initial listing feel slow even though warm calls are ~tens of ms. + This does the cold work up front, off the request path, and swallows all + errors (no bucket/credentials configured yet is fine). + """ + + def _warm() -> None: + try: + settings = get_settings() + bucket = settings.s3_bucket + if not bucket: + return + from app.services import s3_service + + client = s3_service.create_s3_client(settings.aws_profile, settings.aws_region) + # Cheap call that forces credential resolution + a live connection. + client.head_bucket(Bucket=bucket) + except Exception: + # Warmup is best-effort; the real request will surface any error. + pass + + threading.Thread(target=_warm, name="s3-warmup", daemon=True).start() + + def create_app() -> Flask: """Create and configure the Flask application.""" app = Flask(__name__) @@ -226,6 +255,9 @@ def inject_display_name() -> dict[str, str]: # Automatic request/error logging hooks _register_logging_hooks(app) + # Warm the S3 client/credentials so the first file-browser request isn't cold + _start_s3_warmup() + # Start background SSE cleanup thread _start_sse_cleanup() diff --git a/app/config.py b/app/config.py index f4699d2..6440a2b 100644 --- a/app/config.py +++ b/app/config.py @@ -3,9 +3,13 @@ import functools import json import os +import platform +import re import subprocess import sys +import tempfile import tomllib +import uuid from pathlib import Path from typing import Any @@ -14,6 +18,55 @@ # Base directory for the project BASE_DIR = Path(__file__).resolve().parent.parent +# Folder name used under the OS-specific application-data location for logs. +APP_DIR_NAME = "modaq_upload" + + +def get_default_log_directory(app_name: str = APP_DIR_NAME) -> Path: + """Return the OS-appropriate directory for application logs. + + Follows each platform's conventions for per-user application data so logs + live in a writable, persistent, standard location instead of inside the + repository (which breaks when the app is installed read-only or moved): + + - Windows: ``%LOCALAPPDATA%\\\\logs`` + (e.g. ``C:\\Users\\you\\AppData\\Local\\modaq_upload\\logs``) + - macOS: ``~/Library/Logs/`` + - Linux/other POSIX: ``$XDG_STATE_HOME//logs`` + (defaults to ``~/.local/state//logs`` per the XDG Base Directory + spec, which places logs under "state" data) + """ + home = Path.home() + + if sys.platform == "win32": + base = os.environ.get("LOCALAPPDATA") + root = Path(base) if base else home / "AppData" / "Local" + return root / app_name / "logs" + + if sys.platform == "darwin": + return home / "Library" / "Logs" / app_name + + # Linux and other POSIX systems: XDG state directory. + base = os.environ.get("XDG_STATE_HOME") + root = Path(base) if base else home / ".local" / "state" + return root / app_name / "logs" + + +def _is_under_temp_dir(path: Path) -> bool: + """True if ``path`` lives inside the OS temp directory. + + A configured log directory under the system temp dir is almost always a + stale value leaked from a test run (pytest's ``tempfile.mkdtemp``), not a + real log location — production logs must never be written there. + """ + try: + tmp = Path(tempfile.gettempdir()).resolve() + resolved = path.resolve() + return resolved == tmp or tmp in resolved.parents + except OSError: + return False + + # Load environment variables from .env file ENV_FILE = BASE_DIR / ".env" load_dotenv(ENV_FILE) @@ -54,6 +107,44 @@ def get_package_name() -> str: return _read_pyproject_field("name", "modaq-uploader") +def _sanitize_partition_value(value: str) -> str: + """Make a string safe to use as a hive partition value (a path segment). + + Replaces path-hostile characters (slashes, ``=``, and whitespace) with ``-`` so + the value can't break out of its partition directory or confuse hive parsing. + Falls back to ``"unknown"`` when the result is empty. + """ + cleaned = re.sub(r"[/\\=\s]+", "-", value.strip()).strip("-") + return cleaned or "unknown" + + +def get_os_name() -> str: + """Return a partition-safe OS name (e.g. ``Darwin``/``Windows``/``Linux``).""" + return _sanitize_partition_value(platform.system()) + + +def get_os_version() -> str: + """Return a partition-safe OS version (e.g. ``25.4.0``). + + Uses ``platform.release()`` for cross-platform uniformity (on macOS this is the + Darwin kernel version; ``platform.mac_ver()[0]`` would give the product version). + """ + return _sanitize_partition_value(platform.release()) + + +def get_session_partitions() -> list[str]: + """Return hive partition segments identifying this install's log session. + + Combines OS, OS version, and a persistent per-install ID so logs from multiple + machines syncing to one S3 bucket land at distinct, queryable paths. + """ + return [ + f"os={get_os_name()}", + f"os_version={get_os_version()}", + f"session={get_settings().install_id}", + ] + + class Settings: """Manages application settings stored in JSON format.""" @@ -84,7 +175,7 @@ def _load_settings(self) -> None: "s3_bucket": "", "default_upload_folder": "", "display_name": "MODAQ Uploader", - "log_directory": "logs", + "log_directory": str(get_default_log_directory()), "file_categories": [ { "name": "data", @@ -149,6 +240,17 @@ def _load_settings(self) -> None: defaults[key] = value provenance[key] = {"source": "env", "env_var": env_var} + # Repair an empty or temp-dir log_directory (e.g. a path leaked from a + # test run that got persisted to settings.json). Fall back to the + # OS-standard location so logs never silently write to a temp dir. + raw_log_dir = str(defaults.get("log_directory") or "").strip() + resolved_log_dir = ( + Path(raw_log_dir) if Path(raw_log_dir).is_absolute() else BASE_DIR / raw_log_dir + ) + if not raw_log_dir or _is_under_temp_dir(resolved_log_dir): + defaults["log_directory"] = str(get_default_log_directory()) + provenance["log_directory"] = {"source": "builtin"} + self._settings = defaults self._provenance = provenance @@ -212,6 +314,20 @@ def display_name(self) -> str: """Get the display name for the application.""" return str(self._settings["display_name"]) + @property + def install_id(self) -> str: + """Get a stable per-install identifier, generating and persisting it once. + + Stored in the gitignored settings.json so it survives restarts and stays + unique per machine. Used to give each install's logs a distinct S3 path. + """ + existing = self._settings.get("install_id") + if existing: + return str(existing) + new_id = uuid.uuid4().hex[:12] + self.set("install_id", new_id) + return new_id + @property def log_directory(self) -> Path: """Get the log directory path (resolved to absolute path relative to BASE_DIR).""" diff --git a/app/routes/files.py b/app/routes/files.py index 4621bcd..f565154 100644 --- a/app/routes/files.py +++ b/app/routes/files.py @@ -29,12 +29,15 @@ def list_files() -> tuple[Response, int]: Query parameters: prefix: S3 prefix to filter by (default: "") delimiter: Delimiter for grouping (default: "/") + token: Continuation token from a previous response's next_token, to fetch + the next page of a folder with more results than fit in one page Returns: JSON response with folders and files """ prefix = request.args.get("prefix", "") delimiter = request.args.get("delimiter", "/") + token = request.args.get("token") or None try: client = s3_service.create_s3_client( @@ -47,6 +50,7 @@ def list_files() -> tuple[Response, int]: g.settings.s3_bucket, prefix=prefix, delimiter=delimiter, + continuation_token=token, ) if not result["success"]: @@ -103,6 +107,77 @@ def get_file_info() -> tuple[Response, int]: return jsonify({"error": str(e)}), 500 +@files_bp.route("/stats", methods=["GET"]) +def get_prefix_stats() -> tuple[Response, int]: + """Get subfolder and file counts for a folder (one level). + + Query parameters: + prefix: S3 prefix to summarize (default: "" = bucket root) + delimiter: Delimiter for grouping (default: "/") + + Returns: + JSON response with folder_count and file_count + """ + prefix = request.args.get("prefix", "") + delimiter = request.args.get("delimiter", "/") + + try: + client = s3_service.create_s3_client( + g.settings.aws_profile, + g.settings.aws_region, + ) + + result = s3_service.get_prefix_counts( + client, + g.settings.s3_bucket, + prefix=prefix, + delimiter=delimiter, + ) + + if not result["success"]: + return jsonify({"error": result["error"]}), 500 + + return jsonify(result), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + + +@files_bp.route("/download", methods=["GET"]) +def download_file() -> tuple[Response, int]: + """Generate a presigned URL for downloading an S3 object. + + Query parameters: + key: S3 object key + + Returns: + JSON response with a presigned download URL + """ + key = request.args.get("key", "") + if not key: + return jsonify({"error": "Object key required"}), 400 + + try: + client = s3_service.create_s3_client( + g.settings.aws_profile, + g.settings.aws_region, + ) + + result = s3_service.generate_presigned_download_url( + client, + g.settings.s3_bucket, + key, + ) + + if not result["success"]: + return jsonify({"error": result["error"]}), 404 + + return jsonify(result), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + + @files_bp.route("/browse", methods=["GET"]) def browse_local() -> tuple[Response, int]: """Browse local filesystem for folder selection. diff --git a/app/routes/settings.py b/app/routes/settings.py index 541cfe3..2716629 100644 --- a/app/routes/settings.py +++ b/app/routes/settings.py @@ -63,6 +63,10 @@ def update_settings() -> tuple[Response, int]: settings.update(filtered_data) + # Changing the profile/region invalidates any cached S3 client. + if "aws_profile" in filtered_data or "aws_region" in filtered_data: + s3_service.reset_s3_client_cache() + log = get_log_service() log.info( "settings", diff --git a/app/routes/upload.py b/app/routes/upload.py index 419c916..7c3f1ab 100644 --- a/app/routes/upload.py +++ b/app/routes/upload.py @@ -79,10 +79,7 @@ def progress_callback(job: UploadJob) -> None: # Large jobs read per-file results from /api/upload/results (SQLite-backed) # rather than receiving a 10k-row payload here. Small jobs keep the # legacy behavior — frontend merges the full file array directly. - if ( - large_job_threshold is not None - and len(job.files) >= large_job_threshold - ): + if large_job_threshold is not None and len(job.files) >= large_job_threshold: payload = job.to_progress_dict() payload["terminal"] = True sse.send_event(job.job_id, payload) @@ -96,9 +93,7 @@ def progress_callback(job: UploadJob) -> None: def _large_job_threshold() -> int: """Read the live setting for the large-job cutoff.""" - return int( - get_settings().batch_processing.get("large_job_threshold", 1000) - ) + return int(get_settings().batch_processing.get("large_job_threshold", 1000)) @upload_bp.route("/analyze", methods=["POST"]) diff --git a/app/services/log_service.py b/app/services/log_service.py index 1777853..84dff19 100644 --- a/app/services/log_service.py +++ b/app/services/log_service.py @@ -1,7 +1,10 @@ """JSONL logging service for application events. -Writes one JSON object per line to hive-partitioned daily .jsonl files. +Writes one JSON object per line to hive-partitioned daily .jsonl files. Files are +partitioned by date (year/month/day) and then by session (os/os_version/session) so +multiple installs can sync to one S3 bucket without overwriting each other. DuckDB-compatible: SELECT * FROM read_json_auto('logs/json/**/events.jsonl', hive_partitioning=true) +exposes year, month, day, os, os_version, and session as columns. """ import csv @@ -14,7 +17,7 @@ from pathlib import Path from typing import Any -from app.config import get_settings +from app.config import get_session_partitions, get_settings class LogService: @@ -44,17 +47,24 @@ def _get_log_dir(self) -> Path: def _get_hive_dir(self, subdir: str, dt: datetime) -> Path: """Build a hive-partitioned directory path and create it. + The path is partitioned by date and then by this install's session + (OS, OS version, install ID) so logs from multiple machines syncing to one + S3 bucket land at distinct, queryable paths instead of overwriting each other. + Args: subdir: Top-level subdirectory ('json' or 'csv') dt: Datetime to partition by Returns: - Path like logs/json/year=2026/month=02/day=08/ + Path like + logs/json/year=2026/month=02/day=08/os=Darwin/os_version=25.4.0/session=ab12cd34/ """ log_dir = self._get_log_dir() hive_dir = ( log_dir / subdir / f"year={dt.year:04d}" / f"month={dt.month:02d}" / f"day={dt.day:02d}" ) + for segment in get_session_partitions(): + hive_dir = hive_dir / segment hive_dir.mkdir(parents=True, exist_ok=True) return hive_dir @@ -329,14 +339,12 @@ def read_log_entries( dt = datetime.strptime(date, "%Y-%m-%d") except ValueError: return {"entries": [], "total": 0, "offset": offset, "limit": limit} - hive_path = ( - json_dir - / f"year={dt.year:04d}" - / f"month={dt.month:02d}" - / f"day={dt.day:02d}" - / "events.jsonl" + day_dir = ( + json_dir / f"year={dt.year:04d}" / f"month={dt.month:02d}" / f"day={dt.day:02d}" ) - files = [hive_path] if hive_path.exists() else [] + # events.jsonl now lives under session sub-dirs (os=/os_version=/session=), + # and a synced-down bucket may hold several sessions for one day. + files = sorted(day_dir.rglob("events.jsonl")) if day_dir.exists() else [] else: if json_dir.exists(): files = sorted(json_dir.rglob("events.jsonl"), reverse=True) diff --git a/app/services/s3_service.py b/app/services/s3_service.py index f1afa60..68d50ec 100644 --- a/app/services/s3_service.py +++ b/app/services/s3_service.py @@ -9,6 +9,7 @@ from boto3.s3.transfer import TransferConfig from botocore.exceptions import ClientError, NoCredentialsError from mypy_boto3_s3 import S3Client +from mypy_boto3_s3.type_defs import PaginatorConfigTypeDef class UploadCancelledError(Exception): @@ -62,8 +63,20 @@ def get_available_profiles() -> list[str]: return sorted(profiles) +# Cache S3 clients by (profile, region). Building a boto3 Session resolves +# credentials (which for SSO/assume-role can mean a network round-trip), so +# rebuilding one per request added latency to every list/stats/download call. +# boto3 low-level clients are thread-safe for making calls, so a shared instance +# is safe across the dev server's request threads. +_CLIENT_CACHE: dict[tuple[str, str], S3Client] = {} + + def create_s3_client(profile: str, region: str = "us-west-2") -> S3Client: - """Create an S3 client using the specified AWS profile. + """Create (or reuse) an S3 client for the given AWS profile and region. + + Clients are cached by (profile, region) so repeated calls — e.g. the parallel + list/stats requests on the file browser — don't each rebuild a session and + re-resolve credentials. Args: profile: AWS profile name from ~/.aws/credentials or ~/.aws/config @@ -75,11 +88,26 @@ def create_s3_client(profile: str, region: str = "us-west-2") -> S3Client: Raises: NoCredentialsError: If credentials are not found """ + cache_key = (profile, region) + cached = _CLIENT_CACHE.get(cache_key) + if cached is not None: + return cached + session = boto3.Session(profile_name=profile, region_name=region) client: S3Client = session.client("s3") + _CLIENT_CACHE[cache_key] = client return client +def reset_s3_client_cache() -> None: + """Clear the cached S3 clients. + + Call after changing AWS settings so the next request rebuilds the client with + the new profile/region/credentials. + """ + _CLIENT_CACHE.clear() + + def check_file_exists(client: S3Client, bucket: str, key: str) -> bool: """Check if a file exists in S3. @@ -186,40 +214,54 @@ def list_bucket_objects( prefix: str = "", delimiter: str = "/", max_keys: int = 1000, + continuation_token: str | None = None, ) -> dict[str, Any]: - """List objects in an S3 bucket with prefix filtering. + """List objects in an S3 bucket with prefix filtering, one page at a time. + + Returns at most ``max_keys`` items (folders + files combined). When more + results exist, ``next_token`` is a non-null string the caller passes back as + ``continuation_token`` to fetch the next page. This avoids silently truncating + folders that contain more than ``max_keys`` objects. Args: client: S3 client bucket: S3 bucket name prefix: Object key prefix for filtering delimiter: Delimiter for grouping (default: /) - max_keys: Maximum number of keys to return + max_keys: Maximum number of items to return per page + continuation_token: Opaque token from a previous call's ``next_token`` to + resume listing where the last page left off Returns: - Dictionary containing folders and files + Dictionary containing folders, files, and ``next_token`` (None when done) """ folders: list[dict[str, str]] = [] files: list[dict[str, Any]] = [] try: + pagination_config: PaginatorConfigTypeDef = {"MaxItems": max_keys} + if continuation_token: + pagination_config["StartingToken"] = continuation_token + paginator = client.get_paginator("list_objects_v2") page_iterator = paginator.paginate( Bucket=bucket, Prefix=prefix, Delimiter=delimiter, - PaginationConfig={"MaxItems": max_keys}, + PaginationConfig=pagination_config, ) for page in page_iterator: - # Get common prefixes (folders) - for prefix_info in page.get("CommonPrefixes", []): + # Get common prefixes (folders). Note: when resuming from a + # StartingToken these keys may be present but explicitly None, so + # `or []` guards against iterating over None. + for prefix_info in page.get("CommonPrefixes") or []: folder_prefix = prefix_info.get("Prefix", "") folder_name = folder_prefix.rstrip("/").split("/")[-1] folders.append({"name": folder_name, "prefix": folder_prefix}) # Get objects (files) - for obj in page.get("Contents", []): + for obj in page.get("Contents") or []: key = obj.get("Key", "") # Skip the prefix itself if it's listed if key == prefix: @@ -237,12 +279,16 @@ def list_bucket_objects( } ) + # resume_token is None once all results have been returned. + next_token = page_iterator.resume_token + return { "success": True, "bucket": bucket, "prefix": prefix, "folders": folders, "files": files, + "next_token": next_token, "error": None, } except ClientError as e: @@ -252,6 +298,85 @@ def list_bucket_objects( "prefix": prefix, "folders": [], "files": [], + "next_token": None, + "error": str(e), + } + + +# Max entries (subfolders + files) get_prefix_counts looks at. The count is a +# nice-to-have, so it costs exactly ONE list_objects_v2 call (a single page) and +# never enumerates a large level. S3 counts both keys and common prefixes toward +# MaxKeys, so this bounds work whether the level is files, folders, or a mix. +# Capped at S3's 1,000-key page maximum. +STATS_MAX_ITEMS = 1_000 + + +def get_prefix_counts( + client: S3Client, + bucket: str, + prefix: str = "", + delimiter: str = "/", + max_items: int = STATS_MAX_ITEMS, +) -> dict[str, Any]: + """Count subfolders and files directly under a prefix (one level only). + + Uses ``delimiter`` so S3 returns just the immediate level — this never recurses + into subfolders. It makes a SINGLE ``list_objects_v2`` request (one page of up + to ``max_items`` entries) and reads ``IsTruncated`` to know whether there are + more. So it always costs one S3 call no matter how large the level is; when the + level overflows the page, ``capped`` is True and the counts are lower bounds. + + Args: + client: S3 client + bucket: S3 bucket name + prefix: Object key prefix for the folder to summarize + delimiter: Delimiter for grouping (default: /) + max_items: Page size — entries (folders + files) to look at, clamped to + S3's 1,000 maximum; default ``STATS_MAX_ITEMS`` + + Returns: + Dictionary with ``folder_count``, ``file_count``, and ``capped`` for this + level. When ``capped`` is True the true totals are higher than reported. + """ + folder_prefixes: set[str] = set() + file_count = 0 + + try: + max_keys = max(1, min(max_items, 1000)) + response = client.list_objects_v2( + Bucket=bucket, + Prefix=prefix, + Delimiter=delimiter, + MaxKeys=max_keys, + ) + + for prefix_info in response.get("CommonPrefixes") or []: + folder_prefixes.add(prefix_info.get("Prefix", "")) + for obj in response.get("Contents") or []: + key = obj.get("Key", "") + # Skip the prefix placeholder object and any directory markers. + if key == prefix or not key.split("/")[-1]: + continue + file_count += 1 + + # IsTruncated means the level has more entries than fit in this one page. + capped = bool(response.get("IsTruncated", False)) + + return { + "success": True, + "prefix": prefix, + "folder_count": len(folder_prefixes), + "file_count": file_count, + "capped": capped, + "error": None, + } + except ClientError as e: + return { + "success": False, + "prefix": prefix, + "folder_count": 0, + "file_count": 0, + "capped": False, "error": str(e), } @@ -303,6 +428,59 @@ def get_s3_client_from_settings() -> tuple[S3Client, str]: return client, settings.s3_bucket +def generate_presigned_download_url( + client: S3Client, + bucket: str, + key: str, + expires_in: int = 3600, +) -> dict[str, Any]: + """Generate a presigned URL for downloading an S3 object. + + The URL forces a browser download (Content-Disposition: attachment) with the + object's original filename. Presigned URLs let the browser fetch directly from + S3 rather than proxying large files through Flask. + + Args: + client: S3 client + bucket: S3 bucket name + key: S3 object key + expires_in: URL validity in seconds (default: 3600 = 1 hour) + + Returns: + Dictionary with the presigned URL or an error + """ + filename = key.split("/")[-1] + try: + # Confirm the object exists so a missing key returns 404 rather than a + # presigned URL that later fails when the user clicks it. + client.head_object(Bucket=bucket, Key=key) + + url = client.generate_presigned_url( + "get_object", + Params={ + "Bucket": bucket, + "Key": key, + "ResponseContentDisposition": f'attachment; filename="{filename}"', + }, + ExpiresIn=expires_in, + ) + return { + "success": True, + "url": url, + "key": key, + "filename": filename, + "error": None, + } + except ClientError as e: + error_code = e.response["Error"]["Code"] + error_msg = f"File '{filename}' not found" if error_code == "404" else str(e) + return { + "success": False, + "key": key, + "error": error_msg, + } + + def get_object_metadata(client: S3Client, bucket: str, key: str) -> dict[str, Any]: """Get metadata for an S3 object. diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index f2a2fcd..6818fc5 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -17,9 +17,13 @@ async function handleResponse(res: Response): Promise { return res.json() as Promise; } -export async function apiGet(url: string, params?: Record): Promise { +export async function apiGet( + url: string, + params?: Record, + signal?: AbortSignal, +): Promise { const qs = params ? `?${new URLSearchParams(params)}` : ''; - const res = await fetch(`${url}${qs}`); + const res = await fetch(`${url}${qs}`, { signal }); return handleResponse(res); } diff --git a/frontend/src/components/files/FileList.tsx b/frontend/src/components/files/FileList.tsx index 54842c1..52aad89 100644 --- a/frontend/src/components/files/FileList.tsx +++ b/frontend/src/components/files/FileList.tsx @@ -1,5 +1,9 @@ -import type { S3File, S3Folder } from '../../types/api.ts'; -import { FileIcon, FolderIcon } from '../../utils/icons.tsx'; +import { useVirtualizer } from '@tanstack/react-virtual'; +import { useMemo, useRef, useState } from 'react'; +import { apiGet } from '../../api/client.ts'; +import { useAppStore } from '../../stores/appStore.ts'; +import type { S3DownloadResponse, S3File, S3Folder } from '../../types/api.ts'; +import { DownloadIcon, FileIcon, FolderIcon, SpinnerIcon } from '../../utils/icons.tsx'; function formatBytes(bytes: number): string { if (bytes === 0) return '0 B'; @@ -23,9 +27,70 @@ interface FileListProps { folders: S3Folder[]; files: S3File[]; onNavigate: (prefix: string) => void; + hasMore?: boolean; + loadingMore?: boolean; + onLoadMore?: () => void; } -export default function FileList({ folders, files, onNavigate }: FileListProps) { +// Folders render first, then files, in a single virtualized list. +type Row = { type: 'folder'; folder: S3Folder } | { type: 'file'; file: S3File }; + +export default function FileList({ + folders, + files, + onNavigate, + hasMore = false, + loadingMore = false, + onLoadMore, +}: FileListProps) { + const addNotification = useAppStore((s) => s.addNotification); + const [downloadingKey, setDownloadingKey] = useState(null); + const parentRef = useRef(null); + + const rows = useMemo( + () => [ + ...folders.map((folder): Row => ({ type: 'folder', folder })), + ...files.map((file): Row => ({ type: 'file', file })), + ], + [folders, files], + ); + + // Virtualize the rows so folders with thousands of files render only the + // handful currently on screen rather than mounting every row. Rows are a fixed + // single-line height, so no per-row measurement is needed. + const virtualizer = useVirtualizer({ + count: rows.length, + getScrollElement: () => parentRef.current, + estimateSize: () => 49, + overscan: 12, + }); + + const handleDownload = async (file: S3File) => { + setDownloadingKey(file.key); + try { + const data = await apiGet('/api/files/download', { key: file.key }); + if (!data.success) { + addNotification('error', data.error ?? `Failed to download ${file.name}`); + return; + } + // Navigating to the presigned URL triggers the browser download + // (the URL sets Content-Disposition: attachment). + const link = document.createElement('a'); + link.href = data.url; + link.download = file.name; + document.body.appendChild(link); + link.click(); + link.remove(); + } catch (err) { + addNotification( + 'error', + err instanceof Error ? err.message : `Failed to download ${file.name}`, + ); + } finally { + setDownloadingKey(null); + } + }; + if (folders.length === 0 && files.length === 0) { return (
@@ -36,36 +101,86 @@ export default function FileList({ folders, files, onNavigate }: FileListProps) } return ( -
- {folders.map((folder) => ( - - ))} + <> +
+
+ {virtualizer.getVirtualItems().map((vItem) => { + const row = rows[vItem.index]; + return ( +
+ {row.type === 'folder' ? ( + + ) : ( +
+ +
+

+ {row.file.name} +

+
+ + {formatBytes(row.file.size)} + + + {formatDate(row.file.last_modified)} + + +
+ )} +
+ ); + })} +
+
- {files.map((file) => ( -
- -
-

- {file.name} -

-
- {formatBytes(file.size)} - - {formatDate(file.last_modified)} - + {hasMore && ( +
+
- ))} -
+ )} + ); } diff --git a/frontend/src/components/files/S3Browser.tsx b/frontend/src/components/files/S3Browser.tsx index a295dc9..ac2a091 100644 --- a/frontend/src/components/files/S3Browser.tsx +++ b/frontend/src/components/files/S3Browser.tsx @@ -1,9 +1,9 @@ -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { apiGet } from '../../api/client.ts'; -import type { S3File, S3Folder, S3ListResponse } from '../../types/api.ts'; +import type { S3File, S3Folder, S3ListResponse, S3StatsResponse } from '../../types/api.ts'; +import { reportClientEvent } from '../../utils/errorReporter.ts'; import { CloudIcon } from '../../utils/icons.tsx'; import Breadcrumb from '../common/Breadcrumb.tsx'; -import Spinner from '../common/Spinner.tsx'; import FileList from './FileList.tsx'; interface S3BrowserProps { @@ -16,7 +16,13 @@ export default function S3Browser({ bucketName, region }: S3BrowserProps) { const [folders, setFolders] = useState([]); const [files, setFiles] = useState([]); const [loading, setLoading] = useState(true); + const [loadingMore, setLoadingMore] = useState(false); + const [nextToken, setNextToken] = useState(null); const [error, setError] = useState(null); + const [stats, setStats] = useState(null); + // Aborts the in-flight stats request when we navigate away before it returns, + // so stale counts can't land late and overwrite the current folder's note. + const statsAbortRef = useRef(null); // Build breadcrumb items from the current prefix const breadcrumbItems = (() => { @@ -45,16 +51,36 @@ export default function S3Browser({ bucketName, region }: S3BrowserProps) { const fetchObjects = useCallback(async (currentPrefix: string) => { setLoading(true); setError(null); + // Clear the previous folder's rows immediately. Otherwise the parent's + // contents linger under the new breadcrumb until this fetch returns, which + // reads as a stale/"stuck" list showing the wrong folder. + setFolders([]); + setFiles([]); + setNextToken(null); + const startedAt = performance.now(); try { const params: Record = { delimiter: '/' }; if (currentPrefix) params.prefix = currentPrefix; const data = await apiGet('/api/files/list', params); + // Browser-perceived round-trip (network + server). Compare against the + // backend's own duration_ms log to separate S3/network time from handling. + const durationMs = Math.round(performance.now() - startedAt); + reportClientEvent('s3_list_timing', `files/list ${durationMs}ms`, { + endpoint: '/api/files/list', + prefix: currentPrefix || '(root)', + duration_ms: durationMs, + ok: data.success, + folder_count: data.folders?.length ?? 0, + file_count: data.files?.length ?? 0, + has_more: data.next_token != null, + }); if (!data.success) { setError(data.error ?? 'Failed to list objects'); return; } setFolders(data.folders); setFiles(data.files); + setNextToken(data.next_token ?? null); } catch (err) { setError(err instanceof Error ? err.message : 'Failed to fetch files'); } finally { @@ -62,14 +88,111 @@ export default function S3Browser({ bucketName, region }: S3BrowserProps) { } }, []); + // Fetch the next page and append, deduping folders that span page boundaries. + const loadMore = useCallback(async () => { + if (!nextToken || loadingMore) return; + setLoadingMore(true); + try { + const params: Record = { delimiter: '/', token: nextToken }; + if (prefix) params.prefix = prefix; + const data = await apiGet('/api/files/list', params); + if (!data.success) { + setError(data.error ?? 'Failed to load more objects'); + return; + } + setFolders((prev) => { + const seen = new Set(prev.map((f) => f.prefix)); + return [...prev, ...data.folders.filter((f) => !seen.has(f.prefix))]; + }); + setFiles((prev) => { + const seen = new Set(prev.map((f) => f.key)); + return [...prev, ...data.files.filter((f) => !seen.has(f.key))]; + }); + setNextToken(data.next_token ?? null); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load more files'); + } finally { + setLoadingMore(false); + } + }, [nextToken, loadingMore, prefix]); + + // Accurate counts for the current level. Fetched separately because the file + // list is paginated (one page at a time), so its loaded length isn't the total. + const fetchStats = useCallback(async (currentPrefix: string) => { + // Cancel any stats request still running for a previously-viewed folder. + statsAbortRef.current?.abort(); + const controller = new AbortController(); + statsAbortRef.current = controller; + + setStats(null); + const startedAt = performance.now(); + try { + const params: Record = { delimiter: '/' }; + if (currentPrefix) params.prefix = currentPrefix; + const data = await apiGet('/api/files/stats', params, controller.signal); + const durationMs = Math.round(performance.now() - startedAt); + reportClientEvent('s3_stats_timing', `files/stats ${durationMs}ms`, { + endpoint: '/api/files/stats', + prefix: currentPrefix || '(root)', + duration_ms: durationMs, + ok: data.success, + folder_count: data.folder_count, + file_count: data.file_count, + capped: data.capped ?? false, + }); + const valid = + data.success && + typeof data.folder_count === 'number' && + typeof data.file_count === 'number'; + setStats(valid ? data : null); + } catch { + // Aborted (navigated away) or failed — counts are non-critical, so don't + // surface an error. An abort leaves the newer request to set the state. + const durationMs = Math.round(performance.now() - startedAt); + reportClientEvent( + 's3_stats_timing', + controller.signal.aborted + ? `files/stats aborted after ${durationMs}ms` + : `files/stats failed after ${durationMs}ms`, + { + endpoint: '/api/files/stats', + prefix: currentPrefix || '(root)', + duration_ms: durationMs, + aborted: controller.signal.aborted, + }, + ); + if (!controller.signal.aborted) setStats(null); + } + }, []); + useEffect(() => { void fetchObjects(prefix); - }, [prefix, fetchObjects]); + void fetchStats(prefix); + }, [prefix, fetchObjects, fetchStats]); const navigateToPrefix = useCallback((newPrefix: string) => { setPrefix(newPrefix); }, []); + // A plain-language note summarizing the current folder's contents. It appears + // quietly once counting finishes; while counting we render nothing rather than + // a "Counting items…" placeholder, so the note never looks like it's holding + // up the file list (which loads independently and is what actually matters). + const statsNote = (() => { + if (!stats) return null; + // Capped: counting stopped early, so both counts are lower bounds — report a + // single "over N items" rather than implying an exact per-type breakdown. + if (stats.capped) { + const total = stats.folder_count + stats.file_count; + return `Large folder — over ${total.toLocaleString()} items.`; + } + const plural = (n: number, word: string) => + `${n.toLocaleString()} ${word}${n === 1 ? '' : 's'}`; + const folders = plural(stats.folder_count, 'subfolder'); + const files = plural(stats.file_count, 'file'); + return `This folder contains ${folders} and ${files}.`; + })(); + return (
{/* Header: Bucket info + breadcrumb */} @@ -82,15 +205,24 @@ export default function S3Browser({ bucketName, region }: S3BrowserProps) {
+ {statsNote &&

{statsNote}

}
{/* Content */} -
- {loading ? ( -
- +
+ {/* Thin indeterminate bar across the top while a fetch is in flight. + The list stays visible underneath rather than being replaced. */} + {(loading || loadingMore) && ( +
+
- ) : error ? ( + )} + + {error ? (

{error}

+ ) : loading && folders.length === 0 && files.length === 0 ? ( + // First load with nothing to show yet — leave the panel empty (the bar + // signals activity) rather than flashing the "no files" empty state. +
) : ( - + )}
diff --git a/frontend/src/stores/appStore.ts b/frontend/src/stores/appStore.ts index bec8ca4..d762f47 100644 --- a/frontend/src/stores/appStore.ts +++ b/frontend/src/stores/appStore.ts @@ -15,6 +15,41 @@ export interface Notification { message: string; } +// How long an automatic update check stays fresh before we re-run `git fetch`. +const AUTO_UPDATE_CHECK_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours +const AUTO_UPDATE_CHECK_KEY = 'modaq.autoUpdateCheck'; + +interface CachedAutoCheck { + ts: number; + result: UpdateCheckResult; + branchInfo: BranchListResult | null; +} + +/** Return a still-fresh cached update check, or null to trigger a new fetch. */ +function readCachedAutoCheck(): CachedAutoCheck | null { + try { + const raw = localStorage.getItem(AUTO_UPDATE_CHECK_KEY); + if (!raw) return null; + const cached = JSON.parse(raw) as CachedAutoCheck; + if (Date.now() - cached.ts > AUTO_UPDATE_CHECK_TTL_MS) return null; + return cached; + } catch { + return null; + } +} + +function writeCachedAutoCheck( + result: UpdateCheckResult, + branchInfo: BranchListResult | null, +): void { + try { + const payload: CachedAutoCheck = { ts: Date.now(), result, branchInfo }; + localStorage.setItem(AUTO_UPDATE_CHECK_KEY, JSON.stringify(payload)); + } catch { + // Storage unavailable (private mode / quota) — skip caching, no harm. + } +} + interface AppState { // Settings settings: AppSettings | null; @@ -94,12 +129,25 @@ export const useAppStore = create((set, get) => ({ runAutoCheck: async () => { if (get().autoCheckDone) return; set({ autoCheckDone: true }); + + // The update check runs `git fetch` server-side (seconds) and is purely + // informational, so don't run it on every page load. Cache the result in + // localStorage and only re-check once per AUTO_UPDATE_CHECK_TTL_MS. This + // keeps it off the hot path of data pages like /files (which just needs to + // list S3 objects). + const cached = readCachedAutoCheck(); + if (cached) { + set({ autoCheckResult: cached.result, branchInfo: cached.branchInfo }); + return; + } + try { const [checkResult, branchInfo] = await Promise.all([ apiGet('/api/settings/check-updates'), apiGet('/api/settings/branches'), ]); set({ autoCheckResult: checkResult, branchInfo }); + writeCachedAutoCheck(checkResult, branchInfo); // Only pop the modal automatically when an update is actually available if (checkResult.updates_available) { set({ showUpdateModal: true }); diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 286d2b9..64b6ea4 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -344,6 +344,8 @@ export interface S3ListResponse { files: S3File[]; breadcrumbs: S3Breadcrumb[]; prefix?: string; + /** Opaque continuation token; pass back as `token` to load the next page. Null when no more results. */ + next_token?: string | null; error?: string; } @@ -364,6 +366,24 @@ export interface S3Breadcrumb { prefix: string; } +export interface S3DownloadResponse { + success: boolean; + url: string; + key: string; + filename: string; + error?: string; +} + +export interface S3StatsResponse { + success: boolean; + prefix: string; + folder_count: number; + file_count: number; + /** True when file_count hit the server's cap and the real total is higher. */ + capped?: boolean; + error?: string; +} + // ── Settings types ── export type ValueSourceType = 'builtin' | 'default_file' | 'settings_file' | 'env'; diff --git a/frontend/src/utils/errorReporter.ts b/frontend/src/utils/errorReporter.ts index fa3ee45..8977277 100644 --- a/frontend/src/utils/errorReporter.ts +++ b/frontend/src/utils/errorReporter.ts @@ -87,3 +87,20 @@ export function reportClientError(message: string, metadata?: Record, +): void { + report({ + level: 'INFO', + event, + message, + metadata: { ...baseMetadata(), ...metadata }, + }); +} diff --git a/frontend/tests/components/S3Browser.test.tsx b/frontend/tests/components/S3Browser.test.tsx index 7725d2a..73b0eec 100644 --- a/frontend/tests/components/S3Browser.test.tsx +++ b/frontend/tests/components/S3Browser.test.tsx @@ -8,9 +8,35 @@ vi.mock('../../src/api/client.ts', () => ({ apiGet: vi.fn(), })); +// Mock the client-side logger so timing reports don't fire real network calls. +vi.mock('../../src/utils/errorReporter.ts', () => ({ + reportClientEvent: vi.fn(), +})); + import { apiGet } from '../../src/api/client.ts'; +import { reportClientEvent } from '../../src/utils/errorReporter.ts'; const mockApiGet = vi.mocked(apiGet); +const mockReportClientEvent = vi.mocked(reportClientEvent); + +const STATS_RESPONSE = { + success: true, + prefix: '', + folder_count: 2, + file_count: 1, +}; + +// Route apiGet by URL so the /api/files/stats call doesn't consume the queued +// /api/files/list responses. `listResponses` are returned in order per list call. +function mockApi(listResponses: unknown[], stats: unknown = STATS_RESPONSE) { + let listCall = 0; + mockApiGet.mockImplementation((url: string) => { + if (url === '/api/files/stats') return Promise.resolve(stats); + const idx = Math.min(listCall, listResponses.length - 1); + listCall += 1; + return Promise.resolve(listResponses[idx]); + }); +} const MOCK_LIST_RESPONSE = { success: true, @@ -37,20 +63,57 @@ const MOCK_SUBFOLDER_RESPONSE = { describe('S3Browser', () => { beforeEach(() => { mockApiGet.mockReset(); + mockReportClientEvent.mockReset(); }); afterEach(() => { vi.restoreAllMocks(); }); - it('shows loading spinner initially', () => { + it('shows a thin loading bar (not a full spinner) while fetching', () => { mockApiGet.mockReturnValue(new Promise(() => {})); // Never resolves render(); - expect(screen.getByText('Loading files...')).toBeInTheDocument(); + // The list fetch shows the indeterminate top bar, and never the old + // full-panel "Loading files..." spinner. + expect(screen.getByRole('progressbar')).toBeInTheDocument(); + expect(screen.queryByText('Loading files...')).not.toBeInTheDocument(); + }); + + it('clears the previous folder while navigating, then shows the new one', async () => { + const user = userEvent.setup(); + let resolveSecond: ((v: unknown) => void) | undefined; + let listCall = 0; + mockApiGet.mockImplementation((url: string) => { + if (url === '/api/files/stats') return Promise.resolve(STATS_RESPONSE); + listCall += 1; + if (listCall === 1) return Promise.resolve(MOCK_LIST_RESPONSE); + // Second navigation: keep it pending so we can observe the loading state. + return new Promise((resolve) => { + resolveSecond = resolve; + }); + }); + + render(); + + await waitFor(() => { + expect(screen.getByText('year=2024')).toBeInTheDocument(); + }); + + await user.click(screen.getByText('year=2024')); + + // The previous folder's rows are cleared immediately so the parent's + // contents don't linger under the new breadcrumb. The loading bar shows. + expect(screen.queryByText('test.mcap')).not.toBeInTheDocument(); + expect(screen.getByRole('progressbar')).toBeInTheDocument(); + + resolveSecond?.(MOCK_SUBFOLDER_RESPONSE); + await waitFor(() => { + expect(screen.getByText('month=01')).toBeInTheDocument(); + }); }); it('renders bucket name and region', async () => { - mockApiGet.mockResolvedValue(MOCK_LIST_RESPONSE); + mockApi([MOCK_LIST_RESPONSE]); render(); await waitFor(() => { @@ -60,7 +123,7 @@ describe('S3Browser', () => { }); it('renders folders and files from the API response', async () => { - mockApiGet.mockResolvedValue(MOCK_LIST_RESPONSE); + mockApi([MOCK_LIST_RESPONSE]); render(); await waitFor(() => { @@ -73,9 +136,7 @@ describe('S3Browser', () => { it('navigates into a folder when clicked', async () => { const user = userEvent.setup(); - mockApiGet - .mockResolvedValueOnce(MOCK_LIST_RESPONSE) // Initial load - .mockResolvedValueOnce(MOCK_SUBFOLDER_RESPONSE); // After click + mockApi([MOCK_LIST_RESPONSE, MOCK_SUBFOLDER_RESPONSE]); render(); @@ -101,13 +162,137 @@ describe('S3Browser', () => { expect(screen.getByText('Retry')).toBeInTheDocument(); }); - it('shows empty state when no files or folders', async () => { - mockApiGet.mockResolvedValue({ + it('shows a Load more button and appends the next page when clicked', async () => { + const user = userEvent.setup(); + mockApi([ + { + success: true, + folders: [], + files: [ + { name: 'a.mcap', key: 'a.mcap', size: 1024, last_modified: '2024-01-15T10:30:00Z' }, + ], + breadcrumbs: [], + next_token: 'TOKEN_PAGE_2', + }, + { + success: true, + folders: [], + files: [ + { name: 'b.mcap', key: 'b.mcap', size: 2048, last_modified: '2024-01-16T10:30:00Z' }, + ], + breadcrumbs: [], + next_token: null, + }, + ]); + + render(); + + await waitFor(() => { + expect(screen.getByText('a.mcap')).toBeInTheDocument(); + }); + expect(screen.getByText('Load more')).toBeInTheDocument(); + + await user.click(screen.getByText('Load more')); + + // Both pages are now visible and the button is gone (no more pages). + await waitFor(() => { + expect(screen.getByText('b.mcap')).toBeInTheDocument(); + }); + expect(screen.getByText('a.mcap')).toBeInTheDocument(); + expect(screen.queryByText('Load more')).not.toBeInTheDocument(); + + // The second call passed the continuation token. + expect(mockApiGet).toHaveBeenLastCalledWith( + '/api/files/list', + expect.objectContaining({ token: 'TOKEN_PAGE_2' }), + ); + }); + + it('does not show Load more when there are no further pages', async () => { + mockApi([MOCK_LIST_RESPONSE]); + render(); + + await waitFor(() => { + expect(screen.getByText('test.mcap')).toBeInTheDocument(); + }); + expect(screen.queryByText('Load more')).not.toBeInTheDocument(); + }); + + it('shows a note summarizing the folder contents', async () => { + mockApi([MOCK_LIST_RESPONSE], { + success: true, + prefix: '', + folder_count: 2, + file_count: 1240, + }); + render(); + + await waitFor(() => { + expect( + screen.getByText('This folder contains 2 subfolders and 1,240 files.'), + ).toBeInTheDocument(); + }); + }); + + it('singularizes the note for a single file and folder', async () => { + mockApi([MOCK_LIST_RESPONSE], { success: true, - folders: [], - files: [], - breadcrumbs: [], + prefix: '', + folder_count: 1, + file_count: 1, + }); + render(); + + await waitFor(() => { + expect(screen.getByText('This folder contains 1 subfolder and 1 file.')).toBeInTheDocument(); }); + }); + + it('shows an "over N items" note when counting was capped', async () => { + mockApi([MOCK_LIST_RESPONSE], { + success: true, + prefix: '', + folder_count: 2000, + file_count: 0, + capped: true, + }); + render(); + + await waitFor(() => { + expect(screen.getByText('Large folder — over 2,000 items.')).toBeInTheDocument(); + }); + }); + + it('reports list and stats request timings to the client logger', async () => { + mockApi([MOCK_LIST_RESPONSE]); + render(); + + await waitFor(() => { + expect(screen.getByText('test.mcap')).toBeInTheDocument(); + }); + + await waitFor(() => { + const events = mockReportClientEvent.mock.calls.map((c) => c[0]); + expect(events).toContain('s3_list_timing'); + expect(events).toContain('s3_stats_timing'); + }); + + const listCall = mockReportClientEvent.mock.calls.find((c) => c[0] === 's3_list_timing'); + expect(listCall?.[2]).toMatchObject({ + endpoint: '/api/files/list', + duration_ms: expect.any(Number), + }); + }); + + it('shows empty state when no files or folders', async () => { + mockApi([ + { + success: true, + folders: [], + files: [], + breadcrumbs: [], + }, + ]); render(); await waitFor(() => { diff --git a/frontend/tests/setup.ts b/frontend/tests/setup.ts index 0c413f0..8a2d250 100644 --- a/frontend/tests/setup.ts +++ b/frontend/tests/setup.ts @@ -2,3 +2,52 @@ import '@testing-library/jest-dom/vitest'; // Ensure consistent timezone across all test environments process.env.TZ = 'UTC'; + +// jsdom lacks ResizeObserver, which @tanstack/react-virtual uses to learn the +// scroll element's size. A no-op never reports a size, so the virtualizer keeps a +// 0-height viewport and renders nothing. This stub calls back once on observe with +// a non-zero box so virtualized lists render their rows in tests. +if (typeof globalThis.ResizeObserver === 'undefined') { + globalThis.ResizeObserver = class { + private cb: ResizeObserverCallback; + constructor(cb: ResizeObserverCallback) { + this.cb = cb; + } + observe(el: Element) { + this.cb( + [ + { + target: el, + contentRect: el.getBoundingClientRect(), + borderBoxSize: [{ inlineSize: 800, blockSize: 600 }], + contentBoxSize: [{ inlineSize: 800, blockSize: 600 }], + devicePixelContentBoxSize: [{ inlineSize: 800, blockSize: 600 }], + } as unknown as ResizeObserverEntry, + ], + this, + ); + } + unobserve() {} + disconnect() {} + }; +} + +// jsdom reports zero geometry for every element, which makes @tanstack/react-virtual +// measure a 0-height viewport and render no rows. Give elements a non-zero rect so +// virtualized lists render their items in tests. +if (!('__rectStubbed' in HTMLElement.prototype)) { + Object.defineProperty(HTMLElement.prototype, '__rectStubbed', { value: true }); + HTMLElement.prototype.getBoundingClientRect = function getBoundingClientRect() { + return { + width: 800, + height: 600, + top: 0, + left: 0, + bottom: 600, + right: 800, + x: 0, + y: 0, + toJSON() {}, + } as DOMRect; + }; +} diff --git a/settings.default.json b/settings.default.json index 0361c54..5e1afbd 100644 --- a/settings.default.json +++ b/settings.default.json @@ -3,7 +3,6 @@ "aws_region": "us-west-2", "s3_bucket": "", "default_upload_folder": "", - "log_directory": "logs", "file_categories": [ { "name": "data", diff --git a/settings.test.json b/settings.test.json new file mode 100644 index 0000000..f5e05d3 --- /dev/null +++ b/settings.test.json @@ -0,0 +1,6 @@ +{ + "aws_profile": "default", + "aws_region": "us-west-2", + "s3_bucket": "test-bucket", + "default_upload_folder": "" +} diff --git a/tests/conftest.py b/tests/conftest.py index 29a0f14..cd802bb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,23 +12,36 @@ from flask.testing import FlaskClient from app import create_app -from app.config import SETTINGS_FILE + + +@pytest.fixture(autouse=True) +def _reset_s3_client_cache() -> Generator[None, None, None]: + """Clear the cached S3 client around every test. + + The cache persists module-level, so without this a client built under one + test's ``mock_aws`` context (or with one profile) would leak into the next. + """ + from app.services import s3_service + + s3_service.reset_s3_client_cache() + yield + s3_service.reset_s3_client_cache() @pytest.fixture def app() -> Generator[Flask, None, None]: """Create application for testing.""" - # Use a temporary settings file + import app.config as config + + # Isolate settings: copy the committed settings.test.json template to a + # throwaway file, point SETTINGS_FILE at the copy, and reset the singleton so + # the app reads/writes there. Without this, tests that save settings (e.g. + # PUT /api/settings) would clobber the real, gitignored settings.json. We copy + # rather than use the template directly so saves never mutate the template. + test_template = config.BASE_DIR / "settings.test.json" + settings_seed = json.loads(test_template.read_text(encoding="utf-8")) with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump( - { - "aws_profile": "default", - "aws_region": "us-west-2", - "s3_bucket": "test-bucket", - "default_upload_folder": "", - }, - f, - ) + json.dump(settings_seed, f) temp_settings = f.name # Ensure frontend/dist/index.html exists so SPA routes can serve it @@ -42,8 +55,10 @@ def app() -> Generator[Flask, None, None]: with open(index_path, "w") as f: f.write('
') - # Create the app (settings will be loaded from default) - _ = SETTINGS_FILE # Reference to avoid unused import warning + original_settings_file = config.SETTINGS_FILE + original_instance = config.Settings._instance + config.SETTINGS_FILE = Path(temp_settings) + config.Settings._instance = None # force a reload from the temp settings file # Redirect logs to a temp dir so the automatic request/error logging hooks # don't write into the repo's real logs/ directory during tests. @@ -51,7 +66,6 @@ def app() -> Generator[Flask, None, None]: settings = get_settings() log_temp_dir = tempfile.mkdtemp() - original_log_dir = settings._settings.get("log_directory") settings._settings["log_directory"] = log_temp_dir app = create_app() @@ -59,9 +73,9 @@ def app() -> Generator[Flask, None, None]: yield app - # Cleanup - if original_log_dir is not None: - settings._settings["log_directory"] = original_log_dir + # Cleanup: restore the real settings file path and singleton. + config.SETTINGS_FILE = original_settings_file + config.Settings._instance = original_instance os.unlink(temp_settings) if dist_created: os.unlink(index_path) diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..cf09e45 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,179 @@ +"""Tests for configuration: cross-platform log directory and settings sanitizing.""" + +import json +import sys +import tempfile +from pathlib import Path + +import pytest + +from app import config + + +class TestDefaultLogDirectory: + """get_default_log_directory follows each OS's conventions.""" + + def test_macos(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(Path, "home", lambda: Path("/Users/tester")) + + result = config.get_default_log_directory("myapp") + + assert result == Path("/Users/tester/Library/Logs/myapp") + + def test_windows_uses_localappdata(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setenv("LOCALAPPDATA", r"C:\Users\tester\AppData\Local") + + result = config.get_default_log_directory("myapp") + + assert result == Path(r"C:\Users\tester\AppData\Local") / "myapp" / "logs" + + def test_windows_without_localappdata(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.delenv("LOCALAPPDATA", raising=False) + monkeypatch.setattr(Path, "home", lambda: Path(r"C:\Users\tester")) + + result = config.get_default_log_directory("myapp") + + assert result == Path(r"C:\Users\tester") / "AppData" / "Local" / "myapp" / "logs" + + def test_linux_respects_xdg_state_home(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setenv("XDG_STATE_HOME", "/home/tester/.local/state") + + result = config.get_default_log_directory("myapp") + + assert result == Path("/home/tester/.local/state/myapp/logs") + + def test_linux_defaults_without_xdg(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.delenv("XDG_STATE_HOME", raising=False) + monkeypatch.setattr(Path, "home", lambda: Path("/home/tester")) + + result = config.get_default_log_directory("myapp") + + assert result == Path("/home/tester/.local/state/myapp/logs") + + +class TestIsUnderTempDir: + def test_temp_path_detected(self) -> None: + leaked = Path(tempfile.gettempdir()) / "pytest-leak" / "logs" + assert config._is_under_temp_dir(leaked) is True + + def test_non_temp_path_not_detected(self) -> None: + assert config._is_under_temp_dir(Path.home() / "Library" / "Logs" / "app") is False + + +class TestLogDirectorySanitization: + """A temp-dir log_directory leaked into settings.json is repaired on load.""" + + def _load_with_settings( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, data: dict[str, object] + ) -> config.Settings: + settings_file = tmp_path / "settings.json" + settings_file.write_text(json.dumps(data), encoding="utf-8") + monkeypatch.setattr(config, "SETTINGS_FILE", settings_file) + config.Settings._instance = None + return config.Settings() + + def teardown_method(self) -> None: + # Reset the singleton so other tests reload from the real settings file. + config.Settings._instance = None + + def test_temp_log_directory_falls_back_to_default( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + leaked = str(Path(tempfile.gettempdir()) / "tmpXXXX") + settings = self._load_with_settings(monkeypatch, tmp_path, {"log_directory": leaked}) + + assert settings.log_directory == config.get_default_log_directory() + + def test_empty_log_directory_falls_back_to_default( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + settings = self._load_with_settings(monkeypatch, tmp_path, {"log_directory": ""}) + + assert settings.log_directory == config.get_default_log_directory() + + def test_explicit_absolute_log_directory_preserved( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + custom = Path.home() / ".modaq_custom_logs" + settings = self._load_with_settings(monkeypatch, tmp_path, {"log_directory": str(custom)}) + + assert settings.log_directory == custom + + +class TestPartitionSanitization: + """_sanitize_partition_value keeps values safe as a single path segment.""" + + def test_replaces_path_hostile_chars(self) -> None: + assert config._sanitize_partition_value("a/b\\c=d e") == "a-b-c-d-e" + + def test_strips_surrounding_separators(self) -> None: + assert config._sanitize_partition_value(" spaced ") == "spaced" + + def test_empty_falls_back_to_unknown(self) -> None: + assert config._sanitize_partition_value(" ") == "unknown" + + def test_os_helpers_have_no_separators(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(config.platform, "system", lambda: "Mac OS/X") + monkeypatch.setattr(config.platform, "release", lambda: "25.4 .0") + + assert config.get_os_name() == "Mac-OS-X" + assert config.get_os_version() == "25.4-.0" + + +class TestInstallId: + """A persistent per-install ID is generated once and reused thereafter.""" + + def _load_with_settings( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, data: dict[str, object] + ) -> config.Settings: + settings_file = tmp_path / "settings.json" + settings_file.write_text(json.dumps(data), encoding="utf-8") + monkeypatch.setattr(config, "SETTINGS_FILE", settings_file) + config.Settings._instance = None + return config.Settings() + + def teardown_method(self) -> None: + config.Settings._instance = None + + def test_generates_and_persists_when_absent( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + settings = self._load_with_settings(monkeypatch, tmp_path, {"log_directory": str(tmp_path)}) + + generated = settings.install_id + assert generated # non-empty + # Persisted to settings.json + on_disk = json.loads((tmp_path / "settings.json").read_text()) + assert on_disk["install_id"] == generated + + def test_stable_across_reads(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + settings = self._load_with_settings(monkeypatch, tmp_path, {"log_directory": str(tmp_path)}) + assert settings.install_id == settings.install_id + + def test_existing_value_preserved( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + settings = self._load_with_settings( + monkeypatch, tmp_path, {"log_directory": str(tmp_path), "install_id": "preset123456"} + ) + assert settings.install_id == "preset123456" + + def test_session_partitions_shape( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + self._load_with_settings( + monkeypatch, tmp_path, {"log_directory": str(tmp_path), "install_id": "preset123456"} + ) + monkeypatch.setattr(config.platform, "system", lambda: "Linux") + monkeypatch.setattr(config.platform, "release", lambda: "6.1.0") + + assert config.get_session_partitions() == [ + "os=Linux", + "os_version=6.1.0", + "session=preset123456", + ] diff --git a/tests/test_job_storage_integration.py b/tests/test_job_storage_integration.py index 26f3b85..2d3de00 100644 --- a/tests/test_job_storage_integration.py +++ b/tests/test_job_storage_integration.py @@ -67,9 +67,7 @@ def _make_real_files(n: int) -> list[Path]: class TestLargeJobPersistence: - def test_create_job_above_threshold_writes_to_db( - self, isolated_storage: JobStorage - ) -> None: + def test_create_job_above_threshold_writes_to_db(self, isolated_storage: JobStorage) -> None: manager = _make_large_job_manager(threshold=3) paths = _make_real_files(5) try: @@ -87,9 +85,7 @@ def test_create_job_above_threshold_writes_to_db( p.unlink() paths[0].parent.rmdir() - def test_below_threshold_does_not_use_db( - self, isolated_storage: JobStorage - ) -> None: + def test_below_threshold_does_not_use_db(self, isolated_storage: JobStorage) -> None: manager = _make_large_job_manager(threshold=10) paths = _make_real_files(3) try: @@ -103,9 +99,7 @@ def test_below_threshold_does_not_use_db( p.unlink() paths[0].parent.rmdir() - def test_terminal_file_transitions_mirror_to_db( - self, isolated_storage: JobStorage - ) -> None: + def test_terminal_file_transitions_mirror_to_db(self, isolated_storage: JobStorage) -> None: manager = _make_large_job_manager(threshold=3) paths = _make_real_files(4) try: @@ -133,9 +127,7 @@ def test_terminal_file_transitions_mirror_to_db( p.unlink() paths[0].parent.rmdir() - def test_persist_job_terminal_updates_job_row( - self, isolated_storage: JobStorage - ) -> None: + def test_persist_job_terminal_updates_job_row(self, isolated_storage: JobStorage) -> None: manager = _make_large_job_manager(threshold=3) paths = _make_real_files(3) try: @@ -189,9 +181,7 @@ def test_storage_failure_does_not_break_status_transitions( class TestSetFileStatusIgnoresDbWhenFlagOff: - def test_use_db_false_means_no_storage_calls( - self, isolated_storage: JobStorage - ) -> None: + def test_use_db_false_means_no_storage_calls(self, isolated_storage: JobStorage) -> None: """A standalone UploadJob with _use_db=False never touches storage.""" job = UploadJob(job_id="lonely-job") job.files.append(FileUploadState("f1", "/p/f1", 100)) diff --git a/tests/test_log_service.py b/tests/test_log_service.py index 45d6f04..9bae1ae 100644 --- a/tests/test_log_service.py +++ b/tests/test_log_service.py @@ -1,7 +1,9 @@ """Tests for the JSONL log service.""" +import contextlib import csv import json +from collections.abc import Iterator from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -11,6 +13,17 @@ from app.services.log_service import LogService +# Fixed session partition segments used in tests so paths are deterministic and the +# real settings.json / platform values are never touched. +_TEST_SESSION_SEGMENTS = ["os=Darwin", "os_version=25.4.0", "session=testsession01"] + + +def _session_subpath(base: Path) -> Path: + """Append the test session partition segments to a date-partitioned dir.""" + for segment in _TEST_SESSION_SEGMENTS: + base = base / segment + return base + @pytest.fixture def log_service(tmp_path: Path) -> LogService: @@ -35,11 +48,27 @@ def log_service(tmp_path: Path) -> LogService: @pytest.fixture def _mock_settings(log_service: LogService) -> Any: - """Context manager that patches get_settings for the log service.""" - return patch( - "app.services.log_service.get_settings", - return_value=log_service._test_settings_mock, # type: ignore[attr-defined] - ) + """Context manager that patches get_settings + get_session_partitions. + + Patching get_session_partitions keeps log paths deterministic and prevents the + real settings.json / platform values from being read or written during tests. + """ + + @contextlib.contextmanager + def _cm() -> Iterator[None]: + with ( + patch( + "app.services.log_service.get_settings", + return_value=log_service._test_settings_mock, # type: ignore[attr-defined] + ), + patch( + "app.services.log_service.get_session_partitions", + return_value=list(_TEST_SESSION_SEGMENTS), + ), + ): + yield + + return _cm() def _find_event_files(log_dir: Path) -> list[Path]: @@ -368,7 +397,7 @@ def test_hive_dir_structure(self, log_service: LogService, _mock_settings: Any) with _mock_settings: hive_dir = log_service._get_hive_dir("json", dt) - assert hive_dir == log_dir / "json" / "year=2026" / "month=02" / "day=08" + assert hive_dir == _session_subpath(log_dir / "json" / "year=2026" / "month=02" / "day=08") assert hive_dir.exists() def test_hive_dir_csv(self, log_service: LogService, _mock_settings: Any) -> None: @@ -379,7 +408,7 @@ def test_hive_dir_csv(self, log_service: LogService, _mock_settings: Any) -> Non with _mock_settings: hive_dir = log_service._get_hive_dir("csv", dt) - assert hive_dir == log_dir / "csv" / "year=2025" / "month=12" / "day=31" + assert hive_dir == _session_subpath(log_dir / "csv" / "year=2025" / "month=12" / "day=31") assert hive_dir.exists() def test_extract_date_from_hive_path(self) -> None: @@ -403,7 +432,7 @@ def test_log_writes_to_hive_path(self, log_service: LogService, _mock_settings: # Verify the file exists at the hive path now = datetime.now(UTC) - expected_dir = ( + expected_dir = _session_subpath( log_dir / "json" / f"year={now.year:04d}" @@ -416,6 +445,70 @@ def test_log_writes_to_hive_path(self, log_service: LogService, _mock_settings: assert entry["event"] == "test" +class TestSessionPartitioning: + """Tests that logs are partitioned per install session (os/os_version/session).""" + + def test_events_written_under_session_partitions( + self, log_service: LogService, _mock_settings: Any + ) -> None: + """The events.jsonl lands under os=/os_version=/session= sub-dirs.""" + log_dir: Path = log_service._test_settings_mock.log_directory # type: ignore[attr-defined] + + with _mock_settings: + log_service.info("app", "test", "Entry") + + events_file = _find_event_files(log_dir)[0] + path_str = str(events_file) + assert "os=Darwin" in path_str + assert "os_version=25.4.0" in path_str + assert "session=testsession01" in path_str + + def test_distinct_sessions_write_distinct_files(self, log_service: LogService) -> None: + """Two installs (different session segments) don't share an events.jsonl.""" + log_dir: Path = log_service._test_settings_mock.log_directory # type: ignore[attr-defined] + settings_patch = patch( + "app.services.log_service.get_settings", + return_value=log_service._test_settings_mock, # type: ignore[attr-defined] + ) + + with ( + settings_patch, + patch( + "app.services.log_service.get_session_partitions", + return_value=["os=Darwin", "os_version=25.4.0", "session=machineAAAA"], + ), + ): + log_service.info("app", "from_a", "Machine A entry") + + with ( + settings_patch, + patch( + "app.services.log_service.get_session_partitions", + return_value=["os=Darwin", "os_version=25.4.0", "session=machineBBBB"], + ), + ): + log_service.info("app", "from_b", "Machine B entry") + + files = _find_event_files(log_dir) + assert len(files) == 2 + sessions = {p for f in files for p in str(f).split("/") if p.startswith("session=")} + assert sessions == {"session=machineAAAA", "session=machineBBBB"} + + def test_sync_keys_include_session(self, log_service: LogService, _mock_settings: Any) -> None: + """S3 keys carry the session partition so installs don't overwrite each other.""" + mock_client = MagicMock() + + with _mock_settings: + log_service.info("app", "test", "Entry") + log_service.sync_logs_to_s3(mock_client, "test-bucket") + + calls = mock_client.upload_file.call_args_list + assert len(calls) >= 1 + for call in calls: + s3_key = call[0][2] + assert "/session=testsession01/" in s3_key + + class TestJobSaveJsonl: """Tests for per-job JSONL saving.""" @@ -431,7 +524,7 @@ def test_save_job_jsonl_creates_file( with _mock_settings: result_path = log_service.save_job_jsonl(job_id, job_dict, completed_at) - expected_dir = log_dir / "json" / "year=2026" / "month=02" / "day=08" + expected_dir = _session_subpath(log_dir / "json" / "year=2026" / "month=02" / "day=08") assert result_path == expected_dir / f"{job_id}.jsonl" assert result_path.exists() @@ -487,7 +580,7 @@ def test_save_job_csv_creates_file(self, log_service: LogService, _mock_settings with _mock_settings: result_path = log_service.save_job_csv(job_id, mock_job, completed_at) - expected_dir = log_dir / "csv" / "year=2026" / "month=02" / "day=08" + expected_dir = _session_subpath(log_dir / "csv" / "year=2026" / "month=02" / "day=08") assert result_path.parent == expected_dir assert result_path.name.startswith("upload-summary-143025-a1b2c3d4") assert result_path.name.endswith(".csv") diff --git a/tests/test_routes.py b/tests/test_routes.py index 35f58b4..4e47dc7 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -205,6 +205,87 @@ def test_get_file_info_no_key(self, client: FlaskClient) -> None: response = client.get("/api/files/info") assert response.status_code == 400 + @patch("app.routes.files.s3_service") + def test_prefix_stats_success(self, mock_s3: MagicMock, client: FlaskClient) -> None: + """Test the stats endpoint returns folder and file counts.""" + client.put( + "/api/settings", + data=json.dumps({"s3_bucket": "test-bucket"}), + content_type="application/json", + ) + + mock_s3.create_s3_client.return_value = MagicMock() + mock_s3.get_prefix_counts.return_value = { + "success": True, + "prefix": "root/", + "folder_count": 2, + "file_count": 42, + "error": None, + } + + response = client.get("/api/files/stats", query_string={"prefix": "root/"}) + assert response.status_code == 200 + + data = json.loads(response.data) + assert data["success"] is True + assert data["folder_count"] == 2 + assert data["file_count"] == 42 + + def test_download_file_no_key(self, client: FlaskClient) -> None: + """Test download without key parameter returns 400.""" + client.put( + "/api/settings", + data=json.dumps({"s3_bucket": "test-bucket"}), + content_type="application/json", + ) + response = client.get("/api/files/download") + assert response.status_code == 400 + + @patch("app.routes.files.s3_service") + def test_download_file_success(self, mock_s3: MagicMock, client: FlaskClient) -> None: + """Test download returns a presigned URL.""" + client.put( + "/api/settings", + data=json.dumps({"s3_bucket": "test-bucket"}), + content_type="application/json", + ) + + mock_s3.create_s3_client.return_value = MagicMock() + mock_s3.generate_presigned_download_url.return_value = { + "success": True, + "url": "https://s3.example.com/test/file.mcap?sig=abc", + "key": "test/file.mcap", + "filename": "file.mcap", + "error": None, + } + + response = client.get("/api/files/download", query_string={"key": "test/file.mcap"}) + assert response.status_code == 200 + + data = json.loads(response.data) + assert data["success"] is True + assert data["url"].startswith("https://") + assert data["filename"] == "file.mcap" + + @patch("app.routes.files.s3_service") + def test_download_file_not_found(self, mock_s3: MagicMock, client: FlaskClient) -> None: + """Test download of a missing object returns 404.""" + client.put( + "/api/settings", + data=json.dumps({"s3_bucket": "test-bucket"}), + content_type="application/json", + ) + + mock_s3.create_s3_client.return_value = MagicMock() + mock_s3.generate_presigned_download_url.return_value = { + "success": False, + "key": "missing/file.mcap", + "error": "File 'file.mcap' not found", + } + + response = client.get("/api/files/download", query_string={"key": "missing/file.mcap"}) + assert response.status_code == 404 + class TestLogsAPI: """Tests for logs API endpoints.""" diff --git a/tests/test_s3_service.py b/tests/test_s3_service.py index c678aea..39a2980 100644 --- a/tests/test_s3_service.py +++ b/tests/test_s3_service.py @@ -132,6 +132,153 @@ def test_list_bucket_objects_with_prefix(self) -> None: assert result["success"] is True assert result["prefix"] == "folder1/" + def test_list_bucket_objects_single_page(self) -> None: + """A folder smaller than the page size returns no continuation token.""" + with mock_aws(): + client = boto3.client("s3", region_name="us-west-2") + client.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + ) + for i in range(5): + client.put_object(Bucket="test-bucket", Key=f"data/file{i}.mcap", Body=b"x") + + result = s3_service.list_bucket_objects(client, "test-bucket", prefix="data/") + + assert result["success"] is True + assert len(result["files"]) == 5 + assert result["next_token"] is None + + def test_list_bucket_objects_pagination(self) -> None: + """A folder larger than the page size paginates via next_token.""" + with mock_aws(): + client = boto3.client("s3", region_name="us-west-2") + client.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + ) + for i in range(25): + # Zero-pad so keys sort predictably across pages. + client.put_object(Bucket="test-bucket", Key=f"data/file{i:03d}.mcap", Body=b"x") + + # First page: capped at max_keys, with a token for the rest. + page1 = s3_service.list_bucket_objects( + client, "test-bucket", prefix="data/", max_keys=10 + ) + assert page1["success"] is True + assert len(page1["files"]) == 10 + assert page1["next_token"] + + # Second page resumes where the first left off. + page2 = s3_service.list_bucket_objects( + client, + "test-bucket", + prefix="data/", + max_keys=10, + continuation_token=page1["next_token"], + ) + assert len(page2["files"]) == 10 + assert page2["next_token"] + + # Final page drains the remainder and clears the token. + page3 = s3_service.list_bucket_objects( + client, + "test-bucket", + prefix="data/", + max_keys=10, + continuation_token=page2["next_token"], + ) + assert len(page3["files"]) == 5 + assert page3["next_token"] is None + + # All 25 distinct keys are covered with no overlap. + keys = {f["key"] for f in page1["files"] + page2["files"] + page3["files"]} + assert len(keys) == 25 + + def test_get_prefix_counts(self) -> None: + """Counts subfolders and direct files for the current level.""" + with mock_aws(): + client = boto3.client("s3", region_name="us-west-2") + client.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + ) + # Two direct files plus two subfolders (each with nested files). + client.put_object(Bucket="test-bucket", Key="root/a.mcap", Body=b"x") + client.put_object(Bucket="test-bucket", Key="root/b.mcap", Body=b"x") + client.put_object(Bucket="test-bucket", Key="root/sub1/c.mcap", Body=b"x") + client.put_object(Bucket="test-bucket", Key="root/sub2/d.mcap", Body=b"x") + + result = s3_service.get_prefix_counts(client, "test-bucket", prefix="root/") + + assert result["success"] is True + # Direct files only; nested files belong to the subfolders. + assert result["file_count"] == 2 + assert result["folder_count"] == 2 + assert result["capped"] is False + + def test_get_prefix_counts_bounded_to_one_page(self) -> None: + """A level larger than one page is bounded to a single call and capped.""" + with mock_aws(): + client = boto3.client("s3", region_name="us-west-2") + client.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + ) + for i in range(1500): + client.put_object(Bucket="test-bucket", Key=f"data/file{i:04d}.mcap", Body=b"x") + + result = s3_service.get_prefix_counts(client, "test-bucket", prefix="data/") + + assert result["success"] is True + # One page only (no enumeration of all 1500): counts are a lower bound. + assert result["capped"] is True + assert result["file_count"] == 1000 + assert result["folder_count"] == 0 + + def test_get_prefix_counts_caps_large_levels(self) -> None: + """Counting stops at max_items and flags the result as capped.""" + with mock_aws(): + client = boto3.client("s3", region_name="us-west-2") + client.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + ) + for i in range(60): + client.put_object(Bucket="test-bucket", Key=f"data/file{i:04d}.mcap", Body=b"x") + + result = s3_service.get_prefix_counts( + client, "test-bucket", prefix="data/", max_items=25 + ) + + assert result["success"] is True + assert result["file_count"] == 25 + assert result["capped"] is True + + def test_get_prefix_counts_caps_folder_heavy_levels(self) -> None: + """A level that is all subfolders (no files) is bounded too. + + This is the bucket-root case: a files-only cap would never trip and would + page through every folder name. The total-entry cap must catch it. + """ + with mock_aws(): + client = boto3.client("s3", region_name="us-west-2") + client.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + ) + # 60 distinct top-level "folders", zero files at this level. + for i in range(60): + client.put_object(Bucket="test-bucket", Key=f"folder{i:04d}/data.mcap", Body=b"x") + + result = s3_service.get_prefix_counts(client, "test-bucket", prefix="", max_items=25) + + assert result["success"] is True + assert result["file_count"] == 0 + assert result["capped"] is True + # Bounded: we did not enumerate all 60 folders. + assert result["folder_count"] <= 25 + def test_validate_bucket_access_success(self) -> None: """Test validate_bucket_access for accessible bucket.""" with mock_aws(): @@ -196,6 +343,41 @@ def test_get_object_metadata(self) -> None: assert result["success"] is True assert result["size"] == 9 # len("test data") + def test_generate_presigned_download_url(self) -> None: + """Test generating a presigned download URL for an existing object.""" + with mock_aws(): + client = boto3.client("s3", region_name="us-west-2") + client.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + ) + client.put_object(Bucket="test-bucket", Key="test/file.mcap", Body=b"test data") + + result = s3_service.generate_presigned_download_url( + client, "test-bucket", "test/file.mcap" + ) + + assert result["success"] is True + assert result["filename"] == "file.mcap" + assert result["url"].startswith("https://") + assert "test/file.mcap" in result["url"] + + def test_generate_presigned_download_url_missing(self) -> None: + """Test that a missing object returns an error instead of a URL.""" + with mock_aws(): + client = boto3.client("s3", region_name="us-west-2") + client.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + ) + + result = s3_service.generate_presigned_download_url( + client, "test-bucket", "missing/file.mcap" + ) + + assert result["success"] is False + assert "not found" in result["error"] + class TestCreateS3Client: """Tests for create_s3_client function.""" @@ -219,3 +401,25 @@ def test_default_region(self, mock_session: MagicMock) -> None: s3_service.create_s3_client("test-profile") mock_session.assert_called_once_with(profile_name="test-profile", region_name="us-west-2") + + @patch("boto3.Session") + def test_caches_client_per_profile_region(self, mock_session: MagicMock) -> None: + """Repeated calls reuse the cached client instead of rebuilding a session.""" + mock_session.return_value.client.return_value = MagicMock() + + first = s3_service.create_s3_client("p", "us-west-2") + second = s3_service.create_s3_client("p", "us-west-2") + + assert first is second + mock_session.assert_called_once() + + @patch("boto3.Session") + def test_reset_cache_forces_rebuild(self, mock_session: MagicMock) -> None: + """After a reset the next call builds a fresh session.""" + mock_session.return_value.client.return_value = MagicMock() + + s3_service.create_s3_client("p", "us-west-2") + s3_service.reset_s3_client_cache() + s3_service.create_s3_client("p", "us-west-2") + + assert mock_session.call_count == 2 diff --git a/tests/test_sse_infrastructure.py b/tests/test_sse_infrastructure.py index 0fb4caa..cfc0e6a 100644 --- a/tests/test_sse_infrastructure.py +++ b/tests/test_sse_infrastructure.py @@ -115,9 +115,9 @@ def test_throttled_progress_callback_coalesces_to_4_hz() -> None: # We fired hundreds of times but only ~4 Hz should have emitted. assert emit_count > 50, "loop should have run many iterations" max_expected = int(elapsed / SSE_EMIT_INTERVAL_SECONDS) + 2 - assert ( - mock_sse.send_event.call_count <= max_expected - ), f"emits={mock_sse.send_event.call_count} > cap={max_expected}" + assert mock_sse.send_event.call_count <= max_expected, ( + f"emits={mock_sse.send_event.call_count} > cap={max_expected}" + ) def test_throttled_progress_callback_always_emits_terminal() -> None: