From 27f6aa84fe67731d8cf62627918b342d78834e4b Mon Sep 17 00:00:00 2001 From: R-zin Date: Wed, 26 Aug 2026 10:42:13 +0530 Subject: [PATCH] feat: production-grade hardening (persistence, auth, Docker, tests, CI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend: - app/settings.py: pydantic-settings (OSM2NS3_* env prefix, derived data/static dirs) - app/store.py: SQLite job store (WAL, retention pruning, startup recovery of interrupted jobs) replacing the in-memory dict - app/worker.py: PipelineRunner with semaphore concurrency cap; job cancellation that SIGKILLs the process group before task.cancel() (wait_for won't reap children on cancel); cancel-vs-completion race fixed by re-checking the store before marking done - main.py: create_app factory; streamed 1 MiB uploads with size cap; /live + /ready probes; API-key auth off by default (X-API-Key, env-enabled on /jobs/*); CORS fixed from wildcard; SPA StaticFiles mount with deep-link fallback Tests + CI: - 118 tests (90% coverage): models, command builders, store, full API with mocked SUMO, cancellation machinery - CI rewritten: backend job (pytest --cov ≥80% + uvicorn /health smoke) and frontend job (npm ci + tsc/vite build) Frontend: - API-key input persisted to localStorage and sent as X-API-Key - Schema drift fix: configSchema.ts deleted; form generated from live GET /schema (caught real drift: old code shipped lowercase sotl_phase, wire value SOTL_PHASE) - Job cancel button, blob download (no key-in-URL), cancelled status styling, /health -> /ready switch Ops: - Multi-stage Dockerfile (node build -> ubuntu22.04+SUMO+sumo-tools, non-root, /data volume), .dockerignore, docker-compose.yml - black/isort/ruff config in pyproject.toml + one-time reformat; Lint.yml pinned to the same profile Co-Authored-By: Claude Opus 5 (1M context) --- .DS_Store | Bin 8196 -> 0 bytes .dockerignore | 24 + .github/workflows/Lint.yml | 6 +- .github/workflows/github-actions.yml | 47 +- .gitignore | 29 + Dockerfile | 46 + README.md | 85 +- app/__init__.py | 0 app/models.py | 341 ++++--- app/settings.py | 100 +++ app/store.py | 273 ++++++ app/worker.py | 933 ++++++++++++-------- docker-compose.yml | 18 + frontend/.env.development | 1 + frontend/src/api/client.ts | 76 +- frontend/src/api/configSchema.ts | 149 ---- frontend/src/api/schema.ts | 148 ++++ frontend/src/api/types.ts | 3 +- frontend/src/components/ConfigForm.tsx | 7 +- frontend/src/components/FileRow.tsx | 35 +- frontend/src/components/HealthStrip.tsx | 4 +- frontend/src/components/Nav.tsx | 33 + frontend/src/components/PipelineStepper.tsx | 2 +- frontend/src/components/ui/Field.tsx | 16 +- frontend/src/components/ui/StatusTag.tsx | 1 + frontend/src/pages/Dashboard.tsx | 2 +- frontend/src/pages/JobDetail.tsx | 44 +- frontend/src/pages/NewJob.tsx | 22 +- frontend/src/pages/System.tsx | 2 +- frontend/tsconfig.node.tsbuildinfo | 1 - frontend/tsconfig.tsbuildinfo | 1 - frontend/vite.config.d.ts | 2 - frontend/vite.config.js | 6 - main.py | 712 ++++++++------- pyproject.toml | 56 ++ requirements.txt | 8 +- tests/__init__.py | 0 tests/conftest.py | 171 ++++ tests/test_api.py | 405 +++++++++ tests/test_cancel.py | 136 +++ tests/test_commands.py | 280 ++++++ tests/test_models.py | 259 ++++++ tests/test_store.py | 149 ++++ 43 files changed, 3619 insertions(+), 1014 deletions(-) delete mode 100644 .DS_Store create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/settings.py create mode 100644 app/store.py create mode 100644 docker-compose.yml create mode 100644 frontend/.env.development delete mode 100644 frontend/src/api/configSchema.ts create mode 100644 frontend/src/api/schema.ts delete mode 100644 frontend/tsconfig.node.tsbuildinfo delete mode 100644 frontend/tsconfig.tsbuildinfo delete mode 100644 frontend/vite.config.d.ts delete mode 100644 frontend/vite.config.js create mode 100644 pyproject.toml create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_api.py create mode 100644 tests/test_cancel.py create mode 100644 tests/test_commands.py create mode 100644 tests/test_models.py create mode 100644 tests/test_store.py diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index afbd686deb9c7f2b843e39c664f00ff1f0b90908..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMF^dyH7=2?R3D_J6dSWw$wQ$YV>k$zItNQ~wt|R_rC48e zBI-Y|wz1GgY_IZtGjrLQ?5tRchu^@=+syZU`(EBIJDCjtv$dFP0P6ty?1F0-IAof{ zr8nBSFk^{^^_cE#Y~L+s^G2plr~~SNI-m}y1M0xP;sDodY5kUU-$!*+2h@T8(gArs zSa!i8VCB(0Iyks30I|zy*Lb`p2SgjN2v~V!gw6O;qAxXJi(z~@=WWO<0#+VVP`Xbii_|i|g3I9uAWIyD7iMu~|77jHl(8bN$E9`roUQ zPdC!|58Ls)VHfGJmQ(kxO9$6PS;CGcV$G(ygYZ$&-YG`ED*StL|t7kziQ>q&A z?#t^*GGaYHUPgLWvT8l;Ss6XfBo4CF+JHsC%A&Hdfj@P?bO+mmEoQq_JyRvCyk#G@Iz}cXKI`CH=_yyg02W|iW diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8ee759e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,24 @@ +.git +.gitignore +node_modules +frontend/dist +frontend/node_modules +jobs +uploads +outputs +static +*.db +.venv +__pycache__ +*.pyc +*.tsbuildinfo +.idea +.claude +tests +.pytest_cache +.ruff_cache +htmlcov +.coverage +.env +*.md +!frontend/README.md diff --git a/.github/workflows/Lint.yml b/.github/workflows/Lint.yml index 2cb2e95..c21506e 100644 --- a/.github/workflows/Lint.yml +++ b/.github/workflows/Lint.yml @@ -17,10 +17,10 @@ jobs: with: python-version: "3.12" - - name: Install dependencies + - name: Install lint tools run: | python -m pip install --upgrade pip - pip install ruff black isort + pip install "ruff~=0.8" "black~=24.0" "isort~=5.13" - name: Ruff run: ruff check . @@ -29,4 +29,4 @@ jobs: run: black --check . - name: isort - run: isort --check-only . \ No newline at end of file + run: isort --check-only --profile black . diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 16e8382..9c1fd3f 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -2,10 +2,10 @@ name: Test on: [push] jobs: - FastAPI-Installation: + backend: runs-on: ubuntu-latest steps: - - name: CheckOut Code + - name: Check Out Code uses: actions/checkout@v4 - name: Set up Python @@ -16,22 +16,45 @@ jobs: - name: Install dependencies run: | - python -m venv .venv - source .venv/bin/activate python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - pip install uvicorn + pip install -e ".[dev]" + + - name: Run tests with coverage + run: python -m pytest tests/ --cov=app --cov=main --cov-fail-under=80 + - name: Run FastAPI server and test health endpoint run: | - source .venv/bin/activate - uvicorn main:app --host 127.0.0.1 --port 8000 & SERVER_PID=$! # Wait for server to start - sleep 5 + for i in $(seq 1 30); do + curl -sf http://127.0.0.1:8000/live && break || sleep 1 + done + + # SUMO is absent on the runner: /ready must report 503, /health degraded + test "$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8000/ready)" = "503" + curl --fail http://127.0.0.1:8000/health | grep -q '"degraded"' + + kill $SERVER_PID + + frontend: + runs-on: ubuntu-latest + steps: + - name: Check Out Code + uses: actions/checkout@v4 - # Fail the workflow if /health doesn't return a 2xx response - curl --fail http://127.0.0.1:8000/health + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + working-directory: frontend + run: npm ci - kill $SERVER_PID \ No newline at end of file + - name: Build + working-directory: frontend + run: npm run build diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..68b850b --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# macOS +.DS_Store + +# Python +__pycache__/ +*.pyc +.venv/ +.venv312/ +.pytest_cache/ +.ruff_cache/ +htmlcov/ +.coverage +.env + +# Project runtime data +jobs/ +uploads/ +outputs/ +*.db + +# Node / frontend +node_modules/ +frontend/dist/ +*.tsbuildinfo +frontend/vite.config.js +frontend/vite.config.d.ts + +# IDE +.idea/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f5f6900 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +# ── Stage 1: build the frontend ────────────────────────────────────────────── +FROM node:20-slim AS frontend +WORKDIR /fe +COPY frontend/package.json frontend/package-lock.json ./ +RUN npm ci +COPY frontend/ ./ +RUN npm run build + +# ── Stage 2: runtime (Ubuntu + SUMO + backend + built frontend) ───────────── +FROM ubuntu:22.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + python3 python3-pip python3-venv \ + sumo sumo-tools \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Python deps in a venv. +RUN python3 -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" +COPY requirements.txt ./ +RUN pip install --no-cache-dir --upgrade pip \ + && pip install --no-cache-dir -r requirements.txt + +# Backend + built SPA served by FastAPI's StaticFiles mount. +COPY main.py ./ +COPY app/ ./app/ +COPY --from=frontend /fe/dist ./static + +# Non-root user; data (jobs.db + uploads/outputs) lives on a volume. +RUN useradd --create-home --uid 10001 appuser \ + && mkdir -p /data \ + && chown -R appuser:appuser /data /app +USER appuser + +VOLUME /data +ENV OSM2NS3_DATA_DIR=/data \ + OSM2NS3_STATIC_DIR=/app/static \ + SUMO_HOME=/usr/share/sumo + +EXPOSE 8000 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index 0bd341e..9229c8b 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,7 @@ OSM file → netconvert → randomTrips → duarouter → SUMO → tra **Python:** 3.10+ -**Python packages:** -``` -fastapi -uvicorn[standard] -python-multipart -aiofiles -pydantic>=2.0 -``` +**Python packages:** pinned in `requirements.txt` (`fastapi`, `uvicorn[standard]`, `python-multipart`, `aiofiles`, `pydantic`, `pydantic-settings`). **SUMO tools** (must be on `PATH` or set `SUMO_HOME`): | Tool | Purpose | @@ -58,16 +51,8 @@ sudo apt update && sudo apt install sumo sumo-tools ```bash git clone https://github.com/your-username/osm2ns3.git cd osm2ns3 -pip install -r requirements.txt -``` - -**`requirements.txt`** -``` -fastapi>=0.110.0 -uvicorn[standard]>=0.29.0 -python-multipart>=0.0.9 -aiofiles>=23.0.0 -pydantic>=2.0.0 +pip install -r requirements.txt # runtime +pip install -e ".[dev]" # + tests, lint, coverage ``` --- @@ -75,7 +60,7 @@ pydantic>=2.0.0 ## Running ```bash -uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 +uvicorn main:app --reload --host 0.0.0.0 --port 8000 ``` Open **http://localhost:8000/docs** for the interactive Swagger UI. @@ -83,9 +68,27 @@ Open **http://localhost:8000/docs** for the interactive Swagger UI. If SUMO is installed at a non-standard path: ```bash export SUMO_HOME=/opt/sumo -uvicorn app.main:app --reload +uvicorn main:app --reload ``` +### Configuration + +All runtime configuration is via `OSM2NS3_*` environment variables (or a `.env` file): + +| Variable | Default | Purpose | +|----------|---------|---------| +| `OSM2NS3_API_KEY` | *(unset)* | When set, every `/jobs/*` route requires an `X-API-Key` header. Auth is **off by default**. | +| `OSM2NS3_CORS_ORIGINS` | `["http://localhost:5173","http://localhost:8000"]` | JSON list of allowed CORS origins (`allow_credentials=True`). | +| `OSM2NS3_DATA_DIR` | repo root | Where `jobs/`, `uploads/`, `outputs/`, and `jobs.db` live. | +| `OSM2NS3_STATIC_DIR` | `static/` | Built frontend served as an SPA with deep-link fallback. | +| `OSM2NS3_MAX_UPLOAD_MB` | `100` | Upload size cap (413 above it). | +| `OSM2NS3_MAX_CONCURRENT_JOBS` | `2` | Pipeline concurrency (semaphore). | +| `OSM2NS3_JOB_RETENTION_MAX` | `200` | Keep at most N finished job dirs. | +| `OSM2NS3_JOB_RETENTION_DAYS` | `7` | Prune finished jobs older than N days. | +| `OSM2NS3_TIMEOUT_SUMO_S` | `1200` | Hard timeout for the `sumo` stage. | +| `OSM2NS3_LOG_LEVEL` | `INFO` | Root log level. | +| `SUMO_HOME` | *(unset)* | Extra directory searched first for `randomTrips.py`/`traceExporter.py`. | + --- ## Quick Start @@ -182,18 +185,23 @@ curl -O http://localhost:8000/jobs/3f2a1c7e-.../download/mobility.tcl | Method | Endpoint | Description | |--------|----------|-------------| -| `GET` | `/health` | SUMO tool availability check | +| `GET` | `/live` | Liveness probe — always ok while the process serves | +| `GET` | `/ready` | Readiness: 200 when all SUMO tools found, 503 otherwise | +| `GET` | `/health` | `/ready` shape that never returns 5xx (backcompat) | | `GET` | `/schema` | Full JSON Schema for all 74 parameters | | `GET` | `/defaults` | Default values for every parameter | | `POST` | `/jobs` | Submit OSM file + config, returns `job_id` | | `GET` | `/jobs` | List all jobs, newest first | | `GET` | `/jobs/{id}` | Job status + per-stage progress | +| `POST` | `/jobs/{id}/cancel` | Cancel a pending/running job (kills its subprocess tree) | | `GET` | `/jobs/{id}/config` | Exact config used for a job | | `GET` | `/jobs/{id}/files` | List downloadable output files | | `GET` | `/jobs/{id}/download/{file}` | Download an output file | | `GET` | `/jobs/{id}/logs` | Per-stage stdout/stderr logs | | `DELETE` | `/jobs/{id}` | Delete job and all its files | +`/live`, `/ready`, `/health`, `/schema`, `/defaults`, and the SPA are public. Every `/jobs/*` route sits behind the API-key check when `OSM2NS3_API_KEY` is set. + --- ## Parameters @@ -368,18 +376,39 @@ mobility.Install(nodes); ``` osm2ns3/ +├── main.py # FastAPI app factory + route handlers ├── app/ -│ ├── main.py # FastAPI app, all route handlers -│ ├── models.py # Pydantic models: 74 parameters, 14 validators -│ └── pipeline.py # Async pipeline: subprocess + aiofiles throughout -├── jobs/ # Per-job workdirs with intermediate files and .log files -├── uploads/ # Uploaded OSM files (named by job UUID) -├── outputs/ # Final output files served for download -└── requirements.txt +│ ├── settings.py # pydantic-settings (OSM2NS3_* env, derived dirs) +│ ├── models.py # Pydantic models: 74 parameters, cross-stage validators +│ ├── store.py # SQLite job store (WAL, retention pruning, restart recovery) +│ └── worker.py # PipelineRunner: async subprocess pipeline + cancellation +├── frontend/ # React/TS SPA (schema-driven form from GET /schema) +├── static/ # Built SPA served by FastAPI (present in the Docker image) +├── jobs/ # Per-job workdirs with intermediate files and .log files +├── uploads/ # Uploaded OSM files (named by job UUID) +├── outputs/ # Final output files served for download +├── tests/ # 118 tests: models, command builders, store, API, cancel +├── pyproject.toml # project metadata + pytest/black/isort/ruff/coverage config +├── requirements.txt +├── Dockerfile # multi-stage: node build → ubuntu+SUMO+sumo-tools runtime +└── docker-compose.yml ``` --- +## Docker + +A single all-in-one image builds the frontend and runs the backend on an Ubuntu + SUMO base, persisting state to a `/data` volume: + +```bash +docker build -t osm2ns3 . +docker run -p 8000:8000 -v osm2ns3-data:/data osm2ns3 +``` + +or `docker compose up --build`. The SPA is served at `/` with deep links (`/jobs/{id}`) working; set `OSM2NS3_API_KEY` to enable auth inside the container. + +--- + ## Use Case: VANET Research This tool was built for VANET (Vehicular Ad-hoc Network) simulation research where NS-3 is used to evaluate routing protocols (AODV, GPSR, etc.) over realistic urban mobility. A typical research workflow: diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/models.py b/app/models.py index cd97902..97db84e 100644 --- a/app/models.py +++ b/app/models.py @@ -13,87 +13,101 @@ from __future__ import annotations -import math import re from enum import Enum -from typing import List, Literal, Optional from pydantic import BaseModel, Field, field_validator, model_validator - # ────────────────────────────────────────────────────────────── # Shared enums # ────────────────────────────────────────────────────────────── + class OutputFormat(str, Enum): ns_movements = "ns_movements" - tcl = "tcl" - both = "both" + tcl = "tcl" + both = "both" class VehicleClass(str, Enum): - passenger = "passenger" - bus = "bus" - truck = "truck" + passenger = "passenger" + bus = "bus" + truck = "truck" motorcycle = "motorcycle" - bicycle = "bicycle" + bicycle = "bicycle" pedestrian = "pedestrian" - emergency = "emergency" - authority = "authority" - army = "army" - vip = "vip" - hov = "hov" - custom1 = "custom1" - custom2 = "custom2" + emergency = "emergency" + authority = "authority" + army = "army" + vip = "vip" + hov = "hov" + custom1 = "custom1" + custom2 = "custom2" class TLType(str, Enum): - static = "static" - actuated = "actuated" - delay_based = "delay_based" - sotl_phase = "SOTL_PHASE" + static = "static" + actuated = "actuated" + delay_based = "delay_based" + sotl_phase = "SOTL_PHASE" sotl_platoon = "SOTL_PLATOON" class EdgeRemoval(str, Enum): - all = "all" + all = "all" noFringe = "noFringe" class JunctionType(str, Enum): - priority = "priority" - traffic_light = "traffic_light" - right_before_left = "right_before_left" - unregulated = "unregulated" - allway_stop = "allway_stop" - zipper = "zipper" + priority = "priority" + traffic_light = "traffic_light" + right_before_left = "right_before_left" + unregulated = "unregulated" + allway_stop = "allway_stop" + zipper = "zipper" class SpeedMode(str, Enum): - right_of_way = "right_of_way" # bitmask 31 — obey right-of-way - no_checks = "no_checks" # bitmask 32 — ignore all checks - all_checks = "all_checks" # bitmask 31 (same as right_of_way, alias) + right_of_way = "right_of_way" # bitmask 31 — obey right-of-way + no_checks = "no_checks" # bitmask 32 — ignore all checks + all_checks = "all_checks" # bitmask 31 (same as right_of_way, alias) class LaneChangeMode(str, Enum): - default = "default" # 1621 - no_lc = "no_lc" # 0 + default = "default" # 1621 + no_lc = "no_lc" # 0 strategic_only = "strategic" # 256 # Known OSM highway types accepted by netconvert _VALID_HIGHWAY_TYPES = { - "motorway", "motorway_link", "trunk", "trunk_link", - "primary", "primary_link", "secondary", "secondary_link", - "tertiary", "tertiary_link", "residential", "living_street", - "unclassified", "service", "road", "track", - "footway", "cycleway", "path", "steps", "pedestrian", - "bus_guideway", "raceway", "construction", + "motorway", + "motorway_link", + "trunk", + "trunk_link", + "primary", + "primary_link", + "secondary", + "secondary_link", + "tertiary", + "tertiary_link", + "residential", + "living_street", + "unclassified", + "service", + "road", + "track", + "footway", + "cycleway", + "path", + "steps", + "pedestrian", + "bus_guideway", + "raceway", + "construction", } -_BOUNDARY_RE = re.compile( - r"^-?\d+(\.\d+)?,-?\d+(\.\d+)?,-?\d+(\.\d+)?,-?\d+(\.\d+)?$" -) +_BOUNDARY_RE = re.compile(r"^-?\d+(\.\d+)?,-?\d+(\.\d+)?,-?\d+(\.\d+)?,-?\d+(\.\d+)?$") _COLLISION_ACTIONS = {"warn", "teleport", "remove", "none"} @@ -102,6 +116,7 @@ class LaneChangeMode(str, Enum): # Stage 1 — netconvert # ────────────────────────────────────────────────────────────── + class NetconvertParams(BaseModel): """ Controls how the raw OSM file is converted into a SUMO road network. @@ -109,9 +124,17 @@ class NetconvertParams(BaseModel): """ # ── OSM import ────────────────────────────────────────── - osm_highway_types: List[str] = Field( - default=["motorway", "trunk", "primary", "secondary", - "tertiary", "residential", "living_street", "unclassified"], + osm_highway_types: list[str] = Field( + default=[ + "motorway", + "trunk", + "primary", + "secondary", + "tertiary", + "residential", + "living_street", + "unclassified", + ], min_length=1, description=( "OSM highway tag values to import. Must be non-empty. " @@ -148,11 +171,15 @@ class NetconvertParams(BaseModel): ), ) junctions_internal_link_detail: int = Field( - 5, ge=1, le=20, + 5, + ge=1, + le=20, description="Geometry point count for links inside junctions.", ) junctions_corner_detail: int = Field( - 5, ge=1, le=20, + 5, + ge=1, + le=20, description="Geometry point count for junction corner curves.", ) default_junction_type: JunctionType = Field( @@ -160,25 +187,35 @@ class NetconvertParams(BaseModel): description="Default right-of-way rule at intersections.", ) junctions_min_size: float = Field( - 1.5, ge=0.0, le=20.0, + 1.5, + ge=0.0, + le=20.0, description="Minimum junction radius in metres.", ) junctions_limit_turn_speed: float = Field( - 5.5, ge=0.0, le=30.0, + 5.5, + ge=0.0, + le=30.0, description="Cap speed on sharp turns (m/s). 0 = disabled.", ) # ── Lane / road ───────────────────────────────────────── default_lane_width: float = Field( - 3.2, ge=1.0, le=10.0, + 3.2, + ge=1.0, + le=10.0, description="Default lane width in metres.", ) default_speed_limit: float = Field( - 13.9, ge=1.0, le=83.3, + 13.9, + ge=1.0, + le=83.3, description="Default speed limit in m/s (13.9 ≈ 50 km/h, 83.3 ≈ 300 km/h).", ) default_num_lanes: int = Field( - 1, ge=1, le=8, + 1, + ge=1, + le=8, description="Default lane count when OSM data is absent.", ) no_turnarounds: bool = Field( @@ -204,20 +241,24 @@ class NetconvertParams(BaseModel): description="Merge nearby traffic-light junctions into one programme.", ) tl_min_dur: int = Field( - 5, ge=1, le=120, + 5, + ge=1, + le=120, description="Minimum green-phase duration in seconds.", ) tl_max_dur: int = Field( - 50, ge=5, le=300, + 50, + ge=5, + le=300, description="Maximum green-phase duration in seconds.", ) # ── Network cleaning ──────────────────────────────────── - keep_edges_by_vclass: Optional[VehicleClass] = Field( + keep_edges_by_vclass: VehicleClass | None = Field( None, description="Retain only edges permitted for this vehicle class. None = keep all.", ) - remove_edges_by_type: Optional[str] = Field( + remove_edges_by_type: str | None = Field( None, description=( "Comma-separated OSM highway types to discard, e.g. " @@ -253,7 +294,7 @@ class NetconvertParams(BaseModel): @field_validator("osm_highway_types") @classmethod - def validate_highway_types(cls, v: List[str]) -> List[str]: + def validate_highway_types(cls, v: list[str]) -> list[str]: if not v: raise ValueError("osm_highway_types must contain at least one entry.") invalid = [t for t in v if t not in _VALID_HIGHWAY_TYPES] @@ -266,7 +307,7 @@ def validate_highway_types(cls, v: List[str]) -> List[str]: @field_validator("remove_edges_by_type") @classmethod - def validate_remove_types(cls, v: Optional[str]) -> Optional[str]: + def validate_remove_types(cls, v: str | None) -> str | None: if v is None: return v tokens = [t.strip() for t in v.split(",") if t.strip()] @@ -274,13 +315,11 @@ def validate_remove_types(cls, v: Optional[str]) -> Optional[str]: raise ValueError("remove_edges_by_type is empty after splitting on ','.") invalid = [t for t in tokens if t not in _VALID_HIGHWAY_TYPES] if invalid: - raise ValueError( - f"remove_edges_by_type contains unknown types: {invalid}." - ) + raise ValueError(f"remove_edges_by_type contains unknown types: {invalid}.") return ",".join(tokens) @model_validator(mode="after") - def check_projection_mutual_exclusion(self) -> "NetconvertParams": + def check_projection_mutual_exclusion(self) -> NetconvertParams: if self.proj_utm and self.proj_plain_geo: raise ValueError( "proj_utm and proj_plain_geo are mutually exclusive — " @@ -289,7 +328,7 @@ def check_projection_mutual_exclusion(self) -> "NetconvertParams": return self @model_validator(mode="after") - def check_tl_durations(self) -> "NetconvertParams": + def check_tl_durations(self) -> NetconvertParams: if self.tl_min_dur >= self.tl_max_dur: raise ValueError( f"tl_min_dur ({self.tl_min_dur}) must be strictly less than " @@ -302,6 +341,7 @@ def check_tl_durations(self) -> "NetconvertParams": # Stage 2 — randomTrips / route generation # ────────────────────────────────────────────────────────────── + class RandomTripsParams(BaseModel): """ Controls vehicle trip and route generation via SUMO's randomTrips.py. @@ -309,23 +349,28 @@ class RandomTripsParams(BaseModel): """ begin: float = Field( - 0.0, ge=0.0, + 0.0, + ge=0.0, description="Trip generation start time in seconds.", ) end: float = Field( - 3600.0, ge=1.0, + 3600.0, + ge=1.0, description="Trip generation end time in seconds. Must be > begin.", ) period: float = Field( - 2.0, ge=0.1, le=3600.0, + 2.0, + ge=0.1, + le=3600.0, description=( "Mean inter-departure gap in seconds. " "1/period ≈ vehicle insertion rate. " "Ignored when insertion_density is set." ), ) - insertion_density: Optional[float] = Field( - None, ge=0.0, + insertion_density: float | None = Field( + None, + ge=0.0, description=( "Vehicles inserted per hour per km of road. " "Overrides period when provided." @@ -336,22 +381,28 @@ class RandomTripsParams(BaseModel): description="SUMO vehicle class assigned to all generated trips.", ) min_distance: float = Field( - 300.0, ge=0.0, + 300.0, + ge=0.0, description="Minimum Euclidean trip distance in metres.", ) - max_distance: Optional[float] = Field( - None, ge=0.0, + max_distance: float | None = Field( + None, + ge=0.0, description="Maximum trip distance in metres. None = unlimited.", ) fringe_factor: float = Field( - 1.0, ge=0.0, le=100.0, + 1.0, + ge=0.0, + le=100.0, description=( "Bias weight for selecting fringe edges as trip endpoints. " ">1 makes trips more likely to originate/terminate at network edges." ), ) fringe_threshold: float = Field( - 0.0, ge=0.0, le=1.0, + 0.0, + ge=0.0, + le=1.0, description=( "Minimum edge-speed fraction (relative to max network speed) " "to qualify as a fringe edge." @@ -374,14 +425,15 @@ class RandomTripsParams(BaseModel): description="Randomise arrival position along the destination edge.", ) seed: int = Field( - 42, ge=0, + 42, + ge=0, description="Random seed for reproducible trip generation.", ) # ── Validators ────────────────────────────────────────── @model_validator(mode="after") - def check_timeline(self) -> "RandomTripsParams": + def check_timeline(self) -> RandomTripsParams: if self.begin >= self.end: raise ValueError( f"begin ({self.begin}) must be strictly less than end ({self.end})." @@ -389,7 +441,7 @@ def check_timeline(self) -> "RandomTripsParams": return self @model_validator(mode="after") - def check_distance_bounds(self) -> "RandomTripsParams": + def check_distance_bounds(self) -> RandomTripsParams: if self.max_distance is not None and self.max_distance <= self.min_distance: raise ValueError( f"max_distance ({self.max_distance}) must be greater than " @@ -402,6 +454,7 @@ def check_distance_bounds(self) -> "RandomTripsParams": # Stage 3+4 — SUMO simulation # ────────────────────────────────────────────────────────────── + class SumoParams(BaseModel): """ Controls the SUMO microscopic traffic simulation. @@ -409,15 +462,19 @@ class SumoParams(BaseModel): """ begin: float = Field( - 0.0, ge=0.0, + 0.0, + ge=0.0, description="Simulation begin time in seconds.", ) end: float = Field( - 3600.0, ge=1.0, + 3600.0, + ge=1.0, description="Simulation end time in seconds. Must be > begin.", ) step_length: float = Field( - 0.1, ge=0.01, le=10.0, + 0.1, + ge=0.01, + le=10.0, description=( "Simulation time step in seconds. " "0.1 s recommended for VANET (matches 802.11p beacon rate). " @@ -427,44 +484,60 @@ class SumoParams(BaseModel): # ── Vehicle behaviour ──────────────────────────────────── default_max_speed: float = Field( - 50.0, ge=1.0, le=200.0, + 50.0, + ge=1.0, + le=200.0, description="Global maximum vehicle speed cap in m/s.", ) default_accel: float = Field( - 2.6, ge=0.1, le=20.0, + 2.6, + ge=0.1, + le=20.0, description="Default vehicle acceleration in m/s².", ) default_decel: float = Field( - 4.5, ge=0.1, le=20.0, + 4.5, + ge=0.1, + le=20.0, description=( "Default comfortable deceleration in m/s². " "Must be ≤ default_emergency_decel." ), ) default_emergency_decel: float = Field( - 9.0, ge=0.1, le=30.0, + 9.0, + ge=0.1, + le=30.0, description=( "Physical maximum deceleration in m/s² (used in emergencies). " "Must be ≥ default_decel." ), ) default_sigma: float = Field( - 0.5, ge=0.0, le=1.0, + 0.5, + ge=0.0, + le=1.0, description=( "Krauss driver imperfection factor. " "0.0 = perfect driver, 1.0 = maximum randomness." ), ) default_tau: float = Field( - 1.0, ge=0.1, le=10.0, + 1.0, + ge=0.1, + le=10.0, description="Minimum time headway / reaction time in seconds.", ) default_vehicle_length: float = Field( - 5.0, ge=1.0, le=30.0, + 5.0, + ge=1.0, + le=30.0, description="Default vehicle body length in metres.", ) default_min_gap: float = Field( - 2.5, ge=0.0, le=20.0, + 2.5, + ge=0.0, + le=20.0, description="Minimum bumper-to-bumper gap to the vehicle ahead in metres.", ) @@ -499,7 +572,9 @@ class SumoParams(BaseModel): ), ) lateral_resolution: float = Field( - 0.0, ge=0.0, le=5.0, + 0.0, + ge=0.0, + le=5.0, description=( "Sub-lane lateral resolution in metres (0 = standard lane model). " "Enables fine-grained lateral position tracking." @@ -508,7 +583,9 @@ class SumoParams(BaseModel): # ── FCD output ─────────────────────────────────────────── fcd_output_period: float = Field( - 0.1, ge=0.01, le=60.0, + 0.1, + ge=0.01, + le=60.0, description=( "Floating Car Data write interval in seconds. " "Must be a whole multiple of step_length. " @@ -535,7 +612,8 @@ class SumoParams(BaseModel): description="Emit per-step network-wide summary statistics.", ) seed: int = Field( - 42, ge=0, + 42, + ge=0, description="Random seed for reproducible SUMO runs.", ) @@ -551,7 +629,7 @@ def validate_collision_action(cls, v: str) -> str: return v @model_validator(mode="after") - def check_timeline(self) -> "SumoParams": + def check_timeline(self) -> SumoParams: if self.begin >= self.end: raise ValueError( f"begin ({self.begin}) must be strictly less than end ({self.end})." @@ -559,7 +637,7 @@ def check_timeline(self) -> "SumoParams": return self @model_validator(mode="after") - def check_fcd_period_is_multiple_of_step(self) -> "SumoParams": + def check_fcd_period_is_multiple_of_step(self) -> SumoParams: ratio = self.fcd_output_period / self.step_length if abs(ratio - round(ratio)) > 1e-9: raise ValueError( @@ -571,7 +649,7 @@ def check_fcd_period_is_multiple_of_step(self) -> "SumoParams": return self @model_validator(mode="after") - def check_decel_ordering(self) -> "SumoParams": + def check_decel_ordering(self) -> SumoParams: if self.default_decel > self.default_emergency_decel: raise ValueError( f"default_decel ({self.default_decel} m/s²) cannot exceed " @@ -585,6 +663,7 @@ def check_decel_ordering(self) -> "SumoParams": # Stage 5 — traceExporter # ────────────────────────────────────────────────────────────── + class TraceExporterParams(BaseModel): """ Controls conversion of SUMO FCD output to NS-2/NS-3 mobility traces. @@ -610,7 +689,7 @@ class TraceExporterParams(BaseModel): 0.0, description="Y translation applied to all exported coordinates (metres).", ) - boundary: Optional[str] = Field( + boundary: str | None = Field( None, description=( "Spatial filter as 'xMin,yMin,xMax,yMax' (four floats, comma-separated). " @@ -621,17 +700,20 @@ class TraceExporterParams(BaseModel): # ── Temporal ───────────────────────────────────────────── begin: float = Field( - 0.0, ge=0.0, + 0.0, + ge=0.0, description="Export start time in seconds.", ) - end: Optional[float] = Field( + end: float | None = Field( None, description="Export end time in seconds. None = match simulation end.", ) # ── Sampling ───────────────────────────────────────────── sampling_period: float = Field( - 1.0, ge=0.1, le=60.0, + 1.0, + ge=0.1, + le=60.0, description=( "Position update interval in seconds. " "Must be ≥ sumo.fcd_output_period. " @@ -645,14 +727,17 @@ class TraceExporterParams(BaseModel): description="Embed instantaneous speed in each NS-2/NS-3 output line.", ) penetration_rate: float = Field( - 1.0, ge=0.01, le=1.0, + 1.0, + ge=0.01, + le=1.0, description=( "Fraction of vehicles to export (1.0 = all). " "Values < 1.0 model partial OBU/VANET device deployment." ), ) seed: int = Field( - 42, ge=0, + 42, + ge=0, description="Random seed for vehicle sub-sampling when penetration_rate < 1.", ) write_fcd_filtered: bool = Field( @@ -664,7 +749,7 @@ class TraceExporterParams(BaseModel): @field_validator("boundary") @classmethod - def validate_boundary(cls, v: Optional[str]) -> Optional[str]: + def validate_boundary(cls, v: str | None) -> str | None: if v is None: return v if not _BOUNDARY_RE.match(v.strip()): @@ -685,7 +770,7 @@ def validate_boundary(cls, v: Optional[str]) -> Optional[str]: return v.strip() @model_validator(mode="after") - def check_export_timeline(self) -> "TraceExporterParams": + def check_export_timeline(self) -> TraceExporterParams: if self.end is not None and self.end <= self.begin: raise ValueError( f"TraceExporter end ({self.end}) must be greater than begin ({self.begin})." @@ -697,19 +782,21 @@ def check_export_timeline(self) -> "TraceExporterParams": # Top-level job request — cross-stage validation lives here # ────────────────────────────────────────────────────────────── + class ConversionRequest(BaseModel): """Complete pipeline configuration submitted with the OSM file.""" - netconvert: NetconvertParams = Field(default_factory=NetconvertParams) - random_trips: RandomTripsParams = Field(default_factory=RandomTripsParams) - sumo: SumoParams = Field(default_factory=SumoParams) - trace_exporter: TraceExporterParams = Field(default_factory=TraceExporterParams) + + netconvert: NetconvertParams = Field(default_factory=NetconvertParams) + random_trips: RandomTripsParams = Field(default_factory=RandomTripsParams) + sumo: SumoParams = Field(default_factory=SumoParams) + trace_exporter: TraceExporterParams = Field(default_factory=TraceExporterParams) @model_validator(mode="after") - def cross_stage_timeline(self) -> "ConversionRequest": + def cross_stage_timeline(self) -> ConversionRequest: """Warn / reject when stage timelines are obviously inconsistent.""" - rt_end = self.random_trips.end + rt_end = self.random_trips.end sumo_end = self.sumo.end - te_end = self.trace_exporter.end + te_end = self.trace_exporter.end if sumo_end > rt_end: raise ValueError( @@ -725,9 +812,9 @@ def cross_stage_timeline(self) -> "ConversionRequest": return self @model_validator(mode="after") - def cross_stage_sampling(self) -> "ConversionRequest": + def cross_stage_sampling(self) -> ConversionRequest: """Ensure trace sampling resolution is achievable given FCD output rate.""" - fcd_period = self.sumo.fcd_output_period + fcd_period = self.sumo.fcd_output_period sampling_period = self.trace_exporter.sampling_period if sampling_period < fcd_period: raise ValueError( @@ -749,29 +836,31 @@ def cross_stage_sampling(self) -> "ConversionRequest": # Job status types # ────────────────────────────────────────────────────────────── + class StageStatus(str, Enum): pending = "pending" running = "running" - done = "done" - failed = "failed" - skipped = "skipped" + done = "done" + failed = "failed" + cancelled = "cancelled" + skipped = "skipped" # reserved; no pipeline path currently produces it class StageLog(BaseModel): - status: StageStatus - stdout: Optional[str] = None - stderr: Optional[str] = None - duration: Optional[float] = None # wall-clock seconds + status: StageStatus + stdout: str | None = None + stderr: str | None = None + duration: float | None = None # wall-clock seconds class JobStatus(BaseModel): - job_id: str - status: StageStatus - current_stage: Optional[str] = None - stages: dict = Field(default_factory=dict) - stage_logs: dict = Field(default_factory=dict) # stage → StageLog - error: Optional[str] = None - outputs: List[str] = Field(default_factory=list) - created_at: str - completed_at: Optional[str] = None - warnings: List[str] = Field(default_factory=list) \ No newline at end of file + job_id: str + status: StageStatus + current_stage: str | None = None + stages: dict = Field(default_factory=dict) + stage_logs: dict = Field(default_factory=dict) # stage → StageLog + error: str | None = None + outputs: list[str] = Field(default_factory=list) + created_at: str + completed_at: str | None = None + warnings: list[str] = Field(default_factory=list) diff --git a/app/settings.py b/app/settings.py new file mode 100644 index 0000000..abaf9e7 --- /dev/null +++ b/app/settings.py @@ -0,0 +1,100 @@ +""" +Application settings, loaded from environment variables (prefix OSM2NS3_) +and an optional .env file. + +Directory layout is derived from ``data_dir`` — nothing is created at import +time; the FastAPI lifespan handler calls ``ensure_dirs()`` at startup. +""" + +from __future__ import annotations + +import os +from functools import lru_cache +from pathlib import Path + +from pydantic_settings import BaseSettings, SettingsConfigDict + +# Repo root (parent of this file's package directory). +_REPO_ROOT = Path(__file__).resolve().parent.parent + + +class Settings(BaseSettings): + """Runtime configuration for the osm2ns3 service.""" + + model_config = SettingsConfigDict( + env_prefix="OSM2NS3_", + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + ) + + # ── Security ────────────────────────────────────────────── + # API-key auth is DISABLED when api_key is None (local dev default). + # Set OSM2NS3_API_KEY to require an `X-API-Key` header on /jobs routes. + api_key: str | None = None + # JSON list value when set via env: + # OSM2NS3_CORS_ORIGINS='["https://example.com"]' + cors_origins: list[str] = [ + "http://localhost:5173", + "http://localhost:8000", + "http://127.0.0.1:5173", + "http://127.0.0.1:8000", + ] + + # ── Uploads ─────────────────────────────────────────────── + max_upload_mb: int = 100 + + # ── Job control ─────────────────────────────────────────── + max_concurrent_jobs: int = 2 + job_retention_max: int = 200 # prune oldest finished beyond this many + job_retention_days: int = 7 + + # ── Pipeline timeouts (seconds) ─────────────────────────── + timeout_default_s: float = 600.0 + timeout_sumo_s: float = 1200.0 # simulations can be slow on large maps + timeout_trace_export_s: float = 300.0 + + # ── Filesystem ──────────────────────────────────────────── + # jobs/uploads/outputs (and jobs.db) live directly under data_dir. + data_dir: Path = _REPO_ROOT + # Built frontend (Vite dist). Present only in the Docker image by default. + static_dir: Path = _REPO_ROOT / "static" + + # ── SUMO toolchain ──────────────────────────────────────── + # Extra directory searched first for randomTrips.py / traceExporter.py. + # Falls back to the SUMO_HOME environment variable when unset. + sumo_home: str | None = None + + # ── Logging ─────────────────────────────────────────────── + log_level: str = "INFO" + + # ── Derived paths ───────────────────────────────────────── + @property + def jobs_dir(self) -> Path: + return self.data_dir / "jobs" + + @property + def upload_dir(self) -> Path: + return self.data_dir / "uploads" + + @property + def out_dir(self) -> Path: + return self.data_dir / "outputs" + + @property + def db_path(self) -> Path: + return self.data_dir / "jobs.db" + + def ensure_dirs(self) -> None: + for d in (self.jobs_dir, self.upload_dir, self.out_dir): + d.mkdir(parents=True, exist_ok=True) + + def resolve_sumo_home(self) -> str: + return self.sumo_home or os.environ.get("SUMO_HOME", "") + + +@lru_cache +def get_settings() -> Settings: + """Process-wide settings (FastAPI dependency). Bypassed by tests, which + construct Settings directly to avoid lru_cache cross-test pollution.""" + return Settings() diff --git a/app/store.py b/app/store.py new file mode 100644 index 0000000..ad6c622 --- /dev/null +++ b/app/store.py @@ -0,0 +1,273 @@ +""" +SQLite-backed job store. + +Replaces the previous in-memory ``_jobs`` dict: jobs survive server restarts, +and retention pruning is possible. Single-node scope — stdlib ``sqlite3`` +with WAL journaling, one short-lived connection per call, executed via +``asyncio.to_thread`` so the event loop is never blocked. + +On startup, jobs left in ``pending``/``running`` state by a previous +(crash-)shutdown are recovered to ``failed`` — their subprocesses cannot +survive a process restart, so failing with a reason is the honest outcome. +""" + +from __future__ import annotations + +import asyncio +import builtins +import json +import logging +import sqlite3 +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from .models import JobStatus, StageStatus + +logger = logging.getLogger("osm2ns3.store") + +SCHEMA_VERSION = "1" + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS jobs ( + job_id TEXT PRIMARY KEY, + status TEXT NOT NULL, + stage_statuses TEXT NOT NULL, + stage_logs TEXT NOT NULL, + current_stage TEXT, + error TEXT, + outputs TEXT NOT NULL, + warnings TEXT NOT NULL, + config_json TEXT, + created_at TEXT NOT NULL, + completed_at TEXT +); +CREATE TABLE IF NOT EXISTS kv ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +""" + +_FINISHED = ( + StageStatus.done.value, + StageStatus.failed.value, + StageStatus.cancelled.value, +) + + +def _utcnow() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _job_to_row(job: JobStatus, config_json: str | None) -> tuple: + d = job.model_dump(mode="json") + return ( + d["job_id"], + d["status"], + json.dumps(d["stages"]), + json.dumps(d["stage_logs"]), + d["current_stage"], + d["error"], + json.dumps(d["outputs"]), + json.dumps(d["warnings"]), + config_json, + d["created_at"], + d["completed_at"], + ) + + +def _row_to_job(row: sqlite3.Row) -> JobStatus: + return JobStatus.model_validate( + { + "job_id": row["job_id"], + "status": row["status"], + "stages": json.loads(row["stage_statuses"]), + "stage_logs": json.loads(row["stage_logs"]), + "current_stage": row["current_stage"], + "error": row["error"], + "outputs": json.loads(row["outputs"]), + "warnings": json.loads(row["warnings"]), + "created_at": row["created_at"], + "completed_at": row["completed_at"], + } + ) + + +class JobStore: + """Durable job registry. All methods are async; blocking sqlite3 work + runs in the default executor via asyncio.to_thread.""" + + def __init__( + self, db_path: Path, retention_max: int = 200, retention_days: int = 7 + ): + self._path = Path(db_path) + self._retention_max = retention_max + self._retention_days = retention_days + + # ── Sync helpers (run in thread) ───────────────────────── + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self._path, timeout=30) + conn.row_factory = sqlite3.Row + return conn + + # ── Lifecycle ───────────────────────────────────────────── + + async def init(self) -> None: + await asyncio.to_thread(self._init_sync) + recovered = await self._recover_interrupted() + for job_id in recovered: + logger.warning( + "job %s was pending/running at startup — marked failed " + "(interrupted by server restart)", + job_id, + ) + + def _init_sync(self) -> None: + self._path.parent.mkdir(parents=True, exist_ok=True) + with self._connect() as conn: + conn.execute("PRAGMA journal_mode=WAL") + conn.executescript(_SCHEMA) + conn.execute( + "INSERT OR REPLACE INTO kv (key, value) VALUES ('schema_version', ?)", + (SCHEMA_VERSION,), + ) + + async def _recover_interrupted(self) -> builtins.list[str]: + interrupted = [ + j + for j in await self.list() + if j.status in (StageStatus.pending, StageStatus.running) + ] + for job in interrupted: + job.status = StageStatus.failed + job.error = "Interrupted by server restart" + job.completed_at = _utcnow() + for stage, st in job.stages.items(): + if st == StageStatus.running: + job.stages[stage] = StageStatus.failed + await self.upsert(job) + return [j.job_id for j in interrupted] + + # ── CRUD ────────────────────────────────────────────────── + + async def create(self, job: JobStatus, config_json: str | None) -> None: + row = _job_to_row(job, config_json) + + def _op() -> None: + with self._connect() as conn: + conn.execute( + """ + INSERT INTO jobs (job_id, status, stage_statuses, stage_logs, + current_stage, error, outputs, warnings, + config_json, created_at, completed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + row, + ) + + await asyncio.to_thread(_op) + logger.debug("created job row %s", job.job_id) + + async def get(self, job_id: str) -> JobStatus | None: + def _op() -> JobStatus | None: + with self._connect() as conn: + row = conn.execute( + "SELECT * FROM jobs WHERE job_id = ?", (job_id,) + ).fetchone() + return _row_to_job(row) if row else None + + return await asyncio.to_thread(_op) + + async def list(self) -> builtins.list[JobStatus]: + def _op() -> list[JobStatus]: + with self._connect() as conn: + rows = conn.execute("SELECT * FROM jobs").fetchall() + return [_row_to_job(r) for r in rows] + + return await asyncio.to_thread(_op) + + async def upsert(self, job: JobStatus) -> None: + """Full-row rewrite — called on every job state change.""" + + def _op() -> None: + with self._connect() as conn: + conn.execute( + """ + UPDATE jobs SET status = ?, stage_statuses = ?, stage_logs = ?, + current_stage = ?, error = ?, outputs = ?, + warnings = ?, completed_at = ? + WHERE job_id = ? + """, + ( + job.status.value, + json.dumps(job.model_dump(mode="json")["stages"]), + json.dumps(job.model_dump(mode="json")["stage_logs"]), + job.current_stage, + job.error, + json.dumps(job.outputs), + json.dumps(job.warnings), + job.completed_at, + job.job_id, + ), + ) + + await asyncio.to_thread(_op) + + async def get_config(self, job_id: str) -> dict | None: + def _op() -> dict | None: + with self._connect() as conn: + row = conn.execute( + "SELECT config_json FROM jobs WHERE job_id = ?", (job_id,) + ).fetchone() + if not row or row["config_json"] is None: + return None + return json.loads(row["config_json"]) + + return await asyncio.to_thread(_op) + + async def delete(self, job_id: str) -> None: + def _op() -> None: + with self._connect() as conn: + conn.execute("DELETE FROM jobs WHERE job_id = ?", (job_id,)) + + await asyncio.to_thread(_op) + + # ── Retention ───────────────────────────────────────────── + + async def prune(self) -> builtins.list[str]: + """Delete finished jobs beyond retention limits; returns pruned ids + so the caller can remove their files.""" + + def _op() -> list[str]: + with self._connect() as conn: + cutoff = ( + datetime.now(timezone.utc) - timedelta(days=self._retention_days) + ).isoformat() + by_age = conn.execute( + f"SELECT job_id FROM jobs WHERE status IN ({','.join('?' * len(_FINISHED))}) " + "AND completed_at IS NOT NULL AND completed_at < ?", + (*_FINISHED, cutoff), + ).fetchall() + by_count = conn.execute( + f""" + SELECT job_id FROM jobs + WHERE status IN ({','.join('?' * len(_FINISHED))}) + ORDER BY completed_at DESC + LIMIT -1 OFFSET ? + """, + (*_FINISHED, self._retention_max), + ).fetchall() + ids = sorted( + {r["job_id"] for r in by_age} | {r["job_id"] for r in by_count} + ) + if ids: + conn.execute( + f"DELETE FROM jobs WHERE job_id IN ({','.join('?' * len(ids))})", + ids, + ) + return ids + + pruned = await asyncio.to_thread(_op) + if pruned: + logger.info("retention prune removed %d finished job(s)", len(pruned)) + return pruned diff --git a/app/worker.py b/app/worker.py index ef800cd..3babd1c 100644 --- a/app/worker.py +++ b/app/worker.py @@ -2,20 +2,27 @@ Async pipeline executor for OSM → NS-3 conversion. All subprocess calls use asyncio.create_subprocess_exec — no blocking calls -on the event loop. File I/O uses aiofiles. Per-stage logs are persisted to +on the event loop. File I/O uses aiofiles. Job state is persisted to the +SQLite-backed JobStore on every transition, per-stage logs are persisted to disk and returned via the /jobs/{id}/logs endpoint. + +Command builders and XML writers are pure module-level functions (no I/O +beyond what their docstrings say) so they can be imported and unit-tested +directly. Execution state lives in ``PipelineRunner``, which owns the +asyncio task + subprocess registries needed for cancellation. """ from __future__ import annotations import asyncio +import logging +import os import shutil +import signal import time import uuid from datetime import datetime, timezone -from functools import partial from pathlib import Path -from typing import Dict, List, Optional, Tuple import aiofiles import aiofiles.os @@ -34,29 +41,37 @@ TraceExporterParams, VehicleClass, ) +from .settings import Settings +from .store import JobStore -# ── Directory layout ────────────────────────────────────────── -BASE_DIR = Path(__file__).parent.parent -JOBS_DIR = BASE_DIR / "jobs" -UPLOAD_DIR = BASE_DIR / "uploads" -OUT_DIR = BASE_DIR / "outputs" - -for _d in (JOBS_DIR, UPLOAD_DIR, OUT_DIR): - _d.mkdir(parents=True, exist_ok=True) - -# In-memory job store (swap for Redis/SQLite in production) -_jobs: Dict[str, JobStatus] = {} -_job_locks: Dict[str, asyncio.Lock] = {} +logger = logging.getLogger("osm2ns3.worker") -# SUMO tool search paths (extend / override via SUMO_HOME env var) +# SUMO tool search paths (extend / override via Settings.sumo_home +# or the SUMO_HOME env var) _SUMO_TOOL_DIRS = [ "/usr/share/sumo/tools", "/usr/local/share/sumo/tools", ] +STAGE_NAMES = ("netconvert", "random_trips", "duarouter", "sumo", "trace_export") + +# Stages whose `{stage}.log` files are written into the job workdir (the +# old /logs endpoint naming used the raw tool names; kept for backcompat). +LOG_STAGE_ALIASES = { + "netconvert": "netconvert", + "random_trips": "random_trips", + "duarouter": "duarouter", + "sumo": "sumo", + "trace_export": "trace_export", + "traceExporter": "trace_export", # legacy name from the old endpoint +} + +_CANCEL_SENTINEL = "Cancelled by user" + # ── Pure helpers (sync, no I/O) ─────────────────────────────── + def _now() -> str: return datetime.now(timezone.utc).isoformat() @@ -69,7 +84,7 @@ def _lc_mode_int(m: LaneChangeMode) -> int: return {"default": 1621, "no_lc": 0, "strategic": 256}[m.value] -def _find_binary(names: List[str]) -> Optional[str]: +def _find_binary(names: list[str]) -> str | None: """Return the first executable found in PATH from the candidate list.""" for name in names: path = shutil.which(name) @@ -78,122 +93,41 @@ def _find_binary(names: List[str]) -> Optional[str]: return None -def _find_sumo_script(filename: str) -> Optional[str]: - """Locate a SUMO Python tool (randomTrips.py, traceExporter.py, …).""" - import os - sumo_home = os.environ.get("SUMO_HOME", "") +def _find_sumo_script(filename: str, sumo_home: str = "") -> str | None: + """Locate a SUMO Python tool (randomTrips.py, traceExporter.py, …). + + Search order: well-known system dirs, then the provided sumo_home, + then the SUMO_HOME env var. + """ + env_home = os.environ.get("SUMO_HOME", "") candidates = [ *[f"{d}/{filename}" for d in _SUMO_TOOL_DIRS], f"{sumo_home}/tools/{filename}" if sumo_home else None, + f"{env_home}/tools/{filename}" if env_home and env_home != sumo_home else None, ] return next((c for c in candidates if c and Path(c).exists()), None) -# ── Job registry ────────────────────────────────────────────── - -def create_job() -> str: - job_id = str(uuid.uuid4()) - _jobs[job_id] = JobStatus( - job_id=job_id, - status=StageStatus.pending, - stages={ - "netconvert": StageStatus.pending, - "random_trips": StageStatus.pending, - "duarouter": StageStatus.pending, - "sumo": StageStatus.pending, - "trace_export": StageStatus.pending, - }, - stage_logs={}, - created_at=_now(), - ) - _job_locks[job_id] = asyncio.Lock() - return job_id - - -def get_job(job_id: str) -> Optional[JobStatus]: - return _jobs.get(job_id) +def check_tools(sumo_home: str = "") -> dict[str, bool]: + """Availability map for every SUMO tool the pipeline needs. - -def list_jobs() -> List[JobStatus]: - return list(_jobs.values()) - - -def get_full_schema() -> dict: - return ConversionRequest.model_json_schema() - - -def get_defaults() -> dict: - return ConversionRequest().model_dump() - - -# ── Async subprocess primitive ──────────────────────────────── - -async def _run_async( - cmd: List[str], - workdir: Path, - timeout: float = 600.0, -) -> Tuple[int, str, str]: + Reuses the exact lookup logic the pipeline uses, so a `ready` result + genuinely means the pipeline can run. """ - Run a command asynchronously. - Returns (returncode, stdout, stderr). - Raises RuntimeError on timeout or missing executable. - """ - try: - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - cwd=str(workdir), - ) - except FileNotFoundError: - raise RuntimeError(f"Executable not found: {cmd[0]!r}") - - try: - raw_out, raw_err = await asyncio.wait_for( - proc.communicate(), timeout=timeout - ) - except asyncio.TimeoutError: - proc.kill() - await proc.communicate() - raise RuntimeError( - f"Command timed out after {timeout:.0f} s: {' '.join(cmd[:3])!r}…" - ) - - return proc.returncode, raw_out.decode(errors="replace"), raw_err.decode(errors="replace") - - -# ── Async file I/O helpers ──────────────────────────────────── - -async def _write_text(path: Path, content: str) -> None: - async with aiofiles.open(path, "w", encoding="utf-8") as fh: - await fh.write(content) + return { + "netconvert": _find_binary(["netconvert", "netconvert.exe"]) is not None, + "duarouter": _find_binary(["duarouter", "duarouter.exe"]) is not None, + "sumo": _find_binary(["sumo", "sumo.exe"]) is not None, + "traceExporter.py": _find_sumo_script("traceExporter.py", sumo_home) + is not None, + "randomTrips.py": _find_sumo_script("randomTrips.py", sumo_home) is not None, + } -async def _copy_file(src: Path, dst: Path) -> None: - """Non-blocking file copy via aiofiles.""" - async with aiofiles.open(src, "rb") as r: - data = await r.read() - async with aiofiles.open(dst, "wb") as w: - await w.write(data) +# ── Command builders (pure sync, no I/O) ────────────────────── -# ── Job state update ────────────────────────────────────────── - -def _set_stage( - job: JobStatus, - stage: str, - status: StageStatus, - log: Optional[StageLog] = None, -) -> None: - job.stages[stage] = status - job.current_stage = stage - if log: - job.stage_logs[stage] = log.model_dump() - - -# ── Command builders (pure sync, no I/O) ───────────────────── - -def _netconvert_cmd(osm: Path, net_out: Path, p: NetconvertParams) -> List[str]: +def _netconvert_cmd(osm: Path, net_out: Path, p: NetconvertParams) -> list[str]: tool = _find_binary(["netconvert", "netconvert.exe"]) if not tool: raise RuntimeError( @@ -202,32 +136,46 @@ def _netconvert_cmd(osm: Path, net_out: Path, p: NetconvertParams) -> List[str]: ) cmd = [ tool, - "--osm-files", str(osm), - "--output-file", str(net_out), - - "--remove-edges.isolated", str(p.osm_remove_isolated_edges).lower(), - "--keep-edges.by-vclass", p.keep_edges_by_vclass.value - if p.keep_edges_by_vclass else "all", - - "--no-internal-links", str(p.geometry_no_internal_links).lower(), - "--junctions.internal-link-detail", str(p.junctions_internal_link_detail), - "--junctions.corner-detail", str(p.junctions_corner_detail), - "--default.junctions.type", p.default_junction_type.value, - "--junctions.minimum-size", str(p.junctions_min_size), - "--junctions.limit-turn-speed", str(p.junctions_limit_turn_speed), - - "--default.lanewidth", str(p.default_lane_width), - "--default.speed", str(p.default_speed_limit), - "--default.lanenumber", str(p.default_num_lanes), - - "--tls.guess", str(p.tl_guess).lower(), - "--tls.default-type", p.tl_type.value, - "--tls.join", str(p.tl_join).lower(), - "--tls.min-dur", str(p.tl_min_dur), - "--tls.max-dur", str(p.tl_max_dur), - - "--proj.utm", str(p.proj_utm).lower(), - "--osm.highway.type-tag", ",".join(p.osm_highway_types), + "--osm-files", + str(osm), + "--output-file", + str(net_out), + "--remove-edges.isolated", + str(p.osm_remove_isolated_edges).lower(), + "--keep-edges.by-vclass", + p.keep_edges_by_vclass.value if p.keep_edges_by_vclass else "all", + "--no-internal-links", + str(p.geometry_no_internal_links).lower(), + "--junctions.internal-link-detail", + str(p.junctions_internal_link_detail), + "--junctions.corner-detail", + str(p.junctions_corner_detail), + "--default.junctions.type", + p.default_junction_type.value, + "--junctions.minimum-size", + str(p.junctions_min_size), + "--junctions.limit-turn-speed", + str(p.junctions_limit_turn_speed), + "--default.lanewidth", + str(p.default_lane_width), + "--default.speed", + str(p.default_speed_limit), + "--default.lanenumber", + str(p.default_num_lanes), + "--tls.guess", + str(p.tl_guess).lower(), + "--tls.default-type", + p.tl_type.value, + "--tls.join", + str(p.tl_join).lower(), + "--tls.min-dur", + str(p.tl_min_dur), + "--tls.max-dur", + str(p.tl_max_dur), + "--proj.utm", + str(p.proj_utm).lower(), + "--osm.highway.type-tag", + ",".join(p.osm_highway_types), ] if p.osm_no_large_roundabouts: cmd += ["--roundabouts.guess", "false"] @@ -246,24 +194,37 @@ def _netconvert_cmd(osm: Path, net_out: Path, p: NetconvertParams) -> List[str]: return cmd -def _random_trips_cmd(net: Path, trips_out: Path, p: RandomTripsParams) -> List[str]: - script = _find_sumo_script("randomTrips.py") +def _random_trips_cmd( + net: Path, trips_out: Path, p: RandomTripsParams, sumo_home: str = "" +) -> list[str]: + script = _find_sumo_script("randomTrips.py", sumo_home) if not script: raise RuntimeError( "randomTrips.py not found — set SUMO_HOME or install sumo-tools." ) cmd = [ - "python3", script, - "--net-file", str(net), - "--output-trip-file", str(trips_out), - "--begin", str(p.begin), - "--end", str(p.end), - "--period", str(p.period), - "--min-distance", str(p.min_distance), - "--fringe-factor", str(p.fringe_factor), - "--fringe-threshold", str(p.fringe_threshold), - "--seed", str(p.seed), - "--vclass", p.vehicle_class.value, + "python3", + script, + "--net-file", + str(net), + "--output-trip-file", + str(trips_out), + "--begin", + str(p.begin), + "--end", + str(p.end), + "--period", + str(p.period), + "--min-distance", + str(p.min_distance), + "--fringe-factor", + str(p.fringe_factor), + "--fringe-threshold", + str(p.fringe_threshold), + "--seed", + str(p.seed), + "--vclass", + p.vehicle_class.value, ] if p.max_distance is not None: cmd += ["--max-distance", str(p.max_distance)] @@ -282,58 +243,74 @@ def _random_trips_cmd(net: Path, trips_out: Path, p: RandomTripsParams) -> List[ def _duarouter_cmd( net: Path, trips: Path, routes_out: Path, p: RandomTripsParams -) -> List[str]: +) -> list[str]: tool = _find_binary(["duarouter", "duarouter.exe"]) if not tool: raise RuntimeError("duarouter not found — install SUMO.") return [ tool, - "--net-file", str(net), - "--trip-files", str(trips), - "--output-file", str(routes_out), - "--seed", str(p.seed), - "--begin", str(p.begin), - "--end", str(p.end), - "--ignore-errors", "true", + "--net-file", + str(net), + "--trip-files", + str(trips), + "--output-file", + str(routes_out), + "--seed", + str(p.seed), + "--begin", + str(p.begin), + "--end", + str(p.end), + "--ignore-errors", + "true", "--no-step-log", ] -def _sumo_cmd(cfg: Path, vtype: Path) -> List[str]: +def _sumo_cmd(cfg: Path, vtype: Path) -> list[str]: tool = _find_binary(["sumo", "sumo.exe"]) if not tool: raise RuntimeError("sumo not found — install SUMO.") return [ tool, - "--configuration-file", str(cfg), - "--additional-files", str(vtype), + "--configuration-file", + str(cfg), + "--additional-files", + str(vtype), ] def _trace_export_cmds( - fcd: Path, workdir: Path, p: TraceExporterParams -) -> List[Tuple[str, List[str], Path]]: + fcd: Path, workdir: Path, p: TraceExporterParams, sumo_home: str = "" +) -> list[tuple[str, list[str], Path]]: """Return list of (label, cmd, output_path) for traceExporter runs.""" - script = _find_sumo_script("traceExporter.py") + script = _find_sumo_script("traceExporter.py", sumo_home) if not script: raise RuntimeError( "traceExporter.py not found — set SUMO_HOME or install sumo-tools." ) base = [ - "python3", script, - "--fcd-input", str(fcd), - "--begin", str(p.begin), - "--penetration", str(p.penetration_rate), - "--seed", str(p.seed), - "--orig-x", str(p.x_offset), - "--orig-y", str(p.y_offset), + "python3", + script, + "--fcd-input", + str(fcd), + "--begin", + str(p.begin), + "--penetration", + str(p.penetration_rate), + "--seed", + str(p.seed), + "--orig-x", + str(p.x_offset), + "--orig-y", + str(p.y_offset), ] if p.end is not None: base += ["--end", str(p.end)] if p.boundary: base += ["--boundary", p.boundary] - results: List[Tuple[str, List[str], Path]] = [] + results: list[tuple[str, list[str], Path]] = [] if p.output_format in (OutputFormat.ns_movements, OutputFormat.both): out = workdir / "ns_movements" @@ -351,6 +328,20 @@ def _trace_export_cmds( # ── Async file writers ──────────────────────────────────────── + +async def _write_text(path: Path, content: str) -> None: + async with aiofiles.open(path, "w", encoding="utf-8") as fh: + await fh.write(content) + + +async def _copy_file(src: Path, dst: Path) -> None: + """Non-blocking file copy via aiofiles.""" + async with aiofiles.open(src, "rb") as r: + data = await r.read() + async with aiofiles.open(dst, "wb") as w: + await w.write(data) + + async def _write_sumo_cfg( workdir: Path, net: Path, @@ -403,9 +394,7 @@ async def _write_sumo_cfg( return cfg_path -async def _write_vtype_xml( - workdir: Path, p: SumoParams, vc: VehicleClass -) -> Path: +async def _write_vtype_xml(workdir: Path, p: SumoParams, vc: VehicleClass) -> Path: xml = f""" Tuple[str, str]: - """ - Mark stage running, execute the command, persist stdout/stderr to disk, - update the job's StageLog, and return (stdout, stderr). - Raises RuntimeError on non-zero exit or timeout. - """ - _set_stage(job, stage, StageStatus.running) - t0 = time.monotonic() - - rc, stdout, stderr = await _run_async(cmd, workdir, timeout=timeout) - - elapsed = time.monotonic() - t0 - - # Persist log to disk (non-blocking) - log_path = workdir / f"{stage}.log" - await _write_text(log_path, f"=== stdout ===\n{stdout}\n=== stderr ===\n{stderr}\n") - - # Capture SUMO warnings into the job - for line in (stdout + stderr).splitlines(): - if "Warning" in line or "warning" in line: - job.warnings.append(f"[{stage}] {line.strip()}") +# ── Pipeline runner ─────────────────────────────────────────── - log = StageLog( - status=StageStatus.done if rc == 0 else StageStatus.failed, - stdout=stdout[:4000] if stdout else None, # truncate for in-memory store - stderr=stderr[:4000] if stderr else None, - duration=round(elapsed, 3), - ) - if rc != 0: - _set_stage(job, stage, StageStatus.failed, log) - raise RuntimeError( - f"Stage '{stage}' exited with code {rc}.\n" - f"stderr: {stderr[:800]}" - ) - - _set_stage(job, stage, StageStatus.done, log) - return stdout, stderr +class PipelineRunner: + """Owns per-job execution state: task registry, current subprocess set, + concurrency semaphore. Created once at app startup with injected + Settings + JobStore.""" + def __init__(self, settings: Settings, store: JobStore): + self.settings = settings + self.store = store + self._tasks: dict[str, asyncio.Task] = {} + self._procs: dict[str, set[asyncio.subprocess.Process]] = {} + self._semaphore = asyncio.Semaphore(settings.max_concurrent_jobs) -# ── Master pipeline ─────────────────────────────────────────── - -async def run_pipeline( - job_id: str, - osm_path: Path, - config: ConversionRequest, -) -> None: - """ - Execute all five pipeline stages sequentially in the background. - - Stages: - 1. netconvert — OSM → .net.xml - 2. random_trips — .net.xml → trips.xml - 3. duarouter — trips.xml → routes.rou.xml - 4. sumo — routes + net → fcd-output.xml - 5. trace_export — fcd-output.xml → ns_movements / mobility.tcl - """ - workdir = JOBS_DIR / job_id - await aiofiles.os.makedirs(workdir, exist_ok=True) - - job = _jobs[job_id] - job.status = StageStatus.running - - try: - # ── 1. netconvert ──────────────────────────────────── - net_path = workdir / "network.net.xml" - await _run_stage( - job, "netconvert", - _netconvert_cmd(osm_path, net_path, config.netconvert), - workdir, - ) - - # ── 2. randomTrips ─────────────────────────────────── - trips_path = workdir / "trips.xml" - await _run_stage( - job, "random_trips", - _random_trips_cmd(net_path, trips_path, config.random_trips), - workdir, - ) - - # ── 3. duarouter ───────────────────────────────────── - routes_path = workdir / "routes.rou.xml" - await _run_stage( - job, "duarouter", - _duarouter_cmd(net_path, trips_path, routes_path, config.random_trips), - workdir, - ) + # ── Job lifecycle ───────────────────────────────────────── - # ── 4. sumo ────────────────────────────────────────── - fcd_path = workdir / "fcd-output.xml" - tripinfo_path = workdir / "tripinfo.xml" - summary_path = workdir / "summary.xml" - - # Write config files asynchronously before launching SUMO - cfg_path, vtype_path = await asyncio.gather( - _write_sumo_cfg( - workdir, net_path, routes_path, - fcd_path, tripinfo_path, summary_path, - config.sumo, - ), - _write_vtype_xml(workdir, config.sumo, config.random_trips.vehicle_class), + def _new_job(self, job_id: str | None = None) -> JobStatus: + return JobStatus( + job_id=job_id or str(uuid.uuid4()), + status=StageStatus.pending, + stages={name: StageStatus.pending for name in STAGE_NAMES}, + stage_logs={}, + created_at=_now(), ) - await _run_stage( - job, "sumo", - _sumo_cmd(cfg_path, vtype_path), - workdir, - timeout=1200.0, # SUMO can be slow on large networks + async def create_and_launch( + self, osm_path: Path, config: ConversionRequest, job_id: str | None = None + ) -> JobStatus: + """Persist a new job row, then start its pipeline task. When job_id + is provided it must match the pre-saved upload filename.""" + job = self._new_job(job_id) + await self.store.create(job, config.model_dump_json()) + logger.info("job %s created (%s)", job.job_id, osm_path.name) + self._tasks[job.job_id] = asyncio.create_task( + self._guarded_run(job.job_id, osm_path, config) ) - - # ── 5. traceExporter ───────────────────────────────── - _set_stage(job, "trace_export", StageStatus.running) + return job + + async def cancel(self, job_id: str) -> str: + """Cancel a pending/running job: kill its subprocess tree, then its + task. Returns the resulting status string; raises LookupError for an + unknown job and RuntimeError when the job is not in a cancellable + state.""" + job = await self.store.get(job_id) + if not job: + raise LookupError(job_id) + if job.status not in (StageStatus.pending, StageStatus.running): + return job.status.value # not cancellable + + # Durable intent first: the DB records the cancel even if the + # process dies mid-kill. + job.status = StageStatus.cancelled + if job.error is None: + job.error = _CANCEL_SENTINEL + job.completed_at = _now() + for stage, st in job.stages.items(): + if st == StageStatus.running: + job.stages[stage] = StageStatus.cancelled + await self.store.upsert(job) + logger.info("job %s cancelled by user", job_id) + + # Kill the subprocess group BEFORE cancelling the task: + # asyncio.wait_for(proc.communicate()) does not kill the child when + # the awaiting task is cancelled. + for proc in list(self._procs.get(job_id, ())): + self._kill_proc_tree(proc) + + task = self._tasks.get(job_id) + if task is not None: + task.cancel() + return StageStatus.cancelled.value + + async def delete(self, job_id: str) -> None: + await self.store.delete(job_id) + self.cleanup_job_files(job_id) + + def cleanup_job_files(self, job_id: str) -> None: + """Remove workdir, outputs, uploads and the upload file for a job.""" + for d in (self.settings.jobs_dir / job_id, self.settings.out_dir / job_id): + if d.exists(): + shutil.rmtree(d, ignore_errors=True) + for f in self.settings.upload_dir.glob(f"{job_id}*"): + f.unlink(missing_ok=True) + + async def prune_finished(self) -> None: + for job_id in await self.store.prune(): + self.cleanup_job_files(job_id) + + @staticmethod + def _kill_proc_tree(proc: asyncio.subprocess.Process) -> None: + """SIGKILL the whole process group (children included); fall back to + killing just the process where process groups are unavailable.""" + if proc.returncode is not None: + return + try: + os.killpg(proc.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, AttributeError, OSError): + try: + proc.kill() + except ProcessLookupError: + pass + + # ── Task wrapper ────────────────────────────────────────── + + async def _guarded_run( + self, job_id: str, osm_path: Path, config: ConversionRequest + ) -> None: + """Concurrency gate + registry cleanup around the pipeline. This is + the top of our own task tree, so CancelledError is deliberately + swallowed here AFTER cleanup — cancelling this task is the + cancellation mechanism, not an error.""" + try: + async with self._semaphore: + await self._run_pipeline(job_id, osm_path, config) + except asyncio.CancelledError: + logger.info("job %s pipeline task cancelled", job_id) + except Exception: + logger.exception("job %s pipeline task crashed unexpectedly", job_id) + finally: + self._tasks.pop(job_id, None) + self._procs.pop(job_id, None) + await self.prune_finished() + + # ── Async subprocess primitive ──────────────────────────── + + async def _run_async( + self, + job_id: str, + cmd: list[str], + workdir: Path, + timeout: float, + ) -> tuple[int, str, str]: + """Run a command asynchronously. + Returns (returncode, stdout, stderr). + Raises RuntimeError on timeout or missing executable, and + CancelledError when the job is cancelled (after killing the tree).""" + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=str(workdir), + start_new_session=True, + ) + except FileNotFoundError: + raise RuntimeError(f"Executable not found: {cmd[0]!r}") + + self._procs.setdefault(job_id, set()).add(proc) + try: + raw_out, raw_err = await asyncio.wait_for( + proc.communicate(), timeout=timeout + ) + except asyncio.TimeoutError: + self._kill_proc_tree(proc) + await proc.communicate() + raise RuntimeError( + f"Command timed out after {timeout:.0f} s: {' '.join(cmd[:3])!r}…" + ) + except asyncio.CancelledError: + self._kill_proc_tree(proc) + raise + finally: + self._procs.get(job_id, set()).discard(proc) + + rc = proc.returncode + if rc is not None and rc != 0: + logger.warning( + "job %s command %r exited %d", + job_id, + cmd[0], + rc, + ) + return rc, raw_out.decode(errors="replace"), raw_err.decode(errors="replace") + + # ── Job state update + persistence ──────────────────────── + + async def _persist(self, job: JobStatus) -> None: + await self.store.upsert(job) + + async def _set_stage( + self, + job: JobStatus, + stage: str, + status: StageStatus, + log: StageLog | None = None, + ) -> None: + job.stages[stage] = status + job.current_stage = stage + if log: + job.stage_logs[stage] = log.model_dump() + await self._persist(job) + + # ── Per-stage runner with timing + log persistence ──────── + + async def _run_stage( + self, + job: JobStatus, + stage: str, + cmd: list[str], + workdir: Path, + timeout: float, + ) -> tuple[str, str]: + """ + Mark stage running, execute the command, persist stdout/stderr to disk, + update the job's StageLog, and return (stdout, stderr). + Raises RuntimeError on non-zero exit or timeout. + """ + await self._set_stage(job, stage, StageStatus.running) t0 = time.monotonic() - export_specs = _trace_export_cmds(fcd_path, workdir, config.trace_exporter) - - # Run all traceExporter variants concurrently - tasks = [ - _run_async(cmd, workdir, timeout=300.0) - for _, cmd, _ in export_specs - ] - results = await asyncio.gather(*tasks, return_exceptions=True) - - dest_dir = OUT_DIR / job_id - await aiofiles.os.makedirs(dest_dir, exist_ok=True) - - output_files: List[str] = [] - for (label, _, out_path), result in zip(export_specs, results): - if isinstance(result, Exception): - job.warnings.append(f"[trace_export:{label}] {result}") - continue - rc, stdout, stderr = result - if rc != 0: - job.warnings.append( - f"[trace_export:{label}] exit {rc}: {stderr[:300]}" - ) - elif out_path.exists(): - dst = dest_dir / out_path.name - await _copy_file(out_path, dst) - output_files.append(str(dst)) - - # Copy SUMO stats outputs - for extra in (tripinfo_path, summary_path): - if extra.exists(): - dst = dest_dir / extra.name - await _copy_file(extra, dst) - output_files.append(str(dst)) + rc, stdout, stderr = await self._run_async(job.job_id, cmd, workdir, timeout) elapsed = time.monotonic() - t0 - log = StageLog(status=StageStatus.done, duration=round(elapsed, 3)) - _set_stage(job, "trace_export", StageStatus.done, log) - job.outputs = output_files - job.status = StageStatus.done - job.completed_at = _now() - - except RuntimeError as exc: - job.status = StageStatus.failed - job.error = str(exc) - job.completed_at = _now() - - except Exception as exc: - job.status = StageStatus.failed - job.error = f"Unexpected error in pipeline: {exc!r}" - job.completed_at = _now() + # Persist log to disk (non-blocking) + log_path = workdir / f"{stage}.log" + await _write_text( + log_path, f"=== stdout ===\n{stdout}\n=== stderr ===\n{stderr}\n" + ) + # Capture SUMO warnings into the job + for line in (stdout + stderr).splitlines(): + if "Warning" in line or "warning" in line: + job.warnings.append(f"[{stage}] {line.strip()}") + log = StageLog( + status=StageStatus.done if rc == 0 else StageStatus.failed, + stdout=stdout[:4000] if stdout else None, # truncate for the store + stderr=stderr[:4000] if stderr else None, + duration=round(elapsed, 3), + ) + if rc != 0: + await self._set_stage(job, stage, StageStatus.failed, log) + raise RuntimeError( + f"Stage '{stage}' exited with code {rc}.\nstderr: {stderr[:800]}" + ) + + await self._set_stage(job, stage, StageStatus.done, log) + logger.info("job %s stage %s done in %.1fs", job.job_id, stage, elapsed) + return stdout, stderr + + # ── Master pipeline ─────────────────────────────────────── + + async def _run_pipeline( + self, + job_id: str, + osm_path: Path, + config: ConversionRequest, + ) -> None: + """ + Execute all five pipeline stages sequentially. + + Stages: + 1. netconvert — OSM → .net.xml + 2. random_trips — .net.xml → trips.xml + 3. duarouter — trips.xml → routes.rou.xml + 4. sumo — routes + net → fcd-output.xml + 5. trace_export — fcd-output.xml → ns_movements / mobility.tcl + """ + workdir = self.settings.jobs_dir / job_id + await aiofiles.os.makedirs(workdir, exist_ok=True) + + job = await self.store.get(job_id) + if job is None: + logger.error("job %s vanished before pipeline start", job_id) + return + if job.status == StageStatus.cancelled: + return # cancelled while queued on the semaphore + + job.status = StageStatus.running + await self._persist(job) + sumo_home = self.settings.resolve_sumo_home() + + try: + # ── 1. netconvert ──────────────────────────────── + net_path = workdir / "network.net.xml" + await self._run_stage( + job, + "netconvert", + _netconvert_cmd(osm_path, net_path, config.netconvert), + workdir, + timeout=self.settings.timeout_default_s, + ) + + # ── 2. randomTrips ─────────────────────────────── + trips_path = workdir / "trips.xml" + await self._run_stage( + job, + "random_trips", + _random_trips_cmd(net_path, trips_path, config.random_trips, sumo_home), + workdir, + timeout=self.settings.timeout_default_s, + ) + + # ── 3. duarouter ───────────────────────────────── + routes_path = workdir / "routes.rou.xml" + await self._run_stage( + job, + "duarouter", + _duarouter_cmd(net_path, trips_path, routes_path, config.random_trips), + workdir, + timeout=self.settings.timeout_default_s, + ) + + # ── 4. sumo ────────────────────────────────────── + fcd_path = workdir / "fcd-output.xml" + tripinfo_path = workdir / "tripinfo.xml" + summary_path = workdir / "summary.xml" + + # Write config files asynchronously before launching SUMO + cfg_path, vtype_path = await asyncio.gather( + _write_sumo_cfg( + workdir, + net_path, + routes_path, + fcd_path, + tripinfo_path, + summary_path, + config.sumo, + ), + _write_vtype_xml( + workdir, config.sumo, config.random_trips.vehicle_class + ), + ) + + await self._run_stage( + job, + "sumo", + _sumo_cmd(cfg_path, vtype_path), + workdir, + timeout=self.settings.timeout_sumo_s, + ) + + # ── 5. traceExporter ───────────────────────────── + await self._set_stage(job, "trace_export", StageStatus.running) + t0 = time.monotonic() + + export_specs = _trace_export_cmds( + fcd_path, workdir, config.trace_exporter, sumo_home + ) + + # Run all traceExporter variants concurrently + tasks = [ + self._run_async( + job_id, cmd, workdir, self.settings.timeout_trace_export_s + ) + for _, cmd, _ in export_specs + ] + results = await asyncio.gather(*tasks, return_exceptions=True) + + dest_dir = self.settings.out_dir / job_id + await aiofiles.os.makedirs(dest_dir, exist_ok=True) + + output_files: list[str] = [] + for (label, _, out_path), result in zip(export_specs, results): + if isinstance(result, Exception): + job.warnings.append(f"[trace_export:{label}] {result}") + logger.warning( + "job %s trace_export:%s failed: %s", job_id, label, result + ) + continue + rc, stdout, stderr = result + if rc != 0: + job.warnings.append( + f"[trace_export:{label}] exit {rc}: {stderr[:300]}" + ) + elif out_path.exists(): + dst = dest_dir / out_path.name + await _copy_file(out_path, dst) + output_files.append(str(dst)) + + # Copy SUMO stats outputs + for extra in (tripinfo_path, summary_path): + if extra.exists(): + dst = dest_dir / extra.name + await _copy_file(extra, dst) + output_files.append(str(dst)) + + elapsed = time.monotonic() - t0 + log = StageLog(status=StageStatus.done, duration=round(elapsed, 3)) + await self._set_stage(job, "trace_export", StageStatus.done, log) + + # A cancel may have completed all subprocess work before the + # CancelledError was delivered — never clobber it with "done". + latest = await self.store.get(job_id) + if latest is not None and latest.status == StageStatus.cancelled: + raise asyncio.CancelledError() + + job.outputs = output_files + job.status = StageStatus.done + job.completed_at = _now() + await self._persist(job) + logger.info("job %s done (%d output file(s))", job_id, len(output_files)) + + except asyncio.CancelledError: + # DB state was already persisted by cancel(); ensure completion + # markers for races where cancel() ran before the job started. + if job.status != StageStatus.cancelled: + job.status = StageStatus.cancelled + job.error = job.error or _CANCEL_SENTINEL + job.completed_at = job.completed_at or _now() + for stage, st in job.stages.items(): + if st == StageStatus.running: + job.stages[stage] = StageStatus.cancelled + await self._persist(job) + raise # handed to _guarded_run + + except RuntimeError as exc: + if await self._was_cancelled(job_id, job): + raise asyncio.CancelledError() + job.status = StageStatus.failed + job.error = str(exc) + job.completed_at = _now() + await self._persist(job) + logger.warning("job %s failed: %s", job_id, exc) + + except Exception as exc: + if await self._was_cancelled(job_id, job): + raise asyncio.CancelledError() + job.status = StageStatus.failed + job.error = f"Unexpected error in pipeline: {exc!r}" + job.completed_at = _now() + await self._persist(job) + logger.exception("job %s failed unexpectedly", job_id) + + async def _was_cancelled(self, job_id: str, job: JobStatus) -> bool: + """True when a cancel landed while a stage was failing — cancellation + wins over the failure report.""" + latest = await self.store.get(job_id) + return latest is not None and latest.status == StageStatus.cancelled diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5058963 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +services: + osm2ns3: + build: . + ports: + - "8000:8000" + volumes: + - osm2ns3-data:/data + environment: + # Override any OSM2NS3_* setting here (auth is off unless a key is set). + # OSM2NS3_API_KEY: "change-me" + # OSM2NS3_CORS_ORIGINS: '["http://localhost:8000"]' + OSM2NS3_DATA_DIR: /data + OSM2NS3_STATIC_DIR: /app/static + SUMO_HOME: /usr/share/sumo + restart: unless-stopped + +volumes: + osm2ns3-data: diff --git a/frontend/.env.development b/frontend/.env.development new file mode 100644 index 0000000..27786df --- /dev/null +++ b/frontend/.env.development @@ -0,0 +1 @@ +VITE_API_BASE=http://localhost:8000 diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 989f95f..b2dffd4 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,16 +1,40 @@ +import type { JsonSchema } from './schema'; import type { ConversionRequest, - HealthResponse, JobFilesResponse, JobLogsResponse, JobStatus, + ReadyResponse, } from './types'; +/** Same-origin by default (Docker serving); dev sets VITE_API_BASE. */ const API_BASE: string = - (import.meta.env.VITE_API_BASE as string | undefined) || 'http://localhost:8000'; + (import.meta.env.VITE_API_BASE as string | undefined) ?? ''; + +// ── API key (opt-in; server enforces only when OSM2NS3_API_KEY is set) ── + +const KEY_STORAGE = 'osm2ns3.apiKey'; + +let apiKey: string = localStorage.getItem(KEY_STORAGE) ?? ''; + +export function getApiKey(): string { + return apiKey; +} + +export function setApiKey(key: string): void { + apiKey = key; + if (key) localStorage.setItem(KEY_STORAGE, key); + else localStorage.removeItem(KEY_STORAGE); +} async function request(path: string, init?: RequestInit): Promise { - const res = await fetch(`${API_BASE}${path}`, init); + const res = await fetch(`${API_BASE}${path}`, { + ...init, + headers: { + ...(apiKey ? { 'X-API-Key': apiKey } : {}), + ...(init?.headers ?? {}), + }, + }); if (!res.ok) { let detail = ''; try { @@ -19,6 +43,9 @@ async function request(path: string, init?: RequestInit): Promise { } catch { detail = await res.text(); } + if (res.status === 401) { + detail = `Unauthorized — set a valid API key in the nav bar. ${detail}`.trim(); + } throw new Error(detail || `Request failed: ${res.status}`); } if (res.status === 204) return undefined as T; @@ -41,20 +68,37 @@ export const api = { getJobLogs: (id: string) => request(`/jobs/${id}/logs`), getJobConfig: (id: string) => request(`/jobs/${id}/config`), + cancelJob: (id: string) => + request(`/jobs/${id}/cancel`, { method: 'POST' }), deleteJob: (id: string) => request(`/jobs/${id}`, { method: 'DELETE' }), - getHealth: () => request('/health'), + /** + * Readiness is *conveyed* by HTTP status (200 = ready, 503 = degraded), so a + * 503 is a valid answer, not a request failure — fetch directly, never throw. + */ + getReady: async (): Promise => { + try { + const res = await fetch(`${API_BASE}/ready`); + return (await res.json()) as ReadyResponse; + } catch { + return null; // server unreachable entirely + } + }, getDefaults: () => request('/defaults'), - getSchema: () => request('/schema'), + getSchema: () => request('/schema'), - createJob: async ( - osmFile: File, - config: ConversionRequest, - ): Promise => { + createJob: (osmFile: File, config: ConversionRequest) => { const fd = new FormData(); fd.append('osm_file', osmFile); fd.append('config', JSON.stringify(config)); - const res = await fetch(`${API_BASE}/jobs`, { method: 'POST', body: fd }); + return request('/jobs', { method: 'POST', body: fd }); + }, + + /** Authenticated blob download (X-API-Key cannot ride on a plain ). */ + downloadFile: async (id: string, filename: string): Promise => { + const res = await fetch(`${API_BASE}/jobs/${id}/download/${filename}`, { + headers: apiKey ? { 'X-API-Key': apiKey } : {}, + }); if (!res.ok) { let detail = ''; try { @@ -63,8 +107,16 @@ export const api = { } catch { detail = await res.text(); } - throw new Error(detail || `Upload failed: ${res.status}`); + throw new Error(detail || `Download failed: ${res.status}`); } - return (await res.json()) as CreateJobResult; + const blob = await res.blob(); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + a.remove(); + URL.revokeObjectURL(url); }, }; diff --git a/frontend/src/api/configSchema.ts b/frontend/src/api/configSchema.ts deleted file mode 100644 index d3c80c9..0000000 --- a/frontend/src/api/configSchema.ts +++ /dev/null @@ -1,149 +0,0 @@ -export type FieldType = - | 'number' - | 'text' - | 'boolean' - | 'select' - | 'multiselect'; - -export interface FieldDef { - key: string; - label: string; - type: FieldType; - options?: string[]; - min?: number; - max?: number; - step?: number; - /** When true, an empty "(none)" option is allowed (server default null). */ - nullable?: boolean; - group: 'key' | 'advanced'; - placeholder?: string; -} - -export interface SectionDef { - key: string; - title: string; - fields: FieldDef[]; -} - -const HIGHWAY = [ - 'motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', - 'secondary', 'secondary_link', 'tertiary', 'tertiary_link', 'residential', - 'living_street', 'unclassified', 'service', 'road', 'track', 'footway', - 'cycleway', 'path', 'steps', 'pedestrian', 'bus_guideway', 'raceway', - 'construction', -]; -const VCLASS = [ - 'passenger', 'bus', 'truck', 'motorcycle', 'bicycle', 'pedestrian', - 'emergency', 'authority', 'army', 'vip', 'hov', 'custom1', 'custom2', -]; -const JTYPE = [ - 'priority', 'traffic_light', 'right_before_left', 'unregulated', - 'allway_stop', 'zipper', -]; -const TLTYPE = ['static', 'actuated', 'delay_based', 'sotl_phase', 'sotl_platoon']; -const EREM = ['all', 'noFringe']; -const OFMT = ['ns_movements', 'tcl', 'both']; -const SMODE = ['right_of_way', 'no_checks', 'all_checks']; -const LCMODE = ['default', 'no_lc', 'strategic']; -const COLL = ['warn', 'teleport', 'remove', 'none']; - -export const SECTIONS: SectionDef[] = [ - { - key: 'netconvert', - title: 'Netconvert', - fields: [ - { key: 'osm_highway_types', label: 'osm_highway_types', type: 'multiselect', options: HIGHWAY, group: 'key' }, - { key: 'default_speed_limit', label: 'default_speed_limit (m/s)', type: 'number', min: 1, max: 83.3, step: 0.1, group: 'key' }, - { key: 'default_junction_type', label: 'default_junction_type', type: 'select', options: JTYPE, group: 'key' }, - { key: 'tl_type', label: 'tl_type', type: 'select', options: TLTYPE, group: 'key' }, - { key: 'proj_utm', label: 'proj_utm', type: 'boolean', group: 'key' }, - { key: 'osm_remove_isolated_edges', label: 'osm_remove_isolated_edges', type: 'boolean', group: 'advanced' }, - { key: 'osm_no_large_roundabouts', label: 'osm_no_large_roundabouts', type: 'boolean', group: 'advanced' }, - { key: 'osm_oneway_spread', label: 'osm_oneway_spread', type: 'boolean', group: 'advanced' }, - { key: 'geometry_remove_isolated_nodes', label: 'geometry_remove_isolated_nodes', type: 'boolean', group: 'advanced' }, - { key: 'geometry_no_internal_links', label: 'geometry_no_internal_links', type: 'boolean', group: 'advanced' }, - { key: 'junctions_internal_link_detail', label: 'junctions_internal_link_detail', type: 'number', min: 1, max: 20, group: 'advanced' }, - { key: 'junctions_corner_detail', label: 'junctions_corner_detail', type: 'number', min: 1, max: 20, group: 'advanced' }, - { key: 'junctions_min_size', label: 'junctions_min_size (m)', type: 'number', min: 0, max: 20, step: 0.1, group: 'advanced' }, - { key: 'junctions_limit_turn_speed', label: 'junctions_limit_turn_speed (m/s)', type: 'number', min: 0, max: 30, step: 0.1, group: 'advanced' }, - { key: 'default_lane_width', label: 'default_lane_width (m)', type: 'number', min: 1, max: 10, step: 0.1, group: 'advanced' }, - { key: 'default_num_lanes', label: 'default_num_lanes', type: 'number', min: 1, max: 8, group: 'advanced' }, - { key: 'no_turnarounds', label: 'no_turnarounds', type: 'boolean', group: 'advanced' }, - { key: 'no_left_connections', label: 'no_left_connections', type: 'boolean', group: 'advanced' }, - { key: 'tl_guess', label: 'tl_guess', type: 'boolean', group: 'advanced' }, - { key: 'tl_join', label: 'tl_join', type: 'boolean', group: 'advanced' }, - { key: 'tl_min_dur', label: 'tl_min_dur (s)', type: 'number', min: 1, max: 120, group: 'advanced' }, - { key: 'tl_max_dur', label: 'tl_max_dur (s)', type: 'number', min: 5, max: 300, group: 'advanced' }, - { key: 'keep_edges_by_vclass', label: 'keep_edges_by_vclass', type: 'select', options: VCLASS, nullable: true, group: 'advanced' }, - { key: 'remove_edges_by_type', label: 'remove_edges_by_type', type: 'text', placeholder: 'footway,cycleway,path', group: 'advanced' }, - { key: 'keep_fringe', label: 'keep_fringe', type: 'select', options: EREM, group: 'advanced' }, - { key: 'proj_plain_geo', label: 'proj_plain_geo', type: 'boolean', group: 'advanced' }, - ], - }, - { - key: 'random_trips', - title: 'Random Trips', - fields: [ - { key: 'end', label: 'end (s)', type: 'number', min: 1, group: 'key' }, - { key: 'period', label: 'period (s)', type: 'number', min: 0.1, max: 3600, step: 0.1, group: 'key' }, - { key: 'vehicle_class', label: 'vehicle_class', type: 'select', options: VCLASS, group: 'key' }, - { key: 'seed', label: 'seed', type: 'number', min: 0, group: 'key' }, - { key: 'begin', label: 'begin (s)', type: 'number', min: 0, group: 'advanced' }, - { key: 'insertion_density', label: 'insertion_density (/h/km)', type: 'number', min: 0, group: 'advanced' }, - { key: 'min_distance', label: 'min_distance (m)', type: 'number', min: 0, group: 'advanced' }, - { key: 'max_distance', label: 'max_distance (m)', type: 'number', min: 0, group: 'advanced' }, - { key: 'fringe_factor', label: 'fringe_factor', type: 'number', min: 0, max: 100, step: 0.1, group: 'advanced' }, - { key: 'fringe_threshold', label: 'fringe_threshold', type: 'number', min: 0, max: 1, step: 0.01, group: 'advanced' }, - { key: 'validate_routes', label: 'validate_routes', type: 'boolean', group: 'advanced' }, - { key: 'allow_loops', label: 'allow_loops', type: 'boolean', group: 'advanced' }, - { key: 'random_depart_pos', label: 'random_depart_pos', type: 'boolean', group: 'advanced' }, - { key: 'random_arrival_pos', label: 'random_arrival_pos', type: 'boolean', group: 'advanced' }, - ], - }, - { - key: 'sumo', - title: 'SUMO Simulation', - fields: [ - { key: 'end', label: 'end (s)', type: 'number', min: 1, group: 'key' }, - { key: 'step_length', label: 'step_length (s)', type: 'number', min: 0.01, max: 10, step: 0.01, group: 'key' }, - { key: 'fcd_output_period', label: 'fcd_output_period (s)', type: 'number', min: 0.01, max: 60, step: 0.01, group: 'key' }, - { key: 'default_max_speed', label: 'default_max_speed (m/s)', type: 'number', min: 1, max: 200, step: 0.1, group: 'key' }, - { key: 'begin', label: 'begin (s)', type: 'number', min: 0, group: 'advanced' }, - { key: 'default_accel', label: 'default_accel (m/s²)', type: 'number', min: 0.1, max: 20, step: 0.1, group: 'advanced' }, - { key: 'default_decel', label: 'default_decel (m/s²)', type: 'number', min: 0.1, max: 20, step: 0.1, group: 'advanced' }, - { key: 'default_emergency_decel', label: 'default_emergency_decel (m/s²)', type: 'number', min: 0.1, max: 30, step: 0.1, group: 'advanced' }, - { key: 'default_sigma', label: 'default_sigma', type: 'number', min: 0, max: 1, step: 0.01, group: 'advanced' }, - { key: 'default_tau', label: 'default_tau (s)', type: 'number', min: 0.1, max: 10, step: 0.1, group: 'advanced' }, - { key: 'default_vehicle_length', label: 'default_vehicle_length (m)', type: 'number', min: 1, max: 30, step: 0.1, group: 'advanced' }, - { key: 'default_min_gap', label: 'default_min_gap (m)', type: 'number', min: 0, max: 20, step: 0.1, group: 'advanced' }, - { key: 'speed_mode', label: 'speed_mode', type: 'select', options: SMODE, group: 'advanced' }, - { key: 'lanechange_mode', label: 'lanechange_mode', type: 'select', options: LCMODE, group: 'advanced' }, - { key: 'no_internal_links', label: 'no_internal_links', type: 'boolean', group: 'advanced' }, - { key: 'ignore_route_errors', label: 'ignore_route_errors', type: 'boolean', group: 'advanced' }, - { key: 'collision_action', label: 'collision_action', type: 'select', options: COLL, group: 'advanced' }, - { key: 'lateral_resolution', label: 'lateral_resolution (m)', type: 'number', min: 0, max: 5, step: 0.1, group: 'advanced' }, - { key: 'fcd_output_geo', label: 'fcd_output_geo', type: 'boolean', group: 'advanced' }, - { key: 'fcd_filter_shapes', label: 'fcd_filter_shapes', type: 'boolean', group: 'advanced' }, - { key: 'tripinfo_output', label: 'tripinfo_output', type: 'boolean', group: 'advanced' }, - { key: 'summary_output', label: 'summary_output', type: 'boolean', group: 'advanced' }, - { key: 'seed', label: 'seed', type: 'number', min: 0, group: 'advanced' }, - ], - }, - { - key: 'trace_exporter', - title: 'Trace Exporter', - fields: [ - { key: 'output_format', label: 'output_format', type: 'select', options: OFMT, group: 'key' }, - { key: 'sampling_period', label: 'sampling_period (s)', type: 'number', min: 0.1, max: 60, step: 0.1, group: 'key' }, - { key: 'penetration_rate', label: 'penetration_rate', type: 'number', min: 0.01, max: 1, step: 0.01, group: 'key' }, - { key: 'begin', label: 'begin (s)', type: 'number', min: 0, group: 'advanced' }, - { key: 'end', label: 'end (s)', type: 'number', min: 0, group: 'advanced' }, - { key: 'x_offset', label: 'x_offset (m)', type: 'number', group: 'advanced' }, - { key: 'y_offset', label: 'y_offset (m)', type: 'number', group: 'advanced' }, - { key: 'boundary', label: 'boundary (xMin,yMin,xMax,yMax)', type: 'text', placeholder: '0,0,5000,5000', group: 'advanced' }, - { key: 'ns2_include_speed', label: 'ns2_include_speed', type: 'boolean', group: 'advanced' }, - { key: 'write_fcd_filtered', label: 'write_fcd_filtered', type: 'boolean', group: 'advanced' }, - { key: 'seed', label: 'seed', type: 'number', min: 0, group: 'advanced' }, - ], - }, -]; diff --git a/frontend/src/api/schema.ts b/frontend/src/api/schema.ts new file mode 100644 index 0000000..1a0fe76 --- /dev/null +++ b/frontend/src/api/schema.ts @@ -0,0 +1,148 @@ +/** + * Builds form sections from the backend's live JSON Schema (GET /schema), + * so the form can never drift from the Pydantic models. The old hand-written + * configSchema.ts was deleted in favour of this generator. + */ + +export type FieldType = + | 'number' + | 'text' + | 'boolean' + | 'select' + | 'multiselect'; + +export interface FieldDef { + key: string; + label: string; + type: FieldType; + options?: string[]; + min?: number; + max?: number; + step?: number; + /** When true, an empty "(none)" option is allowed (server default null). */ + nullable?: boolean; + group: 'key' | 'advanced'; + placeholder?: string; + /** Schema description, rendered as helper text under the control. */ + description?: string; +} + +export interface SectionDef { + key: string; + title: string; + fields: FieldDef[]; +} + +export type JsonSchema = Record; + +/** Fields the schema alone can't express well — hand overrides by "section.key". */ +const FIELD_OVERRIDES: Record> = { + 'netconvert.osm_highway_types': { group: 'key' }, + 'netconvert.default_speed_limit': { group: 'key' }, + 'netconvert.default_junction_type': { group: 'key' }, + 'netconvert.tl_type': { group: 'key' }, + 'netconvert.proj_utm': { group: 'key' }, + 'random_trips.end': { group: 'key' }, + 'random_trips.period': { group: 'key' }, + 'random_trips.vehicle_class': { group: 'key' }, + 'random_trips.seed': { group: 'key' }, + 'sumo.end': { group: 'key' }, + 'sumo.step_length': { group: 'key' }, + 'sumo.fcd_output_period': { group: 'key' }, + 'sumo.default_max_speed': { group: 'key' }, + 'trace_exporter.output_format': { group: 'key' }, + 'trace_exporter.sampling_period': { group: 'key' }, + 'trace_exporter.penetration_rate': { group: 'key' }, + 'trace_exporter.boundary': { placeholder: '0,0,5000,5000' }, + 'netconvert.remove_edges_by_type': { placeholder: 'footway,cycleway,path' }, +}; + +const SECTION_TITLES: Record = { + netconvert: 'Netconvert', + random_trips: 'Random Trips', + sumo: 'SUMO Simulation', + trace_exporter: 'Trace Exporter', +}; + +function resolveRef(schema: JsonSchema, defs: JsonSchema): JsonSchema { + const ref = schema.$ref; + if (typeof ref === 'string') { + const name = ref.split('/').pop() ?? ''; + const target = defs[name]; + if (target && typeof target === 'object') { + return { ...(target as JsonSchema), description: schema.description ?? (target as JsonSchema).description } as JsonSchema; + } + } + return schema; +} + +/** Split Optional[T] anyOf into [non-null branches, nullable]. */ +function flattenAnyOf(schema: JsonSchema): { type: JsonSchema; nullable: boolean } { + const anyOf = schema.anyOf; + if (Array.isArray(anyOf)) { + const nonNull = anyOf.filter( + (s) => (s as JsonSchema).type !== 'null', + ) as JsonSchema[]; + const nullable = nonNull.length !== anyOf.length; + if (nonNull.length === 1) return { type: nonNull[0], nullable }; + return { type: nonNull[0] ?? {}, nullable }; + } + return { type: schema, nullable: false }; +} + +function fieldFromSchema( + section: string, + key: string, + prop: JsonSchema, + defs: JsonSchema, +): FieldDef { + const { type: raw, nullable } = flattenAnyOf(prop); + const resolved = resolveRef(raw, defs); + const base: FieldDef = { + key, + label: key, + type: 'text', + nullable, + group: 'advanced', + description: (prop.description as string | undefined) ?? undefined, + }; + + // Enum select (either inline or via $ref). + if (Array.isArray(resolved.enum)) { + base.type = 'select'; + base.options = resolved.enum as string[]; + } else if (resolved.type === 'boolean') { + base.type = 'boolean'; + } else if (resolved.type === 'integer' || resolved.type === 'number') { + base.type = 'number'; + if (typeof resolved.minimum === 'number') base.min = resolved.minimum; + if (typeof resolved.maximum === 'number') base.max = resolved.maximum; + base.step = resolved.type === 'integer' ? 1 : undefined; + } else if (resolved.type === 'array') { + // Only the highway-types list today — chip multiselect. + base.type = 'multiselect'; + const items = resolveRef((resolved.items as JsonSchema) ?? {}, defs); + if (Array.isArray(items.enum)) base.options = items.enum as string[]; + } + + return { ...base, ...FIELD_OVERRIDES[`${section}.${key}`] }; +} + +/** sections from ConversionRequest.model_json_schema(). */ +export function buildSections(schema: JsonSchema): SectionDef[] { + const defs = (schema.$defs as JsonSchema) ?? {}; + const props = (schema.properties as JsonSchema) ?? {}; + return Object.entries(props).map(([key, prop]) => { + const section = resolveRef(prop as JsonSchema, defs); + const fields = Object.entries( + (section.properties as JsonSchema) ?? {}, + ).map(([fieldKey, fieldProp]) => + fieldFromSchema(key, fieldKey, fieldProp as JsonSchema, defs), + ); + return { + key, + title: SECTION_TITLES[key] ?? key, + fields, + }; + }); +} diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 5fe72cc..6c95f30 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -3,6 +3,7 @@ export type StageStatus = | 'running' | 'done' | 'failed' + | 'cancelled' | 'skipped'; export type StageKey = @@ -51,7 +52,7 @@ export interface JobLogsResponse { stage_logs: Record; } -export interface HealthResponse { +export interface ReadyResponse { status: 'ready' | 'degraded'; tools: Record; pipeline_ready: boolean; diff --git a/frontend/src/components/ConfigForm.tsx b/frontend/src/components/ConfigForm.tsx index 119d0e0..38e2101 100644 --- a/frontend/src/components/ConfigForm.tsx +++ b/frontend/src/components/ConfigForm.tsx @@ -1,19 +1,20 @@ import { useState } from 'react'; -import { SECTIONS } from '../api/configSchema'; +import type { SectionDef } from '../api/schema'; import type { ConversionRequest } from '../api/types'; import { Field } from './ui/Field'; interface Props { + sections: SectionDef[]; value: ConversionRequest; onChange: (section: keyof ConversionRequest, key: string, val: unknown) => void; } -export function ConfigForm({ value, onChange }: Props) { +export function ConfigForm({ sections, value, onChange }: Props) { const [open, setOpen] = useState>({}); return (
- {SECTIONS.map((section, idx) => { + {sections.map((section, idx) => { const secVal = value[section.key as keyof ConversionRequest] ?? {}; const keyFields = section.fields.filter((f) => f.group === 'key'); const advFields = section.fields.filter((f) => f.group === 'advanced'); diff --git a/frontend/src/components/FileRow.tsx b/frontend/src/components/FileRow.tsx index 656a629..0d7fdae 100644 --- a/frontend/src/components/FileRow.tsx +++ b/frontend/src/components/FileRow.tsx @@ -1,24 +1,45 @@ +import { useState } from 'react'; import type { JobFile } from '../api/types'; import { api } from '../api/client'; import { formatBytes } from '../lib/format'; -export function FileRow({ file }: { file: JobFile }) { +export function FileRow({ file, jobId }: { file: JobFile; jobId: string }) { + const [downloading, setDownloading] = useState(false); + const [error, setError] = useState(null); + + const onDownload = async () => { + setDownloading(true); + setError(null); + try { + await api.downloadFile(jobId, file.filename); + } catch (e) { + setError(e instanceof Error ? e.message : 'Download failed'); + } finally { + setDownloading(false); + } + }; + return ( ); diff --git a/frontend/src/components/HealthStrip.tsx b/frontend/src/components/HealthStrip.tsx index f8b5a29..9360c52 100644 --- a/frontend/src/components/HealthStrip.tsx +++ b/frontend/src/components/HealthStrip.tsx @@ -1,4 +1,4 @@ -import type { HealthResponse } from '../api/types'; +import type { ReadyResponse } from '../api/types'; import { StatusTag } from './ui/StatusTag'; const TOOL_ORDER = [ @@ -9,7 +9,7 @@ const TOOL_ORDER = [ 'randomTrips.py', ]; -export function HealthStrip({ health }: { health: HealthResponse | null }) { +export function HealthStrip({ health }: { health: ReadyResponse | null }) { if (!health) return null; return (
diff --git a/frontend/src/components/Nav.tsx b/frontend/src/components/Nav.tsx index 0037a8f..03d2a72 100644 --- a/frontend/src/components/Nav.tsx +++ b/frontend/src/components/Nav.tsx @@ -1,4 +1,6 @@ +import { useState } from 'react'; import { NavLink } from 'react-router-dom'; +import { getApiKey, setApiKey } from '../api/client'; const LINKS = [ { to: '/', label: 'DASHBOARD', end: true }, @@ -6,6 +8,36 @@ const LINKS = [ { to: '/system', label: 'SYSTEM', end: false }, ]; +function ApiKeyInput() { + const [key, setKey] = useState(getApiKey()); + return ( +
+ { + setKey(e.target.value); + setApiKey(e.target.value); + }} + placeholder="API KEY (IF ENABLED)" + className="font-mono text-[11px] uppercase tracking-[0.1em] bg-white border border-line px-2 py-1.5 w-44 focus:border-ink outline-none rounded-none placeholder:text-muted" + /> + {key && ( + + )} +
+ ); +} + export function Nav() { return (
@@ -16,6 +48,7 @@ export function Nav() {
); } @@ -53,6 +63,7 @@ export function Field({ field, value, onChange }: Props) { ))} +
); } @@ -87,6 +98,7 @@ export function Field({ field, value, onChange }: Props) { ); })} + ); } @@ -110,6 +122,7 @@ export function Field({ field, value, onChange }: Props) { } className={inputCls} /> + ); } @@ -128,6 +141,7 @@ export function Field({ field, value, onChange }: Props) { onChange={(e) => onChange(e.target.value === '' ? null : e.target.value)} className={inputCls} /> + ); } diff --git a/frontend/src/components/ui/StatusTag.tsx b/frontend/src/components/ui/StatusTag.tsx index cf2b9ee..45a4439 100644 --- a/frontend/src/components/ui/StatusTag.tsx +++ b/frontend/src/components/ui/StatusTag.tsx @@ -6,6 +6,7 @@ const STATUS_MAP: Record< running: { label: 'RUNNING', color: 'text-status-running', pulse: true }, done: { label: 'DONE', color: 'text-status-done' }, failed: { label: 'FAILED', color: 'text-status-failed' }, + cancelled: { label: 'CANCELLED', color: 'text-status-skipped' }, skipped: { label: 'SKIPPED', color: 'text-status-skipped' }, }; diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index c032af3..63af40c 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -8,7 +8,7 @@ import { Button } from '../components/ui/Button'; export default function Dashboard() { const navigate = useNavigate(); const jobs = usePolling(() => api.listJobs(), 4000); - const health = usePolling(() => api.getHealth(), 10000); + const health = usePolling(() => api.getReady(), 10000); return (
diff --git a/frontend/src/pages/JobDetail.tsx b/frontend/src/pages/JobDetail.tsx index 87302d5..b4108dc 100644 --- a/frontend/src/pages/JobDetail.tsx +++ b/frontend/src/pages/JobDetail.tsx @@ -23,6 +23,7 @@ export default function JobDetail() { const navigate = useNavigate(); const [tab, setTab] = useState('progress'); const [deleting, setDeleting] = useState(false); + const [cancelling, setCancelling] = useState(false); const [delError, setDelError] = useState(null); const jobQ = usePolling(() => api.getJob(jobId), 2000, !!jobId); @@ -44,6 +45,22 @@ export default function JobDetail() { const job = jobQ.data; const running = job?.status === 'running'; + const cancellable = job?.status === 'running' || job?.status === 'pending'; + + const onCancel = async () => { + if (!window.confirm('Cancel this job? The pipeline subprocess will be killed.')) + return; + setCancelling(true); + setDelError(null); + try { + await api.cancelJob(jobId); + // The poller picks up the cancelled state on its next tick. + } catch (e) { + setDelError(e instanceof Error ? e.message : 'Cancel failed'); + } finally { + setCancelling(false); + } + }; const onDelete = async () => { if (!window.confirm('Delete this job and all its files?')) return; @@ -88,13 +105,24 @@ export default function JobDetail() { {formatDate(job.created_at)} → {formatDate(job.completed_at)}
- +
+ {cancellable && ( + + )} + +
@@ -159,7 +187,7 @@ export default function JobDetail() {
) : ( filesQ.data!.files.map((f) => ( - + )) )} diff --git a/frontend/src/pages/NewJob.tsx b/frontend/src/pages/NewJob.tsx index eee902b..d5f7593 100644 --- a/frontend/src/pages/NewJob.tsx +++ b/frontend/src/pages/NewJob.tsx @@ -1,6 +1,7 @@ import { useEffect, useState } from 'react'; import { useNavigate } from 'react-router-dom'; import { api } from '../api/client'; +import { buildSections, type SectionDef } from '../api/schema'; import type { ConversionRequest } from '../api/types'; import { ConfigForm } from '../components/ConfigForm'; import { Button } from '../components/ui/Button'; @@ -16,13 +17,18 @@ export default function NewJob() { const navigate = useNavigate(); const [file, setFile] = useState(null); const [config, setConfig] = useState(emptyConfig()); + const [sections, setSections] = useState(null); const [submitting, setSubmitting] = useState(false); const [error, setError] = useState(null); useEffect(() => { - api - .getDefaults() - .then(setConfig) + // Schema (form shape) + defaults (values) travel together; the schema is + // the single source of truth so the form can't drift from the backend. + Promise.all([api.getSchema(), api.getDefaults()]) + .then(([schema, defaults]) => { + setSections(buildSections(schema)); + setConfig(defaults); + }) .catch(() => setConfig(emptyConfig())); }, []); @@ -107,10 +113,16 @@ export default function NewJob() { Pipeline Configuration - PRE-FILLED FROM GET /defaults + GENERATED FROM GET /schema + /defaults - + {sections ? ( + + ) : ( +
+ LOADING SCHEMA… +
+ )} {error && ( diff --git a/frontend/src/pages/System.tsx b/frontend/src/pages/System.tsx index 5f4bdb1..1f6d292 100644 --- a/frontend/src/pages/System.tsx +++ b/frontend/src/pages/System.tsx @@ -24,7 +24,7 @@ const ENDPOINTS: { method: string; path: string }[] = [ ]; export default function System() { - const health = usePolling(() => api.getHealth(), 10000); + const health = usePolling(() => api.getReady(), 10000); const h = health.data; return ( diff --git a/frontend/tsconfig.node.tsbuildinfo b/frontend/tsconfig.node.tsbuildinfo deleted file mode 100644 index c5249fc..0000000 --- a/frontend/tsconfig.node.tsbuildinfo +++ /dev/null @@ -1 +0,0 @@ -{"fileNames":["./node_modules/typescript/lib/lib.d.ts","./node_modules/typescript/lib/lib.es5.d.ts","./node_modules/typescript/lib/lib.dom.d.ts","./node_modules/typescript/lib/lib.webworker.importscripts.d.ts","./node_modules/typescript/lib/lib.scripthost.d.ts","./node_modules/typescript/lib/lib.decorators.d.ts","./node_modules/typescript/lib/lib.decorators.legacy.d.ts","./node_modules/@types/estree/index.d.ts","./node_modules/rollup/dist/rollup.d.ts","./node_modules/rollup/dist/parseast.d.ts","./node_modules/vite/types/hmrpayload.d.ts","./node_modules/vite/types/customevent.d.ts","./node_modules/vite/types/hot.d.ts","./node_modules/vite/dist/node/types.d-agj9qkwt.d.ts","./node_modules/esbuild/lib/main.d.ts","./node_modules/source-map-js/source-map.d.ts","./node_modules/postcss/lib/previous-map.d.ts","./node_modules/postcss/lib/input.d.ts","./node_modules/postcss/lib/css-syntax-error.d.ts","./node_modules/postcss/lib/declaration.d.ts","./node_modules/postcss/lib/root.d.ts","./node_modules/postcss/lib/warning.d.ts","./node_modules/postcss/lib/lazy-result.d.ts","./node_modules/postcss/lib/no-work-result.d.ts","./node_modules/postcss/lib/processor.d.ts","./node_modules/postcss/lib/result.d.ts","./node_modules/postcss/lib/document.d.ts","./node_modules/postcss/lib/rule.d.ts","./node_modules/postcss/lib/node.d.ts","./node_modules/postcss/lib/comment.d.ts","./node_modules/postcss/lib/container.d.ts","./node_modules/postcss/lib/at-rule.d.ts","./node_modules/postcss/lib/list.d.ts","./node_modules/postcss/lib/postcss.d.ts","./node_modules/postcss/lib/postcss.d.mts","./node_modules/vite/dist/node/runtime.d.ts","./node_modules/vite/types/importglob.d.ts","./node_modules/vite/types/metadata.d.ts","./node_modules/vite/dist/node/index.d.ts","./node_modules/@babel/types/lib/index.d.ts","./node_modules/@types/babel__generator/index.d.ts","./node_modules/@babel/parser/typings/babel-parser.d.ts","./node_modules/@types/babel__template/index.d.ts","./node_modules/@types/babel__traverse/index.d.ts","./node_modules/@types/babel__core/index.d.ts","./node_modules/@vitejs/plugin-react/dist/index.d.ts","./vite.config.ts","./node_modules/@types/prop-types/index.d.ts","./node_modules/@types/react/global.d.ts","./node_modules/csstype/index.d.ts","./node_modules/@types/react/index.d.ts","./node_modules/@types/react-dom/index.d.ts"],"fileIdsList":[[40],[40,41,42,43,44],[40,42],[51],[48,49,50],[39,45],[31],[29,31],[20,28,29,30,32,34],[18],[21,26,31,34],[17,34],[21,22,25,26,27,34],[21,22,23,25,26,34],[18,19,20,21,22,26,27,28,30,31,32,34],[34],[16,18,19,20,21,22,23,25,26,27,28,29,30,31,32,33],[16,34],[21,23,24,26,27,34],[25,34],[26,27,31,34],[19,29],[9,38],[8,9],[9,10,11,12,13,14,15,35,36,37,38],[11,12,13,14],[11,12,13],[11],[12],[9],[39,46]],"fileInfos":[{"version":"a7297ff837fcdf174a9524925966429eb8e5feecc2cc55cc06574e6b092c1eaa","impliedFormat":1},{"version":"c430d44666289dae81f30fa7b2edebf186ecc91a2d4c71266ea6ae76388792e1","affectsGlobalScope":true,"impliedFormat":1},{"version":"080941d9f9ff9307f7e27a83bcd888b7c8270716c39af943532438932ec1d0b9","affectsGlobalScope":true,"impliedFormat":1},{"version":"80e18897e5884b6723488d4f5652167e7bb5024f946743134ecc4aa4ee731f89","affectsGlobalScope":true,"impliedFormat":1},{"version":"cd034f499c6cdca722b60c04b5b1b78e058487a7085a8e0d6fb50809947ee573","affectsGlobalScope":true,"impliedFormat":1},{"version":"8e7f8264d0fb4c5339605a15daadb037bf238c10b654bb3eee14208f860a32ea","affectsGlobalScope":true,"impliedFormat":1},{"version":"782dec38049b92d4e85c1585fbea5474a219c6984a35b004963b00beb1aab538","affectsGlobalScope":true,"impliedFormat":1},{"version":"751764bb94219b4ce8f5475dc35d3de2e432fea01a0c9610cd7f69ad05e398c6","impliedFormat":1},{"version":"ee70b8037ecdf0de6c04f35277f253663a536d7e38f1539d270e4e916d225a3f","affectsGlobalScope":true,"impliedFormat":1},{"version":"a660aa95476042d3fdcc1343cf6bb8fdf24772d31712b1db321c5a4dcc325434","impliedFormat":1},{"version":"282f98006ed7fa9bb2cd9bdbe2524595cfc4bcd58a0bb3232e4519f2138df811","impliedFormat":1},{"version":"6222e987b58abfe92597e1273ad7233626285bc2d78409d4a7b113d81a83496b","impliedFormat":1},{"version":"cbe726263ae9a7bf32352380f7e8ab66ee25b3457137e316929269c19e18a2be","impliedFormat":1},{"version":"8b96046bf5fb0a815cba6b0880d9f97b7f3a93cf187e8dcfe8e2792e97f38f87","impliedFormat":99},{"version":"bacf2c84cf448b2cd02c717ad46c3d7fd530e0c91282888c923ad64810a4d511","affectsGlobalScope":true,"impliedFormat":1},{"version":"402e5c534fb2b85fa771170595db3ac0dd532112c8fa44fc23f233bc6967488b","impliedFormat":1},{"version":"52dcc257df5119fb66d864625112ce5033ac51a4c2afe376a0b299d2f7f76e4a","impliedFormat":1},{"version":"e5bab5f871ef708d52d47b3e5d0aa72a08ee7a152f33931d9a60809711a2a9a3","impliedFormat":1},{"version":"e16dc2a81595736024a206c7d5c8a39bfe2e6039208ef29981d0d95434ba8fcf","impliedFormat":1},{"version":"cc4a4903fb698ca1d961d4c10dce658aa3a479faf40509d526f122b044eaf6a4","impliedFormat":1},{"version":"19ee8416e6473ed6c7adb868fa796b5653cf0fa2a337658e677eaa0d134388c3","impliedFormat":1},{"version":"1328ab4e442614b28cdb3d4b414cf68325c0da0dca07287a338d0654b7a00261","impliedFormat":1},{"version":"a039dc21f045919f3cbee2ec13812cc6cc3eebc99dae4be00973230f468d19a6","impliedFormat":1},{"version":"3fbe57af01460e49dcd29df55d6931e1672bc6f1be0fb073d11410bc16f9037d","impliedFormat":1},{"version":"f760be449e8562ec5c09bb5187e8e1eabf3c113c0c58cddda53ef8c69f3e2131","impliedFormat":1},{"version":"44325ed13294fce6ab825b82947bbeed2611db7dad9d9135260192f375e5a189","impliedFormat":1},{"version":"e392e8fb5b514eafc585601c1d781485aa6dd6a320e75daf1064a4c6918a1b45","impliedFormat":1},{"version":"46e4a36e8ddbdfb4e7330e11c81c970dc8b218611df9183d39c41c5f8c653b55","impliedFormat":1},{"version":"370bde134aa8c2abc926d0e99d3a4d5d5dba65c6ee65459137e4f02670cbf841","impliedFormat":1},{"version":"6332f565867cf4a740a70e30f31cefba37ef7cebcf74f22eab8d744fde6d193e","impliedFormat":1},{"version":"2977b7884aedc895a1d0c9c210c7cf3272c29d6959a08a6fa3ff71e0aff08175","impliedFormat":1},{"version":"17f2922d41ddd032830a91371c948cd9ce903b35c95adca72271a54584f19b0b","impliedFormat":1},{"version":"3eed76ede2a1a14d7c9bb0a642041282dcc264811139d3dd275c9fe14efc9840","impliedFormat":1},{"version":"354a7f8e1287d9d6b7561bc97fdd8cbc2f7c1dd79e4cb37b942e8a5cfaff1085","impliedFormat":1},{"version":"8d369483f0c2b9ee388129cfdb6a43bc8112b377e86a41884bd06e19ce04f4c1","impliedFormat":99},{"version":"82e687ebd99518bc63ea04b0c3810fb6e50aa6942decd0ca6f7a56d9b9a212a6","impliedFormat":99},{"version":"7f698624bbbb060ece7c0e51b7236520ebada74b747d7523c7df376453ed6fea","impliedFormat":1},{"version":"8f07f2b6514744ac96e51d7cb8518c0f4de319471237ea10cf688b8d0e9d0225","impliedFormat":1},{"version":"257b83faa134d971c738a6b9e4c47e59bb7b23274719d92197580dd662bfafc3","impliedFormat":99},{"version":"556ccd493ec36c7d7cb130d51be66e147b91cc1415be383d71da0f1e49f742a9","impliedFormat":1},{"version":"b6d03c9cfe2cf0ba4c673c209fcd7c46c815b2619fd2aad59fc4229aaef2ed43","impliedFormat":1},{"version":"95aba78013d782537cc5e23868e736bec5d377b918990e28ed56110e3ae8b958","impliedFormat":1},{"version":"670a76db379b27c8ff42f1ba927828a22862e2ab0b0908e38b671f0e912cc5ed","impliedFormat":1},{"version":"13b77ab19ef7aadd86a1e54f2f08ea23a6d74e102909e3c00d31f231ed040f62","impliedFormat":1},{"version":"069bebfee29864e3955378107e243508b163e77ab10de6a5ee03ae06939f0bb9","impliedFormat":1},{"version":"26e0ffceb2198feb1ef460d5d14111c69ad07d44c5a67fd4bfeb74c969aa9afb","impliedFormat":99},{"version":"14501439875c563028a6d28102ab04a36105e6f013b0803a9b5f62690bf45098","signature":"4b96dd19fd2949d28ce80e913412b0026dc421e5bf6c31d87c7b5eb11b5753b4"},{"version":"87d9d29dbc745f182683f63187bf3d53fd8673e5fca38ad5eaab69798ed29fbc","impliedFormat":1},{"version":"eb5b19b86227ace1d29ea4cf81387279d04bb34051e944bc53df69f58914b788","affectsGlobalScope":true,"impliedFormat":1},{"version":"ac51dd7d31333793807a6abaa5ae168512b6131bd41d9c5b98477fc3b7800f9f","impliedFormat":1},{"version":"09ddcfcfbe77a8232d155ca1030005106b1328f6210df43629d0be750da07c16","affectsGlobalScope":true,"impliedFormat":1},{"version":"17ed71200119e86ccef2d96b73b02ce8854b76ad6bd21b5021d4269bec527b5f","impliedFormat":1}],"root":[47],"options":{"allowSyntheticDefaultImports":true,"composite":true,"module":99,"skipLibCheck":true,"strict":true},"referencedMap":[[42,1],[45,2],[41,1],[43,3],[44,1],[52,4],[51,5],[46,6],[32,7],[30,8],[31,9],[19,10],[20,8],[27,11],[18,12],[23,13],[24,14],[29,15],[35,16],[34,17],[17,18],[25,19],[26,20],[21,21],[28,7],[22,22],[10,23],[9,24],[39,25],[36,26],[14,27],[12,28],[13,29],[38,30],[47,31]],"latestChangedDtsFile":"./vite.config.d.ts","version":"5.9.3"} \ No newline at end of file diff --git a/frontend/tsconfig.tsbuildinfo b/frontend/tsconfig.tsbuildinfo deleted file mode 100644 index 4f59de7..0000000 --- a/frontend/tsconfig.tsbuildinfo +++ /dev/null @@ -1 +0,0 @@ -{"root":["./src/app.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/api/client.ts","./src/api/configschema.ts","./src/api/types.ts","./src/components/configform.tsx","./src/components/filerow.tsx","./src/components/healthstrip.tsx","./src/components/jobtable.tsx","./src/components/logblock.tsx","./src/components/nav.tsx","./src/components/pipelinestepper.tsx","./src/components/stageticks.tsx","./src/components/ui/button.tsx","./src/components/ui/card.tsx","./src/components/ui/field.tsx","./src/components/ui/statustag.tsx","./src/hooks/usepolling.ts","./src/lib/format.ts","./src/pages/dashboard.tsx","./src/pages/jobdetail.tsx","./src/pages/newjob.tsx","./src/pages/system.tsx"],"version":"5.9.3"} \ No newline at end of file diff --git a/frontend/vite.config.d.ts b/frontend/vite.config.d.ts deleted file mode 100644 index 340562a..0000000 --- a/frontend/vite.config.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare const _default: import("vite").UserConfig; -export default _default; diff --git a/frontend/vite.config.js b/frontend/vite.config.js deleted file mode 100644 index 08ceb93..0000000 --- a/frontend/vite.config.js +++ /dev/null @@ -1,6 +0,0 @@ -import { defineConfig } from 'vite'; -import react from '@vitejs/plugin-react'; -export default defineConfig({ - plugins: [react()], - server: { port: 5173 }, -}); diff --git a/main.py b/main.py index 17bf209..d58c3b9 100644 --- a/main.py +++ b/main.py @@ -2,80 +2,85 @@ OSM → NS-3 Mobility Trace Conversion API Endpoints: - POST /jobs — Upload OSM + JSON config → start conversion job - GET /jobs — List all jobs - GET /jobs/{id} — Poll job status + stage progress - GET /jobs/{id}/download/{filename} — Download output file - GET /jobs/{id}/logs — Retrieve stage logs - DELETE /jobs/{id} — Remove job and its files - GET /schema — Full JSON Schema for all parameters - GET /defaults — Default configuration values - GET /health — Health check + SUMO tool availability + POST /jobs — Upload OSM + JSON config → start conversion job + GET /jobs — List all jobs + GET /jobs/{id} — Poll job status + stage progress + POST /jobs/{id}/cancel — Cancel a pending/running job + GET /jobs/{id}/download/{fn} — Download output file + GET /jobs/{id}/logs — Retrieve stage logs + DELETE /jobs/{id} — Remove job and its files + GET /schema — Full JSON Schema for all parameters + GET /defaults — Default configuration values + GET /live — Liveness probe + GET /ready — Readiness probe (SUMO tool availability) + GET /health — Health check (backwards-compatible shape) + +Auth: when OSM2NS3_API_KEY is set, every /jobs route requires an +``X-API-Key`` header; system/config endpoints and the SPA stay public. """ from __future__ import annotations -import asyncio import json -import os -import shutil +import logging +import uuid +from contextlib import asynccontextmanager from pathlib import Path -from typing import List, Optional +import aiofiles from fastapi import ( - BackgroundTasks, + APIRouter, Depends, FastAPI, File, Form, + Header, HTTPException, + Request, UploadFile, status, ) from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, JSONResponse +from fastapi.staticfiles import StaticFiles -from app.models import ( - ConversionRequest, - JobStatus, - StageStatus, -) -from app.worker import ( - OUT_DIR, - JOBS_DIR, - UPLOAD_DIR, - create_job, - get_defaults, - get_full_schema, - get_job, - list_jobs, - run_pipeline, -) +from app.models import ConversionRequest, JobStatus, StageStatus +from app.settings import Settings, get_settings +from app.store import JobStore +from app.worker import PipelineRunner, check_tools -# ── App setup ──────────────────────────────────────────────── - -app = FastAPI( - title="OSM → NS-3 Mobility Trace Converter", - description=( - "Convert OpenStreetMap files to NS-3 mobility traces " - "(ns_movements / TCL) via the full SUMO pipeline. " - "Every parameter is exposed." - ), - version="1.0.0", - docs_url="/docs", - redoc_url="/redoc", -) +logger = logging.getLogger("osm2ns3.api") + +_UPLOAD_CHUNK = 1 << 20 # 1 MiB + + +# ── Dependencies ────────────────────────────────────────────── -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) +def get_runner(request: Request) -> PipelineRunner: + return request.app.state.runner + + +def get_store(request: Request) -> JobStore: + return request.app.state.store + + +async def verify_api_key( + request: Request, + x_api_key: str | None = Header(default=None), +) -> None: + """No-op unless an API key is configured. Reads the settings of THE app + handling the request (app.state), not the process-level cache, so the + same process can host apps with different keys.""" + settings = getattr(request.app.state, "settings", None) or get_settings() + if settings.api_key is None: + return + if x_api_key != settings.api_key: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or missing API key. Send it via the X-API-Key header.", + ) -# ── Dependency: parse config from multipart form ────────────── async def parse_config(config: str = Form(default="{}")) -> ConversionRequest: """ @@ -86,272 +91,385 @@ async def parse_config(config: str = Form(default="{}")) -> ConversionRequest: data = json.loads(config) except json.JSONDecodeError as exc: raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=f"config JSON parse error: {exc}", ) try: return ConversionRequest(**data) except Exception as exc: raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=f"config validation error: {exc}", ) -# ── Routes ──────────────────────────────────────────────────── +# ── App factory ─────────────────────────────────────────────── -@app.get("/health", tags=["system"]) -async def health(): - """Check API health and SUMO tool availability on the server.""" - tools = ["netconvert", "duarouter", "sumo"] - availability = {t: shutil.which(t) is not None for t in tools} - # Check traceExporter.py - te_paths = [ - "/usr/share/sumo/tools/traceExporter.py", - "/usr/local/share/sumo/tools/traceExporter.py", - ] - te_available = any(Path(p).exists() for p in te_paths) - availability["traceExporter.py"] = te_available +def create_app(settings: Settings | None = None) -> FastAPI: + settings = settings or get_settings() - rt_paths = [ - "/usr/share/sumo/tools/randomTrips.py", - "/usr/local/share/sumo/tools/randomTrips.py", - ] - rt_available = any(Path(p).exists() for p in rt_paths) - availability["randomTrips.py"] = rt_available - - all_ok = all(availability.values()) - return { - "status": "ready" if all_ok else "degraded", - "tools": availability, - "pipeline_ready": all_ok, - } - - -@app.get("/schema", tags=["config"]) -async def get_schema(): - """ - Return the full JSON Schema for ConversionRequest. - Use this to auto-generate forms or validate configs client-side. - """ - return get_full_schema() - - -@app.get("/defaults", tags=["config"]) -async def get_default_config(): - """Return the default configuration for all pipeline stages.""" - return get_defaults() - - -@app.post("/jobs", status_code=status.HTTP_202_ACCEPTED, tags=["jobs"]) -async def create_conversion_job( - background_tasks: BackgroundTasks, - osm_file: UploadFile = File(..., description="OpenStreetMap .osm or .osm.pbf file"), - config: ConversionRequest = Depends(parse_config), -): - """ - Start an OSM → NS-3 conversion job. - - **Multipart form fields:** - - `osm_file` — the `.osm` file to convert - - `config` — JSON string with pipeline parameters (partial or full). - Omitted fields use defaults. See `/defaults` and `/schema`. - - **Returns:** job_id for polling via `GET /jobs/{id}` - - **Example config (partial — just change what you need):** - ```json - { - "random_trips": { - "end": 1800, - "period": 1.5, - "vehicle_class": "passenger" - }, - "sumo": { - "step_length": 0.1, - "fcd_output_period": 1.0 - }, - "trace_exporter": { - "output_format": "both", - "sampling_period": 1.0, - "penetration_rate": 0.8 - } - } - ``` - """ - if not osm_file.filename: - raise HTTPException(status_code=400, detail="No filename provided.") - - suffix = Path(osm_file.filename).suffix.lower() - if suffix not in (".osm", ".xml", ".pbf"): - raise HTTPException( - status_code=400, - detail=f"Unsupported file type '{suffix}'. Upload a .osm file.", + @asynccontextmanager + async def lifespan(app: FastAPI): + logging.basicConfig( + level=settings.log_level.upper(), + format="%(asctime)s %(levelname)s %(name)s %(message)s", ) + settings.ensure_dirs() + store = JobStore( + settings.db_path, + retention_max=settings.job_retention_max, + retention_days=settings.job_retention_days, + ) + await store.init() # also recovers jobs interrupted by a restart + app.state.settings = settings + app.state.store = store + app.state.runner = PipelineRunner(settings, store) + logger.info( + "osm2ns3 ready (data_dir=%s, auth=%s)", + settings.data_dir, + "on" if settings.api_key else "off", + ) + yield + + app = FastAPI( + title="OSM → NS-3 Mobility Trace Converter", + description=( + "Convert OpenStreetMap files to NS-3 mobility traces " + "(ns_movements / TCL) via the full SUMO pipeline. " + "Every parameter is exposed." + ), + version="1.0.0", + docs_url="/docs", + redoc_url="/redoc", + lifespan=lifespan, + ) - # Save upload - job_id = create_job() - upload_path = UPLOAD_DIR / f"{job_id}{suffix}" - content = await osm_file.read() - upload_path.write_bytes(content) - - # Save config snapshot - cfg_snap = JOBS_DIR / job_id - cfg_snap.mkdir(parents=True, exist_ok=True) - (cfg_snap / "config.json").write_text(config.model_dump_json(indent=2)) - - # Launch pipeline in background - background_tasks.add_task(run_pipeline, job_id, upload_path, config) - - return { - "job_id": job_id, - "message": "Job accepted. Poll GET /jobs/{job_id} for status.", - "config_snapshot": config.model_dump(), - } - - -@app.get("/jobs", response_model=List[JobStatus], tags=["jobs"]) -async def list_all_jobs(): - """List all jobs (most recent first).""" - jobs = list_jobs() - jobs.sort(key=lambda j: j.created_at, reverse=True) - return jobs - - -@app.get("/jobs/{job_id}", response_model=JobStatus, tags=["jobs"]) -async def get_job_status(job_id: str): - """ - Poll the status of a conversion job. - - **Stage order:** - 1. `netconvert` — OSM → SUMO network - 2. `random_trips` — Generate vehicle trips - 3. `duarouter` — Compute routes - 4. `sumo` — Run traffic simulation - 5. `trace_export` — Export NS-3 mobility traces - - **Status values:** `pending` → `running` → `done` | `failed` - """ - job = get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") - return job - - -@app.get("/jobs/{job_id}/config", tags=["jobs"]) -async def get_job_config(job_id: str): - """Retrieve the exact configuration used for a job.""" - job = get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") - cfg_path = JOBS_DIR / job_id / "config.json" - if not cfg_path.exists(): - raise HTTPException(status_code=404, detail="Config not found for this job.") - return json.loads(cfg_path.read_text()) - - -@app.get("/jobs/{job_id}/files", tags=["jobs"]) -async def list_job_files(job_id: str): - """List all output files available for download.""" - job = get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") - - out_dir = OUT_DIR / job_id - if not out_dir.exists(): - return {"job_id": job_id, "files": []} - - files = [] - for f in sorted(out_dir.iterdir()): - files.append({ - "filename": f.name, - "size_bytes": f.stat().st_size, - "download_url": f"/jobs/{job_id}/download/{f.name}", - }) - return {"job_id": job_id, "files": files} - - -@app.get("/jobs/{job_id}/download/{filename}", tags=["jobs"]) -async def download_output(job_id: str, filename: str): - """ - Download an output file. - - Common files produced: - - `ns_movements` — NS-3 Ns2MobilityHelper trace - - `mobility.tcl` — NS-2/NS-3 TCL Setdest trace - - `tripinfo.xml` — Per-vehicle trip statistics - - `summary.xml` — Per-step network statistics - """ - job = get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") - - file_path = OUT_DIR / job_id / filename - if not file_path.exists(): - raise HTTPException(status_code=404, detail=f"File '{filename}' not found.") + app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) - # Sanitise path traversal - try: - file_path.resolve().relative_to((OUT_DIR / job_id).resolve()) - except ValueError: - raise HTTPException(status_code=400, detail="Invalid filename.") - - return FileResponse( - path=str(file_path), - filename=filename, - media_type="application/octet-stream", + # All /jobs routes are behind the (env-enabled) API-key check. + jobs_router = APIRouter( + prefix="/jobs", + tags=["jobs"], + dependencies=[Depends(verify_api_key)], ) + # ── System routes (public) ──────────────────────────────── + + @app.get("/live", tags=["system"]) + async def live(): + """Liveness probe — always ok while the process serves requests.""" + return {"status": "ok"} + + @app.get("/ready", tags=["system"]) + async def ready(): + """Readiness probe — 200 when every SUMO tool the pipeline needs is + available, 503 with the availability map otherwise.""" + tools = check_tools(settings.resolve_sumo_home()) + all_ok = all(tools.values()) + payload = { + "status": "ready" if all_ok else "not_ready", + "tools": tools, + "pipeline_ready": all_ok, + } + # Backwards-compatible alias consumed by the frontend health strip. + payload["tools_available"] = all_ok + return JSONResponse(payload, status_code=status.HTTP_200_OK if all_ok else 503) + + @app.get("/health", tags=["system"]) + async def health(): + """Backwards-compatible health check (same info as /ready, always 200).""" + tools = check_tools(settings.resolve_sumo_home()) + all_ok = all(tools.values()) + return { + "status": "ready" if all_ok else "degraded", + "tools": tools, + "pipeline_ready": all_ok, + } + + @app.get("/schema", tags=["config"]) + async def get_schema(): + """ + Return the full JSON Schema for ConversionRequest. + Use this to auto-generate forms or validate configs client-side. + """ + return ConversionRequest.model_json_schema() + + @app.get("/defaults", tags=["config"]) + async def get_default_config(): + """Return the default configuration for all pipeline stages.""" + return ConversionRequest().model_dump() + + # ── Job routes (API-key gated) ──────────────────────────── + + @jobs_router.post("", status_code=status.HTTP_202_ACCEPTED) + async def create_conversion_job( + osm_file: UploadFile = File( + ..., description="OpenStreetMap .osm or .osm.pbf file" + ), + config: ConversionRequest = Depends(parse_config), + runner: PipelineRunner = Depends(get_runner), + ): + """ + Start an OSM → NS-3 conversion job. + + **Multipart form fields:** + - `osm_file` — the `.osm` file to convert + - `config` — JSON string with pipeline parameters (partial or full). + Omitted fields use defaults. See `/defaults` and `/schema`. + + **Returns:** job_id for polling via `GET /jobs/{id}` + """ + if not osm_file.filename: + raise HTTPException(status_code=400, detail="No filename provided.") + + suffix = Path(osm_file.filename).suffix.lower() + if suffix not in (".osm", ".xml", ".pbf"): + raise HTTPException( + status_code=400, + detail=f"Unsupported file type '{suffix}'. Upload a .osm file.", + ) + + # Stream the upload to disk in chunks with a size cap — never load + # the whole file into memory. The job id is generated up-front so + # the upload filename matches; no DB row exists until the file is + # safely on disk. + job_id = str(uuid.uuid4()) + upload_path = settings.upload_dir / f"{job_id}{suffix}" + limit = settings.max_upload_mb * 1024 * 1024 + size = 0 + try: + async with aiofiles.open(upload_path, "wb") as fh: + while chunk := await osm_file.read(_UPLOAD_CHUNK): + size += len(chunk) + if size > limit: + raise HTTPException( + status_code=status.HTTP_413_CONTENT_TOO_LARGE, + detail=( + f"Upload exceeds the {settings.max_upload_mb} MB limit " + f"(OSM2NS3_MAX_UPLOAD_MB)." + ), + ) + await fh.write(chunk) + except Exception: + upload_path.unlink(missing_ok=True) + raise + finally: + await osm_file.close() + + logger.info("uploaded %s (%d bytes) → job %s", osm_file.filename, size, job_id) + + # Persist the job row (with config snapshot) and launch the pipeline. + await runner.create_and_launch(upload_path, config, job_id=job_id) + + return { + "job_id": job_id, + "message": "Job accepted. Poll GET /jobs/{job_id} for status.", + "config_snapshot": config.model_dump(), + } + + @jobs_router.get("", response_model=list[JobStatus]) + async def list_all_jobs(store: JobStore = Depends(get_store)): + """List all jobs (most recent first).""" + jobs = await store.list() + jobs.sort(key=lambda j: j.created_at, reverse=True) + return jobs + + @jobs_router.get("/{job_id}", response_model=JobStatus) + async def get_job_status(job_id: str, store: JobStore = Depends(get_store)): + """ + Poll the status of a conversion job. + + **Stage order:** + 1. `netconvert` — OSM → SUMO network + 2. `random_trips` — Generate vehicle trips + 3. `duarouter` — Compute routes + 4. `sumo` — Run traffic simulation + 5. `trace_export` — Export NS-3 mobility traces + + **Status values:** `pending` → `running` → `done` | `failed` | `cancelled` + """ + job = await store.get(job_id) + if not job: + raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") + return job + + @jobs_router.post( + "/{job_id}/cancel", + response_model=JobStatus, + status_code=status.HTTP_202_ACCEPTED, + ) + async def cancel_job(job_id: str, runner: PipelineRunner = Depends(get_runner)): + """Cancel a pending or running job (kills its subprocess tree).""" + try: + await runner.cancel(job_id) + except LookupError: + raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") + job = await runner.store.get(job_id) + if job.status != StageStatus.cancelled: + raise HTTPException( + status_code=409, + detail=f"Job is not running (status: {job.status.value}).", + ) + return job + + @jobs_router.get("/{job_id}/config") + async def get_job_config(job_id: str, store: JobStore = Depends(get_store)): + """Retrieve the exact configuration used for a job.""" + job = await store.get(job_id) + if not job: + raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") + cfg = await store.get_config(job_id) + if cfg is None: + raise HTTPException( + status_code=404, detail="Config not found for this job." + ) + return cfg + + @jobs_router.get("/{job_id}/files") + async def list_job_files(job_id: str, store: JobStore = Depends(get_store)): + """List all output files available for download.""" + job = await store.get(job_id) + if not job: + raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") + + out_dir = settings.out_dir / job_id + if not out_dir.exists(): + return {"job_id": job_id, "files": []} + + files = [] + for f in sorted(out_dir.iterdir()): + files.append( + { + "filename": f.name, + "size_bytes": f.stat().st_size, + "download_url": f"/jobs/{job_id}/download/{f.name}", + } + ) + return {"job_id": job_id, "files": files} + + @jobs_router.get("/{job_id}/download/{filename}") + async def download_output( + job_id: str, filename: str, store: JobStore = Depends(get_store) + ): + """ + Download an output file. + + Common files produced: + - `ns_movements` — NS-3 Ns2MobilityHelper trace + - `mobility.tcl` — NS-2/NS-3 TCL Setdest trace + - `tripinfo.xml` — Per-vehicle trip statistics + - `summary.xml` — Per-step network statistics + """ + job = await store.get(job_id) + if not job: + raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") + + file_path = settings.out_dir / job_id / filename + if not file_path.exists(): + raise HTTPException(status_code=404, detail=f"File '{filename}' not found.") + + # Sanitise path traversal + try: + file_path.resolve().relative_to((settings.out_dir / job_id).resolve()) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid filename.") + + return FileResponse( + path=str(file_path), + filename=filename, + media_type="application/octet-stream", + ) -@app.get("/jobs/{job_id}/logs", tags=["jobs"]) -async def get_job_logs(job_id: str): - """ - Retrieve SUMO tool output logs captured during the job. - Useful for debugging parameter choices. - """ - job = get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") - - logs_dir = JOBS_DIR / job_id - logs = {} - for stage in ["netconvert", "duarouter", "sumo", "traceExporter"]: - log_file = logs_dir / f"{stage}.log" - if log_file.exists(): - logs[stage] = log_file.read_text() - else: - logs[stage] = None - - return { - "job_id": job_id, - "warnings": job.warnings, - "stage_logs": logs, - } - - -@app.delete("/jobs/{job_id}", status_code=status.HTTP_204_NO_CONTENT, tags=["jobs"]) -async def delete_job(job_id: str): - """Delete a job and all its associated files.""" - from app.worker import _jobs, _job_locks - job = get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") - - if job.status == StageStatus.running: - raise HTTPException( - status_code=409, - detail="Cannot delete a running job. Wait for it to complete.", + @jobs_router.get("/{job_id}/logs") + async def get_job_logs(job_id: str, store: JobStore = Depends(get_store)): + """ + Retrieve SUMO tool output logs captured during the job. + Useful for debugging parameter choices. + """ + job = await store.get(job_id) + if not job: + raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") + + logs_dir = settings.jobs_dir / job_id + logs = {} + for stage in ["netconvert", "duarouter", "sumo", "traceExporter"]: + # New pipeline writes {stage}.log using stage names; traceExporter + # is kept as the historical alias for trace_export coverage. + for candidate in ( + [logs_dir / f"{stage}.log"] + if stage != "traceExporter" + else [logs_dir / "trace_export.log", logs_dir / "traceExporter.log"] + ): + if candidate.exists(): + logs[stage] = candidate.read_text() + break + else: + logs[stage] = None + + return { + "job_id": job_id, + "warnings": job.warnings, + "stage_logs": logs, + } + + @jobs_router.delete("/{job_id}", status_code=status.HTTP_204_NO_CONTENT) + async def delete_job(job_id: str, runner: PipelineRunner = Depends(get_runner)): + """Delete a job and all its associated files.""" + job = await runner.store.get(job_id) + if not job: + raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") + + if job.status in (StageStatus.pending, StageStatus.running): + raise HTTPException( + status_code=409, + detail="Cannot delete a running job. Cancel it first.", + ) + + await runner.delete(job_id) + logger.info("job %s deleted", job_id) + + app.include_router(jobs_router) + + # ── Static SPA (Docker image ships frontend/dist here) ──── + # Registered LAST so API routes and /docs always win. + if settings.static_dir.exists(): + + @app.exception_handler(404) + async def spa_fallback(request: Request, exc: HTTPException): + """Serve index.html for client-side routes (/jobs/{id} deep links).""" + if request.url.path.startswith( + ( + "/jobs", + "/schema", + "/defaults", + "/health", + "/live", + "/ready", + "/docs", + "/redoc", + "/openapi.json", + ) + ): + return JSONResponse({"detail": "Not Found"}, status_code=404) + index = settings.static_dir / "index.html" + if index.exists(): + return FileResponse(index) + return JSONResponse({"detail": "Not Found"}, status_code=404) + + app.mount( + "/", + StaticFiles(directory=str(settings.static_dir), html=True), + name="spa", ) - # Clean up files - for d in [JOBS_DIR / job_id, OUT_DIR / job_id]: - if d.exists(): - shutil.rmtree(d) + return app - upload_dir = UPLOAD_DIR - for f in upload_dir.glob(f"{job_id}*"): - f.unlink() - _jobs.pop(job_id, None) - _job_locks.pop(job_id, None) \ No newline at end of file +app = create_app() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b15f086 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,56 @@ +[project] +name = "osm2ns3" +version = "1.0.0" +description = "FastAPI service converting OpenStreetMap files to NS-3 mobility traces via the SUMO toolchain" +requires-python = ">=3.10" +dependencies = [ + "fastapi~=0.128.8", + "pydantic~=2.13.4", + "pydantic-settings~=2.5", + "aiofiles~=25.1.0", + "python-multipart~=0.0.20", + "uvicorn[standard]~=0.34", +] + +[project.optional-dependencies] +dev = [ + "pytest~=8.0", + "pytest-asyncio~=0.24", + "httpx~=0.28", + "pytest-cov~=5.0", + "asgi-lifespan~=2.1", + "ruff~=0.8", + "black~=24.0", + "isort~=5.13", +] + +[tool.black] +# Default line length (88); a one-time reformat established this baseline. +extend-exclude = '(\.venv|\.venv312|jobs|uploads|outputs|frontend|static|\.github/scripts)' + +[tool.isort] +profile = "black" +extend_skip = [".venv", ".venv312", "jobs", "uploads", "outputs", "frontend", "static", ".github/scripts"] + +[tool.ruff] +extend-exclude = [".venv", ".venv312", "jobs", "uploads", "outputs", "frontend", "static", ".github/scripts"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "B"] +ignore = [ + "B008", # FastAPI: Depends() in argument defaults is the documented pattern + "B011", # tests: assert False to fail a try/except branch + "B904", # raise-without-from inside FastAPI exception paths + "B905", # zip() strict= is 3.10+ noise for our usage + "E501", # line length left to black's 88 +] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.coverage.run] +source = ["app", "main"] + +[tool.coverage.report] +exclude_lines = ["pragma: no cover", "if __name__ == .__main__.:"] diff --git a/requirements.txt b/requirements.txt index 9819b61..bd688aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ -aiofiles~=25.1.0 -pydantic~=2.13.4 fastapi~=0.128.8 -python-multipart \ No newline at end of file +pydantic~=2.13.4 +pydantic-settings~=2.5 +aiofiles~=25.1.0 +python-multipart~=0.0.20 +uvicorn[standard]~=0.34 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..763f633 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,171 @@ +"""Shared fixtures: no SUMO tools on the test runner — every fixture that +touches the toolchain mocks the lookup and/or the subprocess layer.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest +from asgi_lifespan import LifespanManager +from httpx import ASGITransport, AsyncClient + +import app.worker as worker_mod +from app.settings import Settings +from app.store import JobStore +from app.worker import PipelineRunner +from main import create_app + +OSM_BYTES = b""" + + + +""" + + +@pytest.fixture() +def settings(tmp_path) -> Settings: + """Per-test settings rooted at a tmp dir (bypasses get_settings cache).""" + return Settings( + data_dir=tmp_path, + static_dir=tmp_path / "static-absent", + api_key=None, + max_upload_mb=1, + job_retention_max=200, + job_retention_days=7, + timeout_default_s=5.0, + timeout_sumo_s=5.0, + timeout_trace_export_s=5.0, + max_concurrent_jobs=4, + ) + + +@pytest.fixture(autouse=True) +def no_sumo(monkeypatch): + """Pretend every SUMO binary/script exists at a fixed (mock) path.""" + monkeypatch.setattr( + worker_mod, "_find_binary", lambda names: f"/mock/bin/{names[0]}" + ) + monkeypatch.setattr( + worker_mod, + "_find_sumo_script", + lambda filename, sumo_home="": f"/mock/tools/{filename}", + ) + + +class SubprocessMock: + """Replaces PipelineRunner._run_async; records every invocation. + + mode: "ok" (rc=0), "fail" (rc=1), "slow" (sleeps until cancelled). + on_stage: optional callback receiving (job_id, cmd) per invocation. + """ + + def __init__(self, mode: str = "ok", on_stage=None): + self.mode = mode + self.calls = [] + self.on_stage = on_stage + + async def __call__(self, job_id, cmd, workdir, timeout): + self.calls.append({"job_id": job_id, "cmd": list(cmd), "timeout": timeout}) + if self.on_stage: + self.on_stage(job_id, cmd) + if self.mode == "slow": + await asyncio.sleep(3600) + raise AssertionError("unreachable: slow mock must be cancelled") + if self.mode == "fail": + return 1, "", "mock failure" + # Successful stages must materialise files the pipeline copies. + if "--ns2-output" in cmd: + out = Path(cmd[cmd.index("--ns2-output") + 1]) + out.write_text("mock trace") + Path(workdir / "fcd-output.xml").write_text("") + Path(workdir / "tripinfo.xml").write_text("") + Path(workdir / "summary.xml").write_text("") + return 0, "ok", "" + + +@pytest.fixture() +def mock_subprocess(monkeypatch) -> SubprocessMock: + mock = SubprocessMock(mode="ok") + monkeypatch.setattr(PipelineRunner, "_run_async", mock) + return mock + + +@pytest.fixture() +def store(settings) -> JobStore: + return JobStore( + settings.db_path, + retention_max=settings.job_retention_max, + retention_days=settings.job_retention_days, + ) + + +@pytest.fixture() +def runner(settings, store) -> PipelineRunner: + return PipelineRunner(settings, store) + + +@pytest.fixture() +async def app(settings): + application = create_app(settings) + async with LifespanManager(application): + yield application + + +@pytest.fixture() +async def client(app): + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test" + ) as c: + yield c + + +@pytest.fixture() +def authed_settings(tmp_path) -> Settings: + """Separate app with API-key auth enabled (init kwargs, not mutation — + pydantic-settings resolves every field at construction).""" + return Settings( + data_dir=tmp_path, + static_dir=tmp_path / "static-absent", + api_key="test-secret", + max_upload_mb=1, + ) + + +@pytest.fixture() +async def authed_client(authed_settings): + application = create_app(authed_settings) + async with LifespanManager(application): + async with AsyncClient( + transport=ASGITransport(app=application), base_url="http://test" + ) as c: + yield c + + +async def wait_for_status(client, job_id: str, want: set[str], timeout: float = 5.0): + """Poll GET /jobs/{id} until status ∈ want; returns final payload.""" + deadline = asyncio.get_event_loop().time() + timeout + while True: + r = await client.get(f"/jobs/{job_id}") + r.raise_for_status() + data = r.json() + if data["status"] in want: + return data + if asyncio.get_event_loop().time() > deadline: + raise AssertionError( + f"job {job_id} stuck at {data['status']!r}, wanted {want}" + ) + await asyncio.sleep(0.02) + + +async def post_job( + client, + config: dict | None = None, + filename="map.osm", + content: bytes = OSM_BYTES, + headers: dict | None = None, +): + files = {"osm_file": (filename, content, "application/octet-stream")} + data = {"config": json.dumps(config or {})} + return await client.post("/jobs", files=files, data=data, headers=headers or {}) diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..f6cd6e1 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,405 @@ +"""API integration tests — full ASGI app with SUMO subprocesses mocked out.""" + +import asyncio + +import pytest + +from .conftest import post_job, wait_for_status + +# ── Happy path ──────────────────────────────────────────────── + + +async def test_job_lifecycle_end_to_end(client, mock_subprocess): + r = await post_job( + client, + config={ + "random_trips": {"end": 100.0}, + "sumo": {"end": 90.0, "fcd_output_period": 1.0}, + "trace_exporter": {"sampling_period": 1.0}, + }, + ) + assert r.status_code == 202, r.text + job_id = r.json()["job_id"] + assert r.json()["config_snapshot"]["sumo"]["end"] == 90.0 + + final = await wait_for_status(client, job_id, {"done", "failed"}) + assert final["status"] == "done", final.get("error") + assert all(s == "done" for s in final["stages"].values()) + assert final["completed_at"] is not None + + # All five pipeline stages actually invoked the subprocess layer… + assert len(mock_subprocess.calls) == 6 # 4 stages + 2 exporters (both formats) + + +async def test_outputs_listed_and_downloadable(client, mock_subprocess): + r = await post_job(client) + job_id = r.json()["job_id"] + await wait_for_status(client, job_id, {"done"}) + + files = (await client.get(f"/jobs/{job_id}/files")).json()["files"] + names = {f["filename"] for f in files} + assert {"ns_movements", "mobility.tcl"}.issubset(names) + assert {"tripinfo.xml", "summary.xml"}.issubset(names) + + dl = await client.get(f"/jobs/{job_id}/download/ns_movements") + assert dl.status_code == 200 + assert dl.content # the mock wrote bytes + + +async def test_ns_movements_only_format(client, mock_subprocess): + r = await post_job(client, {"trace_exporter": {"output_format": "ns_movements"}}) + job_id = r.json()["job_id"] + await wait_for_status(client, job_id, {"done"}) + names = { + f["filename"] + for f in (await client.get(f"/jobs/{job_id}/files")).json()["files"] + } + assert "ns_movements" in names + assert "mobility.tcl" not in names + + +# ── Validation / upload guards ─────────────────────────────── + + +async def test_bad_config_json_422(client): + r = await client.post( + "/jobs", + files={"osm_file": ("map.osm", b"", "application/octet-stream")}, + data={"config": "{not json"}, + ) + assert r.status_code == 422 + + +async def test_invalid_config_values_422(client): + r = await post_job( + client, config={"sumo": {"end": 5000.0}} + ) # > random_trips.end=3600 + assert r.status_code == 422 + assert "sumo.end" in r.json()["detail"] + + +async def test_bad_suffix_400(client): + r = await post_job(client, filename="notes.txt") + assert r.status_code == 400 + + +async def test_missing_filename_rejected(client): + # An empty filename is rejected before any job is created — FastAPI's own + # multipart validation (422) or the handler's explicit guard (400). + r = await client.post( + "/jobs", + files={"osm_file": ("", b"", "application/octet-stream")}, + data={"config": "{}"}, + ) + assert r.status_code in (400, 422) + + +async def test_upload_size_limit_413(settings): + """An app with a tiny cap rejects a bigger upload and leaves no junk.""" + from asgi_lifespan import LifespanManager + from httpx import ASGITransport, AsyncClient + + from main import create_app + + settings.max_upload_mb = 0 # 0 MB → any content trips the cap + app = create_app(settings) + async with LifespanManager(app): + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test" + ) as c: + r = await post_job(c, content=b"x" * 1024 * 1024 + b"x") + assert r.status_code == 413 + assert "limit" in r.json()["detail"].lower() + # No partial upload left behind. + leftovers = list(settings.upload_dir.iterdir()) + assert leftovers == [] + + +# ── Cancellation + deletion ─────────────────────────────────── + + +async def test_cancel_running_job(client): + from .conftest import SubprocessMock + + # Slow pipeline keeps the job in-flight until we cancel. + app = client._transport.app + app.state.runner._run_async = SubprocessMock(mode="slow") + + r = await post_job(client) + job_id = r.json()["job_id"] + await wait_for_status(client, job_id, {"running"}) + + c = await client.post(f"/jobs/{job_id}/cancel") + assert c.status_code == 202, c.text + final = c.json() + assert final["status"] == "cancelled" + assert final["stages"]["netconvert"] == "cancelled" + assert final["completed_at"] is not None + + # And it stays cancelled after the task unwinds. + done = await wait_for_status(client, job_id, {"cancelled"}) + assert done["error"] == "Cancelled by user" + + # Cancelled jobs can be deleted. + d = await client.delete(f"/jobs/{job_id}") + assert d.status_code == 204 + gone = await client.get(f"/jobs/{job_id}") + assert gone.status_code == 404 + + +async def test_cancel_unknown_and_finished(client, mock_subprocess): + r = await client.post("/jobs/does-not-exist/cancel") + assert r.status_code == 404 + + r = await post_job(client) + job_id = r.json()["job_id"] + await wait_for_status(client, job_id, {"done"}) + c = await client.post(f"/jobs/{job_id}/cancel") + assert c.status_code == 409 + + +async def test_delete_running_job_409(client): + from .conftest import SubprocessMock + + LongRunner = SubprocessMock(mode="slow") + client._transport.app.state.runner._run_async = LongRunner + r = await post_job(client) + job_id = r.json()["job_id"] + await wait_for_status(client, job_id, {"running"}) + d = await client.delete(f"/jobs/{job_id}") + assert d.status_code == 409 + # Cleanup: cancel so the lingering task unwinds. + await client.post(f"/jobs/{job_id}/cancel") + + +async def test_delete_done_job_removes_files(client, mock_subprocess, settings): + r = await post_job(client) + job_id = r.json()["job_id"] + await wait_for_status(client, job_id, {"done"}) + assert (settings.out_dir / job_id).exists() + + d = await client.delete(f"/jobs/{job_id}") + assert d.status_code == 204 + assert not (settings.out_dir / job_id).exists() + assert list(settings.upload_dir.glob(f"{job_id}*")) == [] + + +# ── Misc endpoints ──────────────────────────────────────────── + + +async def test_list_jobs_most_recent_first(client, mock_subprocess): + ids = [] + for _ in range(3): + ids.append((await post_job(client)).json()["job_id"]) + await asyncio.sleep(0.01) + listing = (await client.get("/jobs")).json() + listed = [j["job_id"] for j in listing] + assert listed[:3] == list(reversed(ids)) + for i in ids: + await wait_for_status(client, i, {"done"}) + + +async def test_get_unknown_job_404(client): + assert (await client.get("/jobs/nope")).status_code == 404 + assert (await client.get("/jobs/nope/files")).status_code == 404 + assert (await client.get("/jobs/nope/logs")).status_code == 404 + assert (await client.get("/jobs/nope/config")).status_code == 404 + + +async def test_download_path_traversal_400(client, mock_subprocess): + r = await post_job(client) + job_id = r.json()["job_id"] + await wait_for_status(client, job_id, {"done"}) + r = await client.get(f"/jobs/{job_id}/download/..%2F..%2Fetc%2Fpasswd") + assert r.status_code in (400, 404) # never a 200 with file bytes + + +async def test_logs_endpoint_shape(client, mock_subprocess): + r = await post_job(client) + job_id = r.json()["job_id"] + await wait_for_status(client, job_id, {"done"}) + body = (await client.get(f"/jobs/{job_id}/logs")).json() + assert set(body["stage_logs"]) == { + "netconvert", + "duarouter", + "sumo", + "traceExporter", + } + assert isinstance(body["warnings"], list) + + +async def test_config_roundtrip(client, mock_subprocess): + cfg = {"random_trips": {"seed": 99, "end": 120.0}, "sumo": {"end": 100.0}} + r = await post_job(client, config=cfg) + job_id = r.json()["job_id"] + await wait_for_status(client, job_id, {"done"}) + got = (await client.get(f"/jobs/{job_id}/config")).json() + assert got["random_trips"]["seed"] == 99 + assert got["sumo"]["end"] == 100.0 + assert got["trace_exporter"]["output_format"] == "both" # defaults filled + + +async def test_schema_and_defaults(client): + schema = (await client.get("/schema")).json() + assert "$defs" in schema and "NetconvertParams" in schema["$defs"] + defaults = (await client.get("/defaults")).json() + assert defaults["random_trips"]["end"] == 3600.0 + + +async def test_system_endpoints(client): + assert (await client.get("/live")).json() == {"status": "ok"} + # no_sumo fixture pretends tools exist → ready 200 + ready = await client.get("/ready") + assert ready.status_code == 200 + assert ready.json()["status"] == "ready" + assert ready.json()["tools_available"] is True + health = (await client.get("/health")).json() + assert health["status"] == "ready" + assert set(health["tools"]) == { + "netconvert", + "duarouter", + "sumo", + "traceExporter.py", + "randomTrips.py", + } + + +async def test_ready_503_when_tools_missing(settings, monkeypatch): + from asgi_lifespan import LifespanManager + from httpx import ASGITransport, AsyncClient + + from main import create_app + + def missing_tools(sumo_home: str = ""): + return { + "netconvert": False, + "duarouter": True, + "sumo": True, + "traceExporter.py": True, + "randomTrips.py": True, + } + + monkeypatch.setattr("main.check_tools", missing_tools) + app = create_app(settings) + async with LifespanManager(app): + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://t" + ) as c: + r = await c.get("/ready") + assert r.status_code == 503 + assert r.json()["status"] == "not_ready" + assert r.json()["tools"]["netconvert"] is False + + +# ── Auth ────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "method,path", + [ + ("GET", "/jobs"), + ("POST", "/jobs"), + ("GET", "/jobs/some-id"), + ("GET", "/jobs/some-id/files"), + ("GET", "/jobs/some-id/logs"), + ("GET", "/jobs/some-id/config"), + ("DELETE", "/jobs/some-id"), + ("POST", "/jobs/some-id/cancel"), + ("GET", "/jobs/some-id/download/x"), + ], +) +async def test_jobs_routes_require_key_when_enabled(authed_client, method, path): + r = await authed_client.request(method, path) + assert r.status_code == 401 + r = await authed_client.request(method, path, headers={"X-API-Key": "wrong"}) + assert r.status_code == 401 + + +async def test_auth_accepts_correct_key(authed_client): + r = await authed_client.get("/jobs", headers={"X-API-Key": "test-secret"}) + assert r.status_code == 200 + + +async def test_public_endpoints_open_with_auth_enabled(authed_client): + for path in ("/live", "/ready", "/health", "/schema", "/defaults", "/docs"): + r = await authed_client.get(path) + assert r.status_code != 401, path + + +async def test_auth_disabled_by_default(client): + assert (await client.get("/jobs")).status_code == 200 + + +# ── Persistence across restarts ─────────────────────────────── + + +async def test_jobs_survive_restart(settings): + """A completed job is still listed after a 'restart' (new app + lifespan + over the same data dir).""" + from asgi_lifespan import LifespanManager + from httpx import ASGITransport, AsyncClient + + from main import create_app + + from .conftest import SubprocessMock + + ok = SubprocessMock(mode="ok") + + app1 = create_app(settings) + async with LifespanManager(app1): + app1.state.runner._run_async = ok + async with AsyncClient( + transport=ASGITransport(app=app1), base_url="http://t" + ) as c: + job_id = (await post_job(c)).json()["job_id"] + await wait_for_status(c, job_id, {"done"}) + + app2 = create_app(settings) # same data_dir = same jobs.db + async with LifespanManager(app2): + async with AsyncClient( + transport=ASGITransport(app=app2), base_url="http://t" + ) as c: + r = await c.get(f"/jobs/{job_id}") + assert r.status_code == 200 + assert r.json()["status"] == "done" + + +async def test_interrupted_running_job_fails_on_restart(settings): + """A row stuck at "running" from a dead process is failed at startup. + + Exactly what a crashed server leaves behind: the DB keeps the last-write + state and no API call ever marks it otherwise.""" + from asgi_lifespan import LifespanManager + from httpx import ASGITransport, AsyncClient + + from app.models import JobStatus, StageStatus + from app.store import JobStore + from main import create_app + + seed_store = JobStore(settings.db_path) + await seed_store.init() + stuck = JobStatus( + job_id="stuck-job", + status=StageStatus.running, + stages={ + "netconvert": StageStatus.done, + "random_trips": StageStatus.done, + "duarouter": StageStatus.done, + "sumo": StageStatus.running, + "trace_export": StageStatus.pending, + }, + created_at="2026-08-26T00:00:00+00:00", + ) + await seed_store.create(stuck, "{}") + + app2 = create_app(settings) + async with LifespanManager(app2): + async with AsyncClient( + transport=ASGITransport(app=app2), base_url="http://t" + ) as c: + r = await c.get("/jobs/stuck-job") + assert r.status_code == 200 + body = r.json() + assert body["status"] == "failed" + assert "Interrupted by server restart" in body["error"] + assert body["stages"]["sumo"] == "failed" diff --git a/tests/test_cancel.py b/tests/test_cancel.py new file mode 100644 index 0000000..0e1e095 --- /dev/null +++ b/tests/test_cancel.py @@ -0,0 +1,136 @@ +"""Focused tests for the cancellation machinery in PipelineRunner.""" + +import asyncio +import signal + +from app.models import ConversionRequest, StageStatus + +from .conftest import OSM_BYTES, SubprocessMock + + +async def _launch(runner, tmp_path, config=None): + upload = tmp_path / "upload.osm" + upload.write_bytes(OSM_BYTES) + job = await runner.create_and_launch(upload, config or ConversionRequest()) + return job + + +async def test_cancel_kills_subprocess_group(runner, store, monkeypatch, tmp_path): + """cancel() must kill the whole process group before task.cancel().""" + killed_pids = [] + + def spy_killpg(pid, sig): + killed_pids.append((pid, sig)) + + monkeypatch.setattr("app.worker.os.killpg", spy_killpg) + + # Slow mock: the pipeline never finishes a stage on its own, so the job + # stays pending/running through the cancel — deterministic regardless of + # how the event loop schedules the freshly-launched background task. + runner._run_async = SubprocessMock(mode="slow") + + class FakeProc: + def __init__(self): + self.pid = 4242 + self.returncode = 0 + + fake = FakeProc() + fake.returncode = None # still running + + await store.init() + job = await _launch(runner, tmp_path) + await store.get(job.job_id) # row materialised + runner._procs.setdefault(job.job_id, set()).add(fake) + + result = await runner.cancel(job.job_id) + assert result == "cancelled" + assert killed_pids == [(4242, signal.SIGKILL)] + + job = await store.get(job.job_id) + assert job.status == StageStatus.cancelled + assert job.error == "Cancelled by user" + + # Task unwinds and registries are cleaned. + await asyncio.sleep(0.05) + assert job.job_id not in runner._tasks + assert job.job_id not in runner._procs + + +async def test_cancel_queued_pending_job(runner, store, tmp_path): + """A job waiting on the concurrency semaphore can be cancelled.""" + runner._semaphore = asyncio.Semaphore(0) # never grants a slot + + # Prevent the pipeline from touching the fake workdir deeply: the slow + # mock never runs because the semaphore blocks first. + runner._run_async = SubprocessMock(mode="slow") + + await store.init() + job = await _launch(runner, tmp_path) + # Give _guarded_run a chance to reach the semaphore. + for _ in range(50): + persisted = await store.get(job.job_id) + if persisted.status != StageStatus.cancelled: + break + await asyncio.sleep(0.01) + + assert (await runner.cancel(job.job_id)) == "cancelled" + job = await store.get(job.job_id) + assert job.status == StageStatus.cancelled + assert all(s == StageStatus.pending for s in job.stages.values()) + + +async def test_cancel_unknown_and_finished(runner, store, mock_subprocess, tmp_path): + await store.init() + try: + await runner.cancel("ghost") + assert False, "expected LookupError" + except LookupError: + pass + + job = await _launch(runner, tmp_path) + # Let the fully-mocked pipeline finish. + for _ in range(500): + j = await store.get(job.job_id) + if j.status == StageStatus.done: + break + await asyncio.sleep(0.01) + else: + raise AssertionError("pipeline did not finish") + + result = await runner.cancel(job.job_id) + assert result == "done" # not cancellable anymore + + +async def test_cancelled_error_survives_guarded_run_cleanly( + runner, store, monkeypatch, tmp_path +): + """_guarded_run swallows CancelledError and leaves no registry entries.""" + runner._run_async = SubprocessMock(mode="slow") + + await store.init() + job = await _launch(runner, tmp_path) + + # Wait for the task to actually be running a stage. + for _ in range(500): + j = await store.get(job.job_id) + if j.status == StageStatus.running: + break + await asyncio.sleep(0.01) + else: + raise AssertionError("job never entered running state") + + task = runner._tasks[job.job_id] + result = await runner.cancel(job.job_id) + assert result == "cancelled" + + await asyncio.sleep(0.05) # let _guarded_run's finally run + assert task.done() + assert not task.cancelled() # _guarded_run swallowed it deliberately + assert job.job_id not in runner._tasks + assert job.job_id not in runner._procs + + j = await store.get(job.job_id) + assert j.status == StageStatus.cancelled + assert j.stages["netconvert"] == StageStatus.cancelled + assert j.stages["random_trips"] == StageStatus.pending + assert j.completed_at is not None diff --git a/tests/test_commands.py b/tests/test_commands.py new file mode 100644 index 0000000..496b5a6 --- /dev/null +++ b/tests/test_commands.py @@ -0,0 +1,280 @@ +"""Tests for the pure command builders and XML writers in app/worker.py. +SUMO lookup is mocked by the autouse no_sumo fixture in conftest.""" + +from pathlib import Path + +from app.models import ( + LaneChangeMode, + NetconvertParams, + RandomTripsParams, + SpeedMode, + SumoParams, + TraceExporterParams, + VehicleClass, +) +from app.worker import ( + _duarouter_cmd, + _lc_mode_int, + _netconvert_cmd, + _random_trips_cmd, + _speed_mode_int, + _sumo_cmd, + _trace_export_cmds, + _write_sumo_cfg, + _write_vtype_xml, + check_tools, +) + +OSM = Path("/in/map.osm") +NET = Path("/w/network.net.xml") + + +def pairs(cmd): + """Dict view of a [--flag, value, ...] argv (flags starting with '--').""" + out = {} + i = 0 + while i < len(cmd): + if cmd[i].startswith("--"): + out[cmd[i]] = ( + cmd[i + 1] + if i + 1 < len(cmd) and not cmd[i + 1].startswith("--") + else None + ) + i += 2 if out[cmd[i]] is not None else 1 + else: + i += 1 + return out + + +# ── netconvert ──────────────────────────────────────────────── + + +def test_netconvert_basics(): + cmd = _netconvert_cmd(OSM, NET, NetconvertParams()) + p = pairs(cmd) + assert cmd[0] == "/mock/bin/netconvert" + assert p["--osm-files"] == str(OSM) + assert p["--output-file"] == str(NET) + assert p["--proj.utm"] == "true" + assert p["--osm.highway.type-tag"] == ",".join(NetconvertParams().osm_highway_types) + # Highway types are joined into ONE argv item (comma-separated). + tag_idx = cmd.index("--osm.highway.type-tag") + assert "," in cmd[tag_idx + 1] + # (type-tag may be the final flag when no conditionals are enabled) + + +def test_netconvert_optional_flags(): + default_cmd = _netconvert_cmd(OSM, NET, NetconvertParams()) + for flag in ( + "--no-turnarounds", + "--no-left-connections", + "--remove-edges.by-type", + "--proj.plain-geo", + "--roundabouts.guess", + "--osm.oneway-spread-right", + "--keep-edges.fringe-edges", + ): + assert flag not in default_cmd + + p = NetconvertParams( + no_turnarounds=True, + no_left_connections=True, + remove_edges_by_type="footway,path", + proj_utm=False, + proj_plain_geo=True, + osm_no_large_roundabouts=True, + osm_oneway_spread=True, + keep_fringe="noFringe", + ) + cmd = _netconvert_cmd(OSM, NET, p) + assert pairs(cmd)["--no-turnarounds"] == "true" + assert pairs(cmd)["--remove-edges.by-type"] == "footway,path" + assert pairs(cmd)["--proj.plain-geo"] == "true" + assert pairs(cmd)["--roundabouts.guess"] == "false" + assert pairs(cmd)["--keep-edges.fringe-edges"] == "false" + + +def test_netconvert_plain_geo_requires_no_utm(): + # proj_utm=True + proj_plain_geo=True is rejected by the model validator + # (mutual exclusion). Only plain_geo alone emits the flag. + plain_only = _netconvert_cmd( + OSM, NET, NetconvertParams(proj_utm=False, proj_plain_geo=True) + ) + assert "--proj.plain-geo" in plain_only + assert pairs(plain_only)["--proj.utm"] == "false" + + +# ── randomTrips ─────────────────────────────────────────────── + + +def test_random_trips_basics(): + cmd = _random_trips_cmd(NET, Path("/w/trips.xml"), RandomTripsParams()) + p = pairs(cmd) + assert cmd[0] == "python3" + assert cmd[1] == "/mock/tools/randomTrips.py" + assert p["--seed"] == "42" + assert "--max-distance" not in p + assert "--insertion-density" not in p + # validate_routes defaults to True → --validate IS present by default + assert "--validate" in cmd + + +def test_random_trips_optional_args(): + params = RandomTripsParams( + max_distance=5000.0, + insertion_density=12.0, + validate_routes=True, + random_depart_pos=True, + random_arrival_pos=True, + allow_loops=True, + ) + cmd = _random_trips_cmd(NET, Path("/w/trips.xml"), params) + p = pairs(cmd) + assert p["--max-distance"] == "5000.0" + assert p["--insertion-density"] == "12.0" + assert "--validate" in cmd + assert "--random-depart-pos" in cmd + assert "--random-arrival-pos" in cmd + assert "--allow-fringe.min-length" in cmd + + +# ── duarouter / sumo ────────────────────────────────────────── + + +def test_duarouter_cmd(): + cmd = _duarouter_cmd( + NET, Path("/w/trips.xml"), Path("/w/routes.rou.xml"), RandomTripsParams() + ) + p = pairs(cmd) + assert cmd[0] == "/mock/bin/duarouter" + assert p["--ignore-errors"] == "true" + assert "--no-step-log" in cmd + + +def test_sumo_cmd(): + cmd = _sumo_cmd(Path("/w/sim.sumocfg"), Path("/w/vtypes.add.xml")) + assert cmd[0] == "/mock/bin/sumo" + assert pairs(cmd)["--configuration-file"] == "/w/sim.sumocfg" + assert pairs(cmd)["--additional-files"] == "/w/vtypes.add.xml" + + +# ── traceExporter ───────────────────────────────────────────── + + +FCD = Path("/w/fcd-output.xml") +WORK = Path("/w") + + +def test_trace_export_both_formats(): + specs = _trace_export_cmds(FCD, WORK, TraceExporterParams()) # both + assert [s[0] for s in specs] == ["ns_movements", "tcl"] + assert specs[0][2].name == "ns_movements" + assert specs[1][2].name == "mobility.tcl" + # ns_movements carries the speed flag by default, tcl does not + assert "--ns2-include-speed" in specs[0][1] + assert "--ns2-include-speed" not in specs[1][1] + + +def test_trace_export_single_formats(): + ns_only = _trace_export_cmds( + FCD, WORK, TraceExporterParams(output_format="ns_movements") + ) + assert len(ns_only) == 1 and ns_only[0][0] == "ns_movements" + tcl_only = _trace_export_cmds(FCD, WORK, TraceExporterParams(output_format="tcl")) + assert len(tcl_only) == 1 and tcl_only[0][0] == "tcl" + + +def test_trace_export_optional_args(): + params = TraceExporterParams(end=500.0, begin=10.0, boundary="0,0,100,100") + cmd = _trace_export_cmds(FCD, WORK, params)[0][1] + p = pairs(cmd) + assert p["--end"] == "500.0" + assert p["--begin"] == "10.0" + assert p["--boundary"] == "0,0,100,100" + base = _trace_export_cmds(FCD, WORK, TraceExporterParams())[0][1] + assert "--end" not in pairs(base) + assert "--boundary" not in pairs(base) + + +# ── Bitmask mappings ────────────────────────────────────────── + + +def test_mode_bitmasks(): + assert _speed_mode_int(SpeedMode.right_of_way) == 31 + assert _speed_mode_int(SpeedMode.no_checks) == 32 + assert _speed_mode_int(SpeedMode.all_checks) == 31 + assert _lc_mode_int(LaneChangeMode.default) == 1621 + assert _lc_mode_int(LaneChangeMode.no_lc) == 0 + assert _lc_mode_int(LaneChangeMode.strategic_only) == 256 + + +# ── XML writers ─────────────────────────────────────────────── + + +async def test_write_sumo_cfg(tmp_path): + # fcd_output_period must be a whole multiple of step_length. + params = SumoParams(step_length=0.5, seed=7, fcd_output_period=0.5) + cfg = await _write_sumo_cfg( + tmp_path, + NET, + Path("/w/routes.rou.xml"), + Path("/w/fcd.xml"), + Path("/w/trip.xml"), + Path("/w/sum.xml"), + params, + ) + text = cfg.read_text() + assert '' in text + assert '' in text + assert "tripinfo-output" in text + assert "summary-output" in text + + sparse = SumoParams( + tripinfo_output=False, summary_output=False, fcd_output_period=0.1 + ) + cfg2 = await _write_sumo_cfg( + tmp_path, + NET, + Path("/w/routes.rou.xml"), + Path("/w/fcd.xml"), + Path("/w/trip.xml"), + Path("/w/sum.xml"), + sparse, + ) + text2 = cfg2.read_text() + assert "tripinfo-output" not in text2 + assert "summary-output" not in text2 + + +async def test_write_vtype_xml(tmp_path): + params = SumoParams(default_decel=4.5, default_emergency_decel=9.0) + path = await _write_vtype_xml(tmp_path, params, VehicleClass.bus) + text = path.read_text() + assert 'vClass="bus"' in text + assert 'speedMode="31"' in text + assert 'lcMode="1621"' in text + assert 'emergencyDecel="9.0"' in text + + +# ── Tool checking ───────────────────────────────────────────── + + +def test_check_tools_all_found(no_sumo): + tools = check_tools("") + assert all(tools.values()) + assert set(tools) == { + "netconvert", + "duarouter", + "sumo", + "traceExporter.py", + "randomTrips.py", + } + + +def test_check_tools_missing(monkeypatch): + import app.worker as worker_mod + + monkeypatch.setattr(worker_mod, "_find_binary", lambda names: None) + monkeypatch.setattr(worker_mod, "_find_sumo_script", lambda f, h="": None) + tools = check_tools("") + assert not any(tools.values()) diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..9d5f29d --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,259 @@ +"""Validation tests for app/models.py — the 74 pipeline parameters and all +cross-field / cross-stage validators. No I/O; everything is in-memory.""" + +import pytest +from pydantic import ValidationError + +from app.models import ( + ConversionRequest, + NetconvertParams, + RandomTripsParams, + SumoParams, + TraceExporterParams, +) + + +def _bad(cls, **kw): + with pytest.raises(ValidationError): + cls(**kw) + + +# ── Parametrised numeric bound sweep (ge/le from Field metadata) ────────── + +BOUND_CASES = [ + # (model, field, below-min, above-max) + (NetconvertParams, "junctions_internal_link_detail", 0, 21), + (NetconvertParams, "junctions_corner_detail", 0, 21), + (NetconvertParams, "junctions_min_size", -0.1, 20.1), + (NetconvertParams, "junctions_limit_turn_speed", -0.1, 30.1), + (NetconvertParams, "default_lane_width", 0.9, 10.1), + (NetconvertParams, "default_speed_limit", 0.9, 83.4), + (NetconvertParams, "default_num_lanes", 0, 9), + (NetconvertParams, "tl_min_dur", 0, 121), + (NetconvertParams, "tl_max_dur", 4, 301), + (RandomTripsParams, "begin", -0.1, None), + (RandomTripsParams, "end", 0.5, None), + (RandomTripsParams, "period", 0.05, 3600.1), + (RandomTripsParams, "insertion_density", -0.1, None), + (RandomTripsParams, "min_distance", -0.1, None), + (RandomTripsParams, "max_distance", -0.1, None), + (RandomTripsParams, "fringe_factor", -0.1, 100.1), + (RandomTripsParams, "fringe_threshold", -0.1, 1.1), + (RandomTripsParams, "seed", -1, None), + (SumoParams, "begin", -0.1, None), + (SumoParams, "end", 0.5, None), + (SumoParams, "step_length", 0.009, 10.1), + (SumoParams, "default_max_speed", 0.9, 200.1), + (SumoParams, "default_accel", 0.09, 20.1), + (SumoParams, "default_decel", 0.09, 20.1), + (SumoParams, "default_emergency_decel", 0.09, 30.1), + (SumoParams, "default_sigma", -0.1, 1.1), + (SumoParams, "default_tau", 0.09, 10.1), + (SumoParams, "default_vehicle_length", 0.9, 30.1), + (SumoParams, "default_min_gap", -0.1, 20.1), + (SumoParams, "lateral_resolution", -0.1, 5.1), + (SumoParams, "fcd_output_period", 0.009, 60.1), + (SumoParams, "seed", -1, None), + (TraceExporterParams, "begin", -0.1, None), + (TraceExporterParams, "sampling_period", 0.05, 60.1), + (TraceExporterParams, "penetration_rate", 0.009, 1.01), + (TraceExporterParams, "seed", -1, None), +] + + +@pytest.mark.parametrize( + "cls,field,bad_low,bad_high", + BOUND_CASES, + ids=[f"{c.__name__}.{f}" for c, f, _, _ in BOUND_CASES], +) +def test_numeric_bounds(cls, field, bad_low, bad_high): + # Values at the boundary must validate (default fill for others). + if bad_low is not None: + _bad(cls, **{field: bad_low}) + if bad_high is not None: + _bad(cls, **{field: bad_high}) + + +def test_highway_types_non_empty(): + _bad(NetconvertParams, osm_highway_types=[]) + + +# ── Enumerations ───────────────────────────────────────────── + + +def test_enum_rejects_unknown_value(): + _bad(RandomTripsParams, vehicle_class="car") + _bad(NetconvertParams, tl_type="banana") + _bad(TraceExporterParams, output_format="gpx") + + +def test_enum_wire_values_accepted(): + """The API/frontend exchange enum *values* (e.g. SOTL_PHASE), not names.""" + assert NetconvertParams(tl_type="SOTL_PHASE").tl_type.value == "SOTL_PHASE" + assert RandomTripsParams(vehicle_class="bus").vehicle_class.value == "bus" + assert ( + TraceExporterParams(output_format="ns_movements").output_format.value + == "ns_movements" + ) + + +# ── Netconvert validators ──────────────────────────────────── + + +def test_highway_type_names_validated(): + _bad(NetconvertParams, osm_highway_types=["motorway", "foo"]) + ok = NetconvertParams(osm_highway_types=["primary", "secondary"]) + assert ok.osm_highway_types == ["primary", "secondary"] + + +def test_remove_edges_by_type_validated_and_normalised(): + _bad(NetconvertParams, remove_edges_by_type="foo,bar") + ok = NetconvertParams(remove_edges_by_type=" footway , path ") + assert ok.remove_edges_by_type == "footway,path" + + +def test_projection_mutual_exclusion(): + _bad(NetconvertParams, proj_utm=True, proj_plain_geo=True) + + +def test_tl_duration_ordering(): + _bad(NetconvertParams, tl_min_dur=30, tl_max_dur=10) + + +# ── RandomTrips validators ─────────────────────────────────── + + +def test_random_trips_timeline(): + _bad(RandomTripsParams, begin=10, end=10) + _bad(RandomTripsParams, begin=20, end=10) + + +def test_random_trips_distance_bounds(): + _bad(RandomTripsParams, min_distance=500, max_distance=100) + + +# ── SUMO validators ────────────────────────────────────────── + + +def test_sumo_timeline(): + _bad(SumoParams, begin=5.0, end=5.0) + + +def test_fcd_period_multiple_of_step_length(): + # 0.3 / 0.2 = 1.5 — not a whole number of steps + _bad(SumoParams, fcd_output_period=0.3, step_length=0.2) + # 0.3 / 0.1 = 3 — exactly divisible + SumoParams(fcd_output_period=0.3, step_length=0.1) + + +def test_decel_ordering(): + _bad(SumoParams, default_decel=10.0, default_emergency_decel=9.0) + SumoParams(default_decel=4.5, default_emergency_decel=9.0) + + +def test_collision_action_choices(): + _bad(SumoParams, collision_action="kill") + for choice in ("warn", "teleport", "remove", "none"): + assert SumoParams(collision_action=choice).collision_action == choice + + +# ── TraceExporter validators ───────────────────────────────── + + +def test_boundary_format(): + _bad(TraceExporterParams, boundary="not-a-boundary") + _bad(TraceExporterParams, boundary="1,2,3") # too few values + # Inverted axes (xMin > xMax) rejected + _bad(TraceExporterParams, boundary="10,0,0,10") + ok = TraceExporterParams(boundary="0,0,10,10") + assert ok.boundary == "0,0,10,10" + + +def test_export_timeline_when_end_set(): + _bad(TraceExporterParams, begin=100, end=50) + + +# ── Cross-stage validators on ConversionRequest ───────────── + + +def test_cross_stage_sumo_vs_random_trips_end(): + _bad(ConversionRequest, sumo={"end": 3600.0}, random_trips={"end": 1000.0}) + + +def test_cross_stage_trace_exporter_vs_sumo_end(): + _bad( + ConversionRequest, + sumo={"end": 100.0}, + random_trips={"end": 1000.0}, + trace_exporter={"end": 200.0}, + ) + # trace_exporter.end=None (match simulation end) is always fine + ConversionRequest( + sumo={"end": 100.0}, random_trips={"end": 1000.0}, trace_exporter={"end": None} + ) + + +def test_cross_stage_sampling_finer_than_fcd(): + _bad( + ConversionRequest, + sumo={"fcd_output_period": 1.0}, + trace_exporter={"sampling_period": 0.5}, + ) + + +def test_cross_stage_sampling_non_integer_ratio(): + _bad( + ConversionRequest, + sumo={"fcd_output_period": 1.0}, + trace_exporter={"sampling_period": 1.5}, + ) + ConversionRequest( + sumo={"fcd_output_period": 0.5}, trace_exporter={"sampling_period": 1.5} + ) + + +# ── Defaults & schema contract ─────────────────────────────── + + +def test_defaults_construct_without_input(): + req = ConversionRequest() + assert req.random_trips.end == 3600.0 + assert req.trace_exporter.output_format.value == "both" + # Round-trip through the wire representation. + ConversionRequest(**req.model_dump()) + + +def test_json_schema_shape_for_form_generator(): + """Locks the schema contract the frontend form generator relies on.""" + schema = ConversionRequest.model_json_schema(mode="validation") + assert set(schema["properties"]) == { + "netconvert", + "random_trips", + "sumo", + "trace_exporter", + } + defs = schema["$defs"] + # Section properties are $ref'ed models… + for prop in schema["properties"].values(): + assert prop["$ref"].startswith("#/$defs/") + # …each a flat property bag. + for section in ( + "NetconvertParams", + "RandomTripsParams", + "SumoParams", + "TraceExporterParams", + ): + props = defs[section]["properties"] + assert props + for name, field in props.items(): + # Field must resolve to a concrete primitive/enum for the form. + if "anyOf" in field: + variants = field["anyOf"] + nulls = [v for v in variants if v.get("type") == "null"] + assert len(nulls) == 1, f"{section}.{name}: anyOf w/o null" + assert ( + len(variants) - len(nulls) == 1 + ), f"{section}.{name}: multi-variant anyOf unsupported by form" + # Enum wire values: TLType serialises values like "SOTL_PHASE". + assert "SOTL_PHASE" in defs["TLType"]["enum"] + assert "passenger" in defs["VehicleClass"]["enum"] diff --git a/tests/test_store.py b/tests/test_store.py new file mode 100644 index 0000000..51d3f1a --- /dev/null +++ b/tests/test_store.py @@ -0,0 +1,149 @@ +"""Unit tests for the SQLite JobStore: round-trips, recovery, retention.""" + +import asyncio +from datetime import datetime, timedelta, timezone + +from app.models import JobStatus, StageStatus + + +def make_job(job_id: str, status: StageStatus = StageStatus.pending) -> JobStatus: + return JobStatus( + job_id=job_id, + status=status, + stages={ + "netconvert": StageStatus.pending, + "random_trips": StageStatus.pending, + "duarouter": StageStatus.pending, + "sumo": StageStatus.pending, + "trace_export": StageStatus.pending, + }, + stage_logs={}, + created_at=datetime.now(timezone.utc).isoformat(), + ) + + +async def test_create_get_roundtrip(store): + job = make_job("j1") + job.warnings = ["w1", "w2"] + job.outputs = ["/outputs/j1/ns_movements"] + job.current_stage = "sumo" + job.stages["netconvert"] = StageStatus.done + job.stage_logs["netconvert"] = { + "status": "done", + "stdout": "out", + "stderr": None, + "duration": 1.2, + } + await store.init() + await store.create(job, '{"sumo": {"seed": 7}}') + + got = await store.get("j1") + assert got is not None + assert got.model_dump() == job.model_dump() + + cfg = await store.get_config("j1") + assert cfg == {"sumo": {"seed": 7}} + + +async def test_unknown_job(store): + await store.init() + assert await store.get("nope") is None + assert await store.get_config("nope") is None + + +async def test_upsert_and_list(store): + await store.init() + job = make_job("j2") + await store.create(job, None) + job.status = StageStatus.done + job.completed_at = datetime.now(timezone.utc).isoformat() + await store.upsert(job) + + jobs = await store.list() + assert [j.job_id for j in jobs] == ["j2"] + assert jobs[0].status == StageStatus.done + assert jobs[0].completed_at is not None + + +async def test_delete(store): + await store.init() + await store.create(make_job("j3"), None) + await store.delete("j3") + assert await store.get("j3") is None + + +async def test_startup_recovery_fails_interrupted_jobs(store): + """pending/running rows from a previous process become failed.""" + await store.init() + running = make_job("j-run", status=StageStatus.running) + running.stages["netconvert"] = StageStatus.done + running.stages["sumo"] = StageStatus.running + waiting = make_job("j-wait", status=StageStatus.pending) + finished = make_job("j-done", status=StageStatus.done) + running.completed_at = None + for j in (running, waiting, finished): + await store.create(j, None) + + # Simulate the next process booting over the same DB file. + recovered = await store._recover_interrupted() + assert set(recovered) == {"j-run", "j-wait"} + + j = await store.get("j-run") + assert j.status == StageStatus.failed + assert j.error == "Interrupted by server restart" + assert j.completed_at is not None + assert j.stages["sumo"] == StageStatus.failed + assert j.stages["netconvert"] == StageStatus.done # untouched + + assert (await store.get("j-done")).status == StageStatus.done + + +async def test_prune_by_age(store): + store._retention_days = 1 + await store.init() + old = make_job("j-old", status=StageStatus.done) + old.completed_at = (datetime.now(timezone.utc) - timedelta(days=3)).isoformat() + fresh = make_job("j-fresh", status=StageStatus.done) + fresh.completed_at = datetime.now(timezone.utc).isoformat() + active = make_job("j-active", status=StageStatus.running) + for j in (old, fresh, active): + await store.create(j, None) + + pruned = await store.prune() + assert pruned == ["j-old"] + assert await store.get("j-old") is None + assert await store.get("j-fresh") is not None + assert await store.get("j-active") is not None + + +async def test_prune_by_count(store): + await store.init() + now = datetime.now(timezone.utc) + for i in range(5): + j = make_job(f"j-{i}", status=StageStatus.done) + j.completed_at = (now + timedelta(seconds=i)).isoformat() + await store.create(j, None) + + store._retention_max = 2 # keep only the 2 newest finished jobs + pruned = await store.prune() + remaining = {j.job_id for j in await store.list()} + assert remaining == {"j-4", "j-3"} + assert set(pruned) == {"j-0", "j-1", "j-2"} + + +async def test_concurrent_upserts(store): + await store.init() + jobs = [make_job(f"cc-{i}") for i in range(10)] + for j in jobs: + await store.create(j, None) + + async def bump(j): + j.status = StageStatus.done + j.warnings.append("x") + await store.upsert(j) + + await asyncio.gather(*[bump(j) for j in jobs]) + listed = await store.list() + assert len(listed) == 10 + assert all(j.status == StageStatus.done for j in listed) + assert all(j.warnings == ["x"] for j in listed)