diff --git a/frontend/.env.example b/frontend/.env.example new file mode 100644 index 00000000..549cfae6 --- /dev/null +++ b/frontend/.env.example @@ -0,0 +1,5 @@ +# Absolute URL of a deployed server (frontend/server), no trailing slash. +# Leave empty for local development: Vite proxies /api to http://localhost:8000. +VITE_API_URL= +# Base path of the built site ("/" for root hosting). +VITE_BASE=/ diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 00000000..52bd796c --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +dist/ +.env.local +__pycache__/ +.pytest_cache/ +package-lock.json diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 00000000..295747d5 --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,123 @@ +# SWMManywhere web frontend + +Contributed by [Zhonghao Zhang](https://github.com/Zhonghao1995). + +![The web frontend after building a small box in Andorra la Vella: parameter panel on the left, synthesised pipes, manholes and outfalls on the map, and a manhole's attributes and simulated flooding series](screenshot.png) + +A browser interface for SWMManywhere: draw a bounding box on a map, adjust the +package's parameters, build, inspect the synthesised network on the map, click any +element for its attributes and simulated time series, and download the model +package. Every build is an ordinary SWMManywhere `config.yml`; the page is a form +for that file, and it shows you the file it used so the run can be reproduced with +`python -m swmmanywhere --config_path=...`. + +Nothing under `src/swmmanywhere` is modified. The folder is self-contained: + +| Path | What it is | +|---|---| +| `src/` | The web app: React, Vite, MapLibre GL, Zustand, Tailwind (TypeScript). | +| `server/` | A small FastAPI service that runs the package for the browser. It calls only the public API (`load_config`, `swmmanywhere`, `run` from `swmmanywhere.swmmanywhere`). | +| `server/tests/` | Contract tests that use the package's bundled Bellinge fixtures; no network, no SWMM. | + +A browser cannot run Python, so the service is the smallest possible bridge: it +writes the config, validates it with `load_config`, runs the synthesis in a worker +thread, runs the SWMM simulation with `run`, and serves the nodes / edges / +subcatchments as GeoJSON (outfalls identified the same way as +`swmmanywhere.utilities.plot_map`) plus per-element time series from the results. + +## Run it + +From the repository root, in the environment where `swmmanywhere` is installed: + +```bash +pip install -e . +pip install -r frontend/server/requirements.txt +cd frontend +uvicorn server.app:app --port 8000 # API on http://localhost:8000 +npm install && npm run dev # UI on http://localhost:5174 (proxies /api) +``` + +For a single-origin deployment build the app once (`npm run build`); the server +then serves `frontend/dist` at `/`. + +The page uses the docs' primary colour (`#00BCD4`, see `docs/custom.css`) as its +only accent and loads the Geist typeface from Google Fonts; drop the `` in +`index.html` for a fully offline build, the system font stack is the fallback. + +| Environment variable | Meaning | Default | +|---|---|---| +| `SWMMANYWHERE_WEB_WORKDIR` | `base_dir` for every build; downloads are reused per bounding box (SWMManywhere's `bbox_N/model_N` layout). | `/swmmanywhere_web` | +| `SWMMANYWHERE_WEB_MAX_KM2` | Largest bounding box accepted. | `5` | +| `SWMMANYWHERE_WEB_ORIGINS` | Comma-separated CORS origins. | `*` | + +Builds run one at a time: the package shares `base_dir` between runs and its logger +is process-global. The first build of a box spends a few minutes downloading +(NASADEM, Overture buildings, OpenStreetMap); rebuilding the same box with other +parameters takes about ten seconds for the documentation's Andorra example (0.6 km²), +including the SWMM run. + +## What the page exposes + +| Page section | Config key | Notes | +|---|---|---| +| Bounding box | `bbox` | Drawn with two clicks or typed; the area limit is enforced server-side. | +| Project | `project` | Folder name under `base_dir`. | +| Parameters | `parameter_overrides` | Rendered from the pydantic models in `swmmanywhere.parameters` (defaults, units, bounds, descriptions). Only changed values are sent. `metric_evaluation` is hidden because the page does not supply a `real` network. | +| Graph functions | `graphfcn_list` | The default list from `defs/demo_config.yml`, editable; the server validates it with `validate_graphfcn_list` via `load_config`. | +| Simulation | `run_model`, `run_settings` | `duration`, `reporting_iters`, `storevars` (the enum from `defs/schema.yml`). | + +Rainfall is the package's bundled demo storm (`defs/storm.dat`), exactly what the +CLI uses when no precipitation file is given. Real-network comparison (`real:`, +metrics), `starting_graph`, and custom modules are not exposed. + +The server additionally range-checks overrides with the package's own pydantic +models before building, because `swmmanywhere()` applies overrides with `setattr` +and only checks the names (issue #379). + +## HTTP surface + +| Method and path | Purpose | +|---|---| +| `GET /api/v1/defaults` | Parameter groups, graph functions, run settings, limits, all read from the package. | +| `POST /api/v1/tasks` | Start a build. Body: `bbox`, `project`, `parameter_overrides`, `graphfcn_list`, `run_model`, `run_settings`. Returns `task_id`. | +| `GET /api/v1/tasks/{id}` | State, stage, progress, last log lines, error. | +| `GET /api/v1/tasks/{id}/config` | The `config.yml` that was validated and run. | +| `GET /api/v1/tasks/{id}/preview` | GeoJSON (EPSG:4326) with `kind` = `junction`, `outfall`, `conduit`, `subcatchment`. | +| `GET /api/v1/tasks/{id}/timeseries?id=` | Simulated series for one SWMM object (`flooding`, `flow`, `depth`, `runoff`). | +| `GET /api/v1/tasks/{id}/result` | Zip of the model directory: `.inp`, nodes, edges, subcatchments, `config.yml`, `results.parquet`. The rain file is included and the `.inp` inside the zip refers to it by name so it opens in EPA SWMM as-is. | + +## Tests + +```bash +cd frontend +pytest server/tests # 8 tests, bundled fixtures only +npm run typecheck +``` + +## Things found while building this + +- **Overture release id.** `prepare_data._get_latest_s3_url` derives the release + from `Path(href).parent` of the STAC catalog's child links. Those hrefs are now + absolute URLs, so the package caches `https:/stac.overturemaps.org/` + and every buildings download fails on a non-existent S3 key. The server seeds + `.cache/overture_release.json` with a well-formed id before each build; the + package itself is unchanged. This wants a one-line fix upstream. +- **`affine` 3.** With `affine>=3` (where `Affine` is no longer a namedtuple), + `pyflwdir.dem.slope`, a numba function SWMManywhere calls with a raster + transform in `geospatial_utilities`, fails with "Cannot determine Numba type of + " during `clip_to_catchments`. `server/requirements.txt` + pins `affine<3`; the package's own dependencies probably want the same pin until + `pyflwdir` supports affine 3. +- **macOS on Apple silicon.** `import pyswmm` is killed by the kernel because + `libomp.dylib` inside the `swmm-toolkit` wheel has an invalid code signature. + Re-signing the wheel's binaries fixes it: + `codesign --force --sign - /swmm/toolkit/*.dylib /swmm/toolkit/*.so`. +- **Paths with spaces.** SWMM cannot read a `[RAINGAGES] FILE` path containing a + space. The server copies the demo storm into its work directory and the + downloaded zip references it by bare name. +- **Subcatchment slope units (question for the maintainers).** `derive_subcatchments` + stores the mean `pyflwdir.dem.slope` gradient, which that function documents as + m/m, and `synthetic_write` copies it unchanged into the `%Slope` column of + `[SUBCATCHMENTS]`, which SWMM reads as a percentage. For the Andorra example the + written values are 0.0–0.9 where 0–89 % would be expected. The map shows the + stored value labelled m/m. diff --git a/frontend/conftest.py b/frontend/conftest.py new file mode 100644 index 00000000..7a859981 --- /dev/null +++ b/frontend/conftest.py @@ -0,0 +1,13 @@ +"""Keep the web service out of the root pytest run unless its extras are installed. + +The repository runs ``pytest --doctest-modules`` from the root, which imports every +module it finds. ``frontend/server`` needs FastAPI (see +``frontend/server/requirements.txt``), which the package's own ``dev`` extra does not +install, so collect it only when FastAPI is importable. +""" + +from __future__ import annotations + +import importlib.util + +collect_ignore = [] if importlib.util.find_spec("fastapi") else ["server"] diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 00000000..5c959fee --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,23 @@ + + + + + + + + + + + SWMManywhere + + +
+ + + diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 00000000..4356464e --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,30 @@ +{ + "name": "swmmanywhere-frontend", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc --noEmit && vite build", + "preview": "vite preview", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "lucide-react": "^1.17.0", + "maplibre-gl": "^5.24.0", + "react": "^19.2.6", + "react-dom": "^19.2.6", + "react-map-gl": "^8.1.1", + "zustand": "^5.0.14" + }, + "devDependencies": { + "@tailwindcss/vite": "^4.3.0", + "@types/geojson": "^7946.0.16", + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^6.0.1", + "tailwindcss": "^4.3.0", + "typescript": "~6.0.2", + "vite": "^8.0.12" + } +} diff --git a/frontend/public/logo.svg b/frontend/public/logo.svg new file mode 100644 index 00000000..2f43b9b9 --- /dev/null +++ b/frontend/public/logo.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/screenshot.png b/frontend/screenshot.png new file mode 100644 index 00000000..37bb540f Binary files /dev/null and b/frontend/screenshot.png differ diff --git a/frontend/server/__init__.py b/frontend/server/__init__.py new file mode 100644 index 00000000..05e358d4 --- /dev/null +++ b/frontend/server/__init__.py @@ -0,0 +1 @@ +"""Web service for the SWMManywhere frontend (see frontend/README.md).""" diff --git a/frontend/server/app.py b/frontend/server/app.py new file mode 100644 index 00000000..5df0d208 --- /dev/null +++ b/frontend/server/app.py @@ -0,0 +1,509 @@ +"""Web service that runs SWMManywhere for the browser frontend in ../src. + +It uses only the package's public Python API (``load_config``, ``swmmanywhere`` and +``run`` from ``swmmanywhere.swmmanywhere``) and never patches anything in ``src/``. +Every build writes the same ``config.yml`` a CLI user would pass to +``python -m swmmanywhere``, validates it with ``load_config`` and runs it; the +frontend is a form for that file. + +Run from the ``frontend/`` directory:: + + uvicorn server.app:app --port 8000 + +Environment: + SWMMANYWHERE_WEB_WORKDIR build directory (default: /swmmanywhere_web) + SWMMANYWHERE_WEB_MAX_KM2 largest bounding box accepted, km² (default: 5) + SWMMANYWHERE_WEB_ORIGINS comma-separated CORS origins (default: *) +""" + +from __future__ import annotations + +import importlib.metadata +import json +import math +import os +import re +import shutil +import tempfile +import threading +import traceback +import uuid +import zipfile +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Callable + +import requests +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, PlainTextResponse +from fastapi.staticfiles import StaticFiles +from jsonschema import ValidationError as SchemaError +from pydantic import BaseModel, Field, ValidationError + +import swmmanywhere +from swmmanywhere import parameters +from swmmanywhere.graph_utilities import graphfcns +from swmmanywhere.logging import logger +from swmmanywhere.swmmanywhere import load_config +from swmmanywhere.swmmanywhere import run as run_simulation +from swmmanywhere.swmmanywhere import swmmanywhere as run_synthesis +from swmmanywhere.utilities import yaml_dump, yaml_load + +from .preview import element_timeseries, model_geojson + +DEFS = Path(swmmanywhere.__file__).parent / "defs" +DIST = Path(__file__).resolve().parent.parent / "dist" +PROJECT_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$") + +# Log lines SWMManywhere emits along the way (swmmanywhere.py, preprocessing.py), +# mapped to a label and a rough percentage for the progress bar. +STAGES: list[tuple[str, str, float]] = [ + ("Running downloads", "Downloading open data", 5), + ("downloading elevation", "Downloading elevation (NASADEM)", 6), + ("downloading buildings", "Downloading buildings (Overture)", 9), + ("downloading network", "Downloading streets (OpenStreetMap)", 12), + ("downloading river", "Downloading rivers (OpenStreetMap)", 15), + ("Iterating graph functions", "Running graph functions", 20), + ("Saving final graph", "Writing the SWMM .inp", 82), + ("initialised in pyswmm", "Running the SWMM simulation", 86), + ("Model run complete", "Simulation complete", 96), +] +GRAPHFCN_DONE = re.compile(r"graphfcn: (\S+) completed") +# `FILE ""` or `FILE ` in a [RAINGAGES] line. +RAINGAGE_FILE = re.compile(r'(FILE\s+)("([^"]+)"|(\S+))') + +# Overture release cache. ``swmmanywhere.prepare_data._get_latest_s3_url`` derives +# the release id as ``Path(href).parent`` of the newest child link in the Overture +# STAC catalog. Those hrefs are absolute URLs now, so it caches +# "https:/stac.overturemaps.org/2026-08-19.0" in ./.cache/overture_release.json and +# every buildings download then fails on a non-existent S3 key. The cache is read +# before the catalog, so seeding it with a well-formed id (fresh timestamp) lets the +# unmodified package download buildings. Drop this once it is fixed upstream. +RELEASE_ID = re.compile(r"\d{4}-\d{2}-\d{2}\.\d+") +OVERTURE_CATALOG = "https://stac.overturemaps.org/catalog.json" + + +def _fetch_overture_catalog() -> dict: + response = requests.get(OVERTURE_CATALOG, timeout=15) + response.raise_for_status() + return response.json() + + +def latest_overture_release(catalog: dict) -> str | None: + """Newest release id among the catalog's child links, e.g. '2026-08-19.0'.""" + ids = [] + for link in catalog.get("links", []): + m = ( + RELEASE_ID.search(str(link.get("href", ""))) + if link.get("rel") == "child" + else None + ) + if m: + ids.append(m.group(0)) + return max(ids) if ids else None + + +def seed_overture_cache( + workdir: Path, fetch_catalog: Callable[[], dict] = _fetch_overture_catalog +) -> str | None: + """Write a well-formed Overture release id to the cache the package reads. + + Offline, the id embedded in an existing (possibly malformed) cache is reused. + """ + cache = workdir / ".cache" / "overture_release.json" + try: + release = latest_overture_release(fetch_catalog()) + except Exception: # noqa: BLE001 — offline is fine if a cache exists + m = RELEASE_ID.search(cache.read_text()) if cache.exists() else None + release = m.group(0) if m else None + if release: + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_text( + json.dumps({"release": release, "timestamp": datetime.now().isoformat()}) + ) + return release + + +class SubmitBody(BaseModel): + """What the frontend posts; everything maps onto a SWMManywhere config key.""" + + bbox: list[float] = Field(min_length=4, max_length=4) + project: str = "swmmanywhere_web" + parameter_overrides: dict[str, dict[str, Any]] = Field(default_factory=dict) + graphfcn_list: list[str] | None = None + run_model: bool = True + run_settings: dict[str, Any] | None = None + + +@dataclass +class Task: + """One build: its validated config, progress and outputs.""" + + id: str + config: dict + config_text: str + run_model: bool + n_graphfcns: int + state: str = "QUEUED" # QUEUED | RUNNING | SUCCEEDED | FAILED + stage: str = "Queued" + progress_pct: float = 0.0 + log: list[str] = field(default_factory=list) + error: str | None = None + model_dir: Path | None = None + results: Path | None = None + preview: dict | None = None + zip_path: Path | None = None + + +def bbox_area_km2(bbox: list[float]) -> float: + """Approximate area of a lon/lat box, enough for the size guard.""" + minx, miny, maxx, maxy = bbox + mid_lat = math.radians((miny + maxy) / 2) + return (maxx - minx) * 111.32 * math.cos(mid_lat) * (maxy - miny) * 110.57 + + +def validate_bbox(bbox: list[float], max_km2: float) -> list[float]: + """Reject a malformed or oversized bounding box with a 422.""" + if not all(math.isfinite(v) for v in bbox): + raise HTTPException(422, "bbox must be four finite numbers.") + minx, miny, maxx, maxy = bbox + if not (-180 <= minx < maxx <= 180 and -90 <= miny < maxy <= 90): + raise HTTPException( + 422, + "bbox must be [min lon, min lat, max lon, max lat] in EPSG:4326 " + "with min < max.", + ) + area = bbox_area_km2(bbox) + if area > max_km2: + raise HTTPException( + 422, + f"Bounding box is {area:.1f} km²; this server accepts at most " + f"{max_km2:g} km².", + ) + return bbox.copy() + + +def validate_overrides(overrides: dict[str, dict[str, Any]]) -> None: + """Range-check overrides with the package's own pydantic parameter models. + + ``swmmanywhere.swmmanywhere`` applies overrides with ``setattr`` and only checks + that the names exist (issue #379), so a web user could otherwise submit values + outside the documented bounds without noticing. + """ + groups = parameters.get_full_parameters() + for category, values in overrides.items(): + if category not in groups: + raise HTTPException( + 422, + f"{category} is not a parameter group. " + f"Must be one of {sorted(groups)}.", + ) + model = groups[category] + current = {k: v for k, v in model.model_dump().items() if v is not None} + try: + type(model).model_validate(current | values) + except ValidationError as exc: + problems = "; ".join( + f"{'.'.join(str(p) for p in e['loc']) or category}: {e['msg']}" + for e in exc.errors() + ) + raise HTTPException(422, f"Invalid {category} override — {problems}") + + +def parameter_groups() -> list[dict]: + """Parameter groups with fields, defaults and bounds, from the package.""" + groups = [] + for name, model in parameters.get_full_parameters().items(): + fields = [ + { + "name": key, + "type": prop.get("type", "unknown"), + "default": prop.get("default"), + "unit": prop.get("unit"), + "description": " ".join((prop.get("description") or "").split()), + "minimum": prop.get("minimum"), + "maximum": prop.get("maximum"), + "exclusiveMaximum": prop.get("exclusiveMaximum"), + } + for key, prop in model.model_json_schema()["properties"].items() + ] + groups.append( + {"name": name, "doc": (type(model).__doc__ or "").strip(), "fields": fields} + ) + return groups + + +def package_zip(task: Task) -> Path: + """Zip the model directory; the rain file is added and referenced by bare name. + + SWMManywhere writes the absolute path of the rain file into ``[RAINGAGES]``, which + only resolves on the machine that built the model. The copy inside the zip points + at the file next to it so the download opens and runs in EPA SWMM as-is. + """ + assert task.model_dir is not None + zip_path = task.model_dir / "model_package.zip" + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as z: + for f in sorted(task.model_dir.iterdir()): + if not f.is_file() or f == zip_path: + continue + if f.suffix != ".inp": + z.write(f, f.name) + continue + lines = [] + for line in f.read_text().splitlines(): + m = RAINGAGE_FILE.search(line) + if m: + path = Path(m.group(3) or m.group(4)) + if path.is_absolute() and path.exists(): + z.write(path, path.name) + line = ( + line[: m.start()] + f'FILE "{path.name}"' + line[m.end() :] + ) + lines.append(line) + z.writestr(f.name, "\n".join(lines) + "\n") + return zip_path + + +def create_app( + *, + synthesis: Callable[[dict], tuple[Path, Any]] = run_synthesis, + simulate: Callable[..., Any] = run_simulation, + prepare: Callable[[Path], Any] = seed_overture_cache, + workdir: Path | str | None = None, + run_inline: bool = False, +) -> FastAPI: + """Build the FastAPI app. + + ``synthesis``, ``simulate`` and ``prepare`` are injectable so tests run without + downloads or SWMM; ``run_inline`` runs tasks synchronously in the request. + """ + workdir = Path( + workdir + or os.environ.get("SWMMANYWHERE_WEB_WORKDIR") + or Path(tempfile.gettempdir()) / "swmmanywhere_web" + ).resolve() + workdir.mkdir(parents=True, exist_ok=True) + # The package's bundled demo storm, copied out so the path never contains spaces + # (SWMM cannot read a [RAINGAGES] FILE path with spaces) and so a future upload has + # an obvious place to go. Same file the CLI falls back to. + precipitation = workdir / "precipitation.dat" + if not precipitation.exists(): + shutil.copyfile(DEFS / "storm.dat", precipitation) + max_km2 = float(os.environ.get("SWMMANYWHERE_WEB_MAX_KM2", "5")) + schema = yaml_load((DEFS / "schema.yml").read_text()) + demo = load_config(validation=False) # package defaults (defs/demo_config.yml) + + app = FastAPI(title="SWMManywhere web") + origins = [ + o.strip() + for o in os.environ.get("SWMMANYWHERE_WEB_ORIGINS", "*").split(",") + if o.strip() + ] + app.add_middleware( + CORSMiddleware, allow_origins=origins, allow_methods=["*"], allow_headers=["*"] + ) + + tasks: dict[str, Task] = {} + lock = threading.Lock() + # One build at a time: SWMManywhere shares the base_dir between builds (it reuses + # downloads per bbox and numbers models), and loguru is process-global. + executor = ThreadPoolExecutor(max_workers=1) + + def on_log(task: Task, message: str) -> None: + line = message.rstrip() + if not line: + return + with lock: + task.log.append(line) + del task.log[:-300] + for needle, label, pct in STAGES: + if needle in line: + task.stage, task.progress_pct = label, pct + m = GRAPHFCN_DONE.search(line) + if m: + done = sum(1 for entry in task.log if GRAPHFCN_DONE.search(entry)) + task.stage = f"Graph function {done}/{task.n_graphfcns}: {m.group(1)}" + task.progress_pct = 20 + 60 * done / max(task.n_graphfcns, 1) + + def run_task(task: Task) -> None: + # SWMManywhere writes temporary DEM tiles and the WhiteboxTools download into + # the current directory; keep those in the workdir. + os.chdir(workdir) + task.state, task.stage = "RUNNING", "Starting" + handler = logger.add( + lambda message: on_log(task, str(message)), + filter=lambda record: True, + format="{message}", + ) + try: + prepare(workdir) + inp, _ = synthesis(task.config) + if inp.suffix != ".inp": + raise RuntimeError( + "SWMManywhere derived no pipes for this bounding box (the graph " + "has no edges). Try a larger or more built-up area." + ) + task.model_dir = inp.parent + (task.model_dir / "config.yml").write_text(task.config_text) + if task.run_model: + task.stage, task.progress_pct = "Running the SWMM simulation", 85 + results = simulate(inp, **task.config["run_settings"]) + task.results = task.model_dir / "results.parquet" + results.to_parquet(task.results) + task.stage, task.progress_pct, task.state = "Complete", 100, "SUCCEEDED" + except Exception as exc: # noqa: BLE001 — any failure must surface in the UI + task.state, task.stage = "FAILED", "Failed" + task.error = f"{type(exc).__name__}: {exc}" + traceback.print_exc() # full trace in the server log + finally: + logger.remove(handler) + + def get_task(task_id: str) -> Task: + task = tasks.get(task_id) + if task is None: + raise HTTPException(404, "Unknown task.") + return task + + @app.get("/api/v1/healthz") + def healthz(): + return { + "status": "ok", + "swmmanywhere": importlib.metadata.version("swmmanywhere"), + } + + @app.get("/api/v1/defaults") + def defaults(): + return { + "version": importlib.metadata.version("swmmanywhere"), + "parameters": parameter_groups(), + "graphfcn_list": demo["graphfcn_list"], + "graphfcns": { + name: (fn.__doc__ or "").strip().splitlines()[0] if fn.__doc__ else "" + for name, fn in sorted(graphfcns.items()) + }, + "run_settings": demo["run_settings"], + "storevars": schema["properties"]["run_settings"]["properties"][ + "storevars" + ]["items"]["enum"], + "max_area_km2": max_km2, + } + + @app.post("/api/v1/tasks", status_code=202) + def submit(body: SubmitBody): + bbox = validate_bbox(body.bbox, max_km2) + if not PROJECT_RE.match(body.project): + raise HTTPException( + 422, "project must be 1–64 letters, digits, '-' or '_'." + ) + validate_overrides(body.parameter_overrides) + + # The config a CLI user would write, then SWMManywhere's own validation. + config: dict[str, Any] = { + "base_dir": str(workdir), + "project": body.project, + "bbox": bbox, + "run_model": body.run_model, + "run_settings": {**demo["run_settings"], **(body.run_settings or {})}, + "address_overrides": {"precipitation": str(precipitation)}, + } + overrides = {k: v for k, v in body.parameter_overrides.items() if v} + if overrides: + config["parameter_overrides"] = overrides + if body.graphfcn_list is not None: + config["graphfcn_list"] = body.graphfcn_list + config_text = yaml_dump(config) + + task_id = uuid.uuid4().hex[:12] + task_dir = workdir / "tasks" / task_id + task_dir.mkdir(parents=True) + config_path = task_dir / "config.yml" + config_path.write_text(config_text) + try: + loaded = load_config(config_path) + except SchemaError as exc: + raise HTTPException( + 422, f"SWMManywhere rejected the configuration: {exc.message}" + ) + except (ValueError, TypeError, AssertionError, FileNotFoundError) as exc: + raise HTTPException(422, f"SWMManywhere rejected the configuration: {exc}") + # The simulation is run here (see run_task) so its results can be served. + loaded["run_model"] = False + + task = Task( + id=task_id, + config=loaded, + config_text=config_text, + run_model=body.run_model, + n_graphfcns=len(loaded.get("graphfcn_list") or demo["graphfcn_list"]), + ) + with lock: + tasks[task_id] = task + if run_inline: + run_task(task) + else: + executor.submit(run_task, task) + return {"task_id": task_id, "status": task.state} + + @app.get("/api/v1/tasks/{task_id}") + def status(task_id: str): + task = get_task(task_id) + with lock: + return { + "state": task.state, + "stage": task.stage, + "progress_pct": task.progress_pct, + "log": task.log[-8:], + "error": task.error, + } + + @app.get("/api/v1/tasks/{task_id}/config") + def config_yaml(task_id: str): + return PlainTextResponse(get_task(task_id).config_text, media_type="text/yaml") + + @app.get("/api/v1/tasks/{task_id}/preview") + def preview(task_id: str): + task = get_task(task_id) + if task.state != "SUCCEEDED" or task.model_dir is None: + raise HTTPException(409, "Preview not ready.") + if task.preview is None: + task.preview = model_geojson(task.model_dir) + return task.preview + + @app.get("/api/v1/tasks/{task_id}/result") + def result(task_id: str): + task = get_task(task_id) + if task.state != "SUCCEEDED" or task.model_dir is None: + raise HTTPException(409, "Result not ready.") + if task.zip_path is None: + task.zip_path = package_zip(task) + return FileResponse( + str(task.zip_path), + media_type="application/zip", + filename="swmmanywhere_model.zip", + ) + + @app.get("/api/v1/tasks/{task_id}/timeseries") + def timeseries(task_id: str, id: str): + task = get_task(task_id) + if task.state != "SUCCEEDED": + raise HTTPException(409, "Results not ready.") + if task.results is None: + raise HTTPException( + 409, "This build did not run the model (run_model was off)." + ) + series = element_timeseries(task.results, id) + if not series: + raise HTTPException(404, f"No simulation results stored for {id}.") + return series + + # Serve the built frontend (npm run build) from the same origin when present. + if DIST.is_dir(): + app.mount("/", StaticFiles(directory=str(DIST), html=True), name="frontend") + + return app + + +app = create_app() diff --git a/frontend/server/preview.py b/frontend/server/preview.py new file mode 100644 index 00000000..55707061 --- /dev/null +++ b/frontend/server/preview.py @@ -0,0 +1,108 @@ +"""Turn a SWMManywhere model directory into GeoJSON and per-element time series. + +Reads the same nodes / edges / subcatchments files ``swmmanywhere.utilities.plot_map`` +reads and identifies outfalls the same way (``plot_basic``), so the map matches what the +package's own folium plot would show. +""" + +from __future__ import annotations + +import math +from contextlib import suppress +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +from shapely.geometry import mapping + +from swmmanywhere.utilities import read_df + + +def _first(model_dir: Path, stem: str) -> Path | None: + return next(iter(sorted(model_dir.glob(f"{stem}.*"))), None) + + +def _clean(value: Any) -> Any: + """JSON-safe scalar: numpy -> python, NaN/inf -> None, anything odd -> str.""" + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, float): + return value if math.isfinite(value) else None + if value is None or isinstance(value, (bool, int, str)): + return value + if isinstance(value, (list, tuple, np.ndarray)): + return [_clean(v) for v in value] + with suppress(TypeError, ValueError): + if pd.isna(value): + return None + return str(value) + + +def _features(df: pd.DataFrame, kind: str, **extra: pd.Series) -> list[dict]: + """One GeoJSON feature per row, every non-geometry column as a property.""" + columns = [c for c in df.columns if c != "geometry"] + features = [] + for i, (_, row) in enumerate(df.iterrows()): + props = {c: _clean(row[c]) for c in columns} + props["kind"] = kind + for name, series in extra.items(): + props[name] = _clean(series.iloc[i]) + features.append( + {"type": "Feature", "geometry": mapping(row.geometry), "properties": props} + ) + return features + + +def model_geojson(model_dir: Path) -> dict: + """GeoJSON (EPSG:4326) of a synthesised model with a ``kind`` per feature. + + kinds: ``junction``, ``outfall``, ``conduit``, ``subcatchment``. Feature ``id`` + is the SWMM object name so it matches ``results.parquet`` (subcatchments are + named ``-sub`` in the .inp, see ``post_processing.synthetic_write``). + """ + nodes_fid, edges_fid = _first(model_dir, "nodes"), _first(model_dir, "edges") + if nodes_fid is None or edges_fid is None: + raise FileNotFoundError("No nodes or edges found in model directory.") + nodes = read_df(nodes_fid).to_crs(4326) + edges = read_df(edges_fid).to_crs(4326) + + node_ids = nodes["id"].astype(str) + # Same rule as swmmanywhere.utilities.plot_basic. + if "outfall" in nodes.columns: + is_outfall = node_ids == nodes["outfall"].astype(str) + else: + is_outfall = ~node_ids.isin(edges["u"].astype(str)) + + features: list[dict] = [] + subs_fid = _first(model_dir, "subcatchments") + if subs_fid is not None: + subs = read_df(subs_fid).to_crs(4326) + outlet = subs["id"].astype(str) + subs = subs.assign(id=outlet + "-sub") + features += _features(subs, "subcatchment", outlet=outlet) + + edges = edges.assign( + id=edges["id"].astype(str), u=edges["u"].astype(str), v=edges["v"].astype(str) + ) + features += _features(edges, "conduit") + nodes = nodes.assign(id=node_ids) + features += _features(nodes.loc[~is_outfall], "junction") + features += _features(nodes.loc[is_outfall], "outfall") + return {"type": "FeatureCollection", "features": features} + + +def element_timeseries( + results_path: Path, element_id: str +) -> dict[str, dict[str, list]]: + """``{variable: {dates: [...iso...], values: [...]}}`` for one SWMM object.""" + df = pd.read_parquet(results_path) + sel = df[df["id"].astype(str) == element_id] + out: dict[str, dict[str, list]] = {} + for variable, grp in sel.groupby("variable"): + grp = grp.sort_values("date") + out[str(variable)] = { + "dates": [pd.Timestamp(d).isoformat() for d in grp["date"]], + "values": [float(v) for v in grp["value"]], + } + return out diff --git a/frontend/server/requirements.txt b/frontend/server/requirements.txt new file mode 100644 index 00000000..e62c84a0 --- /dev/null +++ b/frontend/server/requirements.txt @@ -0,0 +1,9 @@ +# On top of the swmmanywhere package itself (pip install -e . from the repo root). +fastapi>=0.110 +uvicorn>=0.29 +# affine 3 turned Affine into a plain class; pyflwdir's numba-compiled `slope` +# (called from swmmanywhere.geospatial_utilities) cannot type it. Pin until upstream does. +affine<3 +# tests +pytest>=7 +httpx>=0.27 diff --git a/frontend/server/tests/__init__.py b/frontend/server/tests/__init__.py new file mode 100644 index 00000000..c2a37361 --- /dev/null +++ b/frontend/server/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the SWMManywhere web service.""" diff --git a/frontend/server/tests/test_app.py b/frontend/server/tests/test_app.py new file mode 100644 index 00000000..dc9eea87 --- /dev/null +++ b/frontend/server/tests/test_app.py @@ -0,0 +1,269 @@ +"""Contract tests for frontend/server using SWMManywhere's bundled Bellinge fixtures. + +No network and no SWMM run: synthesis and simulation are injected fakes that write +the package's own test data (src/swmmanywhere/defs/bellinge_small_*) into the model +directory. Config validation, GeoJSON conversion, packaging and time-series +extraction are the real code. +""" + +from __future__ import annotations + +import io +import json +import shutil +import zipfile +from datetime import datetime, timedelta +from pathlib import Path + +import pandas as pd +import pytest +from fastapi.testclient import TestClient + +import swmmanywhere +from server.app import create_app, seed_overture_cache +from swmmanywhere.utilities import load_graph, save_graph_to_features + +DEFS = Path(swmmanywhere.__file__).parent / "defs" +CATALOG = { + "links": [ + {"rel": "self", "href": "https://stac.overturemaps.org/catalog.json"}, + { + "rel": "child", + "title": "Latest Overture Release", + "href": "https://stac.overturemaps.org/2026-08-19.0/catalog.json", + }, + { + "rel": "child", + "title": "2026-07-22.0 Overture Release", + "href": "https://stac.overturemaps.org/2026-07-22.0/catalog.json", + }, + ] +} +BBOX = [10.290, 55.318, 10.300, 55.324] # small; the fake synthesis ignores it +NODE = "G72F820" # a node of bellinge_small_graph.json + + +def fake_synthesis(config: dict): + """Write the Bellinge fixtures where SWMManywhere would put a model.""" + model_dir = Path(config["base_dir"]) / config["project"] / "bbox_1" / "model_1" + model_dir.mkdir(parents=True, exist_ok=True) + graph = load_graph(DEFS / "bellinge_small_graph.json") + save_graph_to_features( + graph, + model_dir / "nodes.geojson", + model_dir / "edges.geojson", + graph.graph["crs"], + ) + shutil.copy( + DEFS / "bellinge_small_subcatchments.geojson", + model_dir / "subcatchments.geojson", + ) + rain = config["address_overrides"]["precipitation"] + inp = model_dir / "model_1.inp" + inp.write_text( + "[TITLE]\nfake\n\n[RAINGAGES]\n;;Name Format Interval SCF Source\n" + f'1 INTENSITY 0:15 1.0 FILE "{rain}" 1 MM\n\n[JUNCTIONS]\n' + ) + return inp, None + + +def fake_simulate( + model: Path, reporting_iters: int, duration: int, storevars: list[str] +): + """Return a short flooding series for one node instead of running SWMM.""" + t0 = datetime(2000, 1, 1) + rows = [ + { + "date": t0 + timedelta(minutes=15 * i), + "value": float(i), + "variable": "flooding", + "id": NODE, + } + for i in range(8) + ] + return pd.DataFrame(rows) + + +def no_edges_synthesis(config: dict): + """Mimic SWMManywhere returning the graph file when no pipes were derived.""" + model_dir = Path(config["base_dir"]) / config["project"] / "bbox_1" / "model_1" + model_dir.mkdir(parents=True, exist_ok=True) + return model_dir / "graph.parquet", None + + +@pytest.fixture +def client(tmp_path): + """A test client whose builds run inline on the Bellinge fixtures.""" + app = create_app( + synthesis=fake_synthesis, + simulate=fake_simulate, + prepare=lambda _: None, + workdir=tmp_path, + run_inline=True, + ) + return TestClient(app) + + +def test_overture_cache_is_seeded_with_a_well_formed_release_id(tmp_path): + """The cache the package reads gets a plain release id, online or offline.""" + cache = tmp_path / ".cache" / "overture_release.json" + assert ( + seed_overture_cache(tmp_path, fetch_catalog=lambda: CATALOG) == "2026-08-19.0" + ) + assert json.loads(cache.read_text())["release"] == "2026-08-19.0" + # Offline: the id embedded in the package's malformed cache is reused. + cache.write_text( + json.dumps( + { + "release": "https:/stac.overturemaps.org/2026-07-22.0", + "timestamp": "2026-09-01T00:00:00", + } + ) + ) + + def offline(): + raise ConnectionError("no network") + + assert seed_overture_cache(tmp_path, fetch_catalog=offline) == "2026-07-22.0" + assert json.loads(cache.read_text())["release"] == "2026-07-22.0" + + +def test_defaults_come_from_the_package(client): + """Parameter schema, graphfcn list and run settings are read from the package.""" + d = client.get("/api/v1/defaults").json() + groups = {g["name"]: g for g in d["parameters"]} + assert { + "subcatchment_derivation", + "outfall_derivation", + "topology_derivation", + "hydraulic_design", + } <= set(groups) + lane_width = next( + f + for f in groups["subcatchment_derivation"]["fields"] + if f["name"] == "lane_width" + ) + assert ( + lane_width["default"] == 3.5 + and lane_width["unit"] == "m" + and lane_width["minimum"] == 2.0 + ) + assert ( + d["graphfcn_list"][0] == "assign_id" + and d["graphfcn_list"][-2] == "fix_geometries" + ) + assert "pipe_by_pipe" in d["graphfcns"] + assert "flooding" in d["storevars"] and d["run_settings"]["duration"] == 86400 + + +@pytest.mark.parametrize( + "bbox, fragment", + [([1.0, 2.0, 0.5, 3.0], "min < max"), ([0.0, 0.0, 1.0, 1.0], "km²")], +) +def test_bad_bbox_is_rejected(client, bbox, fragment): + """Reversed corners and oversized boxes are refused before any build starts.""" + r = client.post("/api/v1/tasks", json={"bbox": bbox}) + assert r.status_code == 422 and fragment in r.json()["detail"] + + +def test_parameter_overrides_are_range_checked(client): + """Out-of-range values and unknown groups are refused with the field named.""" + r = client.post( + "/api/v1/tasks", + json={"bbox": BBOX, "parameter_overrides": {"hydraulic_design": {"max_fr": 5}}}, + ) + assert r.status_code == 422 and "max_fr" in r.json()["detail"] + r = client.post( + "/api/v1/tasks", json={"bbox": BBOX, "parameter_overrides": {"nope": {"x": 1}}} + ) + assert r.status_code == 422 and "nope" in r.json()["detail"] + + +def test_unknown_graphfcn_is_rejected_by_swmmanywhere(client): + """The package's own graphfcn validation runs on the submitted list.""" + r = client.post( + "/api/v1/tasks", json={"bbox": BBOX, "graphfcn_list": ["assign_id", "teleport"]} + ) + assert r.status_code == 422 and "teleport" in r.json()["detail"] + + +def test_build_preview_results_and_package(client, tmp_path): + """A build yields status, config, GeoJSON preview, time series and a zip.""" + r = client.post( + "/api/v1/tasks", + json={ + "bbox": BBOX, + "project": "bellinge", + "parameter_overrides": {"outfall_derivation": {"method": "withtopo"}}, + "run_settings": { + "duration": 3600, + "reporting_iters": 10, + "storevars": ["flooding"], + }, + }, + ) + assert r.status_code == 202, r.text + task_id = r.json()["task_id"] + + status = client.get(f"/api/v1/tasks/{task_id}").json() + assert status["state"] == "SUCCEEDED", status + assert status["progress_pct"] == 100 + + cfg = client.get(f"/api/v1/tasks/{task_id}/config").text + assert "bbox:" in cfg and "withtopo" in cfg and "duration: 3600" in cfg + assert (tmp_path / "bellinge" / "bbox_1" / "model_1" / "config.yml").exists() + + fc = client.get(f"/api/v1/tasks/{task_id}/preview").json() + kinds = {f["properties"]["kind"] for f in fc["features"]} + assert kinds == {"junction", "outfall", "conduit", "subcatchment"} + outfalls = [f for f in fc["features"] if f["properties"]["kind"] == "outfall"] + assert len(outfalls) >= 1 + lon, lat = outfalls[0]["geometry"]["coordinates"] + assert 10 < lon < 11 and 55 < lat < 56 # reprojected from EPSG:32632 to lon/lat + sub = next(f for f in fc["features"] if f["properties"]["kind"] == "subcatchment") + assert sub["properties"]["id"].endswith("-sub") and sub["properties"]["outlet"] + node = next(f for f in fc["features"] if f["properties"]["id"] == NODE) + assert node["properties"]["surface_elevation"] == pytest.approx(26.129, abs=1e-3) + + series = client.get( + f"/api/v1/tasks/{task_id}/timeseries", params={"id": NODE} + ).json() + assert list(series) == ["flooding"] and series["flooding"]["values"][-1] == 7.0 + assert ( + client.get( + f"/api/v1/tasks/{task_id}/timeseries", params={"id": "nowhere"} + ).status_code + == 404 + ) + + z = zipfile.ZipFile( + io.BytesIO(client.get(f"/api/v1/tasks/{task_id}/result").content) + ) + names = set(z.namelist()) + assert { + "model_1.inp", + "nodes.geojson", + "edges.geojson", + "config.yml", + "results.parquet", + "precipitation.dat", + } <= names + assert 'FILE "precipitation.dat"' in z.read("model_1.inp").decode() + + +def test_no_edges_fails_with_a_clear_message(tmp_path): + """A graph without edges ends as FAILED with an explanation, not a traceback.""" + app = create_app( + synthesis=no_edges_synthesis, + simulate=fake_simulate, + prepare=lambda _: None, + workdir=tmp_path, + run_inline=True, + ) + client = TestClient(app) + task_id = client.post( + "/api/v1/tasks", json={"bbox": BBOX, "run_model": False} + ).json()["task_id"] + status = client.get(f"/api/v1/tasks/{task_id}").json() + assert status["state"] == "FAILED" and "no pipes" in status["error"] + assert client.get(f"/api/v1/tasks/{task_id}/preview").status_code == 409 diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 00000000..d1351bd7 --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,50 @@ +import { useState } from 'react' +import ControlPanel from './components/ControlPanel' +import MapPanel from './components/MapPanel' + +const MIN_W = 320 +const MAX_W = 680 + +export default function App() { + // Resizable control panel: drag the divider to widen or narrow it. + const [width, setWidth] = useState(400) + + const startResize = (e: React.PointerEvent) => { + e.preventDefault() + const startX = e.clientX + const startW = width + const onMove = (ev: PointerEvent) => { + setWidth(Math.min(MAX_W, Math.max(MIN_W, startW + (ev.clientX - startX)))) + } + const onUp = () => { + window.removeEventListener('pointermove', onMove) + window.removeEventListener('pointerup', onUp) + document.body.style.cursor = '' + document.body.style.userSelect = '' + } + window.addEventListener('pointermove', onMove) + window.addEventListener('pointerup', onUp) + document.body.style.cursor = 'col-resize' + document.body.style.userSelect = 'none' + } + + return ( +
+
+ +
+
+
+
+
+ +
+
+ ) +} diff --git a/frontend/src/components/ControlPanel.tsx b/frontend/src/components/ControlPanel.tsx new file mode 100644 index 00000000..fa9d5580 --- /dev/null +++ b/frontend/src/components/ControlPanel.tsx @@ -0,0 +1,395 @@ +import { useEffect, useMemo, useState } from 'react' +import { ChevronRight, Download, Loader2, LocateFixed, Play, RotateCcw, SquareDashed, Trash2 } from 'lucide-react' +import { bboxAreaKm2, graphfcnLines, useStore } from '../store' +import type { Bbox } from '../types' +import { BTN_PRIMARY, ICON_BTN, INPUT, LABEL } from '../ui' +import ParameterGroup from './ParameterGroup' + +const BBOX_LABELS = ['min lon', 'min lat', 'max lon', 'max lat'] + +// metric_evaluation only applies when a real network is supplied for comparison +// (config `real:`), which this form does not do. +const HIDDEN_GROUPS = new Set(['metric_evaluation']) + +function Section({ title, children }: { title: string; children: React.ReactNode }) { + return ( +
+

{title}

+ {children} +
+ ) +} + +export default function ControlPanel() { + const defaults = useStore((s) => s.defaults) + const defaultsError = useStore((s) => s.defaultsError) + const loadDefaults = useStore((s) => s.loadDefaults) + const bbox = useStore((s) => s.bbox) + const drawing = useStore((s) => s.drawing) + const anchor = useStore((s) => s.anchor) + const startDraw = useStore((s) => s.startDraw) + const cancelDraw = useStore((s) => s.cancelDraw) + const setBbox = useStore((s) => s.setBbox) + const requestFit = useStore((s) => s.requestFit) + const project = useStore((s) => s.project) + const setProject = useStore((s) => s.setProject) + const overrides = useStore((s) => s.overrides) + const setOverride = useStore((s) => s.setOverride) + const resetGroup = useStore((s) => s.resetGroup) + const graphfcnText = useStore((s) => s.graphfcnText) + const setGraphfcnText = useStore((s) => s.setGraphfcnText) + const runModel = useStore((s) => s.runModel) + const setRunModel = useStore((s) => s.setRunModel) + const runSettings = useStore((s) => s.runSettings) + const setRunSettings = useStore((s) => s.setRunSettings) + const job = useStore((s) => s.job) + const submit = useStore((s) => s.submit) + const preview = useStore((s) => s.preview) + const configYaml = useStore((s) => s.configYaml) + const [showLog, setShowLog] = useState(false) + // Typed bbox: the four boxes are free text until all of them parse, so a CLI + // user can paste numbers without drawing first. + const [bboxDraft, setBboxDraft] = useState(['', '', '', '']) + + useEffect(() => { + void loadDefaults() + }, [loadDefaults]) + + useEffect(() => { + if (!bbox) setBboxDraft(['', '', '', '']) + else if (bboxDraft.some((t, i) => Number(t) !== bbox[i])) setBboxDraft(bbox.map(String)) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [bbox]) + + const counts = useMemo(() => { + const c = { subcatchment: 0, junction: 0, outfall: 0, conduit: 0 } + preview?.features.forEach((f) => { + const kind = (f.properties as { kind?: string } | null)?.kind + if (kind && kind in c) c[kind as keyof typeof c]++ + }) + return c + }, [preview]) + + const busy = job.status === 'queued' || job.status === 'running' + const areaKm2 = bbox ? bboxAreaKm2(bbox) : 0 + const tooLarge = !!defaults && areaKm2 > defaults.max_area_km2 + const graphfcnChanged = !!defaults && graphfcnText.trim() !== defaults.graphfcn_list.join('\n') + const storevarChoices = defaults?.storevars ?? runSettings.storevars + + const editBbox = (i: number, text: string) => { + const next = [...bboxDraft] + next[i] = text + setBboxDraft(next) + const nums = next.map(Number) + if (next.every((t) => t.trim() !== '') && nums.every(Number.isFinite)) setBbox(nums as Bbox) + } + + const toggleStorevar = (v: string) => + setRunSettings({ + storevars: runSettings.storevars.includes(v) + ? runSettings.storevars.filter((s) => s !== v) + : [...runSettings.storevars, v], + }) + + return ( +