From 87d0f759bd02db7fc76829533f712f191b5dc39a Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:34:36 +0800 Subject: [PATCH 01/12] feat(server-env): add typed environment configuration (#363) * feat: add typed server environment configuration * docs: remove stale documentation tracker links --- SECURITY.md | 2 +- docs/TROUBLESHOOTING.md | 6 ++-- .../assumptions-and-constraints.md | 4 +-- docs/architecture/system-overview.md | 2 +- docs/architecture/threat-model.md | 4 +-- docs/deployment/environment-config.md | 6 ++-- docs/deployment/maintenance.md | 4 +-- docs/getting-started/docker-setup.md | 2 +- docs/getting-started/server-setup.md | 4 +-- docs/qa/scenario-tooling.md | 6 ++-- server/CLAUDE.md | 4 +-- server/app/api/testing.py | 8 ++--- server/app/env.py | 15 ++++++++ server/app/main.py | 6 ++-- server/app/scripts/seed_db.py | 12 ++++--- server/app/tests/test_env.py | 35 +++++++++++++++++++ 16 files changed, 87 insertions(+), 33 deletions(-) create mode 100644 server/app/env.py create mode 100644 server/app/tests/test_env.py diff --git a/SECURITY.md b/SECURITY.md index 6d6dba43..17e6c885 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,7 +11,7 @@ This is the canonical source of truth for SAPOT's known security-relevant config | Hardcoded MariaDB credentials | `server/app/db_operations/auth.py` (`SQLALCHEMY_DATABASE_URL`) | Now required via `DATABASE_URL` env var; the app raises `RuntimeError` at import time if unset. **Rotate the previously-hardcoded DB password before deploying this fix.** | | Hardcoded JWT secret fallback | `server/app/db_operations/token.py` (`SECRET_KEY`) | The default value has been removed; `JWT_SECRET_KEY` is now required, and the app raises `RuntimeError` at import time if unset. **Rotate to a newly generated secret** (`openssl rand -hex 32`) — the old hardcoded value must be considered compromised since it was committed to source. | | CORS wildcard + credentials | `server/app/main.py` | `allow_origins=["*"]` replaced with an explicit allowlist read from `CORS_ALLOWED_ORIGINS` (comma-separated). The app raises `RuntimeError` at import time if unset. | -| Testing router in production | `server/app/main.py` | `app.include_router(testing.router)` is now gated behind `ENVIRONMENT=development` (see `app/main.py`). The `/testing/*` endpoints (`test-make-admin`, `test-make-rescuer`) are unreachable unless the server is explicitly started in development mode. | +| Testing router in production | `server/app/main.py` | `app.include_router(testing.router)` is gated behind QA-enabled `ENVIRONMENT` values (`development` or `staging`; see `app/main.py`). The `/testing/*` endpoints are unreachable in production. | ## Required environment variables (new) diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index ca0052de..80b46cc3 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -52,13 +52,13 @@ If this fails, check MariaDB is running (`sudo systemctl status mariadb`) and th --- -## `/testing/*` endpoints return 404 in dev +## `/testing/*` endpoints return 404 in development or staging **Symptom:** A test helper endpoint like `/testing/test-make-admin` returns 404 even locally. -**Cause:** The testing router is gated behind `ENVIRONMENT=development` (see the repo-root `SECURITY.md`) — it's unreachable unless that env var is set exactly to `development`. +**Cause:** The testing router is gated behind `ENVIRONMENT=development` or `staging` (see [SECURITY.md](../SECURITY.md#resolved-issues-fixed-in-code)) — it's unreachable unless that env var is set to one of those values exactly. -**Fix:** Set `ENVIRONMENT=development` in `server/.env` for local dev only. **Never** set this in a production deployment. +**Fix:** Set `ENVIRONMENT=development` for local development or `ENVIRONMENT=staging` for a QA deployment. **Never** set either in a production deployment. --- diff --git a/docs/architecture/assumptions-and-constraints.md b/docs/architecture/assumptions-and-constraints.md index ae1845df..a6c5b4ac 100644 --- a/docs/architecture/assumptions-and-constraints.md +++ b/docs/architecture/assumptions-and-constraints.md @@ -58,8 +58,8 @@ See [threat-model.md](threat-model.md#attack-surfaces-explicitly-out-of-scope) f Risks the project has consciously decided to carry rather than fix, reproduced from the threat model's tradeoff table (see that document for status updates): - No LAN segmentation — requires router-level VLAN config not currently documented or automated. -- `testing` router reachable when `ENVIRONMENT=development` — accepted; operational discipline (never deploy with this setting) is the control. -- GSM module DB credentials hardcoded default in `config.py` — open, tracked in the repo-root `SECURITY.md`. +- `testing` router reachable when `ENVIRONMENT=development` or `staging` — accepted; operational discipline (never deploy with either setting in production) is the control. +- GSM module DB credentials hardcoded default in `config.py` — open, tracked in [SECURITY.md](../../SECURITY.md#other-known-gaps-not-yet-resolved). - No remote session/device revocation UI — open. - Optional (not enforced) server-side `PeerKey` signing — open. diff --git a/docs/architecture/system-overview.md b/docs/architecture/system-overview.md index a984d86b..5dc9d4c3 100644 --- a/docs/architecture/system-overview.md +++ b/docs/architecture/system-overview.md @@ -63,7 +63,7 @@ The FastAPI backend. Deployed as a Gunicorn process behind an Nginx reverse prox API routers: `admin`, `auth`, `captive_portal`, `download`, `forgot_password`, `gps`, `gsm`, `keys`, `mikrotik`, `peer_connection`, `ping`, `profile_picture`, `public_chat`, `sync`, `testing` (dev only), `update_info`, `user_keys`, `user_utils`, `verify_email`, `wrapped_key`. -> Note: The `testing` router is only mounted when `ENVIRONMENT=development` (see `app/main.py`); it is excluded from production builds by default. See the repo-root `SECURITY.md` for the fix history. +> Note: The `testing` router is only mounted when `ENVIRONMENT=development` or `staging` (see `app/main.py`); it is excluded from production builds by default. See [SECURITY.md](../../SECURITY.md) for the fix history. --- diff --git a/docs/architecture/threat-model.md b/docs/architecture/threat-model.md index 74a8aaa6..1a24a8e6 100644 --- a/docs/architecture/threat-model.md +++ b/docs/architecture/threat-model.md @@ -118,8 +118,8 @@ flowchart TB | Risk | Status | |---|---| | No LAN segmentation (rescuer/admin/civilian devices share one broadcast domain) | Accepted for now — segmentation requires router-level VLAN config not currently documented or automated. | -| `testing` router reachable when `ENVIRONMENT=development` | Accepted — intentionally dev-gated per C1 in the former documentation audit tracker; operational discipline (never deploy with `ENVIRONMENT=development`) is the control, not code. | -| GSM module DB credentials hardcoded default in `config.py` | Open — tracked in the repo-root `SECURITY.md`. | +| `testing` router reachable when `ENVIRONMENT=development` or `staging` | Accepted — intentionally QA-gated; operational discipline (never deploy with either setting in production) is the control, not code. | +| GSM module DB credentials hardcoded default in `config.py` | Open — tracked in [SECURITY.md](../../SECURITY.md#other-known-gaps-not-yet-resolved). | | No remote session/device revocation UI for end users | Open — see [Device theft](#device-theft). | | Optional (not enforced) server-side `PeerKey` signing | Open — see [E2E encryption design risks](#e2e-encryption-design-risks). Recommend making `SERVER_ED25519_SEED` mandatory in production as a follow-up. | diff --git a/docs/deployment/environment-config.md b/docs/deployment/environment-config.md index ca063b24..5f8f8608 100644 --- a/docs/deployment/environment-config.md +++ b/docs/deployment/environment-config.md @@ -11,14 +11,16 @@ All SAPOT components are configured via environment variables. This document lis | `DATABASE_URL` | None — required, raises `RuntimeError` at import if unset | **MUST** be set (MariaDB connection string) | | `JWT_SECRET_KEY` | None — required, raises `RuntimeError` at import if unset | **MUST** be set — generate a strong random secret | | `CORS_ALLOWED_ORIGINS` | None — required, raises `RuntimeError` at startup if unset | **MUST** be set — comma-separated explicit origin allowlist | -| `ENVIRONMENT` | `production` | Set to `development` to enable the `/testing/*` router; never set to `development` in production | -| `QA_API_TOKEN` | None — required, raises `RuntimeError` at import if unset **when `ENVIRONMENT=development`** | Only relevant in development; the `X-QA-Token` header value `/testing/reset` and `/testing/login-as/{handle}` require | +| `ENVIRONMENT` | `production` | One of `development`, `staging`, or `production`. `development` and `staging` enable the `/testing/*` router; never use either in a production deployment. Any other value raises `ValueError` at import time. | +| `QA_API_TOKEN` | None — required, raises `RuntimeError` at import if unset **when `ENVIRONMENT=development` or `staging`** | Required in QA-enabled environments; the `X-QA-Token` header value protects `/testing/reset` and `/testing/login-as/{handle}` | | `REDIS_URL` | `redis://localhost:6379` | Set if Redis is on a non-default host/port | | `SERVER_ED25519_SEED` | `None` (server key signing disabled if unset) | Set to enable server-signed peer keys | | `GSM_SECRET` | `""` (empty — webhook auth disabled) | Set to a shared secret to authenticate GSM module webhooks | See the repo-root `SECURITY.md` for why `DATABASE_URL`, `JWT_SECRET_KEY`, and `CORS_ALLOWED_ORIGINS` became required. +> **Deployment note:** `ENVIRONMENT` is validated at import time. A typo such as `Development` or `dev` stops the server rather than silently applying production behaviour. Correct the value in the service environment file, then restart the service. + > **Note:** `server/.env.example` has since been synced to include `DATABASE_URL`, `CORS_ALLOWED_ORIGINS`, > `ENVIRONMENT`, and `REDIS_URL` (previously flagged here as missing). It still lists `TLS_CERT`/`TLS_KEY`, > which are **not** read anywhere in `server/app/*.py` (verified via grep for `os.environ`/`os.getenv`) diff --git a/docs/deployment/maintenance.md b/docs/deployment/maintenance.md index 295d601e..5bd29ad7 100644 --- a/docs/deployment/maintenance.md +++ b/docs/deployment/maintenance.md @@ -58,6 +58,6 @@ Never bundle a dependency bump with an unrelated feature change — if it breaks Before standing up SAPOT at a new incident site (fresh hardware, not a restore — see [runbooks.md's disaster recovery](runbooks.md#disaster-recovery--server-hardware-fails-at-incident-site) for that case): 1. Confirm the offline root CA is still valid (see schedule above) and re-issue a server leaf if needed. -2. Confirm all required secrets are set per [environment-config.md](environment-config.md) and the repo-root `SECURITY.md` — the server fails fast at import if `DATABASE_URL`, `JWT_SECRET_KEY`, or `CORS_ALLOWED_ORIGINS` are missing. -3. Confirm `ENVIRONMENT` is **not** set to `development` in the field deployment's env — that gate exists specifically to keep `/testing/*` endpoints out of production (see [TROUBLESHOOTING.md](../TROUBLESHOOTING.md#testing-endpoints-return-404-in-dev)). +2. Confirm all required secrets are set per [environment-config.md](environment-config.md) and [SECURITY.md](../../SECURITY.md) — the server fails fast at import if `DATABASE_URL`, `JWT_SECRET_KEY`, or `CORS_ALLOWED_ORIGINS` are missing. +3. Confirm `ENVIRONMENT` is **not** set to `development` or `staging` in the field deployment's env — those values enable `/testing/*` endpoints (see [TROUBLESHOOTING.md](../TROUBLESHOOTING.md#testing-endpoints-return-404-in-development-or-staging)). 4. Confirm `sapot-db-backup.timer` is enabled (a docker-bundle `install.sh` does this for you; bare-metal is a manual step, see [runbooks.md](runbooks.md#backup-automated)) and run `backup-db.sh` once by hand to take a baseline before real data accumulates only on this host. diff --git a/docs/getting-started/docker-setup.md b/docs/getting-started/docker-setup.md index b2652102..333f1fee 100644 --- a/docs/getting-started/docker-setup.md +++ b/docs/getting-started/docker-setup.md @@ -138,7 +138,7 @@ docker compose ps # is api running, exited, or restarting? docker compose logs api --tail=50 # look for a traceback right before "Application startup failed" ``` - **`api` is running and the 502s stop on their own within a few seconds of `up -d`**: expected. The dev stack gives `api` no healthcheck, so `nginx` starts as soon as the `api` *process* does, which is before Uvicorn finishes importing the app. Just retry. -- **`api` is running and the 502s persist**: read the logs. An unset required env var (`DATABASE_URL`, `JWT_SECRET_KEY`, `CORS_ALLOWED_ORIGINS`, `SERVER_ED25519_SEED`, or `QA_API_TOKEN` when `ENVIRONMENT=development`) raises at import time and the container exits before serving anything. +- **`api` is running and the 502s persist**: read the logs. An unset required env var (`DATABASE_URL`, `JWT_SECRET_KEY`, `CORS_ALLOWED_ORIGINS`, `SERVER_ED25519_SEED`, or `QA_API_TOKEN` when `ENVIRONMENT=development` or `staging`) raises at import time and the container exits before serving anything. An unrecognised `ENVIRONMENT` value also stops the app at import time. **Requests reach `api` but fail on missing tables (`1146 Table ... doesn't exist`).** Migrations were never applied to this `db-data` volume. Run [Apply database migrations](#apply-database-migrations). diff --git a/docs/getting-started/server-setup.md b/docs/getting-started/server-setup.md index 8b3be5ed..47726fa0 100644 --- a/docs/getting-started/server-setup.md +++ b/docs/getting-started/server-setup.md @@ -20,7 +20,7 @@ the variable: | `JWT_SECRET_KEY` | `app/db_operations/token.py` | | `CORS_ALLOWED_ORIGINS` | `app/main.py` | | `SERVER_ED25519_SEED` | `app/db_operations/signing.py` | -| `QA_API_TOKEN` | `app/api/testing.py`, **only** when `ENVIRONMENT=development`, which also mounts the `/testing/*` routes | +| `QA_API_TOKEN` | `app/api/testing.py`, **only** when `ENVIRONMENT=development` or `staging`, which also mounts the `/testing/*` routes | Copy `server/.env.example` to `server/.env` and fill it in. The example ships a usable value for every variable above except the database host, so the only edits a local run needs are the two @@ -33,7 +33,7 @@ JWT_SECRET_KEY= SERVER_ED25519_SEED= CORS_ALLOWED_ORIGINS=http://192.168.1.x:3000 ENVIRONMENT=development -QA_API_TOKEN= +QA_API_TOKEN= ``` > **`.env.example`'s shipped `DATABASE_URL`/`REDIS_URL` point at the Docker Compose service names (`db`/`redis`)** — they only resolve inside the Docker network. Change both to `127.0.0.1`/`localhost` (as above) for this bare-metal path. diff --git a/docs/qa/scenario-tooling.md b/docs/qa/scenario-tooling.md index f3214614..968ac976 100644 --- a/docs/qa/scenario-tooling.md +++ b/docs/qa/scenario-tooling.md @@ -32,9 +32,9 @@ Router: [`server/app/api/testing.py`](../../server/app/api/testing.py). Scenario This surface can reset a database and mint auth tokens without a password, so it's gated defense-in-depth style — see the file header in `testing.py` and `SECURITY.md`: -1. **Import gating** — `app/main.py` only imports this router when `ENVIRONMENT=development`; in a production build the code never loads. +1. **Import gating** — `app/main.py` only imports this router when `ENVIRONMENT=development` or `staging`; in a production build the code never loads. 2. **`require_qa_env()`** — every route re-checks `IS_QA_ENABLED` at request time and 404s otherwise, in case the router is ever mounted somewhere unexpected. -3. **`require_qa_token()`** — `/testing/reset` and `/testing/login-as/{handle}` additionally require an `X-QA-Token` header matching the `QA_API_TOKEN` env var (constant-time compare). `QA_API_TOKEN` has no default and fails fast at import time if unset in a dev environment — mirrors the `JWT_SECRET_KEY` pattern (`SECURITY.md`). +3. **`require_qa_token()`** — `/testing/reset` and `/testing/login-as/{handle}` additionally require an `X-QA-Token` header matching the `QA_API_TOKEN` env var (constant-time compare). `QA_API_TOKEN` has no default and fails fast at import time if unset in a development or staging environment — mirrors the `JWT_SECRET_KEY` pattern (`SECURITY.md`). 4. **Fixed fixture allowlist** — `/testing/login-as/{handle}` only ever mints a token for a handle in the `FIXTURE_HANDLES` set baked into `testing.py`; it can never be used to log in as an arbitrary or real user's account. Set `QA_API_TOKEN` in `server/.env` (see `server/.env.example`); documented alongside other env vars in [`deployment/environment-config.md`](../deployment/environment-config.md). @@ -49,7 +49,7 @@ Set `QA_API_TOKEN` in `server/.env` (see `server/.env.example`); documented alon ## Typical QA workflow -1. Run the stack against `ENVIRONMENT=development` with `QA_API_TOKEN` set (see [Set up an environment to test against](README.md#set-up-an-environment-to-test-against)). +1. Run the stack against `ENVIRONMENT=development` or `staging` with `QA_API_TOKEN` set (see [Set up an environment to test against](README.md#set-up-an-environment-to-test-against)). 2. `POST /testing/reset` to clear the database, then `POST /testing/seed/roles` (or whichever scenario the test plan calls for). 3. In the mobile app, open the debug FAB → Auth section → tap the fixture account you need (e.g. `qa_rescuer`, `qa_admin`). 4. The app wipes local data, logs in as that fixture identity, and restarts — ready to test the role-gated flow without manual registration. diff --git a/server/CLAUDE.md b/server/CLAUDE.md index deeb707a..1f8a3113 100644 --- a/server/CLAUDE.md +++ b/server/CLAUDE.md @@ -10,7 +10,7 @@ FastAPI + SQLModel backend, Python 3.13. Provides auth, sync, WebSocket signalli Entry point `app/main.py` (mounted as `app.main:app`). Request flow: Nginx (TLS) → Gunicorn/Uvicorn workers → FastAPI app → `SlowAPIMiddleware` (rate limiting) → activity-tracking middleware → route handler (`app/api/*.py`) → `SessionDep` (DB session dependency, `app/db_operations/auth.py`) → SQLModel query against MariaDB. -- One router module per feature area in `app/api/`: `auth`, `gps`, `sync`, `admin`, `gsm`, `mikrotik`, `captive_portal`, `keys`, `wrapped_key`, `user_keys`, `peer_connection`, `public_chat`, `profile_picture`, `download`, etc. `app/api/testing.py` is conditionally imported only when `ENVIRONMENT=development` (see `app/main.py`). +- One router module per feature area in `app/api/`: `auth`, `gps`, `sync`, `admin`, `gsm`, `mikrotik`, `captive_portal`, `keys`, `wrapped_key`, `user_keys`, `peer_connection`, `public_chat`, `profile_picture`, `download`, etc. `app/api/testing.py` is conditionally imported only when `ENVIRONMENT=development` or `staging` (see `app/main.py`). - `app/db_operations/*.py` holds DB-facing logic (auth/session handling, GPS manager, sync tokens, router client/metrics, key recovery, etc.) — route handlers call into these rather than querying models directly. - `app/models/*.py` — one SQLModel per file (users, messages, calls, keys, recovery, router, captive_portal, etc.). - Rate limiting is a cross-cutting concern via `slowapi` (`app/limiter.py`, `@limiter.limit(...)` decorators on individual routes) — not middleware-only, it's applied per-endpoint. @@ -53,7 +53,7 @@ Entry point `app/main.py` (mounted as `app.main:app`). Request flow: Nginx (TLS) ## Common Pitfalls - Adding a default/fallback value for a required secret env var — this repo has a documented incident (hardcoded DB credentials, hardcoded JWT secret fallback, CORS wildcard) that this fail-fast pattern exists specifically to prevent. See `../SECURITY.md`. -- Making `app/api/testing.py` reachable outside `ENVIRONMENT=development` — it's conditionally imported in `app/main.py` specifically to keep `test-make-admin`/`test-make-rescuer` out of production. +- Making `app/api/testing.py` reachable outside `ENVIRONMENT=development` or `staging` — it's conditionally imported in `app/main.py` specifically to keep its endpoints out of production. - Hand-editing `../docs/api/openapi/*.yaml`, `../docs/database/tables.md`, or `../docs/database/erd.md` — these are generated; CI fails on drift between them and the source. - Hand-editing `app/version.py` instead of using `./scripts/release.sh server ` (repo root) — `scripts/set_version.py` alone bumps the file but skips the commit/tag/release-notes steps. - Changing an endpoint's request/response shape without checking `../docs/api/openapi/` and the mobile/admin clients that depend on it — this server has no consumers of its own; both other components assume the current contract. diff --git a/server/app/api/testing.py b/server/app/api/testing.py index 50d65a5f..b1dfdfc5 100644 --- a/server/app/api/testing.py +++ b/server/app/api/testing.py @@ -1,4 +1,4 @@ -"""Dev/staging-only QA tooling surface. Never imported outside `ENVIRONMENT=development` +"""Dev/staging-only QA tooling surface. Never imported outside a QA-enabled environment (see `app/main.py`) — every route additionally re-checks that at request time via `require_qa_env`, and the mutating routes require a shared-secret header on top of that. See `SECURITY.md` and the design doc referenced from GH #271-274 for the full rationale. @@ -13,11 +13,9 @@ from app.db_operations.auth import SessionDep, get_user_by_username from app.db_operations.qa_scenarios import SCENARIOS, apply_scenario, reset_database from app.db_operations.token import create_token_pair, get_current_user +from app.env import IS_QA_ENABLED from app.models.users import User, UserPublic -ENVIRONMENT = os.environ.get("ENVIRONMENT", "production") -IS_QA_ENABLED = ENVIRONMENT == "development" - # Fail-fast, no default — mirrors JWT_SECRET_KEY/CORS_ALLOWED_ORIGINS (see SECURITY.md). # Only required when this router is actually reachable; a production import never runs # this module at all (see app/main.py's conditional import). @@ -25,7 +23,7 @@ if IS_QA_ENABLED and not QA_API_TOKEN: raise RuntimeError( "QA_API_TOKEN environment variable is not set. Required whenever " - "ENVIRONMENT=development so /testing/reset and /testing/login-as aren't a " + "ENVIRONMENT=development or staging so /testing/reset and /testing/login-as aren't a " "LAN-wide auth bypass with no secret at all." ) diff --git a/server/app/env.py b/server/app/env.py new file mode 100644 index 00000000..ff122322 --- /dev/null +++ b/server/app/env.py @@ -0,0 +1,15 @@ +"""Validated server deployment environment configuration.""" +import os +from enum import Enum + + +class Env(str, Enum): + development = "development" + staging = "staging" + production = "production" + + +# Deliberately validate at import time. An unrecognised deployment setting must not +# silently fall back to production behaviour. +ENVIRONMENT = Env(os.environ.get("ENVIRONMENT", Env.production.value)) +IS_QA_ENABLED = ENVIRONMENT in (Env.development, Env.staging) diff --git a/server/app/main.py b/server/app/main.py index 21e6f3d3..6aea5d8b 100644 --- a/server/app/main.py +++ b/server/app/main.py @@ -16,6 +16,7 @@ from fastapi import Request from starlette.responses import JSONResponse +from app.env import IS_QA_ENABLED from app.limiter import limiter from app.version import __version__ @@ -24,8 +25,7 @@ from app.db_operations.auth import SessionDep from app.api import auth, forgot_password, verify_email, peer_connection, ping, update_info, sync, profile_picture, gps, admin, public_chat, mikrotik, captive_portal, download -ENVIRONMENT = os.environ.get("ENVIRONMENT", "production") -if ENVIRONMENT == "development": +if IS_QA_ENABLED: from app.api import testing from app.api import keys, wrapped_key, user_keys from app.models.peer_key import PeerKey @@ -255,7 +255,7 @@ async def log_activity(request: Request, call_next): app.include_router(profile_picture.router) app.include_router(gps.router) app.include_router(admin.router) -if ENVIRONMENT == "development": +if IS_QA_ENABLED: app.include_router(testing.router) app.include_router(public_chat.router) app.include_router(gsm.router) diff --git a/server/app/scripts/seed_db.py b/server/app/scripts/seed_db.py index 887cbeb7..875be813 100644 --- a/server/app/scripts/seed_db.py +++ b/server/app/scripts/seed_db.py @@ -13,23 +13,27 @@ cd server ENVIRONMENT=development ./app/venv/bin/python -m app.scripts.seed_db + # A QA deployment can use staging instead. + ENVIRONMENT=staging ./app/venv/bin/python -m app.scripts.seed_db + Idempotent: re-running skips records that already exist (matched by username / conversation title), so it's safe to run after every `docker compose up` without duplicating data. Thin CLI over `app/db_operations/qa_scenarios.py`, which also backs the `/testing/*` HTTP scenario surface — see that module for the actual seeding logic. """ -import os import sys from dotenv import load_dotenv load_dotenv() -if os.environ.get("ENVIRONMENT") != "development": +from app.env import IS_QA_ENABLED + +if not IS_QA_ENABLED: sys.exit( - "Refusing to seed: ENVIRONMENT must be 'development' " - f"(got {os.environ.get('ENVIRONMENT')!r}). This script writes sample data " + "Refusing to seed: ENVIRONMENT must be 'development' or 'staging'. " + "This script writes sample data " "and must never run against a production database." ) diff --git a/server/app/tests/test_env.py b/server/app/tests/test_env.py new file mode 100644 index 00000000..3d865299 --- /dev/null +++ b/server/app/tests/test_env.py @@ -0,0 +1,35 @@ +import importlib + +import pytest + +import app.env as env_module + + +def test_environment_defaults_to_production(monkeypatch): + with monkeypatch.context() as patch: + patch.delenv("ENVIRONMENT", raising=False) + env = importlib.reload(env_module) + assert env.ENVIRONMENT is env.Env.production + assert env.IS_QA_ENABLED is False + importlib.reload(env_module) + + +@pytest.mark.parametrize( + ("value", "expected_qa_enabled"), + [("development", True), ("staging", True), ("production", False)], +) +def test_environment_accepts_known_values(monkeypatch, value, expected_qa_enabled): + with monkeypatch.context() as patch: + patch.setenv("ENVIRONMENT", value) + env = importlib.reload(env_module) + assert env.ENVIRONMENT.value == value + assert env.IS_QA_ENABLED is expected_qa_enabled + importlib.reload(env_module) + + +def test_environment_rejects_unknown_values(monkeypatch): + with monkeypatch.context() as patch: + patch.setenv("ENVIRONMENT", "Development") + with pytest.raises(ValueError, match="Development"): + importlib.reload(env_module) + importlib.reload(env_module) From ed53ea62f2787820f8c824c3b3ec8087a99cdbca Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:46:49 +0800 Subject: [PATCH 02/12] fix(server-env): fail fast when GSM_SECRET is unset #361 --- SECURITY.md | 1 + docs/deployment/environment-config.md | 4 +-- server/app/api/gsm.py | 7 +++-- server/app/tests/test_security_regression.py | 30 ++++++++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 17e6c885..a985f095 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -21,6 +21,7 @@ Set these in the server's env file (see [environment-config.md](docs/deployment/ DATABASE_URL=mysql+pymysql://:@127.0.0.1:3306/sapot_db JWT_SECRET_KEY= CORS_ALLOWED_ORIGINS=http://192.168.0.100:3000 +GSM_SECRET= ``` ## Reporting a vulnerability diff --git a/docs/deployment/environment-config.md b/docs/deployment/environment-config.md index 5f8f8608..7e4604d0 100644 --- a/docs/deployment/environment-config.md +++ b/docs/deployment/environment-config.md @@ -15,9 +15,9 @@ All SAPOT components are configured via environment variables. This document lis | `QA_API_TOKEN` | None — required, raises `RuntimeError` at import if unset **when `ENVIRONMENT=development` or `staging`** | Required in QA-enabled environments; the `X-QA-Token` header value protects `/testing/reset` and `/testing/login-as/{handle}` | | `REDIS_URL` | `redis://localhost:6379` | Set if Redis is on a non-default host/port | | `SERVER_ED25519_SEED` | `None` (server key signing disabled if unset) | Set to enable server-signed peer keys | -| `GSM_SECRET` | `""` (empty — webhook auth disabled) | Set to a shared secret to authenticate GSM module webhooks | +| `GSM_SECRET` | None — required, raises `RuntimeError` at import if unset | **MUST** be set — shared secret for GSM module webhooks | -See the repo-root `SECURITY.md` for why `DATABASE_URL`, `JWT_SECRET_KEY`, and `CORS_ALLOWED_ORIGINS` became required. +See [SECURITY.md](../../SECURITY.md) for why `DATABASE_URL`, `JWT_SECRET_KEY`, `CORS_ALLOWED_ORIGINS`, and `GSM_SECRET` are required. > **Deployment note:** `ENVIRONMENT` is validated at import time. A typo such as `Development` or `dev` stops the server rather than silently applying production behaviour. Correct the value in the service environment file, then restart the service. diff --git a/server/app/api/gsm.py b/server/app/api/gsm.py index 3d2443d4..902bb71f 100644 --- a/server/app/api/gsm.py +++ b/server/app/api/gsm.py @@ -27,6 +27,10 @@ _gsm_http_client: httpx.AsyncClient | None = None logger = logging.getLogger("app") +GSM_SECRET = os.environ.get("GSM_SECRET") +if not GSM_SECRET: + raise RuntimeError("GSM_SECRET environment variable is not set") + def _get_gsm_client() -> httpx.AsyncClient: global _gsm_http_client @@ -69,8 +73,7 @@ class InboundSMSPayload(BaseModel): def _gsm_secret_ok(request: Request) -> bool: - secret = os.getenv("GSM_SECRET", "") - return secret and request.headers.get("X-GSM-Secret") == secret + return request.headers.get("X-GSM-Secret") == GSM_SECRET def _create_sms_conversation(session, convo_id: UUID, user_a_id: UUID, user_b_id: UUID) -> Conversation: diff --git a/server/app/tests/test_security_regression.py b/server/app/tests/test_security_regression.py index 951ec6a1..9b0bde8d 100644 --- a/server/app/tests/test_security_regression.py +++ b/server/app/tests/test_security_regression.py @@ -11,6 +11,10 @@ import time and re-importing it mid-suite is not meaningfully different from a unit test of the guard itself. """ +import os +from pathlib import Path +import subprocess +import sys import uuid import pytest from fastapi import HTTPException @@ -23,6 +27,32 @@ from app.tests.test_db_utils import get_auth_headers +def test_gsm_secret_is_required_at_import_time(): + """GH #244: a missing GSM webhook secret must prevent server startup.""" + env = os.environ.copy() + env.update( + { + "DATABASE_URL": "sqlite:////tmp/gsm-secret-import-test.db", + "JWT_SECRET_KEY": "gsm-secret-import-test", + "CORS_ALLOWED_ORIGINS": "http://testserver", + "SERVER_ED25519_SEED": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + } + ) + env.pop("GSM_SECRET", None) + + result = subprocess.run( + [sys.executable, "-c", "import app.api.gsm"], + cwd=Path(__file__).resolve().parents[2], + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode != 0 + assert "GSM_SECRET environment variable is not set" in result.stderr + + # --------------------------------------------------------------------------- # Shared fixtures # --------------------------------------------------------------------------- From f32f78527fbde5daf19bb213376fd8dbe20b97d4 Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:53:40 +0800 Subject: [PATCH 03/12] fix(mobile-eas): enable debug menu in preview builds (#360) --- mobile-app/sapot-mobile-app/eas.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mobile-app/sapot-mobile-app/eas.json b/mobile-app/sapot-mobile-app/eas.json index 2c3076f2..c22888a7 100644 --- a/mobile-app/sapot-mobile-app/eas.json +++ b/mobile-app/sapot-mobile-app/eas.json @@ -6,7 +6,8 @@ "buildType": "apk" }, "env": { - "APP_VARIANT": "preview" + "APP_VARIANT": "preview", + "EXPO_PUBLIC_DEBUG_MENU": "1" }, "channel": "preview" }, From 330288f31d99e68a06c4bd63110c10b7ea0e4a36 Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:14:07 +0800 Subject: [PATCH 04/12] fix(server-config): replace example secrets (#366) * fix(server-config): replace example secrets * chore(server-config): simplify example cors --- docs/getting-started/docker-setup.md | 6 +++++- docs/getting-started/server-setup.md | 10 +++++----- server/.env.example | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/getting-started/docker-setup.md b/docs/getting-started/docker-setup.md index 333f1fee..32a265ea 100644 --- a/docs/getting-started/docker-setup.md +++ b/docs/getting-started/docker-setup.md @@ -36,6 +36,10 @@ Everything below works from a WSL2 distro's bash shell as-is — use `docker/up. cp server/.env.example server/.env ``` +Before starting the stack, replace the `JWT_SECRET_KEY` and `SERVER_ED25519_SEED` placeholders in +`server/.env`. Generate a separate value for each with `openssl rand -hex 32`; also replace the +other `change-me-*` secrets with values appropriate for your environment. + Optional — override the stack's host-side ports (default: `nginx` 443/80, `admin` 3000, `gsm-fastapi` 8001) by copying the repo-root env file too: @@ -68,7 +72,7 @@ whole `docker compose up` would abort on any machine without the GSM modem attac Without the GSM modem, just run the normal `./docker/up.sh up --build -d` below — `gsm-fastapi` still starts, it just won't have serial access. -See the repo-root `SECURITY.md` for why `DATABASE_URL`, `JWT_SECRET_KEY`, `CORS_ALLOWED_ORIGINS`, and `SERVER_ED25519_SEED` are required at import time. `server/.env.example` ships a working value for each, so the copy above is enough for local dev. +See the repo-root `SECURITY.md` for why `DATABASE_URL`, `JWT_SECRET_KEY`, `CORS_ALLOWED_ORIGINS`, and `SERVER_ED25519_SEED` are required at import time. `server/.env.example` supplies safe defaults only for local service addresses; it never supplies usable secrets. ## Run diff --git a/docs/getting-started/server-setup.md b/docs/getting-started/server-setup.md index 47726fa0..05fdb997 100644 --- a/docs/getting-started/server-setup.md +++ b/docs/getting-started/server-setup.md @@ -22,9 +22,9 @@ the variable: | `SERVER_ED25519_SEED` | `app/db_operations/signing.py` | | `QA_API_TOKEN` | `app/api/testing.py`, **only** when `ENVIRONMENT=development` or `staging`, which also mounts the `/testing/*` routes | -Copy `server/.env.example` to `server/.env` and fill it in. The example ships a usable value for -every variable above except the database host, so the only edits a local run needs are the two -marked below: +Copy `server/.env.example` to `server/.env` and replace every placeholder secret before starting +the server. The example's database and Redis hosts target the Docker Compose network, so change +them for bare-metal use too: ```dotenv DATABASE_URL=mysql+pymysql://sapot:sapot@127.0.0.1:3306/sapot_dev # changed from db:3306 @@ -38,8 +38,8 @@ QA_API_TOKEN= **`.env.example`'s shipped `DATABASE_URL`/`REDIS_URL` point at the Docker Compose service names (`db`/`redis`)** — they only resolve inside the Docker network. Change both to `127.0.0.1`/`localhost` (as above) for this bare-metal path. -> The shipped `JWT_SECRET_KEY` and `SERVER_ED25519_SEED` are committed placeholders. They are fine -> for a laptop, but regenerate both before the server is reachable by anyone else. +> Generate each placeholder value with `openssl rand -hex 32`. Do not reuse a value from the +> template, another environment, or a previous deployment. See [environment-config.md](../deployment/environment-config.md) for the full variable list and the repo-root `SECURITY.md` for why these are required. diff --git a/server/.env.example b/server/.env.example index 6ac08aa5..f2caea74 100644 --- a/server/.env.example +++ b/server/.env.example @@ -1,6 +1,6 @@ GSM_SECRET=change-me-to-a-strong-secret -SERVER_ED25519_SEED=4932ff9c160683dd1097820825f51aecc0d5c066c6b7dbbe61d8c664f0bfa940 -JWT_SECRET_KEY=7a272aa19fd88943207a62115b64f67530731eafd3b79a228f42972a2a51df1e +SERVER_ED25519_SEED= +JWT_SECRET_KEY= # Only read by bare-metal runserver.sh — the Docker stack's certgen/nginx # generate and mount their own certs instead (see docs/getting-started/docker-setup.md). From 5e7518b6ff1939623f08f6195655844309810e50 Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:11:13 +0800 Subject: [PATCH 05/12] feat(server-testing): harden QA route guards (#367) --- SECURITY.md | 5 +- docs/api/README.md | 4 +- docs/api/openapi/system.yaml | 24 ++++ .../assumptions-and-constraints.md | 2 +- docs/architecture/threat-model.md | 2 +- docs/deployment/environment-config.md | 2 +- docs/getting-started/server-setup.md | 2 +- docs/qa/scenario-tooling.md | 26 +++-- postman/flows/admin.postman_collection.json | 2 +- .../authentication.postman_collection.json | 2 +- postman/flows/sync.postman_collection.json | 2 +- server/.env.example | 4 +- server/app/api/testing.py | 49 ++++---- server/app/tests/test_security_regression.py | 107 ++++++++++++++++-- server/app/tests/test_testing_endpoints.py | 19 ++-- 15 files changed, 188 insertions(+), 64 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index a985f095..daf74d31 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,7 +11,7 @@ This is the canonical source of truth for SAPOT's known security-relevant config | Hardcoded MariaDB credentials | `server/app/db_operations/auth.py` (`SQLALCHEMY_DATABASE_URL`) | Now required via `DATABASE_URL` env var; the app raises `RuntimeError` at import time if unset. **Rotate the previously-hardcoded DB password before deploying this fix.** | | Hardcoded JWT secret fallback | `server/app/db_operations/token.py` (`SECRET_KEY`) | The default value has been removed; `JWT_SECRET_KEY` is now required, and the app raises `RuntimeError` at import time if unset. **Rotate to a newly generated secret** (`openssl rand -hex 32`) — the old hardcoded value must be considered compromised since it was committed to source. | | CORS wildcard + credentials | `server/app/main.py` | `allow_origins=["*"]` replaced with an explicit allowlist read from `CORS_ALLOWED_ORIGINS` (comma-separated). The app raises `RuntimeError` at import time if unset. | -| Testing router in production | `server/app/main.py` | `app.include_router(testing.router)` is gated behind QA-enabled `ENVIRONMENT` values (`development` or `staging`; see `app/main.py`). The `/testing/*` endpoints are unreachable in production. | +| Testing router in production | `server/app/main.py`, `server/app/api/testing.py` | The router is imported and mounted only in `development` or `staging`. A router-wide dependency also returns 404 outside those environments if the router is mis-mounted. Every state-changing route requires the `X-QA-Token` shared secret. A production-process regression test exercises every testing path. | ## Required environment variables (new) @@ -24,6 +24,9 @@ CORS_ALLOWED_ORIGINS=http://192.168.0.100:3000 GSM_SECRET= ``` +QA-enabled environments also require `QA_API_TOKEN`. Generate a strong random value and send it +as `X-QA-Token` for state-changing `/testing/*` requests. Production does not load this secret. + ## Reporting a vulnerability This is a LAN-deployed application without a public bug bounty program. Report suspected vulnerabilities directly to the maintainer rather than opening a public GitHub issue. diff --git a/docs/api/README.md b/docs/api/README.md index ba9362b5..f9f15b7f 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -97,8 +97,8 @@ Grouped in [`openapi/system.yaml`](openapi/system.yaml); no dedicated `.md` file | `/ping` | GET | Any | Returns `{"status": "ok", "timestamp": }` — use for latency measurement | | `/static/*` | GET | None | Static file serving (profile pictures, downloads) | | `/download/download-apk` | GET | None | Serves the current mobile app APK build | -| `/testing/test-make-admin` | POST | Any | Dev/testing only — grants the admin role to `username` | -| `/testing/test-make-rescuer` | POST | Any | Dev/testing only — grants the rescuer role to `username` | +| `/testing/test-make-admin` | POST | Authenticated + `X-QA-Token` | Dev/testing only; grants the admin role to `username` | +| `/testing/test-make-rescuer` | POST | Authenticated + `X-QA-Token` | Dev/testing only; grants the rescuer role to `username` | | `/update/profile/` | POST | Any | Updates the calling user's own profile fields | | `/user-utils/current-user-info` | GET | Any | Returns the calling user's own `UserInfo` | | `/user-utils/get-announcements` | GET | Any | Paginated announcements targeted at the calling user | diff --git a/docs/api/openapi/system.yaml b/docs/api/openapi/system.yaml index 8d42f19d..504875df 100644 --- a/docs/api/openapi/system.yaml +++ b/docs/api/openapi/system.yaml @@ -136,6 +136,14 @@ paths: schema: type: string title: Scenario + - name: x-qa-token + in: header + required: false + schema: + anyOf: + - type: string + - type: 'null' + title: X-Qa-Token responses: '200': description: Successful Response @@ -165,6 +173,14 @@ paths: schema: type: string title: Username + - name: x-qa-token + in: header + required: false + schema: + anyOf: + - type: string + - type: 'null' + title: X-Qa-Token responses: '200': description: Successful Response @@ -194,6 +210,14 @@ paths: schema: type: string title: Username + - name: x-qa-token + in: header + required: false + schema: + anyOf: + - type: string + - type: 'null' + title: X-Qa-Token responses: '200': description: Successful Response diff --git a/docs/architecture/assumptions-and-constraints.md b/docs/architecture/assumptions-and-constraints.md index a6c5b4ac..45fdf0dc 100644 --- a/docs/architecture/assumptions-and-constraints.md +++ b/docs/architecture/assumptions-and-constraints.md @@ -58,7 +58,7 @@ See [threat-model.md](threat-model.md#attack-surfaces-explicitly-out-of-scope) f Risks the project has consciously decided to carry rather than fix, reproduced from the threat model's tradeoff table (see that document for status updates): - No LAN segmentation — requires router-level VLAN config not currently documented or automated. -- `testing` router reachable when `ENVIRONMENT=development` or `staging` — accepted; operational discipline (never deploy with either setting in production) is the control. +- `testing` router reachable when `ENVIRONMENT=development` or `staging` is accepted for QA. Production uses conditional mounting, a route-level environment guard, shared-secret authentication on mutations, and regression coverage. - GSM module DB credentials hardcoded default in `config.py` — open, tracked in [SECURITY.md](../../SECURITY.md#other-known-gaps-not-yet-resolved). - No remote session/device revocation UI — open. - Optional (not enforced) server-side `PeerKey` signing — open. diff --git a/docs/architecture/threat-model.md b/docs/architecture/threat-model.md index 1a24a8e6..ec46a341 100644 --- a/docs/architecture/threat-model.md +++ b/docs/architecture/threat-model.md @@ -118,7 +118,7 @@ flowchart TB | Risk | Status | |---|---| | No LAN segmentation (rescuer/admin/civilian devices share one broadcast domain) | Accepted for now — segmentation requires router-level VLAN config not currently documented or automated. | -| `testing` router reachable when `ENVIRONMENT=development` or `staging` | Accepted — intentionally QA-gated; operational discipline (never deploy with either setting in production) is the control, not code. | +| `testing` router reachable when `ENVIRONMENT=development` or `staging` | Accepted for QA. Production is protected by conditional mounting, a route-level environment guard, shared-secret authentication on mutations, and a production-process regression test. | | GSM module DB credentials hardcoded default in `config.py` | Open — tracked in [SECURITY.md](../../SECURITY.md#other-known-gaps-not-yet-resolved). | | No remote session/device revocation UI for end users | Open — see [Device theft](#device-theft). | | Optional (not enforced) server-side `PeerKey` signing | Open — see [E2E encryption design risks](#e2e-encryption-design-risks). Recommend making `SERVER_ED25519_SEED` mandatory in production as a follow-up. | diff --git a/docs/deployment/environment-config.md b/docs/deployment/environment-config.md index 7e4604d0..ac58a7b2 100644 --- a/docs/deployment/environment-config.md +++ b/docs/deployment/environment-config.md @@ -12,7 +12,7 @@ All SAPOT components are configured via environment variables. This document lis | `JWT_SECRET_KEY` | None — required, raises `RuntimeError` at import if unset | **MUST** be set — generate a strong random secret | | `CORS_ALLOWED_ORIGINS` | None — required, raises `RuntimeError` at startup if unset | **MUST** be set — comma-separated explicit origin allowlist | | `ENVIRONMENT` | `production` | One of `development`, `staging`, or `production`. `development` and `staging` enable the `/testing/*` router; never use either in a production deployment. Any other value raises `ValueError` at import time. | -| `QA_API_TOKEN` | None — required, raises `RuntimeError` at import if unset **when `ENVIRONMENT=development` or `staging`** | Required in QA-enabled environments; the `X-QA-Token` header value protects `/testing/reset` and `/testing/login-as/{handle}` | +| `QA_API_TOKEN` | None; raises `RuntimeError` at import if unset when `ENVIRONMENT=development` or `staging` | Required in QA-enabled environments; every state-changing `/testing/*` request must send it in the `X-QA-Token` header | | `REDIS_URL` | `redis://localhost:6379` | Set if Redis is on a non-default host/port | | `SERVER_ED25519_SEED` | `None` (server key signing disabled if unset) | Set to enable server-signed peer keys | | `GSM_SECRET` | None — required, raises `RuntimeError` at import if unset | **MUST** be set — shared secret for GSM module webhooks | diff --git a/docs/getting-started/server-setup.md b/docs/getting-started/server-setup.md index 05fdb997..902b15db 100644 --- a/docs/getting-started/server-setup.md +++ b/docs/getting-started/server-setup.md @@ -20,7 +20,7 @@ the variable: | `JWT_SECRET_KEY` | `app/db_operations/token.py` | | `CORS_ALLOWED_ORIGINS` | `app/main.py` | | `SERVER_ED25519_SEED` | `app/db_operations/signing.py` | -| `QA_API_TOKEN` | `app/api/testing.py`, **only** when `ENVIRONMENT=development` or `staging`, which also mounts the `/testing/*` routes | +| `QA_API_TOKEN` | `app/api/testing.py`, only when `ENVIRONMENT=development` or `staging`; every state-changing `/testing/*` request must send it as `X-QA-Token` | Copy `server/.env.example` to `server/.env` and replace every placeholder secret before starting the server. The example's database and Redis hosts target the Docker Compose network, so change diff --git a/docs/qa/scenario-tooling.md b/docs/qa/scenario-tooling.md index 968ac976..b7065624 100644 --- a/docs/qa/scenario-tooling.md +++ b/docs/qa/scenario-tooling.md @@ -6,13 +6,13 @@ Dev/staging-only tooling that lets a QA tester reset the database to a known sta Router: [`server/app/api/testing.py`](../../server/app/api/testing.py). Scenario builders: [`server/app/db_operations/qa_scenarios.py`](../../server/app/db_operations/qa_scenarios.py) (shared by this router and the `seed_db.py` CLI seeder). -| Endpoint | Method | Purpose | -|---|---|---| -| `/testing/scenarios` | GET | List available scenario names + descriptions | -| `/testing/seed/{scenario}` | POST | Build one scenario's fixture data (additive, doesn't clear existing data) | -| `/testing/reset` | POST | Wipe the database back to empty | -| `/testing/login-as/{handle}` | POST | Mint a JWT pair for a seeded fixture handle, no password required | -| `/testing/test-make-admin`, `/testing/test-make-rescuer` | POST | Promote an existing (real, authenticated) user — pre-existing endpoints, require normal auth | +| Endpoint | Method | Authentication | Purpose | +|---|---|---|---| +| `/testing/scenarios` | GET | QA environment | List available scenario names and descriptions | +| `/testing/seed/{scenario}` | POST | `X-QA-Token` | Build one scenario's fixture data without clearing existing data | +| `/testing/reset` | POST | `X-QA-Token` | Wipe the database and reseed the baseline scenario | +| `/testing/login-as/{handle}` | POST | `X-QA-Token` | Mint a JWT pair for a seeded fixture handle without a password | +| `/testing/test-make-admin`, `/testing/test-make-rescuer` | POST | Normal auth + `X-QA-Token` | Promote an existing user | ### Available scenarios @@ -32,10 +32,12 @@ Router: [`server/app/api/testing.py`](../../server/app/api/testing.py). Scenario This surface can reset a database and mint auth tokens without a password, so it's gated defense-in-depth style — see the file header in `testing.py` and `SECURITY.md`: -1. **Import gating** — `app/main.py` only imports this router when `ENVIRONMENT=development` or `staging`; in a production build the code never loads. -2. **`require_qa_env()`** — every route re-checks `IS_QA_ENABLED` at request time and 404s otherwise, in case the router is ever mounted somewhere unexpected. -3. **`require_qa_token()`** — `/testing/reset` and `/testing/login-as/{handle}` additionally require an `X-QA-Token` header matching the `QA_API_TOKEN` env var (constant-time compare). `QA_API_TOKEN` has no default and fails fast at import time if unset in a development or staging environment — mirrors the `JWT_SECRET_KEY` pattern (`SECURITY.md`). -4. **Fixed fixture allowlist** — `/testing/login-as/{handle}` only ever mints a token for a handle in the `FIXTURE_HANDLES` set baked into `testing.py`; it can never be used to log in as an arbitrary or real user's account. +1. **Import gating:** `app/main.py` only imports and mounts this router when `ENVIRONMENT=development` or `staging`. +2. **`require_qa_env()`:** a router-wide dependency re-checks `IS_QA_ENABLED` at request time and returns 404 if a future change accidentally mounts the router in production. +3. **`require_qa_token()`:** every state-changing route requires an `X-QA-Token` header that matches `QA_API_TOKEN` using a constant-time comparison. The secret has no default and fails fast at import time when a QA environment enables the router. +4. **Production regression:** `test_security_regression.py` starts a production-configured subprocess and asserts that every testing path returns 404 both through the assembled app and through a deliberately mis-mounted router. + +`/testing/login-as/{handle}` also uses a fixed fixture allowlist. It cannot mint a token for an arbitrary account, even when the caller has the QA token. Set `QA_API_TOKEN` in `server/.env` (see `server/.env.example`); documented alongside other env vars in [`deployment/environment-config.md`](../deployment/environment-config.md). @@ -50,7 +52,7 @@ Set `QA_API_TOKEN` in `server/.env` (see `server/.env.example`); documented alon ## Typical QA workflow 1. Run the stack against `ENVIRONMENT=development` or `staging` with `QA_API_TOKEN` set (see [Set up an environment to test against](README.md#set-up-an-environment-to-test-against)). -2. `POST /testing/reset` to clear the database, then `POST /testing/seed/roles` (or whichever scenario the test plan calls for). +2. Send `X-QA-Token` with `POST /testing/reset`, then with `POST /testing/seed/roles` or the scenario the test plan calls for. 3. In the mobile app, open the debug FAB → Auth section → tap the fixture account you need (e.g. `qa_rescuer`, `qa_admin`). 4. The app wipes local data, logs in as that fixture identity, and restarts — ready to test the role-gated flow without manual registration. diff --git a/postman/flows/admin.postman_collection.json b/postman/flows/admin.postman_collection.json index f08e3131..6fc645bf 100644 --- a/postman/flows/admin.postman_collection.json +++ b/postman/flows/admin.postman_collection.json @@ -33,7 +33,7 @@ "name": "Seed roles scenario", "request": { "method": "POST", - "header": [], + "header": [{ "key": "X-QA-Token", "value": "{{qaToken}}" }], "url": { "raw": "{{baseUrl}}/testing/seed/roles", "host": ["{{baseUrl}}"], diff --git a/postman/flows/authentication.postman_collection.json b/postman/flows/authentication.postman_collection.json index f8217291..a925adac 100644 --- a/postman/flows/authentication.postman_collection.json +++ b/postman/flows/authentication.postman_collection.json @@ -37,7 +37,7 @@ "name": "Seed roles scenario", "request": { "method": "POST", - "header": [], + "header": [{ "key": "X-QA-Token", "value": "{{qaToken}}" }], "url": { "raw": "{{baseUrl}}/testing/seed/roles", "host": ["{{baseUrl}}"], diff --git a/postman/flows/sync.postman_collection.json b/postman/flows/sync.postman_collection.json index 9de8ec57..e0b640ab 100644 --- a/postman/flows/sync.postman_collection.json +++ b/postman/flows/sync.postman_collection.json @@ -33,7 +33,7 @@ "name": "Seed baseline scenario", "request": { "method": "POST", - "header": [], + "header": [{ "key": "X-QA-Token", "value": "{{qaToken}}" }], "url": { "raw": "{{baseUrl}}/testing/seed/baseline", "host": ["{{baseUrl}}"], diff --git a/server/.env.example b/server/.env.example index f2caea74..5746eeca 100644 --- a/server/.env.example +++ b/server/.env.example @@ -17,8 +17,8 @@ DATABASE_URL=mysql+pymysql://sapot:sapot@db:3306/sapot_dev ENVIRONMENT=development CORS_ALLOWED_ORIGINS=* -# Required whenever ENVIRONMENT=development — the shared secret /testing/reset and -# /testing/login-as require via the X-QA-Token header (server/app/api/testing.py). +# Required whenever ENVIRONMENT=development or staging. Every state-changing +# /testing route requires it through the X-QA-Token header. QA_API_TOKEN=change-me-to-a-qa-secret # MariaDB container credentials (dev only — do not reuse in production). diff --git a/server/app/api/testing.py b/server/app/api/testing.py index b1dfdfc5..84b9d968 100644 --- a/server/app/api/testing.py +++ b/server/app/api/testing.py @@ -1,7 +1,7 @@ -"""Dev/staging-only QA tooling surface. Never imported outside a QA-enabled environment -(see `app/main.py`) — every route additionally re-checks that at request time via -`require_qa_env`, and the mutating routes require a shared-secret header on top of that. -See `SECURITY.md` and the design doc referenced from GH #271-274 for the full rationale. +"""QA tooling mounted only in development and staging by `app.main`. + +Every route re-checks the environment at request time, and state-changing routes require +a shared-secret header. See `SECURITY.md` and GH #271 through #276 for the rationale. """ import hmac import os @@ -17,14 +17,12 @@ from app.models.users import User, UserPublic # Fail-fast, no default — mirrors JWT_SECRET_KEY/CORS_ALLOWED_ORIGINS (see SECURITY.md). -# Only required when this router is actually reachable; a production import never runs -# this module at all (see app/main.py's conditional import). QA_API_TOKEN = os.environ.get("QA_API_TOKEN") if IS_QA_ENABLED and not QA_API_TOKEN: raise RuntimeError( "QA_API_TOKEN environment variable is not set. Required whenever " - "ENVIRONMENT=development or staging so /testing/reset and /testing/login-as aren't a " - "LAN-wide auth bypass with no secret at all." + "ENVIRONMENT=development or staging so state-changing /testing routes aren't a " + "LAN-wide privilege bypass with no secret at all." ) # Fixture handles the scenario builders in qa_scenarios.py are known to create. @@ -57,11 +55,6 @@ } ) -router = APIRouter( - prefix="/testing", tags=["testing endpoint"], responses={404: {"description": "Not Found"}} -) - - def require_qa_env() -> None: """Layer 2 of the guard: even if this router somehow gets mounted, every route 404s outside development/staging instead of behaving like a live endpoint.""" @@ -70,18 +63,30 @@ def require_qa_env() -> None: def require_qa_token(x_qa_token: Annotated[str | None, Header()] = None) -> None: - """Layer 3: /reset and /login-as additionally require a shared secret so a plain - device on the LAN can't drop the DB or mint an admin token unattended.""" - if not x_qa_token or not hmac.compare_digest(x_qa_token, QA_API_TOKEN): + """Layer 3: mutations require a shared secret so a LAN client cannot alter QA data + or privileges unattended.""" + if ( + QA_API_TOKEN is None + or x_qa_token is None + or not hmac.compare_digest(x_qa_token, QA_API_TOKEN) + ): raise HTTPException(404) -@router.get("/scenarios", dependencies=[Depends(require_qa_env)]) +router = APIRouter( + prefix="/testing", + tags=["testing endpoint"], + dependencies=[Depends(require_qa_env)], + responses={404: {"description": "Not Found"}}, +) + + +@router.get("/scenarios") def list_scenarios(): return {name: scenario.description for name, scenario in SCENARIOS.items()} -@router.post("/seed/{scenario}", dependencies=[Depends(require_qa_env)]) +@router.post("/seed/{scenario}", dependencies=[Depends(require_qa_token)]) def seed_scenario(scenario: str, session: SessionDep): try: summary = apply_scenario(session, scenario) @@ -92,7 +97,7 @@ def seed_scenario(scenario: str, session: SessionDep): @router.post( "/reset", - dependencies=[Depends(require_qa_env), Depends(require_qa_token)], + dependencies=[Depends(require_qa_token)], ) def reset(session: SessionDep): return reset_database(session) @@ -100,7 +105,7 @@ def reset(session: SessionDep): @router.post( "/login-as/{handle}", - dependencies=[Depends(require_qa_env), Depends(require_qa_token)], + dependencies=[Depends(require_qa_token)], response_model=UserPublic, ) def login_as(handle: str, session: SessionDep): @@ -125,7 +130,7 @@ def login_as(handle: str, session: SessionDep): ) -@router.post("/test-make-admin", dependencies=[Depends(require_qa_env)]) +@router.post("/test-make-admin", dependencies=[Depends(require_qa_token)]) def make_user_admin( username: str, session: SessionDep, @@ -138,7 +143,7 @@ def make_user_admin( return {"status": "ok"} -@router.post("/test-make-rescuer", dependencies=[Depends(require_qa_env)]) +@router.post("/test-make-rescuer", dependencies=[Depends(require_qa_token)]) def make_user_rescuer( username: str, session: SessionDep, diff --git a/server/app/tests/test_security_regression.py b/server/app/tests/test_security_regression.py index 9b0bde8d..08547236 100644 --- a/server/app/tests/test_security_regression.py +++ b/server/app/tests/test_security_regression.py @@ -1,20 +1,16 @@ """ -Regression tests for TC-247 and the GH #273 `/testing/*` production guard. +Regression tests for TC-247 and the GH #276 `/testing/*` production guard. TC-247: /gps/ws/monitor/rescuers/{id} must require a valid rescuer token. -GH #273 (`require_qa_env`): every `/testing/*` route must 404 outside -ENVIRONMENT=development, even though this repo's test suite normally runs with -ENVIRONMENT=development (so the router is mounted at all — see main.py). Tested -against the dependency function directly rather than reimporting `app.main` under a -different ENVIRONMENT, since the router-inclusion decision is baked in at process -import time and re-importing it mid-suite is not meaningfully different from a unit -test of the guard itself. +GH #276 (`require_qa_env`): every `/testing/*` route must 404 in a production process, +including when the router is deliberately mis-mounted. """ import os from pathlib import Path import subprocess import sys +import textwrap import uuid import pytest from fastapi import HTTPException @@ -147,11 +143,91 @@ def test_accepts_valid_rescuer_token(self, client: TestClient, rescuer_user: Use # --------------------------------------------------------------------------- -# GH #273 — /testing/* must 404 outside ENVIRONMENT=development +# GH #276: /testing/* must 404 outside a QA-enabled environment # --------------------------------------------------------------------------- class TestQAGuardProductionLockout: + def test_qa_api_token_is_required_at_import_time(self): + env = os.environ.copy() + env.update( + { + "DATABASE_URL": "sqlite:////tmp/qa-token-import-test.db", + "JWT_SECRET_KEY": "qa-token-import-test", + "CORS_ALLOWED_ORIGINS": "http://testserver", + "ENVIRONMENT": "development", + } + ) + env.pop("QA_API_TOKEN", None) + + result = subprocess.run( + [sys.executable, "-c", "import app.api.testing"], + cwd=Path(__file__).resolve().parents[2], + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode != 0 + assert "QA_API_TOKEN environment variable is not set" in result.stderr + + def test_every_testing_path_returns_404_in_production_process(self): + env = os.environ.copy() + env.update( + { + "DATABASE_URL": "sqlite:////tmp/qa-production-lockout-test.db", + "JWT_SECRET_KEY": "qa-production-lockout-test", + "CORS_ALLOWED_ORIGINS": "http://testserver", + "GSM_SECRET": "qa-production-lockout-test", + "SERVER_ED25519_SEED": ( + "0123456789abcdef0123456789abcdef" + "0123456789abcdef0123456789abcdef" + ), + "ENVIRONMENT": "production", + } + ) + env.pop("QA_API_TOKEN", None) + + script = textwrap.dedent( + """ + import re + + from fastapi import FastAPI + from fastapi.routing import APIRoute + from fastapi.testclient import TestClient + + from app.main import app + from app.api import testing + + routes = [route for route in testing.router.routes if isinstance(route, APIRoute)] + + def assert_all_routes_return_404(client): + for route in routes: + path = re.sub(r"{[^}]+}", "fixture", route.path) + for method in route.methods: + response = client.request(method, path) + assert response.status_code == 404, (method, path, response.status_code) + + assert_all_routes_return_404(TestClient(app)) + + mis_mounted_app = FastAPI() + mis_mounted_app.include_router(testing.router) + assert_all_routes_return_404(TestClient(mis_mounted_app)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).resolve().parents[2], + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + def test_require_qa_env_raises_404_when_qa_disabled(self, monkeypatch): from app.api import testing as testing_module @@ -186,8 +262,6 @@ def test_require_qa_token_accepts_correct_token(self): testing_module.require_qa_token(x_qa_token=testing_module.QA_API_TOKEN) # must not raise def test_every_testing_route_depends_on_require_qa_env(self): - """Belt-and-suspenders: catches a route added to testing.py that forgets the - dependency, independent of any single route's behaviour above.""" from app.api import testing as testing_module for route in testing_module.router.routes: @@ -195,3 +269,14 @@ def test_every_testing_route_depends_on_require_qa_env(self): assert testing_module.require_qa_env in dependant_calls, ( f"{route.path} is missing Depends(require_qa_env)" ) + + def test_every_mutating_testing_route_depends_on_require_qa_token(self): + from app.api import testing as testing_module + + for route in testing_module.router.routes: + if route.methods == {"GET"}: + continue + dependant_calls = {dep.call for dep in route.dependant.dependencies} + assert testing_module.require_qa_token in dependant_calls, ( + f"{route.path} is missing Depends(require_qa_token)" + ) diff --git a/server/app/tests/test_testing_endpoints.py b/server/app/tests/test_testing_endpoints.py index 27c3b63d..e2e93cf6 100644 --- a/server/app/tests/test_testing_endpoints.py +++ b/server/app/tests/test_testing_endpoints.py @@ -1,6 +1,6 @@ """HTTP-level coverage for app/api/testing.py's QA scenario/reset/login-as surface -(GH #272, #273). Assumes the shared `client`/`session` fixtures run with -`ENVIRONMENT=development` (see conftest.py, server/.env.example) — the same +(GH #272, #273, #276). Assumes the shared `client`/`session` fixtures run with +`ENVIRONMENT=development` (see conftest.py, server/.env.example), the same precondition every other `/testing/*` test in this suite already relies on. """ from app.api import testing as testing_module @@ -17,12 +17,17 @@ def test_list_scenarios_returns_full_catalog(client): def test_seed_unknown_scenario_returns_404(client): - response = client.post("/testing/seed/does-not-exist") + response = client.post("/testing/seed/does-not-exist", headers=QA_HEADERS) assert response.status_code == 404 -def test_seed_roles_scenario_creates_fixtures(client): +def test_seed_requires_qa_token(client): response = client.post("/testing/seed/roles") + assert response.status_code == 404 + + +def test_seed_roles_scenario_creates_fixtures(client): + response = client.post("/testing/seed/roles", headers=QA_HEADERS) assert response.status_code == 200 body = response.json() assert body["scenario"] == "roles" @@ -30,7 +35,7 @@ def test_seed_roles_scenario_creates_fixtures(client): def test_seed_gps_roles_scenario_creates_multi_role_fixtures(client): - response = client.post("/testing/seed/gps-roles") + response = client.post("/testing/seed/gps-roles", headers=QA_HEADERS) assert response.status_code == 200 body = response.json() assert body["scenario"] == "gps-roles" @@ -39,7 +44,7 @@ def test_seed_gps_roles_scenario_creates_multi_role_fixtures(client): def test_login_as_qa_map_rescuer_mints_usable_tokens(client): - client.post("/testing/seed/gps-roles") + client.post("/testing/seed/gps-roles", headers=QA_HEADERS) response = client.post("/testing/login-as/qa_map_rescuer", headers=QA_HEADERS) assert response.status_code == 200 @@ -73,7 +78,7 @@ def test_login_as_unseeded_fixture_returns_404(client): def test_login_as_mints_usable_tokens(client): - client.post("/testing/seed/roles") + client.post("/testing/seed/roles", headers=QA_HEADERS) response = client.post("/testing/login-as/qa_admin", headers=QA_HEADERS) assert response.status_code == 200 From ccd8c3725e34427cd201e3946b96394bd66b2531 Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:23:28 +0800 Subject: [PATCH 06/12] refactor(mobile-connectivity): remove background signaling task (#364) --- docs/architecture/system-overview.md | 2 +- docs/features/calls/requirements.md | 4 +- mobile-app/sapot-mobile-app/CLAUDE.md | 13 +- mobile-app/sapot-mobile-app/app.config.ts | 12 - .../sapot-mobile-app/app/(drawer)/_layout.tsx | 6 +- .../sapot-mobile-app/docs/ARCHITECTURE.md | 9 +- .../sapot-mobile-app/docs/ENV_CONFIG.md | 13 +- .../sapot-mobile-app/docs/ONBOARDING.md | 2 +- .../docs/READABILITY_AUDIT.md | 2 +- .../sapot-mobile-app/docs/STATE_MANAGEMENT.md | 17 +- mobile-app/sapot-mobile-app/docs/TESTING.md | 2 - .../docs/audits/automation-plan.md | 2 +- .../docs/audits/manual-testing-addendum.md | 10 +- .../docs/audits/regression-suite.md | 8 +- .../docs/audits/test-cases.md | 18 +- .../docs/audits/test-inventory.md | 8 +- .../features/auth/context/auth-context.tsx | 6 +- .../hooks/use-incoming-call-lifecycle.ts | 2 +- .../main-container-initialize.test.ts | 6 - .../shared/core/stores/network-config.ts | 8 - .../shared/core/stores/secure-config.ts | 167 ------------ .../shared/hooks/use-background-task.ts | 123 --------- .../shared/hooks/use-foreground-service.ts | 63 +++++ .../features/shared/main-container.ts | 18 -- mobile-app/sapot-mobile-app/jest-setup.js | 20 +- mobile-app/sapot-mobile-app/package.json | 2 - mobile-app/sapot-mobile-app/pnpm-lock.yaml | 35 --- .../sapot-mobile-app/task/signaling-task.ts | 243 ------------------ 28 files changed, 114 insertions(+), 707 deletions(-) delete mode 100644 mobile-app/sapot-mobile-app/features/shared/hooks/use-background-task.ts create mode 100644 mobile-app/sapot-mobile-app/features/shared/hooks/use-foreground-service.ts delete mode 100644 mobile-app/sapot-mobile-app/task/signaling-task.ts diff --git a/docs/architecture/system-overview.md b/docs/architecture/system-overview.md index 5dc9d4c3..af77e632 100644 --- a/docs/architecture/system-overview.md +++ b/docs/architecture/system-overview.md @@ -29,7 +29,7 @@ The primary user-facing component. An Expo/React Native Android app. - Local database (WatermelonDB/SQLite) with incremental pull/push sync - E2E encryption (NaCl box, per-conversation ECDH keys, at-rest encryption) - Guest-to-authenticated account migration -- Background WebSocket connectivity maintenance (Android background task) +- Foreground-service connectivity while the Android app process remains alive **Key dependencies:** Expo Router, `react-native-webrtc`, `react-native-tcp-socket` (TLS), `react-native-zeroconf`, WatermelonDB, `tweetnacl`, `expo-secure-store`, `@maplibre/maplibre-react-native`. diff --git a/docs/features/calls/requirements.md b/docs/features/calls/requirements.md index aafb4334..8f66c688 100644 --- a/docs/features/calls/requirements.md +++ b/docs/features/calls/requirements.md @@ -49,8 +49,8 @@ SAPOT supports peer-to-peer voice and video calls using WebRTC. The server relay ### FR-CA-04 — Background calls -- Incoming call notifications via `expo-notifications` (foreground) and `expo-background-task` (background). -- The background task maintains the WebSocket connection so the device can receive call signalling while the app is not in the foreground. +- An Android foreground service keeps the current WebSocket connection and call notification handling active while the app process remains alive in the background. +- Force-killing the app stops call signaling. Killed-app notifications require the planned Firebase Cloud Messaging (FCM) replacement. ### FR-CA-05 — Call history diff --git a/mobile-app/sapot-mobile-app/CLAUDE.md b/mobile-app/sapot-mobile-app/CLAUDE.md index e41d5f73..87bb02ce 100644 --- a/mobile-app/sapot-mobile-app/CLAUDE.md +++ b/mobile-app/sapot-mobile-app/CLAUDE.md @@ -6,7 +6,7 @@ Instructions for Claude Code working in the SAPOT mobile app. See root `../../CL React Native / Expo (Expo Router) Android app — the primary client of the SAPOT platform. Provides LAN-first messaging (P2P + WebSocket relay fallback), voice/video calls (WebRTC), live GPS sharing, announcements, and offline-first local storage, so the app keeps working when internet/server connectivity is unavailable. -Stack: Expo, React Native, TypeScript, WatermelonDB (SQLite), WebRTC, `react-native-tcp-socket`, `tweetnacl` (E2E crypto), `expo-background-task`. +Stack: Expo, React Native, TypeScript, WatermelonDB (SQLite), WebRTC, `react-native-tcp-socket`, `tweetnacl` (E2E crypto), `react-native-background-actions`. Use the `app-commands` skill for the full CLI reference beyond the quality-gate commands listed in "When Modifying This Project" below. @@ -48,11 +48,9 @@ Thin injectable wrappers around native modules, for testability: `TcpServerAdapt ### Background connectivity -`expo-background-task` + `expo-task-manager` (`task/signaling-task.ts`). Two coordination mechanisms: -1. **App-alive flag** — `setAppAlive(true)` in `MainContainer.initialize()` tells the background task to stand down; `setAppAlive(false)` on cleanup lets it resume. -2. **Secure storage handoff** — `features/shared/core/stores/secure-config.ts` persists `peerId`, `wsUrl`, TCP host/port, local IP via `expo-secure-store`. `NetworkConfig` writes the updated IP immediately on WiFi change so the background task reads current config on wake. +`features/shared/hooks/use-foreground-service.ts` starts an Android foreground service when the app enters the background. `ConnectionService` and `DiscoveryService` continue running in the existing JavaScript process, so no second transport stack or secure-storage handoff is needed. -The task wakes every 15 minutes (Android minimum) using stored config to maintain connectivity when the app is killed. +Force-killing the app stops connectivity. Firebase Cloud Messaging (FCM) is the planned replacement for killed-app signaling and notification delivery. ## Directory Guide @@ -71,7 +69,6 @@ Features: `announcements`, `auth`, `call`, `chat`, `debug`, `getting-started`, ` - `features/shared/` — cross-feature services, DI containers, connection/crypto/database infrastructure. Check here first before writing a new util/service. - `docs/` — project-specific architecture docs (see Development Conventions — keep in sync). - `test/` — test utilities (builders, factories, mocks); `jest-setup.js` has global mocks for WatermelonDB, TCP sockets, WebRTC, Zeroconf, Expo modules, react-native-paper. -- `task/` — background task registration (`signaling-task.ts`). ## Key Concepts @@ -118,14 +115,14 @@ Before writing any new file, read `docs/ARCHITECTURE.md` (service/adapter landsc - `features/shared/core/database/schema.ts` — WatermelonDB schema (version 11) and column reference. - `features/shared/core/database/migrations.ts` — schema migration history. - `config/runtime.ts` — API/WS base URL resolution per build variant. -- `task/signaling-task.ts` — background connectivity task. +- `features/shared/hooks/use-foreground-service.ts`: Android foreground-service lifecycle. ## Common Pitfalls - Changing `ConnectionService`'s callback wiring to `.bind()` instead of closures — breaks `jest.spyOn` instance replacement in tests. - Editing a shared service/hook without auditing all `features//` consumers first (see Decision Rule 4) — this codebase has many features sharing `features/shared/`. - Introducing a second pattern for something `features/shared/` already solves (a second HTTP client, a second logger, a second encryption helper) instead of extending the existing one. -- Forgetting the background task depends on `secure-config.ts` being written *before* the app is backgrounded — stale secure storage means the background task reconnects to an old IP/peerId. +- Assuming the Android foreground service survives a force-kill. Killed-app delivery requires the planned FCM replacement. - Assuming `server/` changes propagate automatically — the mobile app has its own API client; a backend contract change must be applied here too (see Decision Rule 5). ## When Modifying This Project diff --git a/mobile-app/sapot-mobile-app/app.config.ts b/mobile-app/sapot-mobile-app/app.config.ts index 6618ee21..a0d69aab 100644 --- a/mobile-app/sapot-mobile-app/app.config.ts +++ b/mobile-app/sapot-mobile-app/app.config.ts @@ -259,18 +259,6 @@ export default ({ config }: ConfigContext) => ({ sounds: ["./assets/ringtone.mp3"], }, ], - [ - "expo-background-task", - { - android: { - foregroundService: { - notificationTitle: "App is running", - notificationBody: "Listening for incoming calls...", - notificationColor: "#ffffff", - }, - }, - }, - ], [ "@lovesworking/watermelondb-expo-plugin-sdk-52-plus", { diff --git a/mobile-app/sapot-mobile-app/app/(drawer)/_layout.tsx b/mobile-app/sapot-mobile-app/app/(drawer)/_layout.tsx index 6382d775..43f6d889 100644 --- a/mobile-app/sapot-mobile-app/app/(drawer)/_layout.tsx +++ b/mobile-app/sapot-mobile-app/app/(drawer)/_layout.tsx @@ -16,7 +16,7 @@ import { ServerDownReloginTransition } from "@/features/shared/components/server import { ServerStatusBanner } from "@/features/shared/components/server-status-banner"; import { ZeroconfStatusIndicator } from "@/features/shared/components/zeroconf-status-indicator"; import { HealthProvider, MainContainerProvider, ServerHealthProvider, useAppMode } from "@/features/shared/core/context"; -import { useBackgroundTask } from "@/features/shared/hooks/use-background-task"; +import { useForegroundService } from "@/features/shared/hooks/use-foreground-service"; import { useForegroundSync } from "@/features/shared/hooks/use-foreground-sync"; import { useNotifications } from "@/features/shared/hooks/use-notifications"; import { useZeroconfPublished } from "@/features/shared/hooks/use-zeroconf-published"; @@ -31,7 +31,6 @@ import { Platform, TouchableOpacity, View } from "react-native"; import { Icon, Text, useTheme } from "react-native-paper"; import { PageLoader } from "@/features/shared/components/page-loader"; import { SafeAreaView } from "react-native-safe-area-context"; -import "../../task/signaling-task"; const queryClient = new QueryClient(); function GpsStreamingEffect() { @@ -103,8 +102,7 @@ function HeaderRight() { export default function DrawerLayout() { const { isAuthenticated, loading, isGuest } = useAuth(); const theme = useTheme(); - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { unregister } = useBackgroundTask(); + useForegroundService(); const handledNotifIdRef = useRef(null); const handledNotifTimeoutRef = useRef | null>(null); diff --git a/mobile-app/sapot-mobile-app/docs/ARCHITECTURE.md b/mobile-app/sapot-mobile-app/docs/ARCHITECTURE.md index c5b67bd7..fade9f60 100644 --- a/mobile-app/sapot-mobile-app/docs/ARCHITECTURE.md +++ b/mobile-app/sapot-mobile-app/docs/ARCHITECTURE.md @@ -376,14 +376,11 @@ Server-fetched announcement board — no WatermelonDB, purely React Query. --- -## Background Task +## Android Background Connectivity -On Android, a background task (`task/signaling-task.ts`) maintains WebSocket connectivity when the app is killed. It wakes every 15 minutes (Android minimum). +When the app enters the background on Android, `useForegroundService()` starts `react-native-background-actions`. This keeps the existing JavaScript process alive, allowing `ConnectionService` and `DiscoveryService` to continue owning WebSocket, TCP, and Zeroconf transports without creating duplicate adapters. -Two mechanisms coordinate foreground ↔ background: - -1. **App-alive flag** — `setAppAlive(true)` in `MainContainer.initialize()` tells the background task to stand down. `setAppAlive(false)` on cleanup lets it resume. -2. **Secure storage handoff** — `features/shared/core/stores/secure-config.ts` persists `peerId`, `wsUrl`, TCP host/port, and local IP via `expo-secure-store`. `NetworkConfig` writes the latest IP immediately on WiFi change so the background task always reads fresh config. Background Zeroconf cleanup also uses the adapter-tracked published service name so teardown can unpublish the correct mDNS registration. +The foreground service stops when the app becomes active or its layout unmounts. It does not survive a force-kill. Incoming signaling and notifications after a force-kill require the planned Firebase Cloud Messaging (FCM) replacement. --- diff --git a/mobile-app/sapot-mobile-app/docs/ENV_CONFIG.md b/mobile-app/sapot-mobile-app/docs/ENV_CONFIG.md index 57ab5302..ad6dfa62 100644 --- a/mobile-app/sapot-mobile-app/docs/ENV_CONFIG.md +++ b/mobile-app/sapot-mobile-app/docs/ENV_CONFIG.md @@ -183,20 +183,11 @@ Sensitive runtime config is stored via `expo-secure-store` (not AsyncStorage). Managed in `features/shared/core/stores/secure-config.ts`: -All 18 keys are declared in that file's `KEYS` constant: +All nine keys are declared in that file's `KEYS` constant: | Key | Value | |---|---| -| `peerId` | Current user's ID | -| `wsUrl` | WebSocket server URL | -| `tcpHost` | Peer TCP host | -| `tcpPort` | Peer TCP port | -| `localIp` | Device's current LAN IP | | `access_token` | Current session's JWT (read via `getStoredAccessToken`; written via `saveAccessToken`/cleared via `clearAccessToken`) | -| `appAlive` | App-alive flag the background task checks to decide whether to stand down | -| `username` | Cached profile username | -| `firstName` | Cached profile first name | -| `lastName` | Cached profile last name (optional) | | `syncLastPulledAt` | Last successful sync pull timestamp (Unix ms, stored as string) | | `serverHostOverride` | Dev/QA host override consumed by `config/runtime.ts` (`setRuntimeHostOverride`) | | `appMode` | Persisted transport mode (`auto` / `server` / `lan`) | @@ -206,7 +197,7 @@ All 18 keys are declared in that file's `KEYS` constant: | `recoveryTokenHex` | Recovery session token, hex-encoded | | `guestMigrationState` | Guest→registered-account migration progress state | -This config is also read by the background task (`task/signaling-task.ts`) on Android when the app is killed. +Connection details and profile fields stay in memory while the app runs. They are not copied to secure storage because the Android foreground service keeps the existing JavaScript process alive instead of constructing a second transport stack. `saveAccessToken`/`clearAccessToken` are also used by the gated Auth debug section (`features/debug/services/debug-auth-service.ts`) to inject/clear a fake JWT for testing — see `docs/TESTING.md`. diff --git a/mobile-app/sapot-mobile-app/docs/ONBOARDING.md b/mobile-app/sapot-mobile-app/docs/ONBOARDING.md index eca260e7..f95213da 100644 --- a/mobile-app/sapot-mobile-app/docs/ONBOARDING.md +++ b/mobile-app/sapot-mobile-app/docs/ONBOARDING.md @@ -84,7 +84,7 @@ These are the most complex parts of the codebase. Don't start here — come back | Stores (app state) | `shared/core/stores/` | | API client | `shared/core/api/client.ts` | | Screen routing | `app/` (Expo Router file-based routing) | -| Background connectivity | `task/signaling-task.ts` | +| Android foreground connectivity | `features/shared/hooks/use-foreground-service.ts` | | GPS / location sharing | `features/gps/` (independent WS, not `ConnectionService`) | --- diff --git a/mobile-app/sapot-mobile-app/docs/READABILITY_AUDIT.md b/mobile-app/sapot-mobile-app/docs/READABILITY_AUDIT.md index 377665eb..6be3b144 100644 --- a/mobile-app/sapot-mobile-app/docs/READABILITY_AUDIT.md +++ b/mobile-app/sapot-mobile-app/docs/READABILITY_AUDIT.md @@ -387,7 +387,7 @@ connectionService.setTcpCallbacks({ | Peripheral features (settings, announcements) | 3–5 days | Low complexity, isolated | | Chat / sync flows | 2–3 weeks | Requires understanding DI + service wiring + DB | | Core P2P: connection, signaling, call lifecycle, WebRTC | 4–6 weeks | 290-line function is a hurdle; ADRs help | -| Full confidence (incl. encryption + background tasks) | 6–8 weeks | Crypto stack + background-task coordination needed | +| Full confidence (including encryption + foreground-service lifecycle) | 6–8 weeks | Crypto stack + Android lifecycle coordination needed | | **First meaningful PR touching core services** | 4–6 weeks | Can write code sooner, but confidence takes time | **The docs realistically shave ~30% off what this complexity would otherwise demand.** diff --git a/mobile-app/sapot-mobile-app/docs/STATE_MANAGEMENT.md b/mobile-app/sapot-mobile-app/docs/STATE_MANAGEMENT.md index eba53ba9..daa5882c 100644 --- a/mobile-app/sapot-mobile-app/docs/STATE_MANAGEMENT.md +++ b/mobile-app/sapot-mobile-app/docs/STATE_MANAGEMENT.md @@ -173,13 +173,11 @@ Both `HealthProvider` (`health-context.tsx`) and `ServerHealthProvider` (`server **Risk:** if a service-side state change doesn't emit, the UI is stale. If an event fires out of order (e.g., late `call-ended` after a timeout), the guard refs (`hasTerminated.current`) must prevent double-processing. See [CallContext debugging](#1-callcontext-the-worst). -### d) Connection config in-memory + secure-store -`NetworkConfig` generates a port and reads the IP on init, then keeps both in-memory. On IP change, it immediately writes to secure-store (for the background task to pick up): -- `network-config.ts:54` — `saveLocalIp()` on init -- `network-config.ts:83` — `saveLocalIp()` debounced 3s on change -- `network-config.ts:89–97` — calls `onIpChange` callback after debounce +### d) Connection config in memory -**Risk:** background task reads stale IP if foreground app crashes before the debounce fires. Low risk in practice (3s window), but a race. +`NetworkConfig` generates a port and reads the IP on initialization, then keeps both in memory. On an IP change, it updates `ipAddress` immediately and calls the registered `onIpChange` callback after a three-second debounce so transports can rebind once. + +Connection state is not handed to a second background process. The Android foreground service keeps the existing process and service instances alive while the app is backgrounded. --- @@ -196,10 +194,9 @@ Services emit events; `CallContext` and other components subscribe in `useEffect ### In-memory ↔ secure-store (eager writes) When a user changes a setting or a system value updates, write immediately to secure-store: - `AppModeStore.setMode()` → `saveAppMode()` async -- `NetworkConfig.startWatching()` → `saveLocalIp()` on IP change - `UserService.syncAuthenticatedUser()` → `setItemAsync("userUUID", ...)` -**Cost:** async writes that might fail silently (errors are logged but not propagated). Background task might read fresh values even if write fails. +**Cost:** async writes that might fail silently because errors are logged but not propagated. ### DB ↔ Server (SyncService) `SyncService` polls the server every 60s (or on demand), using `lastPulledAt` (stored in secure-store) to fetch only recent changes. Reads write back to WatermelonDB. See `docs/SYNC.md`. @@ -225,9 +222,7 @@ Similarly for password recovery (`setPendingPassword`). This is **implicit, muta 1. **WatermelonDB** is the offline cache for domain data. UI reads from it regardless of network. Sync updates it when connectivity returns. -2. **secure-store** caches identity (`userUUID`, `username`, profile) and config (tokens, peerId, IP/port) so that: - - The app can boot and authenticate locally if offline - - The background signaling task can read config without waking the main app +2. **secure-store** persists credentials, encryption keys, transport mode, sync progress, and the optional server host override. Authentication state can be rebuilt locally when the server is unavailable. 3. **Offline-auth fallback** (`auth-context.tsx:181–204`): when `refreshSession` fails with a network error, rebuild the session from the local `peers` row. Set `isOfflineWithExpiredToken` to warn the user. diff --git a/mobile-app/sapot-mobile-app/docs/TESTING.md b/mobile-app/sapot-mobile-app/docs/TESTING.md index 2e3f06b2..8ccb2c1e 100644 --- a/mobile-app/sapot-mobile-app/docs/TESTING.md +++ b/mobile-app/sapot-mobile-app/docs/TESTING.md @@ -113,8 +113,6 @@ Global mocks are set up in `jest-setup.js`. These run before every test file. | `react-native-reanimated` | Native animation driver | | `react-native-background-actions` | Native module | | `lottie-react-native` | Native animation module | -| `expo-background-task` | Native module | -| `expo-task-manager` | Native module | | `expo-notifications` | Native module | | `expo-file-system` | Touches the real filesystem (log files) | | `@react-native-documents/picker` | Native file picker | diff --git a/mobile-app/sapot-mobile-app/docs/audits/automation-plan.md b/mobile-app/sapot-mobile-app/docs/audits/automation-plan.md index 94fd829c..5ca7d781 100644 --- a/mobile-app/sapot-mobile-app/docs/audits/automation-plan.md +++ b/mobile-app/sapot-mobile-app/docs/audits/automation-plan.md @@ -122,7 +122,7 @@ describe('ConnectionService', () => { | `GpsLocationService` | `features/gps/services/__tests__/gps-location-service.test.ts` | | `GuestMigrationService` | `features/auth/services/__tests__/guest-migration-service.test.ts` | | `SignalingService` | `features/shared/connection/services/__tests__/signaling-service.test.ts` | -| `SIGNALING_TASK` | `task/__tests__/signaling-task.test.ts` | +| Android foreground-service lifecycle | `features/shared/hooks/__tests__/use-foreground-service.test.ts` | | `use-lockout-timer` | `features/auth/hooks/__tests__/use-lockout-timer.test.ts` | ### Pure functions — highest ROI, lowest effort (< 30 min each) diff --git a/mobile-app/sapot-mobile-app/docs/audits/manual-testing-addendum.md b/mobile-app/sapot-mobile-app/docs/audits/manual-testing-addendum.md index 0d0606a2..11bb7e0d 100644 --- a/mobile-app/sapot-mobile-app/docs/audits/manual-testing-addendum.md +++ b/mobile-app/sapot-mobile-app/docs/audits/manual-testing-addendum.md @@ -199,15 +199,15 @@ All P0 manual tests must pass on all three tiers. P1 tests require at least mid- --- -### MT-024 — Incoming Call While App is Killed +### MT-024: Incoming Call While App is Backgrounded **Priority:** P0 | **Device:** Real Android device -1. Log in, then force-stop the app (swipe away from recents) +1. Log in, then send the app to the background without force-stopping it 2. From a second device, call Device A **Expected:** -- Background task has maintained connectivity +- The foreground service has maintained connectivity - `incoming-call` notification appears with ringtone - Tapping notification opens the app to Incoming Call screen with correct caller info @@ -429,7 +429,7 @@ Run before every release. Record device model and Android version for each row. | 4 | Send + receive message (WebRTC data channel) | Two devices | | | | 5 | Audio call: voice audible both directions | Two devices | | | | 6 | Video call: camera visible on both ends | Two devices | | | -| 7 | Incoming call notification (app killed) | Real device | | | +| 7 | Incoming call notification (app backgrounded, process alive) | Real device | | | | 8 | Cold-start from call notification | Real device | | | | 9 | GPS toggle persists across restart | Any | | | | 10 | Map markers appear (rescuer account) | Any | | | @@ -439,7 +439,7 @@ Run before every release. Record device model and Android version for each row. | 14 | Change password end-to-end | Any | | | | 15 | Recovery key generate + use for reset | Any | | | | 16 | Server Host Override absent in prod build | Prod build | | | -| 17 | Message delivered after 15 min Doze idle | Real device | | | +| 17 | Foreground service keeps connectivity while app is backgrounded | Real device | | | | 18 | LAN chat works with server offline | Two devices | | | | 19 | Camera permission: 3 distinct states | Fresh install | | | | 20 | Mic permission: denied → graceful error on call accept | Real device | | | diff --git a/mobile-app/sapot-mobile-app/docs/audits/regression-suite.md b/mobile-app/sapot-mobile-app/docs/audits/regression-suite.md index 652bab51..ce15aafc 100644 --- a/mobile-app/sapot-mobile-app/docs/audits/regression-suite.md +++ b/mobile-app/sapot-mobile-app/docs/audits/regression-suite.md @@ -79,13 +79,13 @@ Before merging to `main` or tagging a release: | REG-043 | Max 5 retry attempts | Auto-reconnect stops after 5 failures | Jest unit | `features/chat/hooks/__tests__/use-chat-connection.test.ts` | | REG-044 | Exponential backoff timing | Delays: 1s, 1.8s, 3.2s, 5.8s, 10.4s (±20% jitter) | Jest unit | `features/chat/hooks/__tests__/use-chat-connection.test.ts` | -### Background Task & Notifications +### Background Connectivity & Notifications | ID | Name | What It Verifies | Test Type | Target File | |----|------|-----------------|-----------|-------------| -| REG-050 | App-alive prevents duplicate transport | `appAlive=true` → background task skips transport start | Jest unit | `task/__tests__/signaling-task.test.ts` | -| REG-051 | Background task starts transport when killed | `appAlive=false` → TCP + WS + Zeroconf start | Jest unit | `task/__tests__/signaling-task.test.ts` | -| REG-052 | Background call notification fires | `audio-call` → `incoming-call` notification with ringtone | Maestro E2E | `flows/notifications/background-call.yaml` | +| REG-050 | Foreground service starts on background | Background app state starts `react-native-background-actions` | Jest unit | `features/shared/hooks/__tests__/use-foreground-service.test.ts` | +| REG-051 | Foreground service stops on active | Active app state stops `react-native-background-actions` | Jest unit | `features/shared/hooks/__tests__/use-foreground-service.test.ts` | +| REG-052 | Background call notification fires while process is alive | `audio-call` produces an `incoming-call` notification with ringtone | Maestro E2E | `flows/notifications/background-call.yaml` | | REG-053 | Cold-start from notification | `getLastNotificationResponseAsync()` → Incoming Call screen | Maestro E2E | `flows/notifications/cold-start-call.yaml` | | REG-054 | Notification deduplication | Two identical notifications → one navigation in 30 s | Jest unit | `features/shared/connection/services/__tests__/notification-service.test.ts` | diff --git a/mobile-app/sapot-mobile-app/docs/audits/test-cases.md b/mobile-app/sapot-mobile-app/docs/audits/test-cases.md index b95cf6ee..6aa037de 100644 --- a/mobile-app/sapot-mobile-app/docs/audits/test-cases.md +++ b/mobile-app/sapot-mobile-app/docs/audits/test-cases.md @@ -304,18 +304,18 @@ The table below is a quick-reference for precondition states. Full setup procedu --- -## 16. Background Task & Notifications +## 16. Background Connectivity & Notifications | ID | Feature | Scenario | Preconditions | Steps | Expected Result | Priority | Severity | Automate | |----|---------|----------|---------------|-------|-----------------|----------|----------|----------| -| TC-220 | Background Task | Registered on app start | | 1. Launch app | SIGNALING_TASK registered | P0 | Critical | Jest | -| TC-221 | Background Task | App-alive flag prevents duplicate transport | Foreground app | 1. App in foreground | `appAlive=true`; background task skips | P0 | Critical | Jest | -| TC-222 | Background Task | Starts transport when app killed | App killed | 1. Background task fires | TcpServer + WsSignaling + Zeroconf start | P0 | Critical | Jest | -| TC-223 | Background Task | Incoming call notification fires | App killed on real device | 1. Kill app 2. Peer sends audio-call message | `incoming-call` notification appears with ringtone | P0 | Critical | MANUAL (physical device) | -| TC-224 | Background Task | Notification dismissed on call-ended | Notification shown, real device | 1. Leave notification 2. Caller hangs up | Notification dismissed automatically | P0 | Critical | MANUAL (physical device) | +| TC-220 | Foreground Service | Starts when app is backgrounded | Android app active | 1. Send app to background | Foreground-service notification appears; existing transports remain connected | P0 | Critical | Jest + MANUAL | +| TC-221 | Foreground Service | Stops when app becomes active | Foreground service running | 1. Return to app | Foreground-service notification disappears; transports remain owned by `MainContainer` | P0 | Critical | Jest + MANUAL | +| TC-222 | Foreground Service | Stops when drawer layout unmounts | Foreground service running | 1. Log out | Foreground service stops | P0 | Critical | Jest | +| TC-223 | Notifications | Incoming call notification fires while process is alive | App backgrounded on real device | 1. Peer sends audio-call message | `incoming-call` notification appears with ringtone | P0 | Critical | MANUAL (physical device) | +| TC-224 | Notifications | Notification dismissed on call-ended | Notification shown, real device | 1. Leave notification 2. Caller hangs up | Notification dismissed automatically | P0 | Critical | MANUAL (physical device) | | TC-225 | Notifications | Tap call notification → Incoming Call screen | App in background, real device | 1. Receive call notification 2. Tap it | Opens `/call/incoming` with caller info | P0 | Critical | MANUAL (physical device) | | TC-226 | Notifications | Tap message notification → Chat Room | App in background, real device | 1. Receive message notification 2. Tap it | Opens `/chat/[id]` | P1 | High | MANUAL (physical device) | -| TC-227 | Notifications | Cold start from killed app | App killed, real device | 1. Kill app 2. Receive call 3. Tap notification | `getLastNotificationResponseAsync()` used; Incoming Call shown | P0 | Critical | MANUAL (physical device) | +| TC-227 | Notifications | Cold start from existing notification | Call notification already shown | 1. Force-close app 2. Tap the existing notification | `getLastNotificationResponseAsync()` used; Incoming Call shown | P0 | Critical | MANUAL (physical device) | | TC-228 | Notifications | Deduplication prevents double navigation | Duplicate notification | 1. Both foreground + background arrive | Navigation fires once only | P0 | Critical | Jest | --- @@ -368,8 +368,8 @@ The table below is a quick-reference for precondition states. Full setup procedu | ID | Feature | Scenario | Preconditions | Steps | Expected Result | Priority | Severity | Automate | |----|---------|----------|---------------|-------|-----------------|----------|----------|----------| -| TC-270 | App Lifecycle | App-alive flag cleared on kill | App running | 1. Force-close app | `appAlive=false` in secure storage | P0 | Critical | Jest | -| TC-271 | App Lifecycle | IP change updates secure storage | WiFi changes | 1. Switch networks | `NetworkConfig.ipAddress` + port updated in secure storage | P0 | Critical | Jest | +| TC-270 | App Lifecycle | Foreground service stops on layout cleanup | App running in background | 1. Log out | Foreground service stops | P0 | Critical | Jest | +| TC-271 | App Lifecycle | IP change updates in-memory config | WiFi changes | 1. Switch networks | `NetworkConfig.ipAddress` updates and transports rebind after debounce | P0 | Critical | Jest | | TC-272 | App Lifecycle | Debounced IP callback fires once | Multiple rapid IP events | 1. Simulate rapid events | Callback fires once after debounce | P1 | High | Jest | | TC-273 | App Lifecycle | Safe-area insets on all screens | Physical notched device | 1. Navigate all screens on a device with display cutout | No content clipped behind notch or home bar | P1 | High | MANUAL (physical notched device) | | TC-274 | App Lifecycle | Dark mode uses theme colors throughout | Dark mode enabled | 1. Enable dark mode 2. Navigate all screens | No hardcoded white/black colors visible; all colors from theme | P1 | High | MANUAL (physical device) | diff --git a/mobile-app/sapot-mobile-app/docs/audits/test-inventory.md b/mobile-app/sapot-mobile-app/docs/audits/test-inventory.md index e402fb7c..7c600940 100644 --- a/mobile-app/sapot-mobile-app/docs/audits/test-inventory.md +++ b/mobile-app/sapot-mobile-app/docs/audits/test-inventory.md @@ -162,14 +162,12 @@ Scope: React Native / Expo frontend + FastAPI backend | useAnnouncementNewCount | `announcements/hooks/use-announcement-new-count.ts` | None | Missing | AUTOMATED | | format-announcement-date | `announcements/utils/format-announcement-date.ts` | None | Missing | AUTOMATED | -### 10. Background Task (zero coverage) +### 10. Android Foreground Service (zero coverage) | Sub-Feature | File(s) | Test File | Status | Execution Type | Risk | |-------------|---------|-----------|--------|----------------|------| -| SIGNALING_TASK | `task/signaling-task.ts` | None | Missing | AUTOMATED | HIGH | -| App-alive handoff | `MainContainer + signaling-task.ts` | None | Missing | AUTOMATED | HIGH | -| Background task under Android Doze | Real device, battery saver | None | Missing | MANUAL | HIGH | -| Background task after Android LMK kill | Real device, low RAM | None | Missing | MANUAL | HIGH | +| Android foreground-service lifecycle | `hooks/use-foreground-service.ts` | None | Missing | AUTOMATED | HIGH | +| Foreground service under Android Doze | Real device, battery saver | None | Missing | MANUAL | HIGH | --- diff --git a/mobile-app/sapot-mobile-app/features/auth/context/auth-context.tsx b/mobile-app/sapot-mobile-app/features/auth/context/auth-context.tsx index cb1a81ff..2a0d5704 100644 --- a/mobile-app/sapot-mobile-app/features/auth/context/auth-context.tsx +++ b/mobile-app/sapot-mobile-app/features/auth/context/auth-context.tsx @@ -214,9 +214,9 @@ export const AuthProvider = ({ children }: { children: React.ReactNode }) => { authLog.debug("auth › bootstrap start"); setLoading(true); try { - // Check guest status first: `syncGuestUser` also writes `userUUID` - // (so the background task can find it), so a persisted guest looks - // identical to an authenticated user by that key alone. Guest status + // Check guest status first: `syncGuestUser` also writes `userUUID`, + // so a persisted guest looks identical to an authenticated user by + // that key alone. Guest status // is the more specific signal — a guest_user row only exists for // guests — so it must be checked before falling into the // authenticated-user branch below, or a restored guest would diff --git a/mobile-app/sapot-mobile-app/features/call/context/hooks/use-incoming-call-lifecycle.ts b/mobile-app/sapot-mobile-app/features/call/context/hooks/use-incoming-call-lifecycle.ts index 842cc7d9..6adeddc2 100644 --- a/mobile-app/sapot-mobile-app/features/call/context/hooks/use-incoming-call-lifecycle.ts +++ b/mobile-app/sapot-mobile-app/features/call/context/hooks/use-incoming-call-lifecycle.ts @@ -1,6 +1,6 @@ import type { CallService } from "@/features/call/services/call-service"; import type { ConnectionService } from "@/features/shared/connection/services/connection-service"; -import { stopForegroundService } from "@/features/shared/hooks/use-background-task"; +import { stopForegroundService } from "@/features/shared/hooks/use-foreground-service"; import { callLog, uiLog } from "@/features/shared/core/utils/logger"; import * as Notifications from "expo-notifications"; import { useEffect, useRef } from "react"; diff --git a/mobile-app/sapot-mobile-app/features/shared/__tests__/main-container-initialize.test.ts b/mobile-app/sapot-mobile-app/features/shared/__tests__/main-container-initialize.test.ts index 6c8d0670..cbed3f1f 100644 --- a/mobile-app/sapot-mobile-app/features/shared/__tests__/main-container-initialize.test.ts +++ b/mobile-app/sapot-mobile-app/features/shared/__tests__/main-container-initialize.test.ts @@ -12,10 +12,6 @@ jest.mock("@/config/runtime", () => ({ getServerVerifyKey: jest.fn(() => null), })); -jest.mock("@/task/signaling-task", () => ({ - setAppAlive: jest.fn(), -})); - jest.mock("@react-native-community/netinfo", () => ({ __esModule: true, default: { addEventListener: jest.fn(() => jest.fn()) }, @@ -36,8 +32,6 @@ jest.mock("../core/stores/secure-config", () => ({ getStoredAccessToken: jest.fn().mockResolvedValue(null), getMigrationState: jest.fn().mockResolvedValue(null), clearMigrationState: jest.fn().mockResolvedValue(undefined), - saveConnectionConfig: jest.fn().mockResolvedValue(undefined), - saveUserProfile: jest.fn().mockResolvedValue(undefined), })); jest.mock("../crypto/local-encryption-service", () => ({ diff --git a/mobile-app/sapot-mobile-app/features/shared/core/stores/network-config.ts b/mobile-app/sapot-mobile-app/features/shared/core/stores/network-config.ts index 2636d190..205b34f5 100644 --- a/mobile-app/sapot-mobile-app/features/shared/core/stores/network-config.ts +++ b/mobile-app/sapot-mobile-app/features/shared/core/stores/network-config.ts @@ -2,7 +2,6 @@ import { toAppError, captureAppError } from "@/features/shared/core/errors"; import NetInfo from "@react-native-community/netinfo"; import { NetworkInfo } from "react-native-network-info"; import { networkLog } from "../utils/logger"; -import { saveLocalIp, saveLocalPort } from "./secure-config"; networkLog.debug("[network-config] module loaded"); @@ -59,10 +58,6 @@ export class NetworkConfig { } this.ipAddress = ip; - // Persist so background task always reads the latest values - await saveLocalIp(ip); - await saveLocalPort(this.port); - networkLog.info("network › init complete", { hasIp: Boolean(this.ipAddress), }); @@ -104,9 +99,6 @@ export class NetworkConfig { hasNewIp: true, }); this.ipAddress = newIp; - // Persist immediately so background task picks it up on next wake - await saveLocalIp(newIp); - // Notify (debounced) so listeners can re-advertise mDNS / rebind TCP. if (this.ipChangeDebounceTimer) { clearTimeout(this.ipChangeDebounceTimer); diff --git a/mobile-app/sapot-mobile-app/features/shared/core/stores/secure-config.ts b/mobile-app/sapot-mobile-app/features/shared/core/stores/secure-config.ts index bf20af9f..1419478d 100644 --- a/mobile-app/sapot-mobile-app/features/shared/core/stores/secure-config.ts +++ b/mobile-app/sapot-mobile-app/features/shared/core/stores/secure-config.ts @@ -5,16 +5,7 @@ import type { AppMode } from "./app-mode-store"; // ── Keys ─────────────────────────────────────────────────────────────────────── const KEYS = { - PEER_ID: "peerId", - WS_URL: "wsUrl", - TCP_HOST: "tcpHost", - TCP_PORT: "tcpPort", - LOCAL_IP: "localIp", ACCESS_TOKEN: "access_token", - APP_ALIVE: "appAlive", - USERNAME: "username", - FIRST_NAME: "firstName", - LAST_NAME: "lastName", SYNC_LAST_PULLED_AT: "syncLastPulledAt", SERVER_HOST_OVERRIDE: "serverHostOverride", APP_MODE: "appMode", @@ -27,104 +18,8 @@ const KEYS = { // ── Writers ──────────────────────────────────────────────────────────────────── -export const saveConnectionConfig = async (config: { - peerId: string; - wsUrl?: string; - tcpHost?: string; - tcpPort?: number; - localIp?: string; -}) => { - try { - await setItemAsync(KEYS.PEER_ID, config.peerId); - - if (config.wsUrl) { - await setItemAsync(KEYS.WS_URL, config.wsUrl); - } - if (config.tcpHost) { - await setItemAsync(KEYS.TCP_HOST, config.tcpHost); - } - if (config.tcpPort) { - await setItemAsync(KEYS.TCP_PORT, String(config.tcpPort)); - } - if (config.localIp) { - await setItemAsync(KEYS.LOCAL_IP, config.localIp); - } - - backgroundLog.info("secure-config › saved"); - } catch (error) { - backgroundLog.error("secure-config › save failed", { error }); - throw error; - } -}; - -// Individual updaters called by NetworkConfig on IP/port change - -export const saveLocalIp = async (ip: string) => { - try { - await setItemAsync(KEYS.LOCAL_IP, ip); - backgroundLog.info("secure-config › local ip updated"); - } catch (error) { - backgroundLog.error("secure-config › local ip save failed", { error }); - } -}; - -export const saveLocalPort = async (port: number) => { - try { - await setItemAsync(KEYS.TCP_PORT, String(port)); - backgroundLog.info("secure-config › local port updated"); - } catch (error) { - backgroundLog.error("secure-config › local port save failed", { error }); - } -}; - // ── Readers ──────────────────────────────────────────────────────────────────── -export const getStoredPeerId = async (): Promise => { - try { - return (await getItemAsync(KEYS.PEER_ID)) ?? "unknown"; - } catch (error) { - backgroundLog.error("secure-config › read peerId failed", { error }); - return "unknown"; - } -}; - -export const getStoredWsUrl = async (): Promise => { - try { - return (await getItemAsync(KEYS.WS_URL)) ?? undefined; - } catch (error) { - backgroundLog.error("secure-config › read wsUrl failed", { error }); - return undefined; - } -}; - -export const getStoredTcpHost = async (): Promise => { - try { - return (await getItemAsync(KEYS.TCP_HOST)) ?? undefined; - } catch (error) { - backgroundLog.error("secure-config › read tcpHost failed", { error }); - return undefined; - } -}; - -export const getStoredTcpPort = async (): Promise => { - try { - const val = await getItemAsync(KEYS.TCP_PORT); - return val ? parseInt(val, 10) : undefined; - } catch (error) { - backgroundLog.error("secure-config › read tcpPort failed", { error }); - return undefined; - } -}; - -export const getStoredLocalIp = async (): Promise => { - try { - return (await getItemAsync(KEYS.LOCAL_IP)) ?? undefined; - } catch (error) { - backgroundLog.error("secure-config › read localIp failed", { error }); - return undefined; - } -}; - export const getStoredAccessToken = async () => { try { return (await getItemAsync(KEYS.ACCESS_TOKEN)) ?? undefined; @@ -153,68 +48,6 @@ export const clearAccessToken = async (): Promise => { } }; -export const saveAppAlive = async (alive: boolean) => { - try { - await setItemAsync(KEYS.APP_ALIVE, alive ? "1" : "0"); - } catch (error) { - backgroundLog.error("secure-config › save appAlive failed", { error }); - } -}; - -export const getAppAlive = async (): Promise => { - try { - const val = await getItemAsync(KEYS.APP_ALIVE); - return val === "1"; - } catch (error) { - backgroundLog.error("secure-config › read appAlive failed", { error }); - return false; - } -}; - -export const saveUserProfile = async (profile: { - username: string; - firstName: string; - lastName?: string; -}) => { - try { - await setItemAsync(KEYS.USERNAME, profile.username); - await setItemAsync(KEYS.FIRST_NAME, profile.firstName); - if (profile.lastName) { - await setItemAsync(KEYS.LAST_NAME, profile.lastName); - } - backgroundLog.info("secure-config › user profile saved"); - } catch (error) { - backgroundLog.error("secure-config › user profile save failed", { error }); - } -}; - -export const getStoredUsername = async (): Promise => { - try { - return (await getItemAsync(KEYS.USERNAME)) ?? undefined; - } catch (error) { - backgroundLog.error("secure-config › read username failed", { error }); - return undefined; - } -}; - -export const getStoredFirstName = async (): Promise => { - try { - return (await getItemAsync(KEYS.FIRST_NAME)) ?? undefined; - } catch (error) { - backgroundLog.error("secure-config › read firstName failed", { error }); - return undefined; - } -}; - -export const getStoredLastName = async (): Promise => { - try { - return (await getItemAsync(KEYS.LAST_NAME)) ?? undefined; - } catch (error) { - backgroundLog.error("secure-config › read lastName failed", { error }); - return undefined; - } -}; - export const saveServerHostOverride = async (host: string | null) => { try { if (host) { diff --git a/mobile-app/sapot-mobile-app/features/shared/hooks/use-background-task.ts b/mobile-app/sapot-mobile-app/features/shared/hooks/use-background-task.ts deleted file mode 100644 index 1623f76e..00000000 --- a/mobile-app/sapot-mobile-app/features/shared/hooks/use-background-task.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { useEffect } from "react"; -import * as BackgroundTask from "expo-background-task"; -import * as TaskManager from "expo-task-manager"; -import { AppState, AppStateStatus, Platform } from "react-native"; -import BackgroundService from "react-native-background-actions"; -import { backgroundLog } from "../core/utils"; -import { setAppAlive, SIGNALING_TASK } from "@/task/signaling-task"; - -// ── Foreground service task ──────────────────────────────────────────────────── -// Runs inside the Android foreground service to keep the process alive. -// ConnectionService / DiscoveryService continue running in the same JS context — -// no need to re-start them here. - -const foregroundServiceTask = async () => { - await new Promise(() => { - const id = setInterval(() => { - backgroundLog.info("bg › foreground service heartbeat"); - }, 30_000); - - // Promise never resolves — service runs until BackgroundService.stop() is called. - // setInterval is cleared automatically when the process ends. - void id; - }); -}; - -const serviceOptions = { - taskName: "sapot-connectivity", - taskTitle: "App is running", - taskDesc: "Listening for incoming calls...", - taskIcon: { name: "ic_launcher", type: "mipmap" as const }, - color: "#ffffff", - parameters: {}, - foregroundServiceType: ["dataSync"] as Array<"dataSync">, -}; - -// ── Helpers ──────────────────────────────────────────────────────────────────── - -export const startForegroundService = async () => { - try { - if (BackgroundService.isRunning()) return; - await BackgroundService.start(foregroundServiceTask, serviceOptions); - backgroundLog.info("bg › foreground service started"); - } catch (error) { - backgroundLog.error("bg › foreground service start failed", { error }); - } -}; - -export const stopForegroundService = async () => { - try { - if (!BackgroundService.isRunning()) return; - await BackgroundService.stop(); - backgroundLog.info("bg › foreground service stopped"); - } catch (error) { - backgroundLog.error("bg › foreground service stop failed", { error }); - } -}; - -// ── Hook ─────────────────────────────────────────────────────────────────────── - -export const useBackgroundTask = () => { - useEffect(() => { - if (Platform.OS !== "android") return; - - setAppAlive(true); - registerTask(); - - const handleAppStateChange = (nextState: AppStateStatus) => { - if (nextState === "background") { - // App moved to background — start foreground service to keep process alive. - // ConnectionService / DiscoveryService continue running in this JS context. - startForegroundService(); - } else if (nextState === "active") { - // App came back to foreground — foreground service notification no longer needed. - stopForegroundService(); - } - }; - - const subscription = AppState.addEventListener("change", handleAppStateChange); - - return () => { - subscription.remove(); - setAppAlive(false); - stopForegroundService(); - }; - }, []); - - // ── Task registration ──────────────────────────────────────────────────────── - // expo-background-task remains as a 15-minute fallback for force-kill scenarios. - - const registerTask = async () => { - try { - const isRegistered = await TaskManager.isTaskRegisteredAsync(SIGNALING_TASK); - if (!isRegistered) { - await BackgroundTask.registerTaskAsync(SIGNALING_TASK, { - minimumInterval: 15, - }); - backgroundLog.info("bg › task registered"); - } else { - backgroundLog.info("bg › task already registered"); - } - } catch (error) { - backgroundLog.error("bg › task registration failed", { error }); - } - }; - - // ── Unregister on logout ───────────────────────────────────────────────────── - - const unregister = async () => { - try { - await stopForegroundService(); - - const isRegistered = await TaskManager.isTaskRegisteredAsync(SIGNALING_TASK); - if (isRegistered) { - await BackgroundTask.unregisterTaskAsync(SIGNALING_TASK); - backgroundLog.info("bg › task unregistered"); - } - } catch (error) { - backgroundLog.error("bg › task unregister failed", { error }); - } - }; - - return { unregister }; -}; diff --git a/mobile-app/sapot-mobile-app/features/shared/hooks/use-foreground-service.ts b/mobile-app/sapot-mobile-app/features/shared/hooks/use-foreground-service.ts new file mode 100644 index 00000000..80ccf670 --- /dev/null +++ b/mobile-app/sapot-mobile-app/features/shared/hooks/use-foreground-service.ts @@ -0,0 +1,63 @@ +import { useEffect } from "react"; +import { AppState, AppStateStatus, Platform } from "react-native"; +import BackgroundService from "react-native-background-actions"; +import { backgroundLog } from "../core/utils"; + +const foregroundServiceTask = async () => { + await new Promise(() => { + setInterval(() => { + backgroundLog.info("bg › foreground service heartbeat"); + }, 30_000); + }); +}; + +const serviceOptions = { + taskName: "sapot-connectivity", + taskTitle: "App is running", + taskDesc: "Listening for incoming calls...", + taskIcon: { name: "ic_launcher", type: "mipmap" as const }, + color: "#ffffff", + parameters: {}, + foregroundServiceType: ["dataSync"] as Array<"dataSync">, +}; + +export const startForegroundService = async () => { + try { + if (BackgroundService.isRunning()) return; + await BackgroundService.start(foregroundServiceTask, serviceOptions); + backgroundLog.info("bg › foreground service started"); + } catch (error) { + backgroundLog.error("bg › foreground service start failed", { error }); + } +}; + +export const stopForegroundService = async () => { + try { + if (!BackgroundService.isRunning()) return; + await BackgroundService.stop(); + backgroundLog.info("bg › foreground service stopped"); + } catch (error) { + backgroundLog.error("bg › foreground service stop failed", { error }); + } +}; + +export const useForegroundService = () => { + useEffect(() => { + if (Platform.OS !== "android") return; + + const handleAppStateChange = (nextState: AppStateStatus) => { + if (nextState === "background") { + void startForegroundService(); + } else if (nextState === "active") { + void stopForegroundService(); + } + }; + + const subscription = AppState.addEventListener("change", handleAppStateChange); + + return () => { + subscription.remove(); + void stopForegroundService(); + }; + }, []); +}; diff --git a/mobile-app/sapot-mobile-app/features/shared/main-container.ts b/mobile-app/sapot-mobile-app/features/shared/main-container.ts index b35f9dbe..a0b1fc4e 100644 --- a/mobile-app/sapot-mobile-app/features/shared/main-container.ts +++ b/mobile-app/sapot-mobile-app/features/shared/main-container.ts @@ -37,15 +37,12 @@ import { MessageStatusRepository } from "@/features/chat/repositories/message-st import { ChatService } from "@/features/chat/services/chat-service"; import { MessageReceiptManager } from "@/features/chat/services/message-receipt-manager"; import { PublicChatService } from "@/features/chat/services/public-chat-service"; -import { setAppAlive } from "@/task/signaling-task"; import { AuthContainer } from "../auth/auth-container"; import { CallParticipantRepository } from "../call/repositories/call-participant-repository"; import { CallRepository } from "../call/repositories/call-repository"; import { SyncService } from "../sync"; import { getStoredAccessToken, - saveConnectionConfig, - saveUserProfile, getMigrationState, clearMigrationState, } from "./core/stores/secure-config"; @@ -574,18 +571,6 @@ export class MainContainer { } }); this.networkConfig.startWatching(); - - await saveConnectionConfig({ - peerId: this.userContainer.userStore.user.id ?? "unknown", - wsUrl: getWsUrl(), - }); - await saveUserProfile({ - username: this.userContainer.userStore.user.username, - firstName: this.userContainer.userStore.user.firstName, - lastName: this.userContainer.userStore.user.lastName || undefined, - }); - - setAppAlive(true); } /** @@ -610,9 +595,6 @@ export class MainContainer { try { appLog.info("app › cleanup start"); - // Release the lock — background task takes over transport ownership - setAppAlive(false); - this.networkConfig.stopWatching(); this.unsubscribeNetInfo?.(); diff --git a/mobile-app/sapot-mobile-app/jest-setup.js b/mobile-app/sapot-mobile-app/jest-setup.js index 3c5bf322..c548c5aa 100644 --- a/mobile-app/sapot-mobile-app/jest-setup.js +++ b/mobile-app/sapot-mobile-app/jest-setup.js @@ -150,23 +150,7 @@ jest.mock('expo-router', () => { }; }); -// Mock Expo task APIs used by background signaling task -jest.mock('expo-task-manager', () => ({ - defineTask: jest.fn(), - isTaskRegisteredAsync: jest.fn().mockResolvedValue(false), - isTaskDefined: jest.fn().mockReturnValue(true), -})); - -jest.mock('expo-background-task', () => ({ - registerTaskAsync: jest.fn().mockResolvedValue(undefined), - unregisterTaskAsync: jest.fn().mockResolvedValue(undefined), - BackgroundTaskResult: { - Success: 'Success', - Failed: 'Failed', - }, -})); - -// Mock the Android foreground-service wrapper used by use-background-task.ts. +// Mock the Android foreground-service wrapper used by use-foreground-service.ts. // Its real module touches a native event emitter that doesn't exist in the // Node test environment. jest.mock('react-native-background-actions', () => ({ @@ -283,4 +267,4 @@ jest.mock('@nozbe/watermelondb/react', () => { return React.createElement(Component, { ...props, ...extraProps }); } }; -}); \ No newline at end of file +}); diff --git a/mobile-app/sapot-mobile-app/package.json b/mobile-app/sapot-mobile-app/package.json index 38e7ccf5..b9cff3a8 100644 --- a/mobile-app/sapot-mobile-app/package.json +++ b/mobile-app/sapot-mobile-app/package.json @@ -73,7 +73,6 @@ "deepmerge": "^4.3.1", "eslint-plugin-react-hooks": "^7.0.1", "expo": "~54.0.36", - "expo-background-task": "~1.0.10", "expo-build-properties": "~1.0.10", "expo-camera": "~17.0.10", "expo-constants": "~18.0.12", @@ -91,7 +90,6 @@ "expo-secure-store": "~15.0.8", "expo-splash-screen": "~31.0.13", "expo-status-bar": "~3.0.9", - "expo-task-manager": "~14.0.9", "expo-updates": "~29.0.19", "expo-web-browser": "~15.0.11", "input-otp-native": "^0.6.0", diff --git a/mobile-app/sapot-mobile-app/pnpm-lock.yaml b/mobile-app/sapot-mobile-app/pnpm-lock.yaml index d42ea2b0..73d61435 100644 --- a/mobile-app/sapot-mobile-app/pnpm-lock.yaml +++ b/mobile-app/sapot-mobile-app/pnpm-lock.yaml @@ -59,9 +59,6 @@ importers: expo: specifier: ~54.0.36 version: 54.0.36(@babel/core@7.29.7)(@expo/metro-runtime@6.1.2)(expo-router@6.0.24)(react-native@0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0))(react@19.1.0)(typescript@5.9.3) - expo-background-task: - specifier: ~1.0.10 - version: 1.0.10(expo@54.0.36)(react-native@0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0)) expo-build-properties: specifier: ~1.0.10 version: 1.0.10(expo@54.0.36) @@ -113,9 +110,6 @@ importers: expo-status-bar: specifier: ~3.0.9 version: 3.0.9(react-native@0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0))(react@19.1.0) - expo-task-manager: - specifier: ~14.0.9 - version: 14.0.9(expo@54.0.36)(react-native@0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0)) expo-updates: specifier: ~29.0.19 version: 29.0.19(expo@54.0.36)(react-native@0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0))(react@19.1.0) @@ -2626,11 +2620,6 @@ packages: react: '*' react-native: '*' - expo-background-task@1.0.10: - resolution: {integrity: sha512-EbPnuf52Ps/RJiaSFwqKGT6TkvMChv7bI0wF42eADbH3J2EMm5y5Qvj0oFmF1CBOwc3mUhqj63o7Pl6OLkGPZQ==} - peerDependencies: - expo: '*' - expo-build-properties@1.0.10: resolution: {integrity: sha512-mFCZbrbrv0AP5RB151tAoRzwRJelqM7bCJzCkxpu+owOyH+p/rFC/q7H5q8B9EpVWj8etaIuszR+gKwohpmu1Q==} peerDependencies: @@ -2815,12 +2804,6 @@ packages: expo-structured-headers@5.0.0: resolution: {integrity: sha512-RmrBtnSphk5REmZGV+lcdgdpxyzio5rJw8CXviHE6qH5pKQQ83fhMEcigvrkBdsn2Efw2EODp4Yxl1/fqMvOZw==} - expo-task-manager@14.0.9: - resolution: {integrity: sha512-GKWtXrkedr4XChHfTm5IyTcSfMtCPxzx89y4CMVqKfyfROATibrE/8UI5j7UC/pUOfFoYlQvulQEvECMreYuUA==} - peerDependencies: - expo: '*' - react-native: '*' - expo-updates-interface@2.0.0: resolution: {integrity: sha512-pTzAIufEZdVPKql6iMi5ylVSPqV1qbEopz9G6TSECQmnNde2nwq42PxdFBaUEd8IZJ/fdJLQnOT3m6+XJ5s7jg==} peerDependencies: @@ -4904,9 +4887,6 @@ packages: resolution: {integrity: sha512-hpbDzxUY9BFwX+UeBnxv3Sh1q7HFxj48DTmXchNgRa46lO8uj3/1iEn3MiNUYTg1g9ctIqXCCERn8gYZhHC5lQ==} engines: {node: '>=4'} - unimodules-app-loader@6.0.8: - resolution: {integrity: sha512-fqS8QwT/MC/HAmw1NKCHdzsPA6WaLm0dNmoC5Pz6lL+cDGYeYCNdHMO9fy08aL2ZD7cVkNM0pSR/AoNRe+rslA==} - universalify@0.2.0: resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==} engines: {node: '>= 4.0.0'} @@ -8180,13 +8160,6 @@ snapshots: - supports-color - typescript - expo-background-task@1.0.10(expo@54.0.36)(react-native@0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0)): - dependencies: - expo: 54.0.36(@babel/core@7.29.7)(@expo/metro-runtime@6.1.2)(expo-router@6.0.24)(react-native@0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0))(react@19.1.0)(typescript@5.9.3) - expo-task-manager: 14.0.9(expo@54.0.36)(react-native@0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0)) - transitivePeerDependencies: - - react-native - expo-build-properties@1.0.10(expo@54.0.36): dependencies: ajv: 8.20.0 @@ -8405,12 +8378,6 @@ snapshots: expo-structured-headers@5.0.0: {} - expo-task-manager@14.0.9(expo@54.0.36)(react-native@0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0)): - dependencies: - expo: 54.0.36(@babel/core@7.29.7)(@expo/metro-runtime@6.1.2)(expo-router@6.0.24)(react-native@0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0))(react@19.1.0)(typescript@5.9.3) - react-native: 0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0) - unimodules-app-loader: 6.0.8 - expo-updates-interface@2.0.0(expo@54.0.36): dependencies: expo: 54.0.36(@babel/core@7.29.7)(@expo/metro-runtime@6.1.2)(expo-router@6.0.24)(react-native@0.81.5(@babel/core@7.29.7)(@types/react@19.1.17)(react@19.1.0))(react@19.1.0)(typescript@5.9.3) @@ -10919,8 +10886,6 @@ snapshots: unicode-property-aliases-ecmascript@2.2.0: {} - unimodules-app-loader@6.0.8: {} - universalify@0.2.0: {} unpipe@1.0.0: {} diff --git a/mobile-app/sapot-mobile-app/task/signaling-task.ts b/mobile-app/sapot-mobile-app/task/signaling-task.ts deleted file mode 100644 index f5dd0143..00000000 --- a/mobile-app/sapot-mobile-app/task/signaling-task.ts +++ /dev/null @@ -1,243 +0,0 @@ -import { backgroundLog } from "@/features/shared/core/utils/logger"; -import { NetworkConfig } from "@/features/shared/core/stores"; -import { - TcpServerAdapter, - WsSignalingAdapter, - ZeroconfAdapter, -} from "@/features/shared/connection"; -import { - getAppAlive, - getStoredAccessToken, - getStoredFirstName, - getStoredLastName, - getStoredPeerId, - getStoredUsername, - getStoredWsUrl, - saveAppAlive, -} from "@/features/shared/core/stores/secure-config"; -import { CallMessage } from "@/features/shared/types"; -import * as BackgroundTask from "expo-background-task"; -import * as Notifications from "expo-notifications"; -import * as TaskManager from "expo-task-manager"; -import { Service } from "react-native-zeroconf"; - -export const SIGNALING_TASK = "SIGNALING_TASK"; - -// ── App-alive flag ───────────────────────────────────────────────────────────── -// MainContainer flips this true on initialize(), false on cleanup(). -// Prevents background task from spinning up duplicate instances of services -// that ConnectionService and DiscoveryService already own when app is alive. -// -// _isAppAlive is an in-process cache for the foreground JS context. -// The background task runs in a separate JS context so it cannot read this -// variable — it reads the persisted value from secure store instead. - -export const setAppAlive = (alive: boolean) => { - saveAppAlive(alive); // persist across JS context boundary for background task - if (alive) { - // Free the TCP port and WS connection so ConnectionService can claim them. - // Needed when the 15-min background task created bgTcpServer in the same - // process and the main app then opens via notification tap. - stopBackgroundOnlyServices(); - } - backgroundLog.info("bg › app alive flag set", { alive }); -}; - -// ── Background-only singletons ───────────────────────────────────────────────── -// Only instantiated when MainContainer is NOT running. -// When alive: ConnectionService owns TcpServerAdapter + WsSignalingAdapter -// DiscoveryService owns ZeroconfAdapter - -let bgNetworkConfig: NetworkConfig | null = null; -let bgTcpServer: TcpServerAdapter | null = null; -let bgWsAdapter: WsSignalingAdapter | null = null; -let bgZeroconf: ZeroconfAdapter | null = null; - -// ── Incoming call notification ───────────────────────────────────────────────── - -const showIncomingCallNotification = async (message: { - callerName?: string; - type: string; - data: { from_user?: string; from?: string }; -}) => { - try { - await Notifications.scheduleNotificationAsync({ - content: { - title: "📞 Incoming Call", - body: `${message.callerName ?? message.data?.from_user} is calling...`, - sound: "ringtone.mp3", - data: { - type: "incoming_call", - // WS call message shape: { type, data: { from_user } } - // TCP call message shape: { type, data: { from } } - id: message.data?.from_user ?? message.data?.from, - call_type: message.type === "video-call" ? "video" : "audio", - }, - } as Notifications.NotificationContentInput, - trigger: { - channelId: "incoming-call", - } as Notifications.NotificationTriggerInput, - }); - backgroundLog.info("bg › incoming call notification shown"); - } catch (error) { - backgroundLog.error("bg › show notification failed", { error }); - } -}; - -// ── Message handler — mirrors ConnectionService logic ───────────────────────── -// Handles both WS (WsCallMessage) and TCP (Message) shapes - -const handleIncomingMessage = async (message: CallMessage) => { - if (message.type === "audio-call" || message.type === "video-call") { - await showIncomingCallNotification(message); - } - - if (message.type === "call-ended" || message.type === "call-rejected") { - // Dismiss any lingering incoming call notification in the tray - try { - const presented = await Notifications.getPresentedNotificationsAsync(); - for (const n of presented) { - if (n.request.content.data?.type === "incoming_call") { - await Notifications.dismissNotificationAsync(n.request.identifier); - } - } - backgroundLog.info("bg › call ended/rejected — incoming notification dismissed"); - } catch (error) { - backgroundLog.error("bg › dismiss notification failed", { error }); - } - } -}; - -// ── Background-only service lifecycle ───────────────────────────────────────── - -export const startBackgroundOnlyServices = async () => { - // Fresh NetworkConfig every wake — no startWatching() in background - bgNetworkConfig = new NetworkConfig(); - await bgNetworkConfig.initialize(); - - const ip = bgNetworkConfig.ipAddress; - const port = bgNetworkConfig.port; - - backgroundLog.info("bg › network ready", { port }); - - // ── TCP Server ────────────────────────────────────────────────────────────── - // Mirrors TcpServerAdapter usage in ConnectionService.start() - const tcpNeedsRestart = - !bgTcpServer || - bgTcpServer.currentPort !== port || - bgTcpServer.currentIp !== ip; - - if (tcpNeedsRestart) { - bgTcpServer?.stop(); - bgTcpServer = new TcpServerAdapter(); - await bgTcpServer.start(port, ip); - - bgTcpServer.on("data", async (message: CallMessage) => { - await handleIncomingMessage(message); - }); - - backgroundLog.info("bg › tcp server started", { port }); - } - - // ── WebSocket Signaling ───────────────────────────────────────────────────── - // Mirrors WsSignalingAdapter usage in ConnectionService / SignalingService - const wsUrl = await getStoredWsUrl(); - - if (wsUrl && (!bgWsAdapter || !bgWsAdapter.isConnected)) { - bgWsAdapter?.disconnect(); - bgWsAdapter = new WsSignalingAdapter(); - - bgWsAdapter.on("call-message", async (message: CallMessage) => { - await handleIncomingMessage(message); - }); - - bgWsAdapter.on("reconnect-failed", ({ attempts }: { attempts: number }) => { - backgroundLog.warn("bg › ws reconnect failed", { attempts }); - }); - - const token = await getStoredAccessToken(); - - if (!token) return; - - bgWsAdapter.connect({ - baseUrl: wsUrl, - token, - }); - backgroundLog.info("bg › ws adapter connected"); - } - - // ── Zeroconf ──────────────────────────────────────────────────────────────── - // Scan so peers can reach us; publish so we appear to other peers on LAN. - if (!bgZeroconf) { - bgZeroconf = new ZeroconfAdapter(); - - bgZeroconf.on("serviceResolved", async (service: Service) => { - backgroundLog.info("bg › zeroconf service resolved", { - hasId: Boolean(service?.txt?.id), - }); - // No peer registration in background — DiscoveryService handles that - // when the app is alive. We only log here. - }); - - bgZeroconf.startScan(); - backgroundLog.info("bg › zeroconf scan started"); - - // Publish our own service so other LAN peers can discover us - const [peerId, username, firstName, lastName] = await Promise.all([ - getStoredPeerId(), - getStoredUsername(), - getStoredFirstName(), - getStoredLastName(), - ]); - if (peerId && username && firstName) { - bgZeroconf.publishService({ - type: "lanchat", - protocol: "tcp", - domain: "local.", - name: `Device-${Date.now()}`, - port, - txt: { id: peerId, username, firstName, lastName: lastName ?? "" }, - }); - backgroundLog.info("bg › zeroconf service published"); - } - } -}; - -export const stopBackgroundOnlyServices = () => { - try { - bgTcpServer?.stop(); - bgWsAdapter?.disconnect(); - void bgZeroconf?.cleanUp(); - bgTcpServer = null; - bgWsAdapter = null; - bgZeroconf = null; - bgNetworkConfig = null; - backgroundLog.info("bg › background-only services stopped"); - } catch (error) { - backgroundLog.error("bg › stop background services failed", { error }); - } -}; - -// ── Task definition ──────────────────────────────────────────────────────────── - -TaskManager.defineTask(SIGNALING_TASK, async () => { - // Read from secure store — _isAppAlive is always false in this separate JS context. - const appAlive = await getAppAlive(); - backgroundLog.info("bg › task triggered", { appAlive }); - - try { - if (appAlive) { - // ConnectionService + DiscoveryService own all transports. - // Stop any lingering background instances to free ports and listeners. - stopBackgroundOnlyServices(); - backgroundLog.info("bg › app alive, handing off to MainContainer"); - return BackgroundTask.BackgroundTaskResult.Success; - } - - await startBackgroundOnlyServices(); - return BackgroundTask.BackgroundTaskResult.Success; - } catch (error) { - backgroundLog.error("bg › task failed", { error }); - return BackgroundTask.BackgroundTaskResult.Failed; - } -}); From e1ce39ab874f8ad19102251cb305895243f08c36 Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Sat, 15 Aug 2026 20:05:12 +0800 Subject: [PATCH 07/12] fix(gsm): secure and harden SMS delivery (#362) * fix(gsm-queue): bound outbound SMS admission * chore(deploy-gsm): align GSM shutdown grace periods * docs(docs-gsm): document outbound queue saturation * fix(gsm-queue): harden request lifecycle * test(gsm-queue): cover saturation races * fix(deploy-gsm): align runtime configuration * docs(gsm-gateway): correct service contracts * docs(gsm-testing): document queue verification * docs(gsm-deployment): fix callback troubleshooting link * fix(deploy-gsm): restrict direct API to loopback * fix(gsm-gateway): harden SMS delivery lifecycle * fix(server-gsm): preserve gateway failure responses * fix(mobile-gsm): retain failed SMS for retry * docs(docs-gsm): document resilient authenticated delivery * feat(server-testing): seed verified phone fixture --- CLAUDE.md | 2 +- GSM-module/CLAUDE.md | 13 +- GSM-module/GSM-fastapi/.env.example | 5 +- GSM-module/GSM-fastapi/api.py | 78 ++++- GSM-module/GSM-fastapi/config.py | 28 +- GSM-module/GSM-fastapi/database.py | 11 + GSM-module/GSM-fastapi/serial_worker.py | 278 ++++++++++----- GSM-module/GSM-fastapi/sms_handler.py | 55 +-- GSM-module/GSM-fastapi/tests/conftest.py | 7 + .../GSM-fastapi/tests/test_api_queue.py | 173 +++++++++ GSM-module/GSM-fastapi/tests/test_config.py | 48 +++ .../tests/test_database_reconciliation.py | 30 ++ .../GSM-fastapi/tests/test_incoming_sms.py | 52 +++ GSM-module/GSM-fastapi/tests/test_lifespan.py | 40 +++ .../GSM-fastapi/tests/test_serial_worker.py | 301 ++++++++++++++++ SECURITY.md | 9 +- deploy/config/gsm-fastapi.env.example | 3 + deploy/config/nginx.prod.conf | 2 +- deployment-scripts/server-GSM-api.service | 1 + docker-compose.yml | 2 +- docker/nginx.docker.conf | 2 +- docs/TROUBLESHOOTING.md | 4 +- docs/api/gsm-sms.md | 18 +- docs/api/openapi/gsm-sms.yaml | 134 +++++++ .../assumptions-and-constraints.md | 1 - docs/architecture/component-map.md | 2 +- docs/architecture/data-flow.md | 8 +- docs/architecture/threat-model.md | 3 +- docs/deployment/environment-config.md | 11 +- docs/deployment/gsm-module.md | 72 ++-- docs/deployment/maintenance.md | 2 +- docs/deployment/monitoring-logging.md | 13 +- docs/deployment/server.md | 2 +- docs/features/sms-gateway/design.md | 327 +++++------------- docs/features/sms-gateway/requirements.md | 180 ++++++---- docs/features/sms-gateway/testing.md | 187 +++++----- docs/getting-started/docker-setup.md | 8 +- docs/getting-started/gsm-module-setup.md | 9 +- docs/qa/scenario-tooling.md | 1 + .../app/(drawer)/(tabs)/chat/[id].tsx | 6 +- .../app/(drawer)/(tabs)/index.tsx | 13 +- .../settings/account/phone/verify-phone.tsx | 34 +- mobile-app/sapot-mobile-app/docs/API.md | 18 + .../sapot-mobile-app/docs/ARCHITECTURE.md | 3 + .../docs/diagrams/05-sms-flow.md | 4 +- .../auth/api/__tests__/auth.api.test.ts | 45 ++- .../features/auth/api/auth.api.ts | 43 +-- .../features/auth/auth-container.test.ts | 5 + .../features/auth/auth-container.ts | 3 + .../features/auth/hooks/index.ts | 2 +- .../hooks/use-phone-verification-service.ts | 5 + .../services/phone-verification-service.ts | 24 ++ .../__tests__/message-list.test.tsx | 86 ++++- .../features/chat/components/message-list.tsx | 31 +- .../chat/hooks/use-send-message.test.ts | 46 ++- .../features/chat/hooks/use-send-message.ts | 17 +- .../main-container-initialize.test.ts | 1 + .../shared/connection/services/gsm-service.ts | 19 + .../shared/connection/services/index.ts | 1 + .../shared/core/api/__tests__/gsm.api.test.ts | 51 +++ .../features/shared/core/api/gsm.api.ts | 29 +- .../core/errors/__tests__/gsm-error.test.ts | 72 ++++ .../features/shared/core/errors/gsm-error.ts | 96 +++++ .../features/shared/core/errors/index.ts | 7 + .../features/shared/hooks/index.ts | 2 +- .../features/shared/hooks/use-gsm-health.ts | 7 +- .../features/shared/hooks/use-gsm-service.ts | 5 + .../features/shared/main-container.ts | 3 + server/app/api/gsm.py | 105 +++++- server/app/api/testing.py | 1 + server/app/db_operations/qa_scenarios.py | 14 + server/app/tests/test_gsm_health.py | 28 ++ server/app/tests/test_gsm_proxy.py | 247 +++++++++++++ server/app/tests/test_qa_scenarios.py | 11 + server/app/tests/test_testing_endpoints.py | 10 + server/nginx.conf | 2 +- 76 files changed, 2540 insertions(+), 678 deletions(-) create mode 100644 GSM-module/GSM-fastapi/tests/conftest.py create mode 100644 GSM-module/GSM-fastapi/tests/test_api_queue.py create mode 100644 GSM-module/GSM-fastapi/tests/test_config.py create mode 100644 GSM-module/GSM-fastapi/tests/test_database_reconciliation.py create mode 100644 GSM-module/GSM-fastapi/tests/test_incoming_sms.py create mode 100644 GSM-module/GSM-fastapi/tests/test_lifespan.py create mode 100644 GSM-module/GSM-fastapi/tests/test_serial_worker.py create mode 100644 mobile-app/sapot-mobile-app/features/auth/hooks/use-phone-verification-service.ts create mode 100644 mobile-app/sapot-mobile-app/features/auth/services/phone-verification-service.ts create mode 100644 mobile-app/sapot-mobile-app/features/shared/connection/services/gsm-service.ts create mode 100644 mobile-app/sapot-mobile-app/features/shared/core/api/__tests__/gsm.api.test.ts create mode 100644 mobile-app/sapot-mobile-app/features/shared/core/errors/__tests__/gsm-error.test.ts create mode 100644 mobile-app/sapot-mobile-app/features/shared/core/errors/gsm-error.ts create mode 100644 mobile-app/sapot-mobile-app/features/shared/hooks/use-gsm-service.ts create mode 100644 server/app/tests/test_gsm_proxy.py diff --git a/CLAUDE.md b/CLAUDE.md index 5772d520..980152e3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,7 +60,7 @@ Run for each component actually touched — don't assume one component's green b | `server/app/models/` changed | `alembic upgrade head && alembic check` — **run from `server/`** (not `server/app/`) with `DATABASE_URL` set. `alembic check` must report no new operations. Note `pytest` builds its schema with `create_all()` and cannot detect migration drift. | | `mobile-app/sapot-mobile-app/` | `pnpm run testAll` (= test + typecheck + lint + expo-doctor), or the individual `pnpm test` / `pnpm run typecheck` / `pnpm run lint` | | `admin-frontend/sapot-admin/` | `pnpm run lint && pnpm run build` — **no test script exists in this component**; don't claim test coverage that isn't there | -| `GSM-module/` | No automated tests exist — verify manually per `docs/getting-started/gsm-module-setup.md` | +| `GSM-module/` | `pytest` (from `GSM-module/GSM-fastapi/`; serial I/O and database calls are mocked) | If the change is release-relevant (server), `server/app/version.py` must match the git tag per `VERSIONING.md` before tagging — not typically a per-commit concern, but relevant if asked to prepare a release. diff --git a/GSM-module/CLAUDE.md b/GSM-module/CLAUDE.md index 27703143..87aab3e8 100644 --- a/GSM-module/CLAUDE.md +++ b/GSM-module/CLAUDE.md @@ -4,21 +4,21 @@ Instructions for Claude Code working in `GSM-module/` — SAPOT's SMS gateway, b ## Project Overview -Three layers: Arduino firmware talking AT commands to a SIM800L/SIM900 modem over serial; a Python (FastAPI) service on the same machine talking to the Arduino over USB serial; the main `server/` proxying to that Python service over HTTP with a shared secret. There are **two parallel Python implementations** in this directory — they are not both live (see Architecture). +Three layers: Arduino firmware talking AT commands to a SIM800L/SIM900 modem over serial; a Python (FastAPI) service on the same machine talking to the Arduino over USB serial; the main `server/` proxying outbound calls and receiving inbound callbacks over HTTP authenticated with a shared secret. There are **two parallel Python implementations** in this directory; they are not both live (see Architecture). ## Architecture — which implementation is live -**`GSM-fastapi/` is the deployed, current implementation.** Evidence: `docs/deployment/gsm-module.md`, `docs/getting-started/gsm-module-setup.md`, and `docs/features/sms-gateway/README.md` all reference only `GSM-fastapi/`; `server/app/api/gsm.py` proxies to `http://localhost:8001`, and `GSM-fastapi/main.py` hardcodes `uvicorn.run("api:app", port=8001, ...)` — an exact match. +**`GSM-fastapi/` is the deployed, current implementation.** Evidence: `docs/deployment/gsm-module.md`, `docs/getting-started/gsm-module-setup.md`, and `docs/features/sms-gateway/README.md` all reference only `GSM-fastapi/`; `server/app/api/gsm.py` proxies to `http://localhost:8001`, and `GSM-fastapi/main.py` hardcodes `uvicorn.run("api:app", port=8001, ...)`, an exact match. -**`GSM-API/` is a separate, incomplete rewrite — not deployed, not referenced by any doc or by `server/`.** It has in-memory-only session state (no DB persistence), fire-and-forget SMS sends (no delivery confirmation), and an unsynchronized global (`app/gsm/gsm_runtime.py`'s module-level `ser`) shared across threads with no lock. Default to editing `GSM-fastapi/` for SMS-gateway work; only touch `GSM-API/` if a task explicitly asks for it. +**`GSM-API/` is a separate, incomplete rewrite that is not deployed or referenced by `server/`.** It has in-memory-only session state (no DB persistence), fire-and-forget SMS sends (no delivery confirmation), and an unsynchronized global (`app/gsm/gsm_runtime.py`'s module-level `ser`) shared across threads with no lock. Default to editing `GSM-fastapi/` for SMS-gateway work; only touch `GSM-API/` if a task explicitly asks for it. -**Known doc/code mismatch:** `docs/features/sms-gateway/design.md` describes a different wire protocol (`SEND:`/`RECV:`/`ACK:`/`ERR:` frames, endpoints `/gsm/send`/`/gsm/status`, tables `sms_outbox`/`sms_inbound`) that matches neither actual Python service nor the Arduino firmware. Trust the code (`GSM-fastapi/protocol.py` + the `.ino` firmware) over that doc — treat `design.md` as aspirational/stale. +**Documentation source of truth:** `docs/features/sms-gateway/design.md` describes the deployed HTTP routes, queue, lifecycle, and serial flow. Use `GSM-fastapi/protocol.py` and the Arduino firmware as the authoritative definitions for individual serial frames. ### Data flow (GSM-fastapi, the live path) **Inbound:** Arduino emits `SMS_RECEIVED||` over serial → `serial_worker.py`'s `SerialWorker._reader_loop` parses it via `protocol.py` → queued → `api.py`'s async `_inbox_drain()` task offloads to a thread pool → `sms_handler.handle_incoming_sms()` (session/target flow, ban/verified checks against MariaDB) → `database.py`'s `notify_app()` POSTs to the main server's `/gsm/inbound` with an `X-GSM-Secret` header. -**Outbound:** caller (main server or admin frontend's `gsm` page) calls `POST /sms/send` on port 8001 → `SerialWorker.send_sms()` enqueues `SEND_SMS||` and blocks on an `Event` (timeout 60s) → `_sender_loop` writes to serial → Arduino replies `SMS_SENT|`/`SMS_FAILED|` → the reader thread resolves the waiting request. +**Outbound:** the main server calls `POST /sms/send` on port 8001 with `X-GSM-Secret`. The gateway validates the secret before `SerialWorker.send_sms()` atomically admits the request to a bounded FIFO queue or rejects saturation with HTTP 503. The sender writes `SEND_SMS||`, and the reader resolves the request from `SMS_SENT|` or `SMS_FAILED|`. The admin GSM page reads health and message history through the main server. Shutdown rejects queued and active work with `SERVICE_STOPPING`. `SerialWorker` runs two dedicated threads (`_reader_loop`, `_sender_loop`) with proper request/response correlation over the async serial stream, and auto-reconnects every 10s on disconnect. @@ -33,7 +33,7 @@ Three layers: Arduino firmware talking AT commands to a SIM800L/SIM900 modem ove - **Wire protocol** — pipe-delimited lines: `SEND_SMS||` (PC → Arduino), `SMS_RECEIVED||`, `SMS_SENT|`, `SMS_FAILED||`, `GSM_READY`, `NETWORK_OK`/`NETWORK_LOST`, `SIM_MISSING` (Arduino → PC). Implemented identically in `GSM-fastapi/protocol.py`. Any change to this format must be mirrored in the `.ino` firmware's parser/emitter — they are independent implementations of the same contract, not shared code. - **Session/target flow** — inbound SMS starts a session (`NEW`), the sender texts `[target] +63...` to select a recipient (`AWAITING_TARGET` → `ACTIVE`), then messages relay through. `GSM-fastapi` persists this to MariaDB (`SmsSession` table) and checks `banned`/`phone_is_verified` on both sender and target; `GSM-API`'s equivalent is in-memory only and skips those checks. -- **Shared-secret webhook auth** (`X-GSM-Secret` header) — how this service calls back into the main server (`/gsm/inbound`); distinct from the JWT auth used elsewhere in SAPOT (see `../server/CLAUDE.md`). +- **Shared-secret service auth** (`X-GSM-Secret` header) — authenticates main-server calls to `/sms/send` and GSM callbacks to `/gsm/inbound`; distinct from the JWT auth used elsewhere in SAPOT (see `../server/CLAUDE.md`). ## Development Conventions @@ -55,7 +55,6 @@ Three layers: Arduino firmware talking AT commands to a SIM800L/SIM900 modem ove - `GSM-fastapi/sapot.db` is a stale, unused artifact (confirmed in `../docs/database/migrations.md`) — real storage is MariaDB via `config.py`'s `DB_PATH`. Never read from or write to `sapot.db`. - `GSM-API/app/gsm/gsm_runtime.py`'s module-level `ser` is a global shared across threads with no lock — if `GSM-API` is ever revived, this is a live race condition, not a style nit. - `GSM-trial-code.ino` is not wire-compatible with either Python service — never point a deployment at it, even for "quick testing." -- Treating `docs/features/sms-gateway/design.md` as accurate — its protocol/endpoint descriptions don't match the real code (see Architecture). - `server/app/api/gsm.py`'s own code comments refer to "GSM-API" as a generic name for **the GSM service it proxies to** (i.e. the live `GSM-fastapi/`, port 8001) — not the literal `GSM-module/GSM-API/` directory documented above as non-deployed. Don't let those comments override the Architecture section above. ## When Modifying This Project diff --git a/GSM-module/GSM-fastapi/.env.example b/GSM-module/GSM-fastapi/.env.example index dd03f194..ebc8eebb 100644 --- a/GSM-module/GSM-fastapi/.env.example +++ b/GSM-module/GSM-fastapi/.env.example @@ -8,6 +8,9 @@ HOST=127.0.0.1 # to avoid a port collision. PORT=8000 LOG_LEVEL=INFO -SAPOT_API_URL=https://localhost:8000 +# Maximum outbound SMS requests waiting behind the one in-flight request. +# Must be an integer from 1 through 20. +SMS_SEND_QUEUE_MAXSIZE=10 +SAPOT_API_URL=http://localhost:8000 GSM_SECRET=change-me-to-a-strong-secret SMS_BOT_USER_ID= diff --git a/GSM-module/GSM-fastapi/api.py b/GSM-module/GSM-fastapi/api.py index d7bc9b73..6fd2c59d 100644 --- a/GSM-module/GSM-fastapi/api.py +++ b/GSM-module/GSM-fastapi/api.py @@ -27,18 +27,23 @@ """ import asyncio +import hmac import logging import re from contextlib import asynccontextmanager -from typing import Counter, Optional +from typing import Annotated, Counter, Optional -from fastapi import FastAPI, HTTPException, Query, BackgroundTasks +from fastapi import BackgroundTasks, Depends, FastAPI, Header, HTTPException, Query from fastapi.responses import JSONResponse from pydantic import BaseModel, field_validator import database from app_version import __version__ -from serial_worker import SerialWorker +from serial_worker import ( + OutboundQueueFullError, + SerialWorker, + WorkerStoppingError, +) from sms_handler import handle_incoming_sms from config import settings @@ -57,9 +62,19 @@ async def lifespan(app: FastAPI): # Database database.init(settings.db_path) logger.info("Database ready") + orphaned_count = database.fail_orphaned_pending_messages() + if orphaned_count: + logger.warning( + "Marked %d orphaned SMS messages as failed after service restart", + orphaned_count, + ) # Serial worker - _worker = SerialWorker(settings.serial_port, settings.serial_baud) + _worker = SerialWorker( + settings.serial_port, + settings.serial_baud, + settings.sms_send_queue_maxsize, + ) _worker.start() logger.info("Serial worker started on %s", settings.serial_port) @@ -119,7 +134,12 @@ def _process_incoming(event): status="received", ) - reply, forward_number, forward_body = handle_incoming_sms(number, body) + reply, forward_number, forward_body, rejection_reason = handle_incoming_sms( + number, body + ) + + if rejection_reason: + database.update_message_status(msg_id, "rejected", rejection_reason) # Forward to target via SMS if forward_number and forward_body and _worker: @@ -153,6 +173,12 @@ def _send_and_log(from_number: str, to_number: str, body: str): result = _worker.send_sms(to_number, body, timeout=120) status = "sent" if result["ok"] else "failed" database.update_message_status(msg_id, status, result.get("reason")) + except OutboundQueueFullError: + logger.warning("Internal outbound SMS rejected: QUEUE_FULL") + database.update_message_status(msg_id, "failed", "QUEUE_FULL") + except WorkerStoppingError: + logger.info("Internal outbound SMS rejected: SERVICE_STOPPING") + database.update_message_status(msg_id, "failed", "SERVICE_STOPPING") except Exception as e: logger.error("send_sms error: %s", e) database.update_message_status(msg_id, "failed", str(e)) @@ -203,10 +229,21 @@ def validate_phone(cls, v): return v +def require_gsm_secret( + x_gsm_secret: Annotated[ + Optional[str], Header(alias="X-GSM-Secret") + ] = None, +): + if x_gsm_secret is None or not hmac.compare_digest( + x_gsm_secret, settings.gsm_secret + ): + raise HTTPException(status_code=401, detail="Invalid GSM secret") + + # ── Health endpoints ────────────────────────────────────────────────────────── @app.get("/health", tags=["health"]) -def health(): +async def health(): """ Liveness check. Returns 200 if the API is running. Returns 503 if the GSM modem is not ready (so load balancers can react). @@ -257,6 +294,9 @@ def pct(value): "connected": _worker.connected, "last_status": _worker.last_status, "queue_depth": _worker.incoming_queue.qsize(), + "outbound_queue_depth": _worker.outbound_queue_depth, + "outbound_queue_capacity": _worker.outbound_queue_capacity, + "outbound_in_flight": _worker.outbound_in_flight, "port": settings.serial_port, "baud": settings.serial_baud, "total_messages": total, @@ -303,7 +343,11 @@ def status(): # ── SMS endpoints ───────────────────────────────────────────────────────────── -@app.post("/sms/send", tags=["sms"]) +@app.post( + "/sms/send", + tags=["sms"], + dependencies=[Depends(require_gsm_secret)], +) def send_sms(req: SendSMSRequest): """ Send an SMS directly. Blocks until the modem confirms delivery. @@ -323,6 +367,20 @@ def send_sms(req: SendSMSRequest): try: result = _worker.send_sms(req.number, req.body, timeout=60) + except OutboundQueueFullError: + database.update_message_status(msg_id, "failed", "QUEUE_FULL") + raise HTTPException(503, { + "message": "Outbound SMS queue is full", + "reason": "QUEUE_FULL", + "msg_id": msg_id, + }) + except WorkerStoppingError: + database.update_message_status(msg_id, "failed", "SERVICE_STOPPING") + raise HTTPException(503, { + "message": "SMS service is stopping", + "reason": "SERVICE_STOPPING", + "msg_id": msg_id, + }) except RuntimeError as e: database.update_message_status(msg_id, "failed", str(e)) raise HTTPException(503, str(e)) @@ -331,6 +389,12 @@ def send_sms(req: SendSMSRequest): database.update_message_status(msg_id, status, result.get("reason")) if not result["ok"]: + if result["reason"] == "SERVICE_STOPPING": + raise HTTPException(503, { + "message": "SMS service is stopping", + "reason": "SERVICE_STOPPING", + "msg_id": msg_id, + }) raise HTTPException(502, { "message": "SMS delivery failed", "reason": result["reason"], diff --git a/GSM-module/GSM-fastapi/config.py b/GSM-module/GSM-fastapi/config.py index d78c411c..1fd0dcd4 100644 --- a/GSM-module/GSM-fastapi/config.py +++ b/GSM-module/GSM-fastapi/config.py @@ -11,6 +11,24 @@ import os from dotenv import load_dotenv +from serial_worker import MAX_SEND_QUEUE_SIZE + + +def bounded_integer_env(name: str, default: int, maximum: int) -> int: + value = os.environ.get(name) + if value is None: + return default + try: + parsed = int(value) + except ValueError as error: + raise RuntimeError( + f"Environment variable '{name}' must be an integer between 1 and {maximum}." + ) from error + if not 1 <= parsed <= maximum: + raise RuntimeError( + f"Environment variable '{name}' must be an integer between 1 and {maximum}." + ) + return parsed class Settings: @@ -21,12 +39,16 @@ class Settings: # Baud rate — must match PC_BAUD in the Arduino sketch (9600) serial_baud: int = int(os.environ.get("SERIAL_BAUD", "9600")) - # SQLite database file path db_path: str | None = os.environ.get("DB_PATH") if not db_path: raise RuntimeError("Environment variable 'DB_PATH' is not set.") + gsm_secret: str = os.environ.get("GSM_SECRET", "") + + if not gsm_secret: + raise RuntimeError("Environment variable 'GSM_SECRET' is not set.") + # FastAPI host and port host: str = os.environ.get("HOST", "127.0.0.1") port: int = int(os.environ.get("PORT", "8000")) @@ -34,5 +56,9 @@ class Settings: # Logging level log_level: str = os.environ.get("LOG_LEVEL", "INFO") + sms_send_queue_maxsize: int = bounded_integer_env( + "SMS_SEND_QUEUE_MAXSIZE", 10, MAX_SEND_QUEUE_SIZE + ) + settings = Settings() diff --git a/GSM-module/GSM-fastapi/database.py b/GSM-module/GSM-fastapi/database.py index 469ab883..6fce6193 100644 --- a/GSM-module/GSM-fastapi/database.py +++ b/GSM-module/GSM-fastapi/database.py @@ -520,6 +520,17 @@ def update_message_status(msg_id: str, status: str, s.commit() +def fail_orphaned_pending_messages() -> int: + with new_get_session() as s: + result = s.execute( + update(SmsLog) + .where(SmsLog.status == "pending") + .values(status="failed", failure_reason="SERVICE_CRASHED") + ) + s.commit() + return result.rowcount + + def get_messages(limit: int = 50, offset: int = 0, direction: Optional[str] = None, phone: Optional[str] = None) -> dict: with new_get_session() as s: diff --git a/GSM-module/GSM-fastapi/serial_worker.py b/GSM-module/GSM-fastapi/serial_worker.py index 0457f96d..dd2fa080 100644 --- a/GSM-module/GSM-fastapi/serial_worker.py +++ b/GSM-module/GSM-fastapi/serial_worker.py @@ -16,8 +16,8 @@ Incoming SMS events land on worker.incoming_queue as SerialEvent objects. Auto-reconnect: - Serial errors trigger a reconnect loop. Any queued sends are failed - immediately so callers don't hang. The API stays up throughout. + Serial errors fail the active send and trigger a reconnect loop. Waiting + sends remain queued for recovery. The API stays up throughout. """ import logging @@ -34,6 +34,15 @@ logger = logging.getLogger("sapot.serial") RECONNECT_DELAY = 10 # seconds between reconnect attempts +MAX_SEND_QUEUE_SIZE = 20 + + +class OutboundQueueFullError(RuntimeError): + pass + + +class WorkerStoppingError(RuntimeError): + pass @dataclass @@ -43,8 +52,14 @@ class _SendRequest: body: str timeout: float done: threading.Event = field(default_factory=threading.Event) + completion_lock: threading.Lock = field(default_factory=threading.Lock) success: bool = False reason: Optional[str] = None + deadline: float = field(init=False) + write_started: bool = False + + def __post_init__(self): + self.deadline = time.monotonic() + max(0, self.timeout) class SerialWorker: @@ -59,7 +74,12 @@ class SerialWorker: Arduino confirms → reader thread resolves _SendRequest → caller unblocks """ - def __init__(self, port: str, baud: int = 9600): + def __init__(self, port: str, baud: int = 9600, + send_queue_maxsize: int = 10): + if not 1 <= send_queue_maxsize <= MAX_SEND_QUEUE_SIZE: + raise ValueError( + f"send_queue_maxsize must be between 1 and {MAX_SEND_QUEUE_SIZE}" + ) self._port = port self._baud = baud @@ -67,9 +87,15 @@ def __init__(self, port: str, baud: int = 9600): self._ser_lock = threading.Lock() # guards writes to _ser self._stop = threading.Event() + self._lifecycle_lock = threading.Lock() + self._accepting = True + self._stop_lock = threading.Lock() + + self._send_queue: queue.Queue[_SendRequest] = queue.Queue( + maxsize=send_queue_maxsize) - # Outbound queue: send_sms() puts requests here; sender thread consumes - self._send_queue: queue.Queue[_SendRequest] = queue.Queue() + self._active_request: Optional[_SendRequest] = None + self._active_lock = threading.Lock() # The one request currently being sent (set by sender, read by reader) self._in_flight: Optional[_SendRequest] = None @@ -95,11 +121,18 @@ def start(self): self._sender_thread.start() def stop(self): - self._stop.set() - # Unblock sender thread if waiting on empty queue - self._send_queue.put(_SendRequest(number="", body="", timeout=0)) - self._reader_thread.join(timeout=5) - self._sender_thread.join(timeout=5) + with self._stop_lock: + if self._stop.is_set(): + return + + with self._lifecycle_lock: + self._accepting = False + self._stop.set() + + self._drain_queued_requests() + self._fail_active_request("SERVICE_STOPPING") + self._reader_thread.join(timeout=5) + self._sender_thread.join(timeout=5) # ── Public API ──────────────────────────────────────────────────────────── @@ -114,22 +147,46 @@ def send_sms(self, number: str, body: str, Returns {"ok": bool, "reason": str|None} Raises RuntimeError if modem not ready or port not connected. """ - if not self.connected: - raise RuntimeError("Serial port not connected") - if not self.gsm_ready: - raise RuntimeError("GSM modem not ready") - req = _SendRequest(number=number, body=body, timeout=timeout) - self._send_queue.put(req) + with self._lifecycle_lock: + if not self._accepting: + raise WorkerStoppingError("SMS service is stopping") + if not self.connected: + raise RuntimeError("Serial port not connected") + if not self.gsm_ready: + raise RuntimeError("GSM modem not ready") + try: + self._send_queue.put_nowait(req) + req.deadline = time.monotonic() + max(0, req.timeout) + except queue.Full as error: + logger.warning( + "Outbound SMS queue full (depth=%d capacity=%d)", + self._send_queue.qsize(), self.outbound_queue_capacity) + raise OutboundQueueFullError("Outbound SMS queue is full") from error logger.info("SMS enqueued to %s (queue depth %d)", number, self._send_queue.qsize()) - # Block until the sender + reader threads resolve this request - delivered = req.done.wait(timeout=timeout + 5) # +5 s buffer - if not delivered: - return {"ok": False, "reason": "CLIENT_TIMEOUT"} + while not req.done.is_set(): + remaining = max(0, req.deadline - time.monotonic()) + if req.done.wait(timeout=remaining): + break + if not self._timeout_request(req): + req.done.wait(timeout=0.01) return {"ok": req.success, "reason": req.reason} + @property + def outbound_queue_depth(self) -> int: + return self._send_queue.qsize() + + @property + def outbound_queue_capacity(self) -> int: + return self._send_queue.maxsize + + @property + def outbound_in_flight(self) -> bool: + with self._in_flight_lock: + return self._in_flight is not None + # ── Sender thread: one at a time, in order ──────────────────────────────── def _sender_loop(self): @@ -144,55 +201,38 @@ def _sender_loop(self): except queue.Empty: continue - if self._stop.is_set() or not req.number: - break # sentinel or stop + with self._lifecycle_lock: + if not self._accepting: + self._complete_request(req, False, "SERVICE_STOPPING") + continue + with self._active_lock: + self._active_request = req - # Wait until modem is ready (e.g. after reconnect) - deadline = time.time() + req.timeout - while not self.gsm_ready and time.time() < deadline: - time.sleep(0.5) + while (not self.gsm_ready and not req.done.is_set() + and time.monotonic() < req.deadline): + remaining = req.deadline - time.monotonic() + req.done.wait(timeout=min(0.5, max(0, remaining))) - if not self.gsm_ready: - req.success = False - req.reason = "MODEM_NOT_READY" - req.done.set() - logger.warning("SMS to %s dropped — modem not ready", req.number) + if req.done.is_set(): + self._clear_active_request(req) continue - # Register as in-flight BEFORE writing to serial, - # so the reader never misses the SMS_SENT event - with self._in_flight_lock: - self._in_flight = req + if not self.gsm_ready: + self._complete_active_request(req, False, "TIMEOUT") + logger.warning("SMS to %s dropped: modem not ready", req.number) + continue cmd = build_send_sms(req.number, req.body) - try: - with self._ser_lock: - if self._ser and self._ser.is_open: - self._ser.write(cmd.encode("utf-8")) - logger.info("SMS sent to serial: to=%s body=%r", - req.number, req.body) - else: - raise OSError("Serial port not open") - except Exception as e: - logger.error("Serial write failed: %s", e) - with self._in_flight_lock: - self._in_flight = None - req.success = False - req.reason = f"WRITE_ERROR: {e}" - req.done.set() + if not self._write_active_request(req, cmd): + self._clear_active_request(req) continue - # Block here until the reader resolves this request - # (or until the per-SMS timeout expires) - resolved = req.done.wait(timeout=req.timeout) + remaining = max(0, req.deadline - time.monotonic()) + resolved = req.done.wait(timeout=remaining) if not resolved: - with self._in_flight_lock: - if self._in_flight is req: - self._in_flight = None - req.success = False - req.reason = "TIMEOUT" - req.done.set() - logger.error("SMS to %s timed out", req.number) + if self._complete_in_flight(req, False, "TIMEOUT"): + logger.error("SMS to %s timed out", req.number) + self._clear_active_request(req) # ── Reader thread: serial → events ──────────────────────────────────────── @@ -209,18 +249,19 @@ def _reader_loop(self): self.connected = False self.gsm_ready = False - self.last_status = f"disconnected — retrying in {RECONNECT_DELAY}s" + self.last_status = f"disconnected; retrying in {RECONNECT_DELAY}s" logger.warning("Reconnecting in %ds…", RECONNECT_DELAY) # Fail any in-flight request so the sender doesn't hang - self._fail_in_flight("SERIAL_DISCONNECTED") + self._fail_active_request("SERIAL_DISCONNECTED") time.sleep(RECONNECT_DELAY) def _connect_and_read(self): logger.info("Opening %s @ %d baud", self._port, self._baud) try: - ser = serial.Serial(self._port, self._baud, timeout=1) + ser = serial.Serial(self._port, self._baud, timeout=1, + write_timeout=5.0) except serial.SerialException as e: logger.error("Cannot open %s: %s", self._port, e) self.last_status = f"port error: {e}" @@ -282,15 +323,15 @@ def _handle_line(self, line: str): if etype == EventType.NETWORK_LOST: self.gsm_ready = False self.last_status = "network lost" - logger.warning("Network LOST — in-flight SMS will fail") - self._fail_in_flight("NETWORK_LOST") + logger.warning("Network LOST; in-flight SMS will fail") + self._fail_active_request("NETWORK_LOST") return if etype == EventType.SIM_MISSING: self.gsm_ready = False self.last_status = "SIM missing" logger.error("SIM missing") - self._fail_in_flight("SIM_MISSING") + self._fail_active_request("SIM_MISSING") return if etype == EventType.SMS_SENT: @@ -326,20 +367,99 @@ def _resolve_in_flight(self, number: str, success: bool, logger.warning( "SMS_SENT/FAILED number mismatch: expected %s got %s", req.number, number) - # Still resolve it — the Arduino only handles one at a time - self._in_flight = None - - req.success = success - req.reason = reason - req.done.set() + return + self._complete_in_flight(req, success, reason) + + def _write_active_request(self, req: _SendRequest, cmd: str) -> bool: + # Active state serializes failure with the transition to in-flight. + with self._active_lock: + if self._active_request is not req or req.done.is_set(): + return False + if time.monotonic() >= req.deadline: + self._complete_request(req, False, "TIMEOUT") + return False + with self._in_flight_lock: + self._in_flight = req + req.write_started = True + try: + with self._ser_lock: + if not self._ser or not self._ser.is_open: + raise OSError("Serial port not open") + self._ser.write(cmd.encode("utf-8")) + except Exception as error: + logger.error("Serial write failed: %s", error) + self._in_flight = None + self._complete_request(req, False, f"WRITE_ERROR: {error}") + return False + req.deadline = time.monotonic() + max(0, req.timeout) + + logger.info("SMS sent to serial: to=%s body=%r", req.number, req.body) + return True + + def _timeout_request(self, req: _SendRequest) -> bool: + with self._active_lock: + if req.done.is_set(): + return True + if req.write_started: + return False + with self._in_flight_lock: + if self._in_flight is req: + self._in_flight = None + self._complete_request(req, False, "CLIENT_TIMEOUT") + return True - def _fail_in_flight(self, reason: str): - with self._in_flight_lock: - req = self._in_flight + def _fail_active_request(self, reason: str): + with self._active_lock: + req = self._active_request if req is None: return + if not req.done.is_set(): + if not self._complete_in_flight(req, False, reason): + self._complete_request(req, False, reason) + logger.warning("Active SMS to %s failed: %s", req.number, reason) + self._active_request = None + + def _complete_active_request(self, req: _SendRequest, success: bool, + reason: Optional[str]) -> bool: + with self._active_lock: + if self._active_request is not req or req.done.is_set(): + return False + if not self._complete_in_flight(req, success, reason): + self._complete_request(req, success, reason) + self._active_request = None + return True + + def _clear_active_request(self, req: _SendRequest): + with self._active_lock: + if self._active_request is req: + self._active_request = None + + def _complete_in_flight(self, req: _SendRequest, success: bool, + reason: Optional[str]) -> bool: + with self._in_flight_lock: + if self._in_flight is not req: + return False self._in_flight = None - req.success = False - req.reason = reason - req.done.set() - logger.warning("In-flight SMS to %s failed: %s", req.number, reason) + req.success = success + req.reason = reason + req.done.set() + return True + + @staticmethod + def _complete_request(req: _SendRequest, success: bool, + reason: Optional[str]) -> bool: + with req.completion_lock: + if req.done.is_set(): + return False + req.success = success + req.reason = reason + req.done.set() + return True + + def _drain_queued_requests(self): + while True: + try: + req = self._send_queue.get_nowait() + except queue.Empty: + return + self._complete_request(req, False, "SERVICE_STOPPING") diff --git a/GSM-module/GSM-fastapi/sms_handler.py b/GSM-module/GSM-fastapi/sms_handler.py index b439ddce..a16473e1 100644 --- a/GSM-module/GSM-fastapi/sms_handler.py +++ b/GSM-module/GSM-fastapi/sms_handler.py @@ -3,7 +3,7 @@ ────────────── Business logic for incoming SMS messages. -Returns (reply, forward_number, forward_body) — all strings or None. +Returns (reply, forward_number, forward_body, rejection_reason). The caller (api.py _process_incoming) sends them via the serial worker. Message length rule: every constant must stay under 160 chars @@ -68,8 +68,8 @@ def _forward_body(sender_phone: str, body: str) -> str: # ── Types ───────────────────────────────────────────────────────────────────── -ForwardTuple = Tuple[Optional[str], Optional[str], Optional[str]] -# (reply_to_sender, forward_to_number, forward_body) +ForwardTuple = Tuple[Optional[str], Optional[str], Optional[str], Optional[str]] +# (reply_to_sender, forward_to_number, forward_body, rejection_reason) # ── Main entry point ────────────────────────────────────────────────────────── @@ -81,18 +81,28 @@ def handle_incoming_sms(number: str, body: str) -> ForwardTuple: if not sender_user: logger.warning("Account does not exist: %s", number) - return MSG_NO_ACCOUNT, None, None + return MSG_NO_ACCOUNT, None, None, "NO_ACCOUNT" if sender_user.get("banned"): logger.warning("Banned: %s", number) - return "This number has been banned by the system", None, None + return ( + "This number has been banned by the system", + None, + None, + "BANNED_SENDER", + ) if not sender_user.get("phone_is_verified"): logger.warning("Unverified number: %s", number) - return "Please verify your account first.", None, None + return ( + "Please verify your account first.", + None, + None, + "UNVERIFIED_SENDER", + ) sender = database.lookup_number(number) if sender is None: - return MSG_NO_ACCOUNT, None, None + return MSG_NO_ACCOUNT, None, None, "NO_ACCOUNT" session = database.get_session(number) stage = session["stage"] @@ -103,16 +113,16 @@ def handle_incoming_sms(number: str, body: str) -> ForwardTuple: if stage == "NEW": database.update_session(number, stage="AWAITING_TARGET") - return MSG_WELCOME, None, None + return MSG_WELCOME, None, None, None if stage == "AWAITING_TARGET": - return MSG_NEED_TARGET, None, None + return MSG_NEED_TARGET, None, None, None if stage == "ACTIVE": return _do_forward(number, body, session) database.reset_session(number) - return MSG_WELCOME, None, None + return MSG_WELCOME, None, None, None # ── Sub-handlers ────────────────────────────────────────────────────────────── @@ -121,20 +131,20 @@ def _cmd_set_target(number: str, body: str) -> ForwardTuple: # body is like "[target] +639281234567" or "[target]" (no arg) parts = body.split(None, 1) if len(parts) < 2 or not parts[1].strip(): - return MSG_NO_ARG, None, None + return MSG_NO_ARG, None, None, None target_phone = parts[1].strip() if not target_phone.startswith("+") or not target_phone[1:].isdigit(): - return MSG_INVALID_FMT, None, None + return MSG_INVALID_FMT, None, None, None # Sender cannot target themselves if target_phone == number: - return "You cannot set yourself as the target.", None, None + return "You cannot set yourself as the target.", None, None, None target = database.lookup_number(target_phone) if target is None: - return MSG_TARGET_NOT_FOUND, None, None + return MSG_TARGET_NOT_FOUND, None, None, None database.update_session( number, @@ -143,7 +153,7 @@ def _cmd_set_target(number: str, body: str) -> ForwardTuple: target_username=target["username"], ) - return _msg_target_set(target["username"], target_phone), None, None + return _msg_target_set(target["username"], target_phone), None, None, None def _do_forward(sender_phone: str, body: str, session: dict) -> ForwardTuple: @@ -154,18 +164,23 @@ def _do_forward(sender_phone: str, body: str, session: dict) -> ForwardTuple: if not target_user: logger.warning("Target does not exist: %s", target_phone) - return f"Target {target_phone} does not exist.", sender_phone, None + return f"Target {target_phone} does not exist.", sender_phone, None, None if target_user.get("banned"): logger.warning("Banned: %s", target_phone) - return f"This number ({target_phone}) has been banned by the system.", None, None + return ( + f"This number ({target_phone}) has been banned by the system.", + None, + None, + None, + ) if not target_user.get("phone_is_verified"): logger.warning("Unverified number: %s", target_phone) - return f"Target {target_phone} is not verified.", None, None + return f"Target {target_phone} is not verified.", None, None, None if not target_phone: database.reset_session(sender_phone) - return MSG_WELCOME, None, None + return MSG_WELCOME, None, None, None ok = database.notify_app(sender_phone, target_phone, body) logger.info("notify_app result: %s (sender=%s target=%s)", ok, sender_phone, target_phone) @@ -173,4 +188,4 @@ def _do_forward(sender_phone: str, body: str, session: dict) -> ForwardTuple: # Build clean forward body for the target's SMS fwd = _forward_body(sender_phone, body) - return _msg_forwarded(target_username), target_phone, fwd + return _msg_forwarded(target_username), target_phone, fwd, None diff --git a/GSM-module/GSM-fastapi/tests/conftest.py b/GSM-module/GSM-fastapi/tests/conftest.py new file mode 100644 index 00000000..e2e8a64b --- /dev/null +++ b/GSM-module/GSM-fastapi/tests/conftest.py @@ -0,0 +1,7 @@ +import os +import sys +from pathlib import Path + +os.environ.setdefault("DB_PATH", "sqlite:///./test-gsm.db") +os.environ.setdefault("GSM_SECRET", "test-gsm-secret") +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) diff --git a/GSM-module/GSM-fastapi/tests/test_api_queue.py b/GSM-module/GSM-fastapi/tests/test_api_queue.py new file mode 100644 index 00000000..72804f74 --- /dev/null +++ b/GSM-module/GSM-fastapi/tests/test_api_queue.py @@ -0,0 +1,173 @@ +import asyncio +import threading + +from fastapi.testclient import TestClient +import httpx +import pytest + +import api +from config import settings +from serial_worker import ( + MAX_SEND_QUEUE_SIZE, + OutboundQueueFullError, + WorkerStoppingError, +) + +AUTH_HEADERS = {"X-GSM-Secret": settings.gsm_secret} + + +@pytest.fixture(autouse=True) +def reset_worker(): + previous = api._worker + yield + api._worker = previous + + +class RejectingWorker: + gsm_ready = True + connected = True + last_status = "ready" + outbound_queue_depth = 1 + outbound_queue_capacity = 1 + outbound_in_flight = True + + def __init__(self, error): + self.error = error + + def send_sms(self, *_args, **_kwargs): + raise self.error + + +def test_queue_full_returns_nested_503_and_updates_log(monkeypatch): + updates = [] + monkeypatch.setattr(api.database, "log_message", lambda **_kwargs: "message-id") + monkeypatch.setattr(api.database, "update_message_status", lambda *args: updates.append(args)) + api._worker = RejectingWorker(OutboundQueueFullError()) + + response = TestClient(api.app).post( + "/sms/send", + json={"number": "+639171234567", "body": "message"}, + headers=AUTH_HEADERS, + ) + + assert response.status_code == 503 + assert response.json()["detail"] == { + "message": "Outbound SMS queue is full", + "reason": "QUEUE_FULL", + "msg_id": "message-id", + } + assert updates == [("message-id", "failed", "QUEUE_FULL")] + + +def test_stopping_returns_nested_503_and_updates_log(monkeypatch): + updates = [] + monkeypatch.setattr(api.database, "log_message", lambda **_kwargs: "message-id") + monkeypatch.setattr(api.database, "update_message_status", lambda *args: updates.append(args)) + api._worker = RejectingWorker(WorkerStoppingError()) + + response = TestClient(api.app).post( + "/sms/send", + json={"number": "+639171234567", "body": "message"}, + headers=AUTH_HEADERS, + ) + + assert response.status_code == 503 + assert response.json()["detail"]["reason"] == "SERVICE_STOPPING" + assert updates == [("message-id", "failed", "SERVICE_STOPPING")] + + +def test_detailed_health_keeps_inbound_queue_depth_and_adds_outbound_fields(monkeypatch): + class Worker(RejectingWorker): + def __init__(self): + self.incoming_queue = __import__("queue").Queue() + self.incoming_queue.put(object()) + + api._worker = Worker() + monkeypatch.setattr(api.database, "get_messages", lambda **_kwargs: {"messages": [], "total": 0}) + + response = TestClient(api.app).get("/health/detailed") + + assert response.status_code == 200 + assert response.json()["queue_depth"] == 1 + assert response.json()["outbound_queue_depth"] == 1 + assert response.json()["outbound_queue_capacity"] == 1 + assert response.json()["outbound_in_flight"] is True + + +def test_maximum_capacity_still_rejects_and_serves_health(monkeypatch): + class SaturatingWorker: + gsm_ready = True + connected = True + last_status = "ready" + + def __init__(self, capacity): + self.capacity = capacity + self.admitted = 0 + self.lock = threading.Lock() + self.release = threading.Event() + + def send_sms(self, *_args, **_kwargs): + with self.lock: + if self.admitted >= self.capacity: + raise OutboundQueueFullError() + self.admitted += 1 + self.release.wait(timeout=5) + return {"ok": True, "reason": None} + + worker = SaturatingWorker(capacity=MAX_SEND_QUEUE_SIZE + 1) + api._worker = worker + monkeypatch.setattr(api.database, "log_message", lambda **_kwargs: "message-id") + monkeypatch.setattr(api.database, "update_message_status", lambda *_args: None) + + async def exercise_saturation(): + transport = httpx.ASGITransport(app=api.app) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + payload = {"number": "+639171234567", "body": "message"} + admitted = [asyncio.create_task(client.post( + "/sms/send", json=payload, headers=AUTH_HEADERS + )) + for _ in range(worker.capacity)] + + for _ in range(100): + with worker.lock: + if worker.admitted == worker.capacity: + break + await asyncio.sleep(0.01) + + rejected = await asyncio.wait_for( + client.post("/sms/send", json=payload, headers=AUTH_HEADERS), + timeout=1, + ) + health = await asyncio.wait_for(client.get("/health"), timeout=1) + worker.release.set() + completed = await asyncio.gather(*admitted) + + return rejected, health, completed + + rejected, health, completed = asyncio.run(exercise_saturation()) + + assert rejected.status_code == 503 + assert rejected.json()["detail"]["reason"] == "QUEUE_FULL" + assert health.status_code == 200 + assert all(response.status_code == 200 for response in completed) + + +@pytest.mark.parametrize("headers", [{}, {"X-GSM-Secret": "wrong-secret"}]) +def test_send_rejects_missing_or_invalid_secret_before_side_effects( + monkeypatch, headers +): + calls = [] + monkeypatch.setattr( + api.database, "log_message", lambda **_kwargs: calls.append("logged") + ) + api._worker = RejectingWorker(AssertionError("worker must not be called")) + + response = TestClient(api.app).post( + "/sms/send", + json={"number": "+639171234567", "body": "message"}, + headers=headers, + ) + + assert response.status_code == 401 + assert response.json() == {"detail": "Invalid GSM secret"} + assert calls == [] diff --git a/GSM-module/GSM-fastapi/tests/test_config.py b/GSM-module/GSM-fastapi/tests/test_config.py new file mode 100644 index 00000000..036d4639 --- /dev/null +++ b/GSM-module/GSM-fastapi/tests/test_config.py @@ -0,0 +1,48 @@ +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from config import bounded_integer_env + + +def test_bounded_integer_env_uses_default_when_missing(monkeypatch): + monkeypatch.delenv("TEST_QUEUE_SIZE", raising=False) + + assert bounded_integer_env("TEST_QUEUE_SIZE", 10, 20) == 10 + + +@pytest.mark.parametrize("value", ["0", "-1", "", "ten", "21"]) +def test_bounded_integer_env_rejects_invalid_values(monkeypatch, value): + monkeypatch.setenv("TEST_QUEUE_SIZE", value) + + with pytest.raises(RuntimeError, match="TEST_QUEUE_SIZE.*between 1 and 20"): + bounded_integer_env("TEST_QUEUE_SIZE", 10, 20) + + +@pytest.mark.parametrize("value", ["1", "4", "20"]) +def test_bounded_integer_env_accepts_value_in_range(monkeypatch, value): + monkeypatch.setenv("TEST_QUEUE_SIZE", value) + + assert bounded_integer_env("TEST_QUEUE_SIZE", 10, 20) == int(value) + + +def test_config_rejects_missing_gsm_secret(tmp_path): + env = os.environ.copy() + env["DB_PATH"] = "sqlite:///test.db" + env.pop("GSM_SECRET", None) + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[1]) + + result = subprocess.run( + [sys.executable, "-c", "import config"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode != 0 + assert "Environment variable 'GSM_SECRET' is not set." in result.stderr diff --git a/GSM-module/GSM-fastapi/tests/test_database_reconciliation.py b/GSM-module/GSM-fastapi/tests/test_database_reconciliation.py new file mode 100644 index 00000000..b884e81f --- /dev/null +++ b/GSM-module/GSM-fastapi/tests/test_database_reconciliation.py @@ -0,0 +1,30 @@ +import database + + +def test_fail_orphaned_pending_messages_marks_only_pending_rows(tmp_path): + database.init(f"sqlite:///{tmp_path / 'gsm.db'}") + pending_id = database.log_message( + direction="OUT", + from_number="API", + to_number="+639171234567", + body="pending message", + ) + received_id = database.log_message( + direction="IN", + from_number="+639171234568", + to_number="SERVER", + body="received message", + status="received", + ) + + assert database.fail_orphaned_pending_messages() == 1 + + messages = { + message["id"]: message + for message in database.get_messages(limit=10)["messages"] + } + assert messages[pending_id]["status"] == "failed" + assert messages[pending_id]["failure_reason"] == "SERVICE_CRASHED" + assert messages[received_id]["status"] == "received" + assert messages[received_id]["failure_reason"] is None + assert database.fail_orphaned_pending_messages() == 0 diff --git a/GSM-module/GSM-fastapi/tests/test_incoming_sms.py b/GSM-module/GSM-fastapi/tests/test_incoming_sms.py new file mode 100644 index 00000000..c9348813 --- /dev/null +++ b/GSM-module/GSM-fastapi/tests/test_incoming_sms.py @@ -0,0 +1,52 @@ +from types import SimpleNamespace + +import pytest + +import api +import sms_handler + + +@pytest.mark.parametrize( + ("sender", "expected_reason"), + [ + (None, "NO_ACCOUNT"), + ({"banned": True, "phone_is_verified": True}, "BANNED_SENDER"), + ({"banned": False, "phone_is_verified": False}, "UNVERIFIED_SENDER"), + ], +) +def test_handle_incoming_sms_reports_sender_rejection_reason( + monkeypatch, sender, expected_reason +): + monkeypatch.setattr( + sms_handler.database, "get_user_by_phone", lambda _number: sender + ) + + reply, forward_number, forward_body, rejection_reason = ( + sms_handler.handle_incoming_sms("+639171234567", "help") + ) + + assert reply + assert forward_number is None + assert forward_body is None + assert rejection_reason == expected_reason + + +def test_process_incoming_persists_rejection_reason(monkeypatch): + updates = [] + monkeypatch.setattr(api.database, "log_message", lambda **_kwargs: "message-id") + monkeypatch.setattr( + api.database, + "update_message_status", + lambda *args: updates.append(args), + ) + monkeypatch.setattr( + api, + "handle_incoming_sms", + lambda _number, _body: (None, None, None, "BANNED_SENDER"), + ) + + api._process_incoming( + SimpleNamespace(number="+639171234567", body="blocked message") + ) + + assert updates == [("message-id", "rejected", "BANNED_SENDER")] diff --git a/GSM-module/GSM-fastapi/tests/test_lifespan.py b/GSM-module/GSM-fastapi/tests/test_lifespan.py new file mode 100644 index 00000000..60b0e72d --- /dev/null +++ b/GSM-module/GSM-fastapi/tests/test_lifespan.py @@ -0,0 +1,40 @@ +import asyncio +import queue + +import api + + +def test_lifespan_reconciles_pending_messages_before_starting_worker(monkeypatch): + events = [] + + class FakeWorker: + def __init__(self, *_args): + events.append("worker_created") + self.incoming_queue = queue.Queue() + + def start(self): + events.append("worker_started") + + def stop(self): + events.append("worker_stopped") + + monkeypatch.setattr(api.database, "init", lambda _path: events.append("db_ready")) + monkeypatch.setattr( + api.database, + "fail_orphaned_pending_messages", + lambda: events.append("pending_reconciled") or 2, + ) + monkeypatch.setattr(api, "SerialWorker", FakeWorker) + + async def run_lifespan(): + async with api.lifespan(api.app): + assert events == [ + "db_ready", + "pending_reconciled", + "worker_created", + "worker_started", + ] + + asyncio.run(run_lifespan()) + + assert events[-1] == "worker_stopped" diff --git a/GSM-module/GSM-fastapi/tests/test_serial_worker.py b/GSM-module/GSM-fastapi/tests/test_serial_worker.py new file mode 100644 index 00000000..db82f464 --- /dev/null +++ b/GSM-module/GSM-fastapi/tests/test_serial_worker.py @@ -0,0 +1,301 @@ +import threading +import time + +import pytest + +from serial_worker import ( + OutboundQueueFullError, + SerialWorker, + WorkerStoppingError, + _SendRequest, +) + + +def ready_worker(capacity=1): + worker = SerialWorker("fake", send_queue_maxsize=capacity) + worker.connected = True + worker.gsm_ready = True + return worker + + +def test_capacity_rejects_without_serial_write(): + worker = ready_worker() + worker._send_queue.put(_SendRequest("+639171234567", "first", 1)) + + with pytest.raises(OutboundQueueFullError): + worker.send_sms("+639171234568", "second", timeout=0) + + assert worker.outbound_queue_depth == 1 + assert worker.outbound_in_flight is False + + +def test_capacity_excludes_registered_in_flight_request(): + worker = ready_worker() + in_flight = _SendRequest("+639171234567", "first", 1) + with worker._in_flight_lock: + worker._in_flight = in_flight + + worker._send_queue.put_nowait(_SendRequest("+639171234568", "second", 1)) + + assert worker.outbound_in_flight is True + assert worker.outbound_queue_depth == 1 + assert worker.outbound_queue_capacity == 1 + + +def test_stop_drains_waiting_requests_without_sentinel(): + worker = ready_worker() + request = _SendRequest("+639171234567", "queued", 1) + worker._send_queue.put_nowait(request) + + class JoinedThread: + def join(self, timeout): + assert timeout == 5 + + worker._reader_thread = JoinedThread() + worker._sender_thread = JoinedThread() + + worker.stop() + + assert request.done.is_set() + assert request.reason == "SERVICE_STOPPING" + assert worker.outbound_queue_depth == 0 + + +def test_stop_fails_active_request(): + worker = ready_worker() + request = _SendRequest("+639171234567", "active", 1) + with worker._active_lock: + worker._active_request = request + + class JoinedThread: + def join(self, timeout): + assert timeout == 5 + + worker._reader_thread = JoinedThread() + worker._sender_thread = JoinedThread() + + worker.stop() + + assert request.done.is_set() + assert request.reason == "SERVICE_STOPPING" + assert worker.outbound_in_flight is False + + +def test_admission_rejects_after_shutdown_cutoff(): + worker = ready_worker() + with worker._lifecycle_lock: + worker._accepting = False + + with pytest.raises(WorkerStoppingError): + worker.send_sms("+639171234567", "message", timeout=0) + + +def test_in_flight_completion_is_exact_once(): + worker = ready_worker() + request = _SendRequest("+639171234567", "message", 1) + with worker._in_flight_lock: + worker._in_flight = request + + assert worker._complete_in_flight(request, True, None) is True + assert worker._complete_in_flight(request, False, "TIMEOUT") is False + assert request.success is True + assert request.reason is None + + +def test_failure_before_write_prevents_late_serial_send(): + writes = [] + + class FakeSerial: + is_open = True + + def write(self, payload): + writes.append(payload) + + worker = ready_worker() + worker.gsm_ready = False + worker._ser = FakeSerial() + request = _SendRequest("+639171234567", "message", 1) + worker._send_queue.put_nowait(request) + sender = threading.Thread(target=worker._sender_loop) + sender.start() + + for _ in range(100): + with worker._active_lock: + if worker._active_request is request: + break + time.sleep(0.01) + + worker._fail_active_request("NETWORK_LOST") + worker.gsm_ready = True + time.sleep(0.1) + worker._stop.set() + sender.join(timeout=2) + + assert request.done.is_set() + assert request.reason == "NETWORK_LOST" + assert writes == [] + + +def test_queued_request_timed_out_by_caller_is_never_written(): + writes = [] + + class FakeSerial: + is_open = True + + def write(self, payload): + writes.append(payload) + + worker = ready_worker(capacity=2) + worker._ser = FakeSerial() + first = _SendRequest("+639171234567", "first", 10) + worker._send_queue.put_nowait(first) + sender = threading.Thread(target=worker._sender_loop) + sender.start() + + for _ in range(100): + if writes: + break + time.sleep(0.01) + + result = worker.send_sms("+639171234568", "second", timeout=0.01) + worker._complete_in_flight(first, True, None) + + for _ in range(100): + if worker.outbound_queue_depth == 0: + break + time.sleep(0.01) + worker._stop.set() + sender.join(timeout=2) + + assert result == {"ok": False, "reason": "CLIENT_TIMEOUT"} + assert writes == [b"SEND_SMS|+639171234567|first\n"] + assert sender.is_alive() is False + + +def test_stop_does_not_overwrite_completed_queue_timeout(): + worker = ready_worker() + + result = worker.send_sms("+639171234568", "message", timeout=0.01) + request = worker._send_queue.get_nowait() + worker._send_queue.put_nowait(request) + + class JoinedThread: + def join(self, timeout): + assert timeout == 5 + + worker._reader_thread = JoinedThread() + worker._sender_thread = JoinedThread() + worker.stop() + + assert result == {"ok": False, "reason": "CLIENT_TIMEOUT"} + assert request.reason == "CLIENT_TIMEOUT" + + +def test_write_crossing_admission_deadline_waits_for_confirmation(): + writes = [] + write_started = threading.Event() + release_write = threading.Event() + + class BlockingSerial: + is_open = True + + def write(self, payload): + write_started.set() + release_write.wait(timeout=1) + writes.append(payload) + + worker = ready_worker() + worker._ser = BlockingSerial() + sender = threading.Thread(target=worker._sender_loop) + sender.start() + result = {} + + def send(): + result.update( + worker.send_sms("+639171234568", "message", timeout=0.05) + ) + + caller = threading.Thread(target=send) + caller.start() + assert write_started.wait(timeout=1) + time.sleep(0.06) + release_write.set() + worker._resolve_in_flight("+639171234568", True, None) + caller.join(timeout=1) + worker._stop.set() + sender.join(timeout=2) + + assert result == {"ok": True, "reason": None} + assert writes == [b"SEND_SMS|+639171234568|message\n"] + assert caller.is_alive() is False + assert sender.is_alive() is False + + +def test_stale_confirmation_cannot_complete_request_before_write(): + writes = [] + + class FakeSerial: + is_open = True + + def write(self, payload): + writes.append(payload) + + worker = ready_worker() + worker.gsm_ready = False + worker._ser = FakeSerial() + request = _SendRequest("+639171234568", "message", 1) + worker._send_queue.put_nowait(request) + sender = threading.Thread(target=worker._sender_loop) + sender.start() + + for _ in range(100): + with worker._active_lock: + if worker._active_request is request: + break + time.sleep(0.01) + + worker._resolve_in_flight("+639171234567", True, None) + assert request.done.is_set() is False + + worker.gsm_ready = True + for _ in range(100): + if writes: + break + time.sleep(0.01) + worker._resolve_in_flight(request.number, True, None) + worker._stop.set() + sender.join(timeout=2) + + assert writes == [b"SEND_SMS|+639171234568|message\n"] + assert request.success is True + assert sender.is_alive() is False + + +@pytest.mark.parametrize("capacity", [0, 21]) +def test_constructor_rejects_capacity_outside_threadpool_safe_range(capacity): + with pytest.raises(ValueError, match="between 1 and 20"): + SerialWorker("fake", send_queue_maxsize=capacity) + + +def test_serial_connection_uses_finite_write_timeout(monkeypatch): + captured = {} + + class FakeSerial: + is_open = True + in_waiting = 0 + + def __init__(self, *args, **kwargs): + captured.update(kwargs) + + def read(self, _): + worker._stop.set() + return b"" + + def close(self): + pass + + worker = SerialWorker("fake") + monkeypatch.setattr("serial_worker.serial.Serial", FakeSerial) + worker._connect_and_read() + + assert captured["write_timeout"] == 5.0 diff --git a/SECURITY.md b/SECURITY.md index daf74d31..27b6aeaf 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -9,9 +9,11 @@ This is the canonical source of truth for SAPOT's known security-relevant config | Issue | Location | Fix | |---|---|---| | Hardcoded MariaDB credentials | `server/app/db_operations/auth.py` (`SQLALCHEMY_DATABASE_URL`) | Now required via `DATABASE_URL` env var; the app raises `RuntimeError` at import time if unset. **Rotate the previously-hardcoded DB password before deploying this fix.** | +| Hardcoded GSM MariaDB credentials | `GSM-module/GSM-fastapi/config.py` (`db_path`) | Now required via `DB_PATH`; the GSM service raises `RuntimeError` at import time if unset. **Rotate the previously-hardcoded DB password before deploying this fix.** | | Hardcoded JWT secret fallback | `server/app/db_operations/token.py` (`SECRET_KEY`) | The default value has been removed; `JWT_SECRET_KEY` is now required, and the app raises `RuntimeError` at import time if unset. **Rotate to a newly generated secret** (`openssl rand -hex 32`) — the old hardcoded value must be considered compromised since it was committed to source. | | CORS wildcard + credentials | `server/app/main.py` | `allow_origins=["*"]` replaced with an explicit allowlist read from `CORS_ALLOWED_ORIGINS` (comma-separated). The app raises `RuntimeError` at import time if unset. | | Testing router in production | `server/app/main.py`, `server/app/api/testing.py` | The router is imported and mounted only in `development` or `staging`. A router-wide dependency also returns 404 outside those environments if the router is mis-mounted. Every state-changing route requires the `X-QA-Token` shared secret. A production-process regression test exercises every testing path. | +| Unauthenticated direct GSM sends | `GSM-module/GSM-fastapi/api.py`, `server/app/api/gsm.py` | `GSM_SECRET` is required by both services. The main server sends it as `X-GSM-Secret`, and the gateway validates it before logging or queueing `POST /sms/send`. | ## Required environment variables (new) @@ -24,6 +26,12 @@ CORS_ALLOWED_ORIGINS=http://192.168.0.100:3000 GSM_SECRET= ``` +Set this in `/etc/sapot/gsm.env` before starting the GSM service: + +```dotenv +DB_PATH=mysql+pymysql://:@127.0.0.1:3306/sapot_db +GSM_SECRET= +``` QA-enabled environments also require `QA_API_TOKEN`. Generate a strong random value and send it as `X-QA-Token` for state-changing `/testing/*` requests. Production does not load this secret. @@ -35,7 +43,6 @@ This is a LAN-deployed application without a public bug bounty program. Report s | Gap | Location | Risk | |---|---|---| -| GSM module DB credentials | Hardcoded default in `GSM-module/GSM-fastapi/config.py` (`db_path`) | Same class of risk as the server's DB URL; not yet env-var-only. | | Optional (not enforced) server-side `PeerKey` signing | `SERVER_ED25519_SEED` env var | If unset, a compromised server can MITM new conversations by substituting public keys. See [docs/architecture/threat-model.md](docs/architecture/threat-model.md#e2e-encryption-design-risks). | | No remote session/device revocation UI | Mobile app | A stolen, already-unlocked device has an unbounded access window until an admin manually suspends the account. See [docs/architecture/threat-model.md](docs/architecture/threat-model.md#device-theft). | diff --git a/deploy/config/gsm-fastapi.env.example b/deploy/config/gsm-fastapi.env.example index 38586915..42aecf81 100644 --- a/deploy/config/gsm-fastapi.env.example +++ b/deploy/config/gsm-fastapi.env.example @@ -4,6 +4,9 @@ DB_PATH=mysql+pymysql://sapot:__FROM_SERVER_MYSQL_PASSWORD__@db:3306/sapot HOST=0.0.0.0 PORT=8001 LOG_LEVEL=INFO +# Maximum outbound SMS requests waiting behind the one in-flight request. +# Must be an integer from 1 through 20. +SMS_SEND_QUEUE_MAXSIZE=10 SAPOT_API_URL=https://nginx GSM_SECRET=__FROM_SERVER_GSM_SECRET__ SMS_BOT_USER_ID= diff --git a/deploy/config/nginx.prod.conf b/deploy/config/nginx.prod.conf index 80a80688..68adf794 100644 --- a/deploy/config/nginx.prod.conf +++ b/deploy/config/nginx.prod.conf @@ -11,6 +11,6 @@ server { location ~ ^/(data|fonts|sprites)/ { proxy_pass http://tileserver:8080; proxy_set_header Host $host; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } location /admin { proxy_pass http://admin:3000; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; } location /ws/ { proxy_pass http://api:8000; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; proxy_read_timeout 86400s; proxy_send_timeout 86400s; } - location / { proxy_set_header Authorization $http_authorization; proxy_pass_header Authorization; proxy_pass http://api:8000; proxy_set_header Host $host; proxy_read_timeout 135s; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_redirect http:// https://; gzip on; gzip_types application/json text/plain; gzip_min_length 256; } + location / { proxy_set_header Authorization $http_authorization; proxy_pass_header Authorization; proxy_pass http://api:8000; proxy_set_header Host $host; proxy_read_timeout 155s; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_redirect http:// https://; gzip on; gzip_types application/json text/plain; gzip_min_length 256; } } server { listen 80; listen [::]:80; return 301 https://$host$request_uri; } diff --git a/deployment-scripts/server-GSM-api.service b/deployment-scripts/server-GSM-api.service index b445e8ef..72b17198 100644 --- a/deployment-scripts/server-GSM-api.service +++ b/deployment-scripts/server-GSM-api.service @@ -7,6 +7,7 @@ User=sapot Group=sapot WorkingDirectory=/home/sapot/YLP-software/GSM-module/GSM-fastapi ExecStart=/home/sapot/YLP-software/GSM-module/GSM-fastapi/run-api.sh +EnvironmentFile=/etc/sapot/gsm.env Restart=always # Automatic restart logic diff --git a/docker-compose.yml b/docker-compose.yml index 0eaab2eb..b4142ae1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -136,7 +136,7 @@ services: db: condition: service_healthy ports: - - "${GSM_FASTAPI_PORT:-8001}:8001" + - "127.0.0.1:${GSM_FASTAPI_PORT:-8001}:8001" volumes: db-data: diff --git a/docker/nginx.docker.conf b/docker/nginx.docker.conf index 73fd9111..20cdb345 100644 --- a/docker/nginx.docker.conf +++ b/docker/nginx.docker.conf @@ -102,7 +102,7 @@ server { proxy_pass http://api:8000; proxy_set_header Host $host; - proxy_read_timeout 135s; + proxy_read_timeout 155s; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; # ← tells backend it's HTTPS diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 80b46cc3..d0d740df 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -72,9 +72,9 @@ If this fails, check MariaDB is running (`sudo systemctl status mariadb`) and th --- -## GSM module and server can't authenticate each other +## Server rejects GSM inbound callbacks -**Symptom:** SMS send/receive fails; server logs show a rejected `X-GSM-Secret` header, or the GSM module logs show the reverse. +**Symptom:** Inbound SMS forwarding fails and the server logs show a rejected `X-GSM-Secret` header. **Cause:** `GSM_SECRET` differs between the two components' env files. diff --git a/docs/api/gsm-sms.md b/docs/api/gsm-sms.md index ed0553d5..3d7d942d 100644 --- a/docs/api/gsm-sms.md +++ b/docs/api/gsm-sms.md @@ -41,6 +41,22 @@ Webhook endpoint the GSM hardware gateway calls when it receives an SMS. Protect --- -The GSM module's own standalone hardware-facing API (separate service) is documented in [`docs/deployment/gsm-module.md`](../deployment/gsm-module.md). +The GSM module's own standalone hardware-facing API is documented in [`docs/deployment/gsm-module.md`](../deployment/gsm-module.md). Its `POST /sms/send` route requires the same `X-GSM-Secret` shared secret that protects the main server's inbound webhook. + +## Gateway failure contract + +The main server preserves synchronous gateway failures for `/gsm/sms/send`, `/gsm/request`, `/gsm/resend`, and `/gsm/contact-unknown-user`. Queue saturation returns HTTP 503: + +```json +{ + "detail": { + "message": "Outbound SMS queue is full", + "reason": "QUEUE_FULL", + "msg_id": "" + } +} +``` + +An unreachable gateway also returns HTTP 503 with `reason: "GATEWAY_UNAVAILABLE"`. Clients should keep rejected messages available for manual retry and should not report verification or onboarding SMS as sent. See [gsm-sms.yaml](openapi/gsm-sms.yaml) for exact field-level request/response schemas, or the live server's `/docs` / `/openapi.json`. diff --git a/docs/api/openapi/gsm-sms.yaml b/docs/api/openapi/gsm-sms.yaml index 4e7dee7e..9656fe9b 100644 --- a/docs/api/openapi/gsm-sms.yaml +++ b/docs/api/openapi/gsm-sms.yaml @@ -31,6 +31,21 @@ paths: schema: {} '404': description: Not Found + '502': + description: The modem rejected the SMS or delivery confirmation timed out. + content: + application/json: + schema: + $ref: '#/components/schemas/GsmFailureResponse' + '503': + description: The GSM gateway is unavailable, stopping, or at queue capacity. + content: + application/json: + schema: + anyOf: + - $ref: '#/components/schemas/GsmFailureResponse' + - $ref: '#/components/schemas/GsmHealthUnavailableResponse' + title: Response 503 Contact Unknown User Gsm Contact Unknown User Post '422': description: Validation Error content: @@ -51,6 +66,15 @@ paths: schema: {} '404': description: Not Found + '503': + description: The GSM gateway is unavailable or reports degraded health. + content: + application/json: + schema: + anyOf: + - $ref: '#/components/schemas/GsmHealthResponse' + - $ref: '#/components/schemas/GsmHealthUnavailableResponse' + title: Response 503 Gsm Health Gsm Health Get security: - OAuth2PasswordBearer: [] /gsm/health/detailed: @@ -68,6 +92,15 @@ paths: schema: {} '404': description: Not Found + '503': + description: The GSM gateway is unavailable or reports degraded health. + content: + application/json: + schema: + anyOf: + - $ref: '#/components/schemas/GsmHealthResponse' + - $ref: '#/components/schemas/GsmHealthUnavailableResponse' + title: Response 503 Gsm Health Detailed Gsm Health Detailed Get security: - OAuth2PasswordBearer: [] /gsm/inbound: @@ -405,6 +438,21 @@ paths: schema: {} '404': description: Not Found + '502': + description: The modem rejected the SMS or delivery confirmation timed out. + content: + application/json: + schema: + $ref: '#/components/schemas/GsmFailureResponse' + '503': + description: The GSM gateway is unavailable, stopping, or at queue capacity. + content: + application/json: + schema: + anyOf: + - $ref: '#/components/schemas/GsmFailureResponse' + - $ref: '#/components/schemas/GsmHealthUnavailableResponse' + title: Response 503 Request Phone Verification Gsm Request Post '422': description: Validation Error content: @@ -428,6 +476,21 @@ paths: schema: {} '404': description: Not Found + '502': + description: The modem rejected the SMS or delivery confirmation timed out. + content: + application/json: + schema: + $ref: '#/components/schemas/GsmFailureResponse' + '503': + description: The GSM gateway is unavailable, stopping, or at queue capacity. + content: + application/json: + schema: + anyOf: + - $ref: '#/components/schemas/GsmFailureResponse' + - $ref: '#/components/schemas/GsmHealthUnavailableResponse' + title: Response 503 Resend Phone Code Gsm Resend Post security: - OAuth2PasswordBearer: [] /gsm/sms/messages: @@ -514,6 +577,21 @@ paths: schema: {} '404': description: Not Found + '502': + description: The modem rejected the SMS or delivery confirmation timed out. + content: + application/json: + schema: + $ref: '#/components/schemas/GsmFailureResponse' + '503': + description: The GSM gateway is unavailable, stopping, or at queue capacity. + content: + application/json: + schema: + anyOf: + - $ref: '#/components/schemas/GsmFailureResponse' + - $ref: '#/components/schemas/GsmHealthUnavailableResponse' + title: Response 503 Send Sms Gsm Sms Send Post '422': description: Validation Error content: @@ -579,6 +657,62 @@ paths: - OAuth2PasswordBearer: [] components: schemas: + GsmFailureDetail: + properties: + message: + type: string + title: Message + reason: + type: string + title: Reason + msg_id: + anyOf: + - type: string + - type: 'null' + title: Msg Id + type: object + required: + - message + - reason + title: GsmFailureDetail + GsmFailureResponse: + properties: + detail: + $ref: '#/components/schemas/GsmFailureDetail' + type: object + required: + - detail + title: GsmFailureResponse + GsmHealthResponse: + properties: + status: + type: string + title: Status + gsm_ready: + type: boolean + title: Gsm Ready + connected: + type: boolean + title: Connected + detail: + type: string + title: Detail + type: object + required: + - status + - gsm_ready + - connected + - detail + title: GsmHealthResponse + GsmHealthUnavailableResponse: + properties: + detail: + type: string + title: Detail + type: object + required: + - detail + title: GsmHealthUnavailableResponse HTTPValidationError: properties: detail: diff --git a/docs/architecture/assumptions-and-constraints.md b/docs/architecture/assumptions-and-constraints.md index 45fdf0dc..e45201f4 100644 --- a/docs/architecture/assumptions-and-constraints.md +++ b/docs/architecture/assumptions-and-constraints.md @@ -59,7 +59,6 @@ Risks the project has consciously decided to carry rather than fix, reproduced f - No LAN segmentation — requires router-level VLAN config not currently documented or automated. - `testing` router reachable when `ENVIRONMENT=development` or `staging` is accepted for QA. Production uses conditional mounting, a route-level environment guard, shared-secret authentication on mutations, and regression coverage. -- GSM module DB credentials hardcoded default in `config.py` — open, tracked in [SECURITY.md](../../SECURITY.md#other-known-gaps-not-yet-resolved). - No remote session/device revocation UI — open. - Optional (not enforced) server-side `PeerKey` signing — open. diff --git a/docs/architecture/component-map.md b/docs/architecture/component-map.md index 9ce25e4a..b65500e5 100644 --- a/docs/architecture/component-map.md +++ b/docs/architecture/component-map.md @@ -86,7 +86,7 @@ For the security trust boundaries overlaid on this same topology (which zones ar | `/static/` | Filesystem | Served directly by Nginx; 30-day cache | | `/tiles/` | `http://127.0.0.1:8080` | Tileserver styles; prefix stripped by trailing `/` on `proxy_pass` | | `/data/`, `/fonts/`, `/sprites/` | `http://127.0.0.1:8080` | TileServer GL assets referenced by its absolute style URLs | -| `/` (all other) | `http://127.0.0.1:8000` | Standard proxy; 135 s read timeout | +| `/` (all other) | `http://127.0.0.1:8000` | Standard proxy; 155 s read timeout | HTTP (port 80) redirects to HTTPS with 301. diff --git a/docs/architecture/data-flow.md b/docs/architecture/data-flow.md index a68d9e97..d58884e8 100644 --- a/docs/architecture/data-flow.md +++ b/docs/architecture/data-flow.md @@ -112,8 +112,8 @@ sequenceDiagram participant C as Carrier network participant P as Recipient phone - A->>S: POST /gsm/send-sms (target user has no app presence) - S->>G: POST http://localhost:8001/sms/send (X-GSM-Secret header) + A->>S: POST /gsm/sms/send with JWT + S->>G: POST http://localhost:8001/sms/send G->>M: AT command: send SMS (serial_worker.py) M->>C: SMS PDU C->>P: SMS delivered @@ -122,11 +122,11 @@ sequenceDiagram P->>C: SMS reply C->>M: SMS PDU M->>G: serial_worker reads modem, sms_handler.py parses - G->>S: POST /gsm/inbound-sms (X-GSM-Secret header) + G->>S: POST /gsm/inbound with X-GSM-Secret S->>S: resolve sender/target user, create/append SMS conversation ``` -The server and GSM module authenticate each other with a shared `GSM_SECRET` header (`X-GSM-Secret`), not a user session token — see [environment-config.md](../deployment/environment-config.md). +The main server authenticates user-facing outbound requests with a JSON Web Token (JWT). Calls across the server and GSM service boundary use `X-GSM-Secret` in both directions: the main server sends it to `/sms/send`, and the GSM service sends it to `/gsm/inbound`. Network restriction to the host or trusted Compose network remains an additional boundary; see [environment-config.md](../deployment/environment-config.md). --- diff --git a/docs/architecture/threat-model.md b/docs/architecture/threat-model.md index ec46a341..6c4afd22 100644 --- a/docs/architecture/threat-model.md +++ b/docs/architecture/threat-model.md @@ -66,7 +66,7 @@ flowchart TB | mDNS/Zeroconf discovery | Broadcasts peer presence and connection info on the LAN; unauthenticated by design (mDNS has no auth mechanism) | | Captive portal | The first thing an unauthenticated device interacts with; controls initial network admission | | Admin frontend | Higher-privilege surface — user management, announcements, network config | -| GSM module webhook (`/gsm/inbound`) | Authenticated via shared secret (`GSM_SECRET`/`X-GSM-Secret`); reachable from the server, and from the GSM module's own network segment | +| GSM service HTTP boundary (`/sms/send`, `/gsm/inbound`) | Authenticated in both directions via shared secret (`GSM_SECRET`/`X-GSM-Secret`); reachable from the server and the GSM module's network segment | | MariaDB, Redis | Server-internal; in scope only via server compromise (not directly LAN-reachable in the documented deployment) | ## Attack surfaces explicitly out of scope @@ -119,7 +119,6 @@ flowchart TB |---|---| | No LAN segmentation (rescuer/admin/civilian devices share one broadcast domain) | Accepted for now — segmentation requires router-level VLAN config not currently documented or automated. | | `testing` router reachable when `ENVIRONMENT=development` or `staging` | Accepted for QA. Production is protected by conditional mounting, a route-level environment guard, shared-secret authentication on mutations, and a production-process regression test. | -| GSM module DB credentials hardcoded default in `config.py` | Open — tracked in [SECURITY.md](../../SECURITY.md#other-known-gaps-not-yet-resolved). | | No remote session/device revocation UI for end users | Open — see [Device theft](#device-theft). | | Optional (not enforced) server-side `PeerKey` signing | Open — see [E2E encryption design risks](#e2e-encryption-design-risks). Recommend making `SERVER_ED25519_SEED` mandatory in production as a follow-up. | diff --git a/docs/deployment/environment-config.md b/docs/deployment/environment-config.md index ac58a7b2..c1c826a6 100644 --- a/docs/deployment/environment-config.md +++ b/docs/deployment/environment-config.md @@ -45,7 +45,8 @@ GSM_SECRET= > **Note:** `GSM-module/` also contains a separate, undocumented `GSM-API/` directory with its own > app code and a committed `.env.example` (`SAPOT_API_URL`, `GSM_SECRET` only). It is not referenced -> by any doc, systemd unit, or setup guide in this repo — `GSM-fastapi/` is the deployed component +> by any doc, systemd unit, or setup guide in this repo. `GSM-fastapi/` is the current implementation +> and intended deployment target. > (see [gsm-module.md](gsm-module.md) and [gsm-module-setup.md](../getting-started/gsm-module-setup.md)). > Not resolved as part of this pass; flagged for a follow-up doc/architecture decision. @@ -53,13 +54,14 @@ GSM_SECRET= |---|---|---| | `SERIAL_PORT` | `/dev/ttyACM0` | USB serial device path | | `SERIAL_BAUD` | `9600` | Serial baud rate | -| `DB_PATH` | `mysql+pymysql://sapot:sapot@localhost:3306/sapot_db` (hardcoded default in `config.py`) | MariaDB connection string | +| `DB_PATH` | None; startup raises `RuntimeError` when unset | MariaDB connection string; required | | `HOST` | `127.0.0.1` | FastAPI bind host | | `PORT` | `8000` (code default in `config.py`), but **not actually read** — `GSM-fastapi/main.py` hardcodes `uvicorn.run(..., port=8001, ...)` regardless of this variable. The service always listens on `8001` in practice, which is what avoids colliding with the main SAPOT server on `127.0.0.1:8000` — not the `PORT` variable. | Not a real configuration knob today — see `GSM-module/CLAUDE.md`'s "Common Pitfalls" | | `LOG_LEVEL` | `INFO` | Python logging level (`config.py`) | | `SAPOT_API_URL` | `http://localhost:8000` | Base URL the GSM module uses to call back into the SAPOT server (`database.py`) — must match wherever the server actually listens | -| `GSM_SECRET` | `""` (empty — webhook auth disabled) | Shared secret sent as `X-GSM-Secret` on both directions of the server↔GSM webhook calls (`database.py`). **Must match the server's `GSM_SECRET`** (see above) | +| `GSM_SECRET` | None; startup raises `RuntimeError` when unset | Shared secret validated for server calls to `/sms/send` and sent by the GSM module on `/gsm/inbound` callbacks. **Must match the server's `GSM_SECRET`** (see above) | | `SMS_BOT_USER_ID` | unset | User ID the GSM module attributes inbound SMS-originated messages to, when the sender can't be resolved to a registered user (`database.py`) | +| `SMS_SEND_QUEUE_MAXSIZE` | `10` | Maximum waiting outbound requests. Integers from `1` through `20` are accepted; other values fail startup. The upper bound leaves capacity in FastAPI's default 40-thread worker pool so overload requests can reach the non-blocking admission check. | ### Recommended production `gsm.env` @@ -70,9 +72,10 @@ DB_PATH=mysql+pymysql://:@127.0.0.1:3306/sapot_db HOST=127.0.0.1 PORT=8001 # harmless to set, but has no real effect — main.py always binds 8001 LOG_LEVEL=INFO -SAPOT_API_URL=https:// +SAPOT_API_URL=http://127.0.0.1:8000 GSM_SECRET= SMS_BOT_USER_ID= +SMS_SEND_QUEUE_MAXSIZE=10 ``` --- diff --git a/docs/deployment/gsm-module.md b/docs/deployment/gsm-module.md index f0394c2e..49128ca5 100644 --- a/docs/deployment/gsm-module.md +++ b/docs/deployment/gsm-module.md @@ -1,6 +1,6 @@ # GSM Module Deployment -The GSM module (`GSM-module/GSM-fastapi/`) is a FastAPI application that bridges an Arduino-connected SIM800L/SIM900 GSM modem to the SAPOT server's SMS webhook. It exposes SMS send/receive endpoints and forwards inbound SMS to the main server. +The GSM module (`GSM-module/GSM-fastapi/`) is a FastAPI application that bridges an Arduino-connected SIM800L/SIM900 GSM modem to the SAPOT server's SMS webhook. It exposes send, message-history, and health endpoints, then forwards serially received SMS to the main server. --- @@ -16,10 +16,12 @@ The GSM module (`GSM-module/GSM-fastapi/`) is a FastAPI application that bridges ```bash cd GSM-module/GSM-fastapi/ -python3 -m venv venv -source venv/bin/activate +nix develop --command python -m venv venv +nix develop pip install -r requirements.txt -SERIAL_PORT=/dev/ttyACM0 python3 main.py +cp .env.example .env +# Edit .env and set DB_PATH, GSM_SECRET, SAPOT_API_URL, and the serial device. +python3 main.py ``` Or use the helper script: @@ -28,15 +30,11 @@ Or use the helper script: bash run-api.sh ``` -`main.py` starts FastAPI on `settings.host` (from `config.py`, default `127.0.0.1`) — **but the port is hardcoded to `8001`** in the `uvicorn.run(...)` call, not read from `settings.port`/`PORT`. Setting `PORT` has no effect on the bound port (it only affects the startup log line, which will report the wrong port — see `GSM-module/CLAUDE.md`'s "Common Pitfalls"). `HOST` is honored via `config.py`. +`main.py` starts FastAPI on `settings.host` (from `config.py`, default `127.0.0.1`), but the port is hardcoded to `8001` in the `uvicorn.run(...)` call. It does not read `settings.port` or `PORT`. Setting `PORT` only changes the startup log line. `HOST` is honored via `config.py`. ### Docker (dev/test alternative) -The root `docker-compose.yml` (see [docker-setup.md](../getting-started/docker-setup.md)) -includes a `gsm-fastapi` service alongside the rest of the stack. It passes through the host's -`/dev/ttyACM0` device, so it only starts successfully on a machine with the modem attached — set -`HOST=0.0.0.0` inside the container (already set in the compose service) so the published port is -actually reachable from outside the container. +The root `docker-compose.yml` (see [docker-setup.md](../getting-started/docker-setup.md)) includes a `gsm-fastapi` service alongside the rest of the stack. The base file does not pass through `/dev/ttyACM0`, which lets development stacks start without GSM hardware. Add `docker-compose.gsm-hardware.yml` when the modem is attached. The service listens on all interfaces inside its container, validates `X-GSM-Secret` for direct sends, and publishes port 8001 only on host loopback as an additional network boundary. --- @@ -45,46 +43,58 @@ actually reachable from outside the container. | Variable | Default | Purpose | |---|---|---| | `SERIAL_PORT` | `/dev/ttyACM0` | USB serial device for the Arduino/GSM modem | +| `SMS_SEND_QUEUE_MAXSIZE` | `10` | Maximum outbound requests waiting behind the one in-flight request. Accepts `1` through `20`. | | `SERIAL_BAUD` | `9600` | Serial baud rate | -| `DB_PATH` | `mysql+pymysql://sapot:sapot@localhost:3306/sapot_db` | Database connection (hardcoded default — override in production) | +| `DB_PATH` | None | Required database connection URL; startup fails when unset | | `HOST` | `127.0.0.1` | FastAPI bind host | -| `PORT` | `8000` in `config.py`, but **not actually used** — `main.py` hardcodes port `8001` regardless of this variable | Documented for completeness only; do not rely on it to change the bound port | +| `PORT` | `8000` in `config.py`, but not used for binding | `main.py` always binds port `8001`; do not rely on this setting | +| `SAPOT_API_URL` | `http://localhost:8000` | Base URL for authenticated inbound callbacks to the main server | +| `GSM_SECRET` | None | Required at startup; must match the main server value | -> **Security note:** `DB_PATH` has a hardcoded default with plaintext credentials. Always set it explicitly in production. See [secrets-management.md](secrets-management.md). +> **Security note:** Set `DB_PATH` and `GSM_SECRET` explicitly before startup. Never deploy the placeholder credentials from `.env.example`. See [secrets-management.md](secrets-management.md). --- ## Database -The module ships a pre-seeded SQLite development database at `GSM-module/GSM-fastapi/sapot.db`. Replace it with an empty database or configure `DB_PATH` to point to the production MariaDB instance before deploying. +The committed `GSM-module/GSM-fastapi/sapot.db` file is stale and unused. Set `DB_PATH` to the production MariaDB instance before deploying. --- ## Production systemd -Create `/etc/systemd/system/server-GSM-api.service`: - -```ini -[Unit] -Description=SAPOT GSM API -After=network.target - -[Service] -WorkingDirectory=/home/sapot/YLP-software/GSM-module/GSM-fastapi -ExecStart=/home/sapot/YLP-software/GSM-module/GSM-fastapi/venv/bin/python3 main.py -Restart=always -User=sapot -EnvironmentFile=/etc/sapot/gsm.env - -[Install] -WantedBy=multi-user.target -``` +The tracked unit loads `/etc/sapot/gsm.env` through `EnvironmentFile=`. Provision that restricted file from the committed example before installing the unit. The tracked unit is not installed automatically. ```bash +sudo install -d -m 0700 -o sapot -g sapot /etc/sapot +sudo install -m 0600 -o sapot -g sapot \ + GSM-module/GSM-fastapi/.env.example /etc/sapot/gsm.env +sudoedit /etc/sapot/gsm.env +sudo cp deployment-scripts/server-GSM-api.service /etc/systemd/system/server-GSM-api.service +sudo systemctl daemon-reload sudo systemctl enable server-GSM-api sudo systemctl start server-GSM-api ``` +This is a manual deployment step. Repository updates do not install the unit or refresh `/etc/sapot/gsm.env`; repeat the copy and restart the service when either artifact changes. + +## Outbound capacity and overload + +The intended deployment accepts 10 waiting outbound requests and one active serial request by default. Configure `SMS_SEND_QUEUE_MAXSIZE` from `1` through `20` before startup to change the waiting capacity. The upper bound keeps enough of FastAPI's default 40-thread worker pool available to reject overload. When the queue is full, `POST /sms/send` returns HTTP 503 with `QUEUE_FULL`; callers should use bounded backoff and must not retry in a tight loop. + +## Queue diagnostics + +| Field | Meaning | +|---|---| +| `outbound_queue_depth` | Accepted requests waiting for the sender | +| `outbound_queue_capacity` | Configured maximum waiting requests | +| `outbound_in_flight` | Whether a request is being written or awaiting modem confirmation | +| `queue_depth` | Existing inbound queue depth, not outbound capacity | + +## Shutdown behavior + +Shutdown closes admission before draining waiting work. Queued and active requests resolve with `SERVICE_STOPPING`, allowing blocked callers to return without extending the service manager's normal stop budget. + --- ## Serial port permissions diff --git a/docs/deployment/maintenance.md b/docs/deployment/maintenance.md index 5bd29ad7..e2c7e9e5 100644 --- a/docs/deployment/maintenance.md +++ b/docs/deployment/maintenance.md @@ -28,7 +28,7 @@ Each component owns its own dependency file — there is no repo-wide update mec | `server/` | `requirements.txt` | Schema is Alembic-managed ([ADR 0007](../adr/0007-alembic-for-server-migrations.md)) — a dependency bump that changes SQLModel/SQLAlchemy/DB-driver behavior can shift what autogenerate emits, so re-run `alembic check` and follow [runbooks.md](runbooks.md#applying-schema-migrations-alembic) if it touches schema. Pin `alembic` itself deliberately. | | `mobile-app/sapot-mobile-app/` | `package.json` | Expo SDK bumps need `expo-doctor` (`pnpm run testAll` includes it) — do not hand-edit `pnpm-lock.yaml` | | `admin-frontend/sapot-admin/` | `package.json` | `pnpm run lint && pnpm run build` after any bump — no test script exists in this component | -| `GSM-module/GSM-fastapi/` | `requirements.txt` | No automated tests — verify manually per [gsm-module-setup.md](../getting-started/gsm-module-setup.md) after any bump | +| `GSM-module/GSM-fastapi/` | `requirements.txt` | Run `cd GSM-module/GSM-fastapi && pytest`; serial I/O and database calls are mocked | | Nix flakes (per component) | `flake.lock` | Never hand-edit; only `nix flake update` should touch it | Never bundle a dependency bump with an unrelated feature change — if it breaks something, you want to be able to tell which caused it. diff --git a/docs/deployment/monitoring-logging.md b/docs/deployment/monitoring-logging.md index a8d75e25..cc44c9ba 100644 --- a/docs/deployment/monitoring-logging.md +++ b/docs/deployment/monitoring-logging.md @@ -62,11 +62,22 @@ A second background thread (`expire_announcements_loop`) periodically marks anno The GSM module logs to `GSM-module/GSM-fastapi/sapot.log`. Rotate or clear this file periodically in production. +When outbound admission returns `QUEUE_FULL`, the service logs the outbound queue depth and configured capacity at warning level. The saturation warning contains no SMS content. Operators should check modem readiness and throughput, allow the queue to drain, and investigate callers before increasing capacity; raising the limit increases waiting time and worker occupancy. + --- ## Health checks -No dedicated health-check endpoints are documented. The Nginx proxy (port 443 → Gunicorn :8000) can be used as a liveness check: +The GSM service exposes dedicated liveness and diagnostic endpoints: + +```bash +curl http://127.0.0.1:8001/health +curl http://127.0.0.1:8001/health/detailed +``` + +The first route remains asynchronous during outbound saturation. The detailed route includes modem state and outbound queue depth, capacity, and in-flight state. + +For the main server, the Nginx proxy (port 443 → Gunicorn :8000) can be used as a liveness check: ```bash curl -k https://localhost/auth/exists?identifier=probe@example.com diff --git a/docs/deployment/server.md b/docs/deployment/server.md index 17235a40..85940358 100644 --- a/docs/deployment/server.md +++ b/docs/deployment/server.md @@ -140,7 +140,7 @@ If you are pointing the server at a database created *before* Alembic was adopte |---|---| | `/ws/` | WebSocket proxy; no read timeout (86400 s) | | `/static/` | Filesystem; 30-day cache | -| `/` | Standard proxy; 135 s read timeout | +| `/` | Standard proxy; 155 s read timeout | Port 80 redirects to HTTPS (301). TLS 1.2/1.3, cipher `HIGH:!aNULL:!MD5`. diff --git a/docs/features/sms-gateway/design.md b/docs/features/sms-gateway/design.md index 7b204b9a..3db8906e 100644 --- a/docs/features/sms-gateway/design.md +++ b/docs/features/sms-gateway/design.md @@ -1,279 +1,142 @@ -# SMS Gateway — Design +# SMS Gateway: Design ## Overview -The SMS gateway is a separate FastAPI microservice (`GSM-module/GSM-fastapi/`) that bridges the main server and an Arduino-based GSM module over a serial port. The main server calls the GSM API to send SMS; the Arduino forwards inbound SMS back to the main server via a webhook. +The SMS gateway connects SAPOT to an Arduino-controlled GSM modem. The main server sends HTTP requests to the GSM FastAPI service, which serializes outbound SMS commands over USB. Inbound modem events travel back through the GSM service to the main server. -This feature is server-mediated; it has no P2P path. +The deployed implementation is `GSM-module/GSM-fastapi/`. The parallel `GSM-module/GSM-API/` directory is incomplete and is not part of this design. ---- +## Why is the gateway a separate service? -## Architecture +Serial communication is stateful and permits only one outbound command at a time. Keeping it outside the main server gives one process ownership of the serial port and prevents concurrent HTTP requests from interleaving modem commands. -``` -Main Server (FastAPI) - ├── POST /gsm/otp/request ──► generate OTP, store in phone_verification - │ └► POST /gsm/send ──────────────────────────────►┐ - │ │ - └── POST /gsm/inbound ◄── (webhook with X-GSM-Secret) ◄────────────────────┤ - │ - GSM FastAPI Service │ - GSM-module/GSM-fastapi/ - ├── serial_worker.py - ├── sms_handler.py - └── protocol.py - │ serial /dev/ttyACM0 - ▼ - Arduino Uno - SIM800L / SIM900 -``` +The boundary also lets the main server remain available when the modem disconnects. The gateway reports readiness and delivery failures without making GSM hardware a dependency of the main API process. + +## How do the components interact? ```mermaid sequenceDiagram - participant Main as Main Server - participant GSM as GSM FastAPI service - participant Ard as Arduino / modem - - Note over Main,Ard: Outbound SMS - Main->>GSM: POST /gsm/send { phone, message } - GSM->>GSM: encode SEND:: frame - GSM->>Ard: serial write (SEND frame) - Ard-->>GSM: ACK: or ERR:: - GSM->>GSM: mark sms_outbox delivered/failed - - Note over Main,Ard: Inbound SMS - Ard->>GSM: serial RECV:: - GSM->>Main: POST /gsm/inbound { from, body }
header X-GSM-Secret - Main->>Main: verify X-GSM-Secret, process inbound SMS + participant Client + participant Main as Main server + participant GSM as GSM FastAPI + participant Arduino + + Client->>Main: POST /gsm/sms/send + Main->>GSM: POST /sms/send with X-GSM-Secret + GSM->>GSM: log pending and admit to bounded queue + GSM->>Arduino: SEND_SMS|number|body + Arduino-->>GSM: SMS_SENT|number or SMS_FAILED|number|reason + GSM-->>Main: HTTP result + + Arduino->>GSM: SMS_RECEIVED|number|body + GSM->>GSM: apply session and target rules + GSM->>Main: POST /gsm/inbound with X-GSM-Secret ``` ---- - -## GSM FastAPI Service — `GSM-module/GSM-fastapi/` - -### `serial_worker.py` +The main server authenticates the user-facing `/gsm/sms/send` route with a JSON Web Token (JWT). It authenticates its direct gateway call with the same shared `GSM_SECRET` used for callbacks. The GSM service validates `X-GSM-Secret` before logging or queueing a send, which prevents another container on the internal network from occupying the serial modem. -Owns the serial connection lifecycle: +## How does outbound admission work? -- Opens `/dev/ttyACM0` at 9600 baud on startup. -- Runs a background thread that reads lines from the serial port. -- Forwards inbound lines to `sms_handler.handle_inbound(line)`. -- Exposes `serial_send(command: str)` for outbound AT commands. -- Reconnects automatically if the serial port closes unexpectedly. +`SerialWorker` owns a bounded first-in, first-out queue. `SMS_SEND_QUEUE_MAXSIZE` configures between 1 and 20 waiting requests, with a default of 10. One additional request may be active in the sender. -### `protocol.py` - -Defines the serial communication protocol between the GSM service and the Arduino: +Admission uses `put_nowait()` while the lifecycle lock is held. A full queue raises `OutboundQueueFullError`, and `POST /sms/send` returns HTTP 503: +```json +{ + "detail": { + "message": "Outbound SMS queue is full", + "reason": "QUEUE_FULL", + "msg_id": "" + } +} ``` -Outbound (service → Arduino): - SEND::\n -Inbound (Arduino → service): - RECV::\n - ACK:\n - ERR::\n -``` +The upper limit leaves worker threads available for overload responses and other synchronous FastAPI routes. `GET /health` is asynchronous, so liveness remains responsive while admitted sends wait for modem results. -The protocol layer encodes/decodes these frames and validates that all required fields are present before passing to the handler. +Each request receives a pre-write deadline when it is admitted. Queue wait and modem-readiness wait share that deadline. If the caller reaches it first, it marks the request complete while synchronized with the active-to-in-flight transition, so the sender cannot write that request later. -### `sms_handler.py` +Starting the serial write moves the request to a separate confirmation deadline. The caller follows that transition even when the serial write crosses the pre-write deadline, so it cannot report a retryable queue timeout after bytes have reached the Arduino. -Processes both directions: +## How is one SMS sent? -**Outbound flow:** +The sender owns one active request at a time: -```python -async def send_sms(phone: str, message: str) -> str: - message_id = uuid4().hex - frame = protocol.encode_send(phone, message, message_id) - serial_worker.serial_send(frame) - db.insert_pending(message_id, phone, message) - return message_id # returned to caller for tracking -``` - -**Inbound flow:** - -```python -def handle_inbound(line: str): - frame = protocol.decode(line) - if frame.type == "RECV": - # POST to main server webhook - requests.post( - f"{MAIN_SERVER_URL}/gsm/inbound", - json={"from": frame.from_number, "body": frame.message_body}, - headers={"X-GSM-Secret": GSM_SECRET}, - timeout=5 - ) - elif frame.type == "ACK": - db.mark_delivered(frame.message_id) - elif frame.type == "ERR": - db.mark_failed(frame.message_id, frame.detail) -``` +1. Dequeue and register the request as active. +2. Wait for modem readiness within the remaining admission deadline. +3. Atomically transition the request to in-flight while writing `SEND_SMS||`. +4. Start a fresh confirmation deadline after the serial write completes. +5. Wait for `SMS_SENT` or `SMS_FAILED` within the confirmation deadline. +6. Complete the matching caller and let the API update `sms_log`. -### GSM FastAPI Routes +The serial connection has a five-second write timeout. Reader events cannot complete a request before its serial write begins. Queue depth excludes the active request. -| Method | Path | Description | -|--------|------------|----------------------------------------| -| POST | /gsm/send | Accept send request; enqueue via serial | -| GET | /gsm/status| Return service health and serial state | +Shutdown closes admission, drains waiting work, and resolves active work with `SERVICE_STOPPING`. This avoids blocking on a sentinel when the bounded queue is full. -`/gsm/send` is called by the main server; it is not exposed to mobile clients directly. +## What is the serial protocol? -### Storage — `sapot.db` +`GSM-fastapi/protocol.py` and the production Arduino firmware are the sources of truth. -The GSM service uses a local SQLite database for outbox state: +```text +Python to Arduino: + SEND_SMS||\n -| Table | Purpose | -|--------------|--------------------------------------------| -| sms_outbox | Pending and delivered outbound messages | -| sms_inbound | Log of received inbound messages | - -In production, `DB_PATH` environment variable points to a MariaDB connection string to replace SQLite. - ---- - -## Main Server — OTP Flow - -### `POST /gsm/otp/request` - -```python -otp = generate_otp(6) # cryptographically random 6-digit string -expires_at = datetime.utcnow() + timedelta(minutes=10) -db.upsert(PhoneVerification(phone=phone, otp_hash=bcrypt(otp), expires_at=expires_at, used=False)) -gsm_api.send(phone=phone, message=f"Your SAPOT code is {otp}. Valid for 10 minutes.") +Arduino to Python: + GSM_READY + NETWORK_OK + NETWORK_LOST + SIM_MISSING + SMS_RECEIVED||\n + SMS_SENT|\n + SMS_FAILED||\n + LOG|\n ``` -`phone_verification` table: - -| Column | Type | Notes | -|------------|----------|---------------------------------| -| id | UUID | | -| phone | string | E.164 format | -| otp_hash | string | bcrypt hash; never store raw OTP| -| expires_at | datetime | UTC | -| used | boolean | Set true after successful verify| -| created_at | datetime | | - -### `POST /gsm/otp/verify` - -```python -row = db.query(PhoneVerification).filter( - phone=phone, - used=False, - expires_at > datetime.utcnow() -).order_by(created_at.desc()).first() - -if not row or not bcrypt.check(otp, row.otp_hash): - raise HTTPException(401) - -row.used = True -db.commit() -``` - -```mermaid -sequenceDiagram - participant User - participant Main as Main Server - participant GSM as GSM FastAPI service - participant Ard as Arduino / modem - - User->>Main: POST /gsm/otp/request { phone } - Main->>Main: generate 6-digit OTP, bcrypt hash,
upsert phone_verification (expires_at +10min) - Main->>GSM: POST /gsm/send { phone, "Your SAPOT code is..." } - GSM->>Ard: serial SEND frame - Ard-->>User: SMS delivered - - User->>Main: POST /gsm/otp/verify { phone, otp } - Main->>Main: lookup unused, unexpired phone_verification row - alt otp matches hash - Main->>Main: mark used = true - Main-->>User: 200 OK - else no match / expired / not found - Main-->>User: 401 - end -``` - -### `POST /gsm/inbound` (Webhook) - -```python -@router.post("/gsm/inbound") -async def receive_inbound(request: Request, payload: InboundSmsPayload): - secret = request.headers.get("X-GSM-Secret") - if secret != GSM_SECRET: - raise HTTPException(401) - # process inbound SMS: store, parse commands, etc. -``` - -`GSM_SECRET` is loaded from the environment at startup; the application refuses to start if it is not set. - ---- - -## Webhook Authentication - -``` -GSM FastAPI service ──POST /gsm/inbound──► Main Server - X-GSM-Secret: -``` - -- `GSM_SECRET` is a shared secret set as an environment variable on both services. -- The main server rejects any `/gsm/inbound` request where the header is absent or does not match. -- In production this secret should be at least 32 random bytes, base64-encoded. - ---- - -## Rate Limiting +Message bodies may contain pipe characters. `parse_line()` preserves them for `SMS_RECEIVED`. Newlines in outbound bodies are replaced with spaces by `build_send_sms()`. -OTP endpoints use Slowapi on the main server: +## How are inbound messages handled? -| Endpoint | Limit | -|-----------------------|-------------------| -| `/gsm/otp/request` | 1 per 60 s per IP | -| `/gsm/otp/resend` | 1 per 60 s per IP | -| `/gsm/otp/verify` | 5 per 60 s per IP | +The reader places `SMS_RECEIVED` events on `incoming_queue`. The API lifespan task passes each event to `handle_incoming_sms()`, which applies registration, ban, phone-verification, session, and target checks. ---- +The handler can return a reply to the sender and a forwarded message for the selected target. Both use the same bounded outbound queue. `database.notify_app()` also calls the main server's `POST /gsm/inbound` route with `X-GSM-Secret`. -## Dependencies +## What is persisted? -| Component | Purpose | -|------------------------|-----------------------------------------------| -| pyserial | Serial port communication with Arduino | -| FastAPI (GSM service) | HTTP API for send/status endpoints | -| SQLite / MariaDB | GSM service outbox state | -| Slowapi | Rate limiting on OTP endpoints (main server) | -| bcrypt | OTP hashing in `phone_verification` | +The GSM service uses the database configured by required `DB_PATH`. ---- +| Table | Responsibility | +|---|---| +| `sms_log` | Inbound and outbound audit rows, delivery status, and failure reason | +| `sms_session` | Per-phone conversation stage and selected target | +| Shared user and conversation tables | Lookup and delivery integration with the main server | -## Non-goals +The committed `sapot.db` file is stale and is not used by the deployed service. -- Not a two-way in-app messaging replacement — SMS is a fallback for OTP delivery and reaching users without the app installed, not a full-featured SMS inbox/thread UI. -- No multi-modem/multi-line support — the current design assumes a single serial-attached modem (`serial_worker.py` owns one connection); sending to multiple numbers concurrently is serialized through that one channel. -- No delivery-status UI beyond `db.mark_delivered`/`db.mark_failed` — there is no user-facing "message delivered/read" indicator for SMS, unlike in-app messages. -- Not encrypted — SMS content is plaintext by the nature of the SMS protocol; see [messaging design's SMS fallback note](../messaging/design.md#sms-fallback). +## How does startup recover interrupted work? -## Failure handling +After database initialization and before constructing `SerialWorker`, the lifespan calls `fail_orphaned_pending_messages()`. One database update changes every `pending` `sms_log` row to `failed` with `SERVICE_CRASHED`. -- **Serial port closes unexpectedly:** `serial_worker.py` reconnects automatically; any AT command in flight when the port closes is presumed lost — `sms_handler.py`'s outbox (`sms_outbox` table, `pending`/`delivered`/`failed` states) is the source of truth for what still needs resending, but automatic resend of `pending` rows after a reconnect is not described in the current design — worth confirming as a follow-up. -- **`ERR` frame from the Arduino:** `handle_inbound` marks the corresponding outbox row `failed` with the modem's detail code — the main server's OTP flow surfaces this as an OTP-send failure rather than silently leaving the user waiting. -- **Webhook call to the main server fails** (network blip, main server down): `requests.post(...)` to `/gsm/inbound` has a 5s timeout; a failed webhook call means an inbound SMS is acknowledged to the modem but never reaches the main server — there is no retry/dead-letter queue for this today. -- **`GSM_SECRET` mismatch or missing:** the main server rejects the webhook with 401; the GSM service has no visibility into *why* it was rejected beyond the HTTP status. -- **GSM module fully unreachable from the main server:** phone OTP requests fail; per [account-recovery design](../account-recovery/design.md#failure-handling), other recovery/verification methods remain usable. +The gateway does not re-queue these rows. A crash can happen after the modem transmits an SMS but before the process records its confirmation, so replay could deliver duplicate emergency messages. -## Performance impact +## How are failures reported? -- SMS delivery latency is bounded by the modem's own network round-trip (cellular network, typically seconds) — orders of magnitude slower than in-app message delivery; the OTP flow's 10-minute expiry window is sized to tolerate this. -- The serial channel is a single sequential bottleneck — `serial_send` calls queue behind whatever is currently in flight on `/dev/ttyACM0`, so send throughput is capped by modem + serial round-trip time, not by the FastAPI service itself. +| Failure | Result | +|---|---| +| Queue at capacity | HTTP 503 with `QUEUE_FULL`; no serial write | +| Worker stopping | HTTP 503 with `SERVICE_STOPPING` | +| Serial port or modem unavailable before admission | HTTP 503 | +| Serial write error | HTTP 502 with a reason beginning `WRITE_ERROR:` | +| Modem reports failure or confirmation times out | HTTP 502 with the modem or timeout reason | +| Caller deadline expires while waiting in the queue | Request fails and is never written later | +| Main server cannot reach the GSM service | Main server health route returns HTTP 503 | -## Scalability +The main server preserves the gateway status and error detail for synchronous SMS operations. Its 135-second read timeout covers the gateway's 125-second worst case plus HTTP overhead. Nginx allows 155 seconds so the one-second pool, five-second connect, five-second write, and 135-second read phase limits all fit inside the outer proxy limit. The main server permits 22 GSM connections, enough for 21 admitted gateway requests plus one request that observes `QUEUE_FULL`. Further requests fail pool admission within one second and never reach the gateway later. -- Designed for low SMS volume (OTPs and occasional fallback messages), not bulk SMS — a single serial-attached modem has a hard throughput ceiling unsuitable for high-volume sending. -- `sms_outbox`/`sms_inbound` grow unboundedly with no documented retention policy; the sqlite-vs-MariaDB storage note in [migrations.md](../../database/migrations.md#gsm-module-database-note) means production data must be in the MariaDB path, not the stale committed `sapot.db`. +The mobile app maps `QUEUE_FULL` to a busy message, keeps a rejected chat message as `not_sent`, and offers its existing manual resend action. Phone verification, resend, and first-contact screens remain in place and show the gateway failure instead of reporting success. -## Acceptance criteria +## Security and deployment assumptions -- An OTP requested via `/gsm/otp/request` is delivered as an SMS and successfully verified via `/gsm/otp/verify` within its 10-minute validity window. -- An unauthenticated (`X-GSM-Secret` mismatch or missing) call to `/gsm/inbound` is rejected with 401 and has no side effects. -- A failed outbound send is reflected in `sms_outbox` as `failed`, not left indefinitely `pending`. -- OTP endpoints enforce their documented rate limits (`1 per 60s` for request/resend, `5 per 60s` for verify). +- Set `DB_PATH` and `GSM_SECRET` in restricted environment files. Bare-metal systemd deployments use `/etc/sapot/gsm.env`. +- The main server and GSM service check `X-GSM-Secret` on both directions of their HTTP integration. +- Keep port 8001 restricted to the host or trusted Compose network as an additional boundary. +- SMS content is plaintext on the carrier network and should not be treated as end-to-end encrypted. +- The design supports one serial modem. Multi-modem failover and bulk SMS are out of scope. diff --git a/docs/features/sms-gateway/requirements.md b/docs/features/sms-gateway/requirements.md index 298e58d6..61bdd05f 100644 --- a/docs/features/sms-gateway/requirements.md +++ b/docs/features/sms-gateway/requirements.md @@ -1,92 +1,144 @@ -# SMS Gateway — Requirements +# SMS Gateway: Requirements ## Overview -The SMS gateway bridges the main server and an Arduino-based GSM module over a serial connection, letting the server send and receive SMS for OTP delivery and rescuer-initiated outreach to users without app connectivity. +The SMS gateway must let SAPOT send and receive SMS through one serial-attached modem without allowing HTTP load to create an unbounded in-memory backlog. ---- +These requirements describe the deployed `GSM-module/GSM-fastapi/` service and its HTTP integration with the main server. -## User Stories +## User outcomes -| ID | As a… | I want to… | So that… | -|--------|----------|---------------------------------------------------------|-----------------------------------------------------------------| -| SG-01 | rescuer | send an SMS to a registered user's phone number | I can reach them even if they are not connected to the LAN | -| SG-02 | user | receive a one-time password via SMS | I can verify my phone number or recover my account | -| SG-03 | user | resend an OTP if I did not receive it | I am not locked out due to delivery failure | -| SG-04 | system | receive inbound SMS from the GSM module | Users can send text commands or replies back to the system | -| SG-05 | admin | see whether the GSM module is online | I can confirm SMS delivery capability before relying on it | +| ID | User | Outcome | +|---|---|---| +| SG-01 | Rescuer | Send an SMS to a registered phone number | +| SG-02 | User | Receive phone-verification and recovery codes | +| SG-03 | User | Send an SMS reply through the gateway | +| SG-04 | Administrator | Observe modem readiness and queue saturation | ---- +## Functional requirements -## Functional Requirements +### FR-SG-01: Direct outbound SMS -### FR-SG-01 — Outbound SMS +`POST /sms/send` accepts: -- `POST /gsm/send` accepts `{ phone: string, message: string }`. -- Requires a valid user or admin JWT (rescuer role for sending arbitrary SMS; system for OTP). -- The GSM API forwards the message to the Arduino over the serial port (`/dev/ttyACM0`). -- The Arduino commands the SIM800L / SIM900 module to send the SMS. -- Response: `{ success: true, message_id: string }` on success; error detail on failure. +```json +{ + "number": "+639171234567", + "body": "message" +} +``` -### FR-SG-02 — OTP Request +The request must include `X-GSM-Secret` matching the required `GSM_SECRET` +configuration. Missing or invalid credentials return HTTP 401 before the +gateway creates a log row or admits work to the serial queue. -- `POST /gsm/otp/request` accepts `{ phone: string, purpose: "verification" | "recovery" }`. -- The main server generates a 6-digit OTP, stores it in `phone_verification` table with a 10-minute TTL, and calls the GSM API `POST /gsm/send` to deliver it. -- A phone number may request at most one OTP per 60 seconds (rate-limited by Slowapi). -- Response: `{ success: true, expires_in: 600 }`. +The number must use E.164 format. The submitted body must not exceed 160 characters and must remain nonempty after trimming. -### FR-SG-03 — OTP Verify +A successful modem confirmation returns HTTP 200: -- `POST /gsm/otp/verify` accepts `{ phone: string, otp: string }`. -- Looks up the most recent non-expired `phone_verification` row for the phone number. -- Returns 200 `{ verified: true }` if the OTP matches and has not expired. -- Returns 401 if the OTP is wrong. -- Returns 401 if the OTP has expired. -- Marks the row as used after a successful verification (prevents replay). +```json +{ + "ok": true, + "msg_id": "", + "to": "+639171234567" +} +``` -### FR-SG-04 — OTP Resend +A modem failure, write failure, or confirmation timeout returns HTTP 502 and records the failure in `sms_log`. -- `POST /gsm/otp/resend` accepts `{ phone: string }`. -- Invalidates any existing OTP for that phone and generates a new one. -- Subject to the same 60-second rate limit as `/gsm/otp/request`. +### FR-SG-02: Bounded outbound admission -### FR-SG-05 — Inbound SMS +- One outbound request may be active in the serial sender. +- `SMS_SEND_QUEUE_MAXSIZE` permits 1 through 20 additional waiting requests and defaults to 10. +- Waiting requests retain first-in, first-out order. +- Admission must not block when the queue is full. +- Work beyond capacity must never reach the serial port. +- The pre-write timeout starts at admission, not when the request reaches the front of the queue. +- A waiting request whose caller-visible deadline expires must never be written later. +- Once a serial write starts, the caller must wait for modem confirmation or the post-write confirmation timeout instead of reporting the pre-write timeout. +- Saturated requests must be logged as failed and return HTTP 503 with `reason: "QUEUE_FULL"`. -- The Arduino receives inbound SMS from the SIM module and forwards it over serial to the GSM FastAPI service. -- The GSM FastAPI service POSTs the SMS content to the main server webhook: `POST /gsm/inbound`. -- The webhook is authenticated with a shared `GSM_SECRET` environment variable (sent as `X-GSM-Secret` header). -- A request without the correct `GSM_SECRET` is rejected with 401. -- The main server processes the inbound SMS content (e.g. parse OTP replies, store message). +```json +{ + "detail": { + "message": "Outbound SMS queue is full", + "reason": "QUEUE_FULL", + "msg_id": "" + } +} +``` -### FR-SG-06 — Hardware +### FR-SG-03: Lifecycle cutoff -- GSM module: Arduino Uno (or compatible) connected to a SIM800L or SIM900 GSM shield. -- Serial interface: `/dev/ttyACM0` at 9600 baud. -- The GSM FastAPI service runs as a separate systemd unit: `server-GSM-api.service`. -- The service is co-located on the same server host as the main FastAPI application. +`SerialWorker.stop()` must close admission atomically. Waiting and active requests must resolve with `SERVICE_STOPPING`, and a new request after the cutoff must receive HTTP 503. -### FR-SG-07 — GSM Module Storage +A request completed before the cutoff must not be overwritten. A request failed before its serial write must never be written after modem recovery. -- The GSM FastAPI service uses a local SQLite database (`sapot.db`) for state (pending outbox, delivery receipts). -- In production this is replaced by a MariaDB connection configured via the `DB_PATH` environment variable. +### FR-SG-04: Health and diagnostics ---- +- `GET /health` must remain responsive while synchronous send requests occupy worker threads. +- `GET /health` returns modem readiness and serial connection state. +- `GET /health/detailed` reports inbound queue depth, outbound waiting depth, outbound capacity, and whether a serial request is in flight. +- Queue saturation must log depth and capacity without including SMS content in the saturation warning. -## Non-Functional Requirements +### FR-SG-05: Serial protocol -| ID | Requirement | -|----------|-----------------------------------------------------------------------------| -| NFR-SG-01 | SMS delivery attempt must complete or fail within 30 seconds | -| NFR-SG-02 | OTP must expire after exactly 10 minutes | -| NFR-SG-03 | Inbound webhook must respond within 5 seconds to avoid Arduino timeout | -| NFR-SG-04 | Tests must never use a real serial port or SIM module | -| NFR-SG-05 | `GSM_SECRET` must be set via environment variable; never hardcoded | +The service must send and receive these frames: ---- +```text +SEND_SMS|| +SMS_RECEIVED|| +SMS_SENT| +SMS_FAILED|| +GSM_READY +NETWORK_OK +NETWORK_LOST +SIM_MISSING +``` -## Out of Scope +Only one request may await a modem confirmation. A confirmation received before a new request starts its serial write must not complete that request. -- MMS support. -- SMS delivery receipts from the carrier network. -- Multi-SIM failover. -- Direct SIM module management via the admin UI (v1 is send/receive only). +### FR-SG-06: Inbound SMS + +- The serial reader must enqueue `SMS_RECEIVED` events for application processing. +- `handle_incoming_sms()` must apply the registered-user, banned-user, verified-phone, session, and target rules. +- Sender eligibility failures must set the inbound `sms_log` row to `rejected` with `NO_ACCOUNT`, `BANNED_SENDER`, or `UNVERIFIED_SENDER` as the failure reason. +- The GSM service must call the main server's `POST /gsm/inbound` route with `X-GSM-Secret` when forwarding into the app. +- Failed callbacks are logged. Automatic callback retry is not required. + +### FR-SG-07: Configuration and storage + +- `DB_PATH` is required. Startup must raise `RuntimeError` when it is missing. +- `GSM_SECRET` is required. Startup must raise `RuntimeError` when it is missing. +- Invalid `SMS_SEND_QUEUE_MAXSIZE` values must fail startup. +- `sms_log` stores inbound and outbound audit records. +- `sms_session` stores per-phone relay state. +- Before starting `SerialWorker`, startup must change every orphaned `pending` log row to `failed` with `SERVICE_CRASHED`. +- Startup reconciliation must not re-queue orphaned messages because the modem may have transmitted them before the prior process stopped. +- The committed `sapot.db` file must not be used as the deployment datastore. + +### FR-SG-08: Main server integration + +- The user-facing `/gsm/sms/send` route remains on the main server and requires its normal JWT authentication. +- The main server calls the direct gateway at `http://localhost:8001/sms/send` with `X-GSM-Secret`. +- The direct gateway is a trusted local service and must not be exposed to untrusted networks. +- The main server must preserve gateway HTTP 502 and 503 failures for user-facing send, verification, resend, and first-contact requests. +- The mobile app must retain rejected chat messages as `not_sent` and distinguish queue saturation from a generic delivery failure. + +## Non-functional requirements + +| ID | Requirement | +|---|---| +| NFR-SG-01 | Memory used by outbound waiting work is bounded by the configured queue | +| NFR-SG-02 | The default 40-thread FastAPI worker pool retains headroom at maximum queue capacity | +| NFR-SG-03 | `GET /health` does not wait on serial I/O or synchronous endpoint capacity | +| NFR-SG-04 | Serial writes time out after five seconds | +| NFR-SG-05 | Automated tests never open a real serial device or contact a real modem | +| NFR-SG-06 | Production secrets are supplied through restricted environment files | + +## Out of scope + +- Bulk SMS and multi-modem scheduling +- Multimedia Messaging Service (MMS) +- Carrier delivery or read receipts +- Automatic retry of failed main-server callbacks diff --git a/docs/features/sms-gateway/testing.md b/docs/features/sms-gateway/testing.md index d9e6f54c..256d0109 100644 --- a/docs/features/sms-gateway/testing.md +++ b/docs/features/sms-gateway/testing.md @@ -1,136 +1,113 @@ -# SMS Gateway — Testing +# SMS Gateway: Testing -## Strategy +## Overview -| Layer | Tooling | Scope | -|-------------|----------------------------------|--------------------------------------------------------------------| -| Unit | pytest | `protocol.py` encode/decode, `sms_handler` outbound/inbound logic | -| Integration | pytest + HTTPX + in-memory SQLite| GSM FastAPI endpoints, main server OTP endpoints, webhook handler | -| Hardware | **Never tested against real HW** | Serial port and SIM module are always mocked | +The GSM FastAPI tests verify queue admission, lifecycle races, API responses, and configuration without opening a real serial port or connecting to the production database. Hardware behavior still requires a modem smoke test because unit tests cannot prove Arduino timing or carrier delivery. ---- +## Prerequisites -## Coverage Targets +Run each touched component in its pinned environment. -| Area | Target | -|-----------------------------------|--------| -| `protocol.py` encode/decode | 100% | -| `sms_handler` outbound path | 100% | -| `sms_handler` inbound path | 100% | -| OTP request / verify / resend | 100% | -| Webhook authentication | 100% | -| Rate limiting enforcement | 90%+ | -| Overall SMS gateway coverage | ≥ 80% | +GSM gateway: ---- +```bash +cd GSM-module/GSM-fastapi +nix develop +pytest +``` -## Mocking Rules +Main server proxy contract: -- **Serial port** — mock `serial_worker.serial_send`; never open a real `/dev/ttyACM0`. Use `unittest.mock.patch`. -- **GSM API HTTP calls** — mock `requests.post` in `sms_handler.handle_inbound`; never hit the real main server. -- **Main server → GSM API calls** — mock the GSM API `POST /gsm/send` with `respx` or `responses`; never hit the real GSM service. -- **Database** — use in-memory SQLite for both the GSM service (`sapot.db`) and main server tests. -- **Time** — use `freezegun` for OTP expiry assertions. -- **GSM_SECRET** — set via `os.environ` in fixture setup; use a fixed test value `"test-gsm-secret-value"`. -- **OTP generation** — mock `generate_otp` to return a fixed value (`"123456"`) in integration tests for predictable assertions. +```bash +cd server +nix develop +cd app +pytest tests/test_gsm_health.py tests/test_gsm_proxy.py +``` ---- +Mobile full component gate, including GSM error handling and dependency injection wiring: -## Test Cases +```bash +cd mobile-app +nix develop +cd sapot-mobile-app +pnpm run testAll +``` -### `protocol.py` — Unit +`tests/conftest.py` supplies test `DB_PATH` and `GSM_SECRET` values before importing application settings. Tests that reach API handlers replace database operations with fakes, so they do not create or mutate production records. -| Scenario | Expected result | -|----------|-----------------| -| `encode_send("+639171234567", "hello", "msg-1")` | Returns `"SEND:+639171234567:hello:msg-1\n"` | -| `decode("RECV:+639171234567:test message\n")` | Returns frame with `type="RECV"`, `from_number="+639171234567"`, `message_body="test message"` | -| `decode("ACK:msg-1\n")` | Returns frame with `type="ACK"`, `message_id="msg-1"` | -| `decode("ERR:101:module timeout\n")` | Returns frame with `type="ERR"`, `code="101"`, `detail="module timeout"` | -| `decode("INVALID\n")` | Raises `ProtocolError` | -| Message body containing colon character | Encoded and decoded without truncation | +## What is covered? -### GSM Service — `POST /gsm/send` (Integration) +| File | Responsibility | +|---|---| +| `tests/test_config.py` | Default queue capacity, valid range, and startup rejection | +| `tests/test_serial_worker.py` | Queue capacity, admission deadlines, lifecycle cutoff, exact-once completion, pre-write races, and serial write timeout | +| `tests/test_api_queue.py` | HTTP 503 contracts, message-log updates, diagnostics, worker-pool headroom, and health responsiveness | +| `tests/test_database_reconciliation.py` | Idempotent startup recovery of orphaned pending log rows | +| `tests/test_incoming_sms.py` | Sender rejection reason codes and inbound log status updates | +| `tests/test_lifespan.py` | Reconciliation ordering before serial worker startup | +| `server/app/tests/test_gsm_proxy.py` | Main-server shared-secret header, status preservation, and timeout headroom for chat, verification, resend, and first-contact requests | +| `mobile-app/sapot-mobile-app/features/shared/core/errors/__tests__/gsm-error.test.ts` | Typed `QUEUE_FULL` parsing and user-visible error messages | +| `mobile-app/sapot-mobile-app/features/chat/components/__tests__/message-list.test.tsx` | Manual resend rejection and `not_sent` restoration | +| `mobile-app/sapot-mobile-app/features/auth/auth-container.test.ts` | Phone-verification service construction | +| `mobile-app/sapot-mobile-app/features/shared/__tests__/main-container-initialize.test.ts` | GSM service construction within the runtime container | -| Scenario | Expected result | -|----------|-----------------| -| Valid `{ phone, message }` payload | `serial_worker.serial_send` called with correct encoded frame; row inserted in `sms_outbox` with status `pending`; response `{ success: true, message_id }` | -| Arduino ACK received via serial | `sms_outbox` row updated to `delivered` | -| Arduino ERR received via serial | `sms_outbox` row updated to `failed` with error detail | -| Serial port unavailable at startup | Service returns 503 on `/gsm/send`; error logged | -| Missing `phone` field | Returns 422 | -| `message` exceeds 160 characters | Returns 400 or splits into multiple frames depending on config | +## How is serial I/O isolated? -### Main Server — OTP Request (Integration) +Most worker tests do not start the reader or sender threads. Tests that need a serial write assign a small fake object to `worker._ser`. The connection test replaces `serial_worker.serial.Serial` before calling `_connect_and_read()`. -| Scenario | Expected result | -|----------|-----------------| -| `POST /gsm/otp/request` with valid phone, purpose `"verification"` | `phone_verification` row created with hashed OTP; `expires_at = now + 10 min`; GSM API send called | -| Second request within 60 s for same phone | Returns 429 (rate limit) | -| Request after 60 s | New OTP generated; old row marked superseded | -| GSM API send fails | Returns 503; `phone_verification` row not created | -| Missing `phone` field | Returns 422 | +No test may rely on `/dev/ttyACM0`, a SIM card, or a carrier network. A test that starts a thread must signal it to stop and join it before returning. -### Main Server — OTP Verify (Integration) +## Queue and lifecycle cases | Scenario | Expected result | -|----------|-----------------| -| Correct OTP within expiry window | Returns 200 `{ verified: true }`; `phone_verification.used` set to `true` | -| Correct OTP reused after first verify | Returns 401 (row is marked `used`) | -| Wrong OTP | Returns 401 | -| OTP expired (`expires_at` in past) | Returns 401 | -| No OTP row exists for phone | Returns 401 | -| `POST /gsm/otp/verify` 6 times in 60 s | 6th request returns 429 (rate limit) | +|---|---| +| Waiting queue reaches configured capacity | Next admission raises `OutboundQueueFullError` without a serial write | +| Capacity is outside 1 through 20 | Configuration or worker construction fails | +| Shutdown begins with waiting work | Waiting requests complete with `SERVICE_STOPPING` | +| Shutdown begins with active work | Active request completes with `SERVICE_STOPPING` | +| Network loss completes active work before writing | Recovery does not write the failed request | +| Caller deadline expires while a request is queued | The request returns `CLIENT_TIMEOUT` and is never written | +| Serial write crosses the admission deadline | The caller waits for modem confirmation instead of returning the pre-write timeout | +| Stale confirmation arrives before a new write | New request remains pending and is written normally | +| Two completions race for one request | First completion wins | +| Serial connection opens | PySerial receives the configured five-second write timeout | -### Main Server — OTP Resend (Integration) +## API saturation cases -| Scenario | Expected result | -|----------|-----------------| -| `POST /gsm/otp/resend` for phone with existing OTP | Old row invalidated; new `phone_verification` row created; GSM API send called with new OTP | -| Resend within 60 s of last request | Returns 429 | +The saturation test starts 21 blocking send requests, representing 20 waiting requests plus one active request. It then verifies that: -### Inbound Webhook — Main Server (Integration) +1. A further `POST /sms/send` reaches the handler and returns HTTP 503 with `QUEUE_FULL`. +2. `GET /health` returns while those sends remain blocked. +3. Releasing the admitted requests lets every pending HTTP request complete. -| Scenario | Expected result | -|----------|-----------------| -| `POST /gsm/inbound` with correct `X-GSM-Secret` | Returns 200; inbound SMS processed | -| `POST /gsm/inbound` with wrong secret | Returns 401; SMS not processed | -| `POST /gsm/inbound` with missing `X-GSM-Secret` header | Returns 401 | -| `POST /gsm/inbound` with valid secret and OTP-reply body | OTP reply matched to pending verification; user notified | +This covers the thread-pool exhaustion described by issue #252. A test with only a rejecting fake would verify the response shape but would not prove that the rejection handler can still obtain a worker thread. -### GSM Service — Inbound SMS Path (Unit) +## Manual modem smoke test -| Scenario | Expected result | -|----------|-----------------| -| `handle_inbound("RECV:+639171234567:hello\n")` | `requests.post` called to main server `/gsm/inbound` with correct payload and `X-GSM-Secret` header | -| Main server webhook call times out | Error logged; no retry in v1; service continues | -| `handle_inbound("ACK:msg-1\n")` | `sms_outbox` row for `msg-1` updated to `delivered` | -| `handle_inbound("ERR:101:timeout\n")` | Matching outbox row updated to `failed` | -| Serial line that does not parse | `ProtocolError` caught; error logged; service continues | +Run this only on a host with the configured Arduino and SIM: ---- +1. Set `DB_PATH`, `GSM_SECRET`, `SERIAL_PORT`, and `SMS_SEND_QUEUE_MAXSIZE` in the host environment file. +2. Start the GSM service and wait for `GSM_READY`. +3. Check liveness: -## Test File Locations + ```bash + curl http://127.0.0.1:8001/health + ``` -``` -GSM-module/GSM-fastapi/ - tests/ - test_protocol.py - test_sms_handler.py - test_gsm_api.py - -server/ - tests/ - test_gsm_otp.py - test_gsm_inbound_webhook.py -``` +4. Send one SMS to a controlled test number: -## Important: No Real Hardware in CI + ```bash + curl -X POST http://127.0.0.1:8001/sms/send \ + -H 'Content-Type: application/json' \ + -H 'X-GSM-Secret: ' \ + -d '{"number":"+639171234567","body":"SAPOT GSM smoke test"}' + ``` -All CI runs must set: -``` -MOCK_SERIAL=true -GSM_SECRET=test-gsm-secret-value -DB_PATH=:memory: -``` +5. Confirm the API response, `sms_log` status, Arduino event, and receipt on the test phone. + +## Limitations -Any test that attempts to open `/dev/ttyACM0` or make an outbound HTTP call to a non-mocked host must fail the test suite with a clear error message. +- Automated tests do not prove USB permissions, modem readiness, SIM balance, signal quality, or carrier delivery. +- API tests mock message-log persistence; they do not validate the MariaDB schema. +- Real-hardware testing must use a controlled phone number and must not run in shared CI. diff --git a/docs/getting-started/docker-setup.md b/docs/getting-started/docker-setup.md index 32a265ea..0c092eae 100644 --- a/docs/getting-started/docker-setup.md +++ b/docs/getting-started/docker-setup.md @@ -90,8 +90,9 @@ that includes: (through `nginx`), **not** at the bare `/`, which 404s. - `tileserver`: offline map tiles. Not published to the host: `docker-compose.yml` only `expose`s port 8080 on the internal network, so reach it at `https://localhost/tiles/` through `nginx`. -- `gsm-fastapi`: the SMS gateway, `http://localhost:8001` (starts without the GSM modem; add - `docker-compose.gsm-hardware.yml` per the [Configure](#configure) section above for real SMS) +- `gsm-fastapi`: the SMS gateway, `http://localhost:8001` on host loopback only (starts without the + GSM modem; add `docker-compose.gsm-hardware.yml` per the [Configure](#configure) section above for + real SMS) `nginx` declares `depends_on` on `admin` and `tileserver` (both proxied by `nginx.docker.conf` as static upstreams, which nginx resolves at config-load time and refuses to start without). Naming @@ -184,7 +185,8 @@ both running at once: directory name, so a worktree gets its own containers, network, and `db-data` volume automatically — no shared state with the main checkout's stack. - **Host ports are not automatically isolated.** Two stacks (main checkout + a worktree, or two - worktrees) both bind `443`/`80`/`3000`/`8001` on the host by default, so bringing up a + worktrees) both bind `443`/`80`/`3000`/`8001` on the host by default. GSM port 8001 binds only to + loopback, while the other published services keep their configured interfaces. Bringing up a second stack while the first is still running fails with "port is already allocated". If you want them running concurrently, give the worktree its own `.env` (root-level, copied from `.env.example`) with different port values, e.g.: diff --git a/docs/getting-started/gsm-module-setup.md b/docs/getting-started/gsm-module-setup.md index 331f9183..5141c188 100644 --- a/docs/getting-started/gsm-module-setup.md +++ b/docs/getting-started/gsm-module-setup.md @@ -41,14 +41,15 @@ cp .env.example .env | Variable | Default | Notes | |---|---|---| -| `DB_PATH` | none, **required** | SQLModel URL for the server's MariaDB, e.g. `mysql+pymysql://sapot:sapot@127.0.0.1:3306/sapot_dev`. `config.py` raises `RuntimeError` at import if unset. (Its source comment says "SQLite"; that is stale; the deployed value is MariaDB.) | -| `GSM_SECRET` | `""` | Must match the server's `GSM_SECRET`. The two components authenticate each other's webhook calls with it via the `X-GSM-Secret` header. Left empty, the server rejects this gateway's calls. | +| `DB_PATH` | none, **required** | SQLModel URL for the server's MariaDB, e.g. `mysql+pymysql://sapot:sapot@127.0.0.1:3306/sapot_dev`. `config.py` raises `RuntimeError` at import if unset. | +| `GSM_SECRET` | None | Required at startup and must match the server's `GSM_SECRET`. Both services send it as `X-GSM-Secret` when calling the other service. | | `SAPOT_API_URL` | `http://localhost:8000` | Base URL of the SAPOT server this gateway forwards inbound SMS to (`POST /gsm/inbound`). The Docker service overrides it to `https://nginx`. | | `SERIAL_PORT` | `/dev/ttyACM0` | Serial device the Arduino is on. `COM3`-style on Windows. | | `SERIAL_BAUD` | `9600` | Must match `PC_BAUD` in the Arduino sketch. | | `HOST` | `127.0.0.1` | Bind address. The Docker service overrides this to `0.0.0.0`. | | `PORT` | `8000` | **Not read.** See [Run](#run) below. | | `LOG_LEVEL` | `INFO` | | +| `SMS_SEND_QUEUE_MAXSIZE` | `10` | Maximum waiting outbound SMS requests. Values from `1` through `20` preserve FastAPI worker capacity; startup fails for other values. | | `SMS_BOT_USER_ID` | unset | UUID of the "SMS Bot" user in the SAPOT database. Inbound SMS is written into the app's conversation/message tables as coming from this user, so create it once on the server and paste the UUID here. | See [environment-config.md](../deployment/environment-config.md) for the cross-component view. @@ -82,6 +83,6 @@ quickest way to confirm the `.env` was picked up. ## Next -- [docker-setup.md](docker-setup.md) — the server must have a matching `GSM_SECRET` set for inbound/outbound SMS to authenticate. +- [docker-setup.md](docker-setup.md): the server must have a matching `GSM_SECRET` for inbound GSM callbacks. - [data-flow.md](../architecture/data-flow.md#sms-fallback) — the end-to-end SMS fallback flow diagram. -- [TROUBLESHOOTING.md](../TROUBLESHOOTING.md#gsm-module-and-server-cant-authenticate-each-other): when the two sides reject each other's webhooks. +- [TROUBLESHOOTING.md](../TROUBLESHOOTING.md#server-rejects-gsm-inbound-callbacks): when the server rejects the gateway's callback secret. diff --git a/docs/qa/scenario-tooling.md b/docs/qa/scenario-tooling.md index b7065624..058e4c22 100644 --- a/docs/qa/scenario-tooling.md +++ b/docs/qa/scenario-tooling.md @@ -24,6 +24,7 @@ Router: [`server/app/api/testing.py`](../../server/app/api/testing.py). Scenario | `large` | `qa_large` + many peers/messages/GPS points, for list perf and sync-cursor testing | | `banned` | `qa_banned` with an active `BannedUser` row | | `locked-out` | `qa_locked` with a `LoginAttempt` row at the lockout threshold | +| `verified-phone` | `qa_phone_verified` with a `PhoneVerified` row, for phone-verification-gated flows | | `announcements` | Active + expired announcements across all priorities and audiences | | `gps-track` | `qa_gps` with a 60-point location history along a route | | `calls` | `qa_calls_a` / `qa_calls_b` with completed/missed/rejected call rows | diff --git a/mobile-app/sapot-mobile-app/app/(drawer)/(tabs)/chat/[id].tsx b/mobile-app/sapot-mobile-app/app/(drawer)/(tabs)/chat/[id].tsx index e9416c3d..873354c6 100644 --- a/mobile-app/sapot-mobile-app/app/(drawer)/(tabs)/chat/[id].tsx +++ b/mobile-app/sapot-mobile-app/app/(drawer)/(tabs)/chat/[id].tsx @@ -661,7 +661,11 @@ const ChatRoom = () => { {conversationId ? ( - + ) : ( No messages yet diff --git a/mobile-app/sapot-mobile-app/app/(drawer)/(tabs)/index.tsx b/mobile-app/sapot-mobile-app/app/(drawer)/(tabs)/index.tsx index 42f34aae..f665ea44 100644 --- a/mobile-app/sapot-mobile-app/app/(drawer)/(tabs)/index.tsx +++ b/mobile-app/sapot-mobile-app/app/(drawer)/(tabs)/index.tsx @@ -6,13 +6,13 @@ import { ChatRoomSource } from "@/features/chat/types"; import { toInternationalPhone } from "@/features/auth/utils/validation"; import { ChatList, useChats } from "@/features/chat"; import { useChatService } from "@/features/chat/hooks/use-chat-service"; -import { contactUnknownUser } from "@/features/shared/core/api/gsm.api"; import { AppSnackbar } from "@/features/shared/components/app-snackbar"; import PeerList from "@/features/shared/components/peer-list"; import { Peer } from "@/features/shared/core/database"; import { useConnectionService, useDiscoveryService, + useGsmService, usePeerService, useToast, } from "@/features/shared/hooks"; @@ -21,6 +21,7 @@ import { useGsmHealth } from "@/features/shared/hooks/use-gsm-health"; import { useSyncService } from "@/features/shared/hooks/use-sync-service"; import { useUserStore } from "@/features/shared/hooks/use-user-store"; import { uiLog } from "@/features/shared/core/utils/logger"; +import { getGsmErrorMessage } from "@/features/shared/core/errors"; import { useHeaderHeight } from "@react-navigation/elements"; import { LinearGradient } from "expo-linear-gradient"; import { useRouter } from "expo-router"; @@ -94,6 +95,7 @@ export default function Chat() { const connectionService = useConnectionService(); const peerService = usePeerService(); const chatService = useChatService(); + const gsmService = useGsmService(); useEffect(() => { uiLog.debug("[Chat] useEffect triggered, deps:", { @@ -252,7 +254,7 @@ export default function Chat() { setContacting(true); try { const phone = toInternationalPhone(targetPhone.trim()); - const res = await contactUnknownUser(phone); + const res = await gsmService.contactUnknownUser(phone); await peerService.upsertPeer({ id: res.user_id, username: phone, @@ -284,9 +286,12 @@ export default function Chat() { source: ChatRoomSource.PEER, }, }); - } catch { + } catch (error) { showError( - "Failed to contact user. Check phone number format (+63...)." + getGsmErrorMessage( + error, + "Failed to contact user. Check phone number format (+63...)." + ) ); } finally { setContacting(false); diff --git a/mobile-app/sapot-mobile-app/app/(drawer)/settings/account/phone/verify-phone.tsx b/mobile-app/sapot-mobile-app/app/(drawer)/settings/account/phone/verify-phone.tsx index 260edbe2..5a07e999 100644 --- a/mobile-app/sapot-mobile-app/app/(drawer)/settings/account/phone/verify-phone.tsx +++ b/mobile-app/sapot-mobile-app/app/(drawer)/settings/account/phone/verify-phone.tsx @@ -1,18 +1,13 @@ import { SETTINGS_ROUTES } from "@/config/routes"; -import { useUserService } from "@/features/auth"; -import { - checkGsmHealth, - migratePhoneUserApi, - requestPhoneVerification, - resendVerificationCodePhone, - verifyCodePhone, -} from "@/features/auth/api/auth.api"; +import { usePhoneVerificationService, useUserService } from "@/features/auth"; import { toInternationalPhone } from "@/features/auth/utils/validation"; import { useRecoveryKeySetup } from "@/features/auth/hooks/use-recovery-key-setup"; import { VerificationCodeContent } from "@/features/settings"; import AppSnackbar from "@/features/shared/components/app-snackbar"; import { useSyncService } from "@/features/shared/hooks/use-sync-service"; +import { useGsmService } from "@/features/shared/hooks"; import { uiLog } from "@/features/shared/core/utils/logger"; +import { getGsmErrorMessage } from "@/features/shared/core/errors"; import { router, useLocalSearchParams } from "expo-router"; import { useEffect, useState } from "react"; import { View } from "react-native"; @@ -35,6 +30,8 @@ export default function VerifyPhone() { variant: "neutral", }); const userService = useUserService(); + const gsmService = useGsmService(); + const phoneVerificationService = usePhoneVerificationService(); const { setupPhoneBlob } = useRecoveryKeySetup(); const syncService = useSyncService(); @@ -50,8 +47,8 @@ export default function VerifyPhone() { const sendCode = async () => { setIsSending(true); setSendFailed(false); - const gsmOnline = await checkGsmHealth(); - if (!gsmOnline) { + const gsmHealth = await gsmService.getHealth().catch(() => null); + if (!gsmHealth?.gsm_ready) { setSnackbar({ visible: true, message: @@ -64,7 +61,7 @@ export default function VerifyPhone() { } try { setIsSending(false); - await requestPhoneVerification( + await phoneVerificationService.requestVerification( phone ? toInternationalPhone(phone) : undefined, reauth_token || undefined ); @@ -76,7 +73,10 @@ export default function VerifyPhone() { setSendFailed(true); setSnackbar({ visible: true, - message: "Failed to send verification code. Please try again.", + message: getGsmErrorMessage( + error, + "Failed to send verification code. Please try again." + ), variant: "error", }); } @@ -91,14 +91,14 @@ export default function VerifyPhone() { setCodeError(undefined); try { - await verifyCodePhone(code); + await phoneVerificationService.verifyCode(code); await userService.updateAuthenticatedUser({ phoneNumber: toInternationalPhone(phone), phoneNumberVerified: true, }); await setupPhoneBlob(phone); try { - const migration = await migratePhoneUserApi(); + const migration = await phoneVerificationService.migratePhoneUser(); if (migration.migrated) { uiLog.info("[VerifyPhone] ghost user migrated", { ghostUserId: migration.ghost_user_id, @@ -127,10 +127,12 @@ export default function VerifyPhone() { setCodeError(undefined); try { - await resendVerificationCodePhone(); + await phoneVerificationService.resendCode(); } catch (error) { uiLog.error("[VerifyPhone] Error resending code", { error }); - setCodeError("Failed to resend code. Please try again."); + setCodeError( + getGsmErrorMessage(error, "Failed to resend code. Please try again.") + ); } }; diff --git a/mobile-app/sapot-mobile-app/docs/API.md b/mobile-app/sapot-mobile-app/docs/API.md index 670b0823..79d59109 100644 --- a/mobile-app/sapot-mobile-app/docs/API.md +++ b/mobile-app/sapot-mobile-app/docs/API.md @@ -848,6 +848,8 @@ directly. { "status": "string", "gsm_ready": "boolean", "connected": "boolean", "detail": "string" } ``` +The server preserves the gateway's HTTP 503 status when the modem is not ready. Callers treat that response as unavailable rather than relying only on the JSON status field. + `gsm_ready` (modem registered on the network) and `connected` (API can reach the GSM service) fail independently — surface them separately rather than collapsing to one "offline" state. @@ -862,6 +864,19 @@ independently — surface them separately rather than collapsing to one "offline { "msg_id": "string", "ok": "boolean", "to": "string" } ``` +**Response `503` when the outbound queue is full:** +```json +{ + "detail": { + "message": "Outbound SMS queue is full", + "reason": "QUEUE_FULL", + "msg_id": "string" + } +} +``` + +The chat screen marks the local message `not_sent`, shows that the SMS service is busy, and keeps the manual resend action available. + --- ### `POST /gsm/contact-unknown-user` — SMS an Arbitrary Number @@ -879,6 +894,7 @@ independently — surface them separately rather than collapsing to one "offline ``` `is_sapot_user` reports whether the number already belongs to a registered account. +If the onboarding SMS is rejected, the endpoint preserves the gateway error and the app does not display its success confirmation. --- @@ -900,6 +916,8 @@ independently — surface them separately rather than collapsing to one "offline { "message": "string" } ``` +Phone verification request and resend calls preserve HTTP 503 gateway failures. The verification screen remains retryable and displays a busy or unavailable message. + --- ### `POST /gsm/migrate-phone-user` — Claim a Ghost Phone Account diff --git a/mobile-app/sapot-mobile-app/docs/ARCHITECTURE.md b/mobile-app/sapot-mobile-app/docs/ARCHITECTURE.md index fade9f60..d6e1ce64 100644 --- a/mobile-app/sapot-mobile-app/docs/ARCHITECTURE.md +++ b/mobile-app/sapot-mobile-app/docs/ARCHITECTURE.md @@ -21,6 +21,7 @@ and takes no arguments: - `guestUserRepository` — guest profile row - `guestMigrationService` — guest→auth conversion - `userService` — login/logout; `MainContainer` injects the `CleanUpService` into it so logout purges local data +- `phoneVerificationService`: phone verification request/resend, code verification, and ghost-user migration. Screens access it through `usePhoneVerificationService()`; GSM availability comes from the shared `GsmService`. ### MainContainer (`features/shared/main-container.ts`) @@ -142,6 +143,8 @@ Crypto stack: `tweetnacl` + `tweetnacl-util`, `@noble/hashes`, `expo-crypto`, `r | `NotificationService` | Local incoming-call notifications via `expo-notifications`. Constructed inline in `main-container.ts` and passed to `ConnectionService`; not exposed as a container field. | | `CallMessageRouter` | Pure decision layer for inbound call messages. Maps a `CallMessage` + busy/active state to a `CallRouterResult` (`emit` / suppress), keeping glare handling out of `ConnectionService`. | | `PublicChatService` | Server-relayed public chat over `WsSignalingAdapter`, with history loaded from `GET /public-chat`. Independent of the P2P chat path. | +| `GsmService` | Owned by `MainContainer`. Reads GSM health, sends chat SMS, and sends first-contact onboarding requests through the GSM API. UI and chat hooks access it through `useGsmService()` so screens do not call API modules directly. | +| `PhoneVerificationService` | Owned by `AuthContainer`. Coordinates phone verification request/resend, code verification, and ghost-user migration through the auth API module. GSM health remains centralized in `GsmService`. | --- diff --git a/mobile-app/sapot-mobile-app/docs/diagrams/05-sms-flow.md b/mobile-app/sapot-mobile-app/docs/diagrams/05-sms-flow.md index bb2822cf..6df903d2 100644 --- a/mobile-app/sapot-mobile-app/docs/diagrams/05-sms-flow.md +++ b/mobile-app/sapot-mobile-app/docs/diagrams/05-sms-flow.md @@ -16,7 +16,9 @@ flowchart TD F --> Z([End]) E -->|Yes| G[Forward message to server] G --> H[Server sends message to GSM module] - H --> I[GSM module transmits SMS over cellular network] + H --> Q{Outbound queue has capacity?} + Q -->|No| K[Keep message as not sent and display busy error] + Q -->|Yes| I[GSM module transmits SMS over cellular network] I --> J{Transmission successful?} J -->|No| K[Display send failure] K --> Z diff --git a/mobile-app/sapot-mobile-app/features/auth/api/__tests__/auth.api.test.ts b/mobile-app/sapot-mobile-app/features/auth/api/__tests__/auth.api.test.ts index 8a96327f..8fca017b 100644 --- a/mobile-app/sapot-mobile-app/features/auth/api/__tests__/auth.api.test.ts +++ b/mobile-app/sapot-mobile-app/features/auth/api/__tests__/auth.api.test.ts @@ -1,4 +1,10 @@ -import { loginAsFixtureApi, resetPasswordApi } from "../auth.api"; +import { GsmGatewayError } from "@/features/shared/core/errors/gsm-error"; +import { + loginAsFixtureApi, + requestPhoneVerification, + resendVerificationCodePhone, + resetPasswordApi, +} from "../auth.api"; jest.mock("@/features/shared", () => ({ apiClient: { post: jest.fn(), get: jest.fn() }, @@ -56,3 +62,40 @@ describe("loginAsFixtureApi", () => { ); }); }); + +describe("GSM verification API", () => { + const queueFullError = { + response: { + status: 503, + data: { + detail: { + message: "Outbound SMS queue is full", + reason: "QUEUE_FULL", + msg_id: "sms-log-id", + }, + }, + }, + }; + + it("wraps verification saturation as a typed GSM gateway error", async () => { + mockPost.mockRejectedValue(queueFullError); + + await expect( + requestPhoneVerification("+639171234567") + ).rejects.toMatchObject({ + name: "GsmGatewayError", + status: 503, + reason: "QUEUE_FULL", + } satisfies Partial); + }); + + it("wraps resend saturation as a typed GSM gateway error", async () => { + mockPost.mockRejectedValue(queueFullError); + + await expect(resendVerificationCodePhone()).rejects.toMatchObject({ + name: "GsmGatewayError", + status: 503, + reason: "QUEUE_FULL", + } satisfies Partial); + }); +}); diff --git a/mobile-app/sapot-mobile-app/features/auth/api/auth.api.ts b/mobile-app/sapot-mobile-app/features/auth/api/auth.api.ts index 9fa0c3c0..f2d7c773 100644 --- a/mobile-app/sapot-mobile-app/features/auth/api/auth.api.ts +++ b/mobile-app/sapot-mobile-app/features/auth/api/auth.api.ts @@ -1,5 +1,5 @@ import { QA_API_TOKEN } from "@/config/debug"; -import { toAppError } from "@/features/shared/core/errors"; +import { toAppError, toGsmGatewayError } from "@/features/shared/core/errors"; import { apiClient } from "@/features/shared"; import { apiLog } from "@/features/shared/core/utils/logger"; import { AxiosResponse } from "axios"; @@ -327,32 +327,23 @@ export const verifyCodeEmail = async (code: string) => { return res.data; }; -export const checkGsmHealth = async (): Promise => { - try { - apiLog.info("[AuthApi] Calling /gsm/health"); - const res = await apiClient.get<{ status: string }>("/gsm/health"); - apiLog.info("[AuthApi] GSM health response", { status: res.status }); - return res.status === 200; - } catch (error) { - const appErr = toAppError(error, "auth"); - apiLog.warn("[AuthApi] GSM health check failed", appErr); - return false; - } -}; - export const requestPhoneVerification = async ( phoneNumber?: string, reauthToken?: string ) => { apiLog.info("[AuthApi] Calling /gsm/request", { hasPhoneNumber: Boolean(phoneNumber) }); - const res = await apiClient.post<{ message: string }>( - "/gsm/request", - { phone_number: phoneNumber }, - reauthToken ? { headers: { "X-Reauth-Token": reauthToken } } : undefined - ); + try { + const res = await apiClient.post<{ message: string }>( + "/gsm/request", + { phone_number: phoneNumber }, + reauthToken ? { headers: { "X-Reauth-Token": reauthToken } } : undefined + ); - apiLog.info("[AuthApi] Response received", { status: res.status }); - return res.data; + apiLog.info("[AuthApi] Response received", { status: res.status }); + return res.data; + } catch (error) { + throw toGsmGatewayError(error); + } }; export const verifyCodePhone = async (code: string) => { @@ -396,10 +387,14 @@ export const fetchTermsContent = async (): Promise => { export const resendVerificationCodePhone = async () => { apiLog.info("[AuthApi] Calling /gsm/resend"); - const res = await apiClient.post<{ message: string }>("/gsm/resend"); + try { + const res = await apiClient.post<{ message: string }>("/gsm/resend"); - apiLog.info("[AuthApi] Response received", { status: res.status }); - return res.data; + apiLog.info("[AuthApi] Response received", { status: res.status }); + return res.data; + } catch (error) { + throw toGsmGatewayError(error); + } }; export const migratePhoneUserApi = async (): Promise<{ diff --git a/mobile-app/sapot-mobile-app/features/auth/auth-container.test.ts b/mobile-app/sapot-mobile-app/features/auth/auth-container.test.ts index 128941bf..6afc3408 100644 --- a/mobile-app/sapot-mobile-app/features/auth/auth-container.test.ts +++ b/mobile-app/sapot-mobile-app/features/auth/auth-container.test.ts @@ -20,6 +20,9 @@ jest.mock("../shared/peer/guest-user-repository", () => ({ jest.mock("../shared/connection/services/user-service", () => ({ UserService: jest.fn().mockImplementation(() => ({ initialize: jest.fn() })), })); +jest.mock("./services/phone-verification-service", () => ({ + PhoneVerificationService: jest.fn().mockImplementation(() => ({})), +})); describe("AuthContainer", () => { it("constructs dependencies", () => { @@ -34,6 +37,8 @@ describe("AuthContainer", () => { expect(shared.UserStore).toHaveBeenCalledTimes(1); expect(shared.GuestUserRepository).toHaveBeenCalledWith(shared.database); expect(shared.UserService).toHaveBeenCalledTimes(1); + const { PhoneVerificationService } = require("./services/phone-verification-service"); + expect(PhoneVerificationService).toHaveBeenCalledTimes(1); expect(container).toBeInstanceOf(AuthContainer); }); diff --git a/mobile-app/sapot-mobile-app/features/auth/auth-container.ts b/mobile-app/sapot-mobile-app/features/auth/auth-container.ts index 2452cd7d..68dc1568 100644 --- a/mobile-app/sapot-mobile-app/features/auth/auth-container.ts +++ b/mobile-app/sapot-mobile-app/features/auth/auth-container.ts @@ -8,6 +8,7 @@ import { SessionStore } from "../shared/core/stores/session-store"; import { UserStore } from "../shared/core/stores/user-store"; import { authLog } from "../shared/core/utils/logger"; import { GuestMigrationService } from "./services/guest-migration-service"; +import { PhoneVerificationService } from "./services/phone-verification-service"; authLog.debug("[auth-container] module loaded"); @@ -17,6 +18,7 @@ export class AuthContainer { readonly peerRepository: PeerRepository; readonly guestUserRepository: GuestUserRepository; readonly guestMigrationService: GuestMigrationService; + readonly phoneVerificationService: PhoneVerificationService; readonly userStore: UserStore; readonly sessionStore: SessionStore; private initPromise?: Promise; @@ -33,6 +35,7 @@ export class AuthContainer { this.guestMigrationService = new GuestMigrationService( this.guestUserRepository ); + this.phoneVerificationService = new PhoneVerificationService(); this.userService = new UserService( this.userStore, diff --git a/mobile-app/sapot-mobile-app/features/auth/hooks/index.ts b/mobile-app/sapot-mobile-app/features/auth/hooks/index.ts index f687661d..9c506750 100644 --- a/mobile-app/sapot-mobile-app/features/auth/hooks/index.ts +++ b/mobile-app/sapot-mobile-app/features/auth/hooks/index.ts @@ -13,4 +13,4 @@ export * from "./use-validate-identifier"; export * from "./use-verify-question"; export * from "./use-verify-recovery-key"; export * from "./use-password-reset-key-recovery"; - +export * from "./use-phone-verification-service"; diff --git a/mobile-app/sapot-mobile-app/features/auth/hooks/use-phone-verification-service.ts b/mobile-app/sapot-mobile-app/features/auth/hooks/use-phone-verification-service.ts new file mode 100644 index 00000000..9b8e22c5 --- /dev/null +++ b/mobile-app/sapot-mobile-app/features/auth/hooks/use-phone-verification-service.ts @@ -0,0 +1,5 @@ +import { useAuthContainer } from "./use-auth-container"; + +export function usePhoneVerificationService() { + return useAuthContainer().phoneVerificationService; +} diff --git a/mobile-app/sapot-mobile-app/features/auth/services/phone-verification-service.ts b/mobile-app/sapot-mobile-app/features/auth/services/phone-verification-service.ts new file mode 100644 index 00000000..bdb44464 --- /dev/null +++ b/mobile-app/sapot-mobile-app/features/auth/services/phone-verification-service.ts @@ -0,0 +1,24 @@ +import { + migratePhoneUserApi, + requestPhoneVerification, + resendVerificationCodePhone, + verifyCodePhone, +} from "@/features/auth/api/auth.api"; + +export class PhoneVerificationService { + requestVerification(phone?: string, reauthToken?: string) { + return requestPhoneVerification(phone, reauthToken); + } + + verifyCode(code: string) { + return verifyCodePhone(code); + } + + resendCode() { + return resendVerificationCodePhone(); + } + + migratePhoneUser() { + return migratePhoneUserApi(); + } +} diff --git a/mobile-app/sapot-mobile-app/features/chat/components/__tests__/message-list.test.tsx b/mobile-app/sapot-mobile-app/features/chat/components/__tests__/message-list.test.tsx index 75b100e2..4431eead 100644 --- a/mobile-app/sapot-mobile-app/features/chat/components/__tests__/message-list.test.tsx +++ b/mobile-app/sapot-mobile-app/features/chat/components/__tests__/message-list.test.tsx @@ -1,6 +1,11 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { render } from "@testing-library/react-native"; +import { fireEvent, render, waitFor } from "@testing-library/react-native"; import React from "react"; +import { GsmGatewayError } from "@/features/shared/core/errors/gsm-error"; + +const mockTryResendMessage = jest.fn(); +const mockUpdateMessageStatus = jest.fn(); +const mockSendSmsToUser = jest.fn(); // Mock withObservables to synchronously unwrap simple observables returned // by the mapping function so the enhanced components receive plain values. @@ -31,7 +36,8 @@ jest.mock("@nozbe/watermelondb/react", () => { jest.mock("@/features/chat/hooks/use-chat-service", () => ({ useChatService: () => ({ - tryResendMessage: jest.fn(), + tryResendMessage: mockTryResendMessage, + updateMessageStatus: mockUpdateMessageStatus, }), })); @@ -49,6 +55,9 @@ jest.mock("@/features/shared/hooks", () => ({ }, }), useReducedMotion: () => false, + useGsmService: () => ({ + sendSmsToUser: mockSendSmsToUser, + }), })); jest.mock("@/features/shared/hooks/use-user-store", () => ({ @@ -68,10 +77,6 @@ jest.mock("@/features/call", () => ({ useInformCall: () => jest.fn(), })); -jest.mock("@/features/shared/core/api/gsm.api", () => ({ - sendSmsToUser: jest.fn().mockResolvedValue({ ok: true }), -})); - // Track query args so tests can assert the newest-first + pagination fix // for issue #142 (chat only showed the oldest 100 messages). const messagesQueryCalls: unknown[][] = []; @@ -103,6 +108,16 @@ jest.mock("@/features/shared", () => { // Newest-first order, matching a `Q.sortBy("created_at", Q.desc)` query. const messagesNewestFirst = [ + { + ...makeMessage("sms-retry", "2024-01-03T00:00:00Z"), + messageType: "sms", + content: "Retry SMS", + sender: { + id: "current-user", + observe: () => of({ username: "Current user" }), + }, + _raw: { sender: "current-user" }, + }, makeMessage("msg-2", "2024-01-02T00:00:00Z"), makeMessage("msg-1", "2024-01-01T00:00:00Z"), ]; @@ -117,7 +132,7 @@ jest.mock("@/features/shared", () => { // Return RxJS observables for observes so the // withObservables HOC receives expected observable inputs. observe: () => (table === "messages" ? of(messagesNewestFirst) : of([])), - observeWithColumns: () => of([{ status: "sent" }]), + observeWithColumns: () => of([{ status: "not_sent" }]), }; }, }; @@ -144,11 +159,18 @@ describe("MessageList", () => { beforeEach(() => { messagesQueryCalls.length = 0; + jest.clearAllMocks(); + mockUpdateMessageStatus.mockResolvedValue(undefined); + mockSendSmsToUser.mockResolvedValue({ ok: true }); }); it("renders messages", async () => { const { findByText } = render( - + ); expect(await findByText(/Content msg-2/)).toBeTruthy(); @@ -158,7 +180,11 @@ describe("MessageList", () => { const { Q } = require("@nozbe/watermelondb"); const { findByText } = render( - + ); await findByText(/Content msg-2/); @@ -169,7 +195,11 @@ describe("MessageList", () => { it("opens a conversation on the newest message", async () => { const { findByText, getAllByText } = render( - + ); await findByText(/Content msg-2/); @@ -178,4 +208,40 @@ describe("MessageList", () => { expect(rendered[0].props.children).toContain("msg-2"); expect(rendered[1].props.children).toContain("msg-1"); }); + + it("shows the queue-full error when manual SMS resend is rejected", async () => { + mockSendSmsToUser.mockRejectedValue( + new GsmGatewayError({ + status: 503, + reason: "QUEUE_FULL", + message: "Outbound SMS queue is full", + }) + ); + const showError = jest.fn(); + const { findByText } = render( + + ); + + fireEvent.press(await findByText("Resend")); + + await waitFor(() => { + expect(showError).toHaveBeenCalledWith( + "SMS service is busy. Please try again shortly." + ); + }); + expect(mockUpdateMessageStatus).toHaveBeenNthCalledWith( + 1, + "sms-retry", + "sending" + ); + expect(mockUpdateMessageStatus).toHaveBeenNthCalledWith( + 2, + "sms-retry", + "not_sent" + ); + }); }); diff --git a/mobile-app/sapot-mobile-app/features/chat/components/message-list.tsx b/mobile-app/sapot-mobile-app/features/chat/components/message-list.tsx index 1a757e1c..c396225f 100644 --- a/mobile-app/sapot-mobile-app/features/chat/components/message-list.tsx +++ b/mobile-app/sapot-mobile-app/features/chat/components/message-list.tsx @@ -24,7 +24,7 @@ import { } from "@/features/shared"; import { MessageType } from "@/features/shared/core/database/model/Message"; import { CallType } from "@/features/shared/core/database/model/Call"; -import { useMainContainer, useReducedMotion } from "@/features/shared/hooks"; +import { useGsmService, useMainContainer, useReducedMotion } from "@/features/shared/hooks"; import { ECDH_PREFIX } from "@/features/chat/repositories/message-repository"; import { useUserStore } from "@/features/shared/hooks/use-user-store"; import { MessageStatusType } from "@/features/shared/core/database/model/MessageStatus"; @@ -33,7 +33,7 @@ import { uiLog } from "@/features/shared/core/utils/logger"; import { useChatService } from "@/features/chat/hooks/use-chat-service"; import { useTheme } from "react-native-paper"; import { useInformCall } from "@/features/call"; -import { sendSmsToUser } from "@/features/shared/core/api/gsm.api"; +import { getGsmErrorMessage } from "@/features/shared/core/errors"; import { toLocalPhone } from "@/features/auth/utils/validation"; uiLog.debug("[message-list] module loaded"); @@ -63,10 +63,12 @@ const MessageListWithData = enhanceMessages( ({ messages, peerId, + showError, onLoadOlderMessages, }: { messages: Message[]; peerId: string; + showError: (message: string) => void; onLoadOlderMessages: () => void; }) => { const hasUserScrolledRef = useRef(false); @@ -91,7 +93,13 @@ const MessageListWithData = enhanceMessages( seenIdsRef.current!.add(item.id); if (reducedMotion || !isNewMessage) { - return ; + return ( + + ); } return ( @@ -100,7 +108,11 @@ const MessageListWithData = enhanceMessages( Easing.bezier(...motion.easing.standard) )} > - + ); }} @@ -129,9 +141,11 @@ const MessageListWithData = enhanceMessages( const MessageList = ({ conversationId, peerId, + showError, }: { conversationId: string; peerId: string; + showError: (message: string) => void; }) => { const [messageLimit, setMessageLimit] = useState(MESSAGE_PAGE_SIZE); @@ -148,6 +162,7 @@ const MessageList = ({ key={conversationId} conversationId={conversationId} peerId={peerId} + showError={showError} messageLimit={messageLimit} onLoadOlderMessages={handleLoadOlderMessages} /> @@ -354,6 +369,7 @@ type MessageListItemProps = { guestSender?: GuestUser | null; status: MessageStatus[]; peerId: string; + showError: (message: string) => void; }; const useDecryptedContent = (message: Message): string => { @@ -379,6 +395,7 @@ const MessageListItemInner = memo( guestSender, status, peerId, + showError, }: MessageListItemProps) => { const statusObj = status?.[0]; const senderName = getSenderName(sender ?? guestSender); @@ -387,6 +404,7 @@ const MessageListItemInner = memo( const theme = useTheme(); const isCurrentUserMessage = message.sender?.id === userStore.user?.id; const chatService = useChatService(); + const gsmService = useGsmService(); const peerService = usePeerService(); const [isResending, setIsResending] = useState(false); const { callRepository } = useMainContainer(); @@ -412,7 +430,7 @@ const MessageListItemInner = memo( try { if (message.messageType === MessageType.SMS) { await chatService.updateMessageStatus(message.id, MessageStatusType.SENDING); - const res = await sendSmsToUser(peerId, content); + const res = await gsmService.sendSmsToUser(peerId, content); const status = res.ok ? MessageStatusType.DELIVERED : MessageStatusType.NOT_SENT; @@ -431,6 +449,9 @@ const MessageListItemInner = memo( uiLog.warn("[message-list] resend failed", { peerId, err }); if (message.messageType === MessageType.SMS) { await chatService.updateMessageStatus(message.id, MessageStatusType.NOT_SENT).catch((error) => uiLog.warn("[message-list] reset message status failed", { error })); + showError( + getGsmErrorMessage(err, "SMS could not be delivered. Please try again.") + ); } } finally { setIsResending(false); diff --git a/mobile-app/sapot-mobile-app/features/chat/hooks/use-send-message.test.ts b/mobile-app/sapot-mobile-app/features/chat/hooks/use-send-message.test.ts index 462dcc67..4c191cf4 100644 --- a/mobile-app/sapot-mobile-app/features/chat/hooks/use-send-message.test.ts +++ b/mobile-app/sapot-mobile-app/features/chat/hooks/use-send-message.test.ts @@ -1,8 +1,11 @@ import { renderHook, act } from "@testing-library/react-native"; +import { GsmGatewayError } from "@/features/shared/core/errors/gsm-error"; import { useSendMessage } from "./use-send-message"; -jest.mock("@/features/shared/core/api/gsm.api", () => ({ - sendSmsToUser: jest.fn().mockResolvedValue({ ok: true }), +const mockSendSmsToUser = jest.fn(); + +jest.mock("@/features/shared/hooks", () => ({ + useGsmService: () => ({ sendSmsToUser: mockSendSmsToUser }), })); jest.mock("@/features/shared/core/utils/logger", () => ({ @@ -44,6 +47,15 @@ function makeParams(overrides: Record = {}) { } describe("useSendMessage", () => { + beforeEach(() => { + jest.clearAllMocks(); + mockSendSmsToUser.mockResolvedValue({ + ok: true, + msg_id: "sms-log-id", + to: "+639171234567", + }); + }); + it("returns synchronously before sendChatMessage resolves", () => { let resolveDeferred!: (v: { conversationId: string; messageId: string }) => void; const deferred = new Promise<{ conversationId: string; messageId: string }>( @@ -137,4 +149,34 @@ describe("useSendMessage", () => { expect(params.showError).toHaveBeenCalledWith("Failed to send message"); }); + + it("shows a busy message and marks SMS not sent when the queue is full", async () => { + mockSendSmsToUser.mockRejectedValue( + new GsmGatewayError({ + status: 503, + reason: "QUEUE_FULL", + message: "Outbound SMS queue is full", + messageId: "sms-log-id", + }) + ); + const params = makeParams({ isSmsMode: true }); + const { result } = renderHook(() => useSendMessage(params)); + + act(() => { + result.current(); + }); + + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + + expect(params.chatService.updateMessageStatus).toHaveBeenCalledWith( + "sms-1", + "not_sent" + ); + expect(params.showError).toHaveBeenCalledWith( + "SMS service is busy. Please try again shortly." + ); + }); }); diff --git a/mobile-app/sapot-mobile-app/features/chat/hooks/use-send-message.ts b/mobile-app/sapot-mobile-app/features/chat/hooks/use-send-message.ts index 02fdac21..96fdce03 100644 --- a/mobile-app/sapot-mobile-app/features/chat/hooks/use-send-message.ts +++ b/mobile-app/sapot-mobile-app/features/chat/hooks/use-send-message.ts @@ -1,8 +1,9 @@ import { useCallback } from "react"; -import { sendSmsToUser } from "@/features/shared/core/api/gsm.api"; import { MessageStatusType } from "@/features/shared/core/database/model/MessageStatus"; +import { getGsmErrorMessage } from "@/features/shared/core/errors"; import { uiLog } from "@/features/shared/core/utils/logger"; import { ChatService } from "@/features/chat/services/chat-service"; +import { useGsmService } from "@/features/shared/hooks"; type Params = { message: string; @@ -27,6 +28,8 @@ export function useSendMessage({ peerId, showError, }: Params): () => void { + const gsmService = useGsmService(); + return useCallback(() => { const textToSend = message.trim(); if (!textToSend) return; @@ -42,7 +45,7 @@ export function useSendMessage({ void chatService .sendSmsChannelMessage(textToSend) .then(({ messageId: smsMessageId }) => { - sendSmsToUser(peerId, textToSend) + gsmService.sendSmsToUser(peerId, textToSend) .then((res) => { const status = res.ok ? MessageStatusType.DELIVERED @@ -50,11 +53,16 @@ export function useSendMessage({ chatService.updateMessageStatus(smsMessageId, status).catch((error) => uiLog.warn("use-send-message › SMS status update failed", { error })); if (!res.ok) showError("SMS could not be delivered"); }) - .catch(() => { + .catch((error) => { chatService .updateMessageStatus(smsMessageId, MessageStatusType.NOT_SENT) .catch((error) => uiLog.warn("use-send-message › SMS not_sent status update failed", { error })); - showError("Message sent, but SMS delivery failed."); + showError( + getGsmErrorMessage( + error, + "Message sent, but SMS delivery failed." + ) + ); }); }) .catch((error) => { @@ -85,5 +93,6 @@ export function useSendMessage({ isSmsConversation, peerId, showError, + gsmService, ]); } diff --git a/mobile-app/sapot-mobile-app/features/shared/__tests__/main-container-initialize.test.ts b/mobile-app/sapot-mobile-app/features/shared/__tests__/main-container-initialize.test.ts index cbed3f1f..94cc3a1b 100644 --- a/mobile-app/sapot-mobile-app/features/shared/__tests__/main-container-initialize.test.ts +++ b/mobile-app/sapot-mobile-app/features/shared/__tests__/main-container-initialize.test.ts @@ -97,6 +97,7 @@ jest.mock("../connection/services", () => ({ republish: jest.fn().mockResolvedValue(undefined), destroy: jest.fn().mockResolvedValue(undefined), })), + GsmService: jest.fn().mockImplementation(() => ({})), SignalingService: jest.fn().mockImplementation(() => ({})), WebrtcSessionManager: jest.fn().mockImplementation(() => ({ getWebrtcAdapter: jest.fn(), diff --git a/mobile-app/sapot-mobile-app/features/shared/connection/services/gsm-service.ts b/mobile-app/sapot-mobile-app/features/shared/connection/services/gsm-service.ts new file mode 100644 index 00000000..b194df05 --- /dev/null +++ b/mobile-app/sapot-mobile-app/features/shared/connection/services/gsm-service.ts @@ -0,0 +1,19 @@ +import { + contactUnknownUser, + getGsmHealth, + sendSmsToUser, +} from "@/features/shared/core/api/gsm.api"; + +export class GsmService { + getHealth() { + return getGsmHealth(); + } + + sendSmsToUser(userId: string, message: string) { + return sendSmsToUser(userId, message); + } + + contactUnknownUser(targetPhoneNumber: string) { + return contactUnknownUser(targetPhoneNumber); + } +} diff --git a/mobile-app/sapot-mobile-app/features/shared/connection/services/index.ts b/mobile-app/sapot-mobile-app/features/shared/connection/services/index.ts index 0ced7d7b..cc9a1c0b 100644 --- a/mobile-app/sapot-mobile-app/features/shared/connection/services/index.ts +++ b/mobile-app/sapot-mobile-app/features/shared/connection/services/index.ts @@ -6,6 +6,7 @@ export { CallMediaService } from "./call-media-service"; export * from "./clean-up-service"; export { ConnectionService } from "./connection-service"; export { DiscoveryService } from "./discovery-service"; +export { GsmService } from "./gsm-service"; export { NotificationService } from "./notification-service"; export * from "./service-interfaces"; export { SignalingService } from "./signaling-service"; diff --git a/mobile-app/sapot-mobile-app/features/shared/core/api/__tests__/gsm.api.test.ts b/mobile-app/sapot-mobile-app/features/shared/core/api/__tests__/gsm.api.test.ts new file mode 100644 index 00000000..9b8ecf10 --- /dev/null +++ b/mobile-app/sapot-mobile-app/features/shared/core/api/__tests__/gsm.api.test.ts @@ -0,0 +1,51 @@ +import { GsmGatewayError } from "../../errors/gsm-error"; +import { apiClient } from "../client"; +import { contactUnknownUser, sendSmsToUser } from "../gsm.api"; + +jest.mock("../client", () => ({ + apiClient: { + post: jest.fn(), + }, +})); + +const mockedApiClient = apiClient as jest.Mocked; + +const queueFullError = { + response: { + status: 503, + data: { + detail: { + message: "Outbound SMS queue is full", + reason: "QUEUE_FULL", + msg_id: "sms-log-id", + }, + }, + }, +}; + +describe("GSM API", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("wraps send saturation as a typed GSM gateway error", async () => { + mockedApiClient.post.mockRejectedValue(queueFullError); + + await expect(sendSmsToUser("user-id", "message")).rejects.toMatchObject({ + name: "GsmGatewayError", + status: 503, + reason: "QUEUE_FULL", + messageId: "sms-log-id", + } satisfies Partial); + }); + + it("wraps first-contact saturation as a typed GSM gateway error", async () => { + mockedApiClient.post.mockRejectedValue(queueFullError); + + await expect(contactUnknownUser("+639171234567")).rejects.toMatchObject({ + name: "GsmGatewayError", + status: 503, + reason: "QUEUE_FULL", + } satisfies Partial); + }); +}); diff --git a/mobile-app/sapot-mobile-app/features/shared/core/api/gsm.api.ts b/mobile-app/sapot-mobile-app/features/shared/core/api/gsm.api.ts index 3688f5f9..5b2faa7f 100644 --- a/mobile-app/sapot-mobile-app/features/shared/core/api/gsm.api.ts +++ b/mobile-app/sapot-mobile-app/features/shared/core/api/gsm.api.ts @@ -1,4 +1,5 @@ import { apiClient } from "@/features/shared/core/api/client"; +import { toGsmGatewayError } from "@/features/shared/core/errors"; import { apiLog } from "@/features/shared/core/utils/logger"; export type GsmHealthResponse = { @@ -32,20 +33,28 @@ export const sendSmsToUser = async ( message: string ): Promise => { apiLog.debug("api › gsm sms send", { userId }); - const res = await apiClient.post("/gsm/sms/send", null, { - params: { user_id: userId, message }, - }); - return res.data; + try { + const res = await apiClient.post("/gsm/sms/send", null, { + params: { user_id: userId, message }, + }); + return res.data; + } catch (error) { + throw toGsmGatewayError(error); + } }; export const contactUnknownUser = async ( targetPhoneNumber: string ): Promise => { apiLog.debug("api › gsm contact unknown user", { targetPhoneNumber }); - const res = await apiClient.post( - "/gsm/contact-unknown-user", - null, - { params: { target_phone_number: targetPhoneNumber } } - ); - return res.data; + try { + const res = await apiClient.post( + "/gsm/contact-unknown-user", + null, + { params: { target_phone_number: targetPhoneNumber } } + ); + return res.data; + } catch (error) { + throw toGsmGatewayError(error); + } }; diff --git a/mobile-app/sapot-mobile-app/features/shared/core/errors/__tests__/gsm-error.test.ts b/mobile-app/sapot-mobile-app/features/shared/core/errors/__tests__/gsm-error.test.ts new file mode 100644 index 00000000..f2b1e5d8 --- /dev/null +++ b/mobile-app/sapot-mobile-app/features/shared/core/errors/__tests__/gsm-error.test.ts @@ -0,0 +1,72 @@ +import { getGsmErrorMessage, getGsmFailure } from "../gsm-error"; + +describe("GSM gateway errors", () => { + it("recognizes a queue saturation response", () => { + const error = { + response: { + status: 503, + data: { + detail: { + message: "Outbound SMS queue is full", + reason: "QUEUE_FULL", + msg_id: "sms-log-id", + }, + }, + }, + }; + + expect(getGsmFailure(error)).toEqual({ + status: 503, + reason: "QUEUE_FULL", + message: "Outbound SMS queue is full", + messageId: "sms-log-id", + }); + expect(getGsmErrorMessage(error, "fallback")).toBe( + "SMS service is busy. Please try again shortly." + ); + }); + + it("recognizes a service shutdown response", () => { + const error = { + response: { + status: 503, + data: { + detail: { + message: "SMS service is stopping", + reason: "SERVICE_STOPPING", + msg_id: "sms-log-id", + }, + }, + }, + }; + + expect(getGsmErrorMessage(error, "fallback")).toBe( + "SMS service is restarting. Please try again shortly." + ); + }); + + it("recognizes an unavailable modem response", () => { + const error = { + response: { + status: 503, + data: { detail: "GSM modem not ready" }, + }, + }; + + expect(getGsmFailure(error)).toEqual({ + status: 503, + reason: "GATEWAY_UNAVAILABLE", + message: "GSM modem not ready", + }); + expect(getGsmErrorMessage(error, "fallback")).toBe( + "SMS service is unavailable. Please try again later." + ); + }); + + it("uses the supplied fallback for unrelated errors", () => { + expect(getGsmFailure(new Error("network failed"))).toBeUndefined(); + expect(getGsmErrorMessage(new Error("network failed"), "fallback")).toBe( + "fallback" + ); + }); +}); diff --git a/mobile-app/sapot-mobile-app/features/shared/core/errors/gsm-error.ts b/mobile-app/sapot-mobile-app/features/shared/core/errors/gsm-error.ts new file mode 100644 index 00000000..ede2798d --- /dev/null +++ b/mobile-app/sapot-mobile-app/features/shared/core/errors/gsm-error.ts @@ -0,0 +1,96 @@ +import { AppError } from "./app-error"; +import { toAppError } from "./to-app-error"; + +export interface GsmFailure { + status: number; + reason: string; + message: string; + messageId?: string; +} + +export class GsmGatewayError extends AppError { + readonly status: number; + readonly reason: string; + readonly messageId?: string; + + constructor(failure: GsmFailure, cause?: unknown) { + super(failure.message, "network", "medium", cause); + this.name = "GsmGatewayError"; + this.status = failure.status; + this.reason = failure.reason; + this.messageId = failure.messageId; + } +} + +function asRecord(value: unknown): Record | undefined { + return typeof value === "object" && value !== null + ? (value as Record) + : undefined; +} + +export function getGsmFailure(error: unknown): GsmFailure | undefined { + if (error instanceof GsmGatewayError) { + return { + status: error.status, + reason: error.reason, + message: error.message, + messageId: error.messageId, + }; + } + + const response = asRecord(asRecord(error)?.response); + const status = response?.status; + const data = asRecord(response?.data); + const detail = data?.detail; + const detailRecord = asRecord(detail); + + if (typeof status !== "number") return undefined; + + if (detailRecord) { + const reason = detailRecord.reason; + const message = detailRecord.message; + const messageId = detailRecord.msg_id; + if (typeof reason !== "string" || typeof message !== "string") { + return undefined; + } + return { + status, + reason, + message, + messageId: typeof messageId === "string" ? messageId : undefined, + }; + } + + if (status === 503 && typeof detail === "string") { + return { + status, + reason: "GATEWAY_UNAVAILABLE", + message: detail, + }; + } + + return undefined; +} + +export function toGsmGatewayError(error: unknown): AppError { + const failure = getGsmFailure(error); + return failure + ? new GsmGatewayError(failure, error) + : toAppError(error, "network"); +} + +export function getGsmErrorMessage(error: unknown, fallback: string): string { + const failure = getGsmFailure(error); + if (!failure) return fallback; + + if (failure.reason === "QUEUE_FULL") { + return "SMS service is busy. Please try again shortly."; + } + if (failure.reason === "SERVICE_STOPPING") { + return "SMS service is restarting. Please try again shortly."; + } + if (failure.status === 503) { + return "SMS service is unavailable. Please try again later."; + } + return fallback; +} diff --git a/mobile-app/sapot-mobile-app/features/shared/core/errors/index.ts b/mobile-app/sapot-mobile-app/features/shared/core/errors/index.ts index 9042aeb5..58fa5dd3 100644 --- a/mobile-app/sapot-mobile-app/features/shared/core/errors/index.ts +++ b/mobile-app/sapot-mobile-app/features/shared/core/errors/index.ts @@ -4,3 +4,10 @@ export { toAppError } from "./to-app-error"; export { KeyInitError, toKeyInitError } from "./key-init-error"; export type { KeyInitErrorCode } from "./key-init-error"; export { captureAppError } from "./sentry-capture"; +export { + GsmGatewayError, + getGsmErrorMessage, + getGsmFailure, + toGsmGatewayError, +} from "./gsm-error"; +export type { GsmFailure } from "./gsm-error"; diff --git a/mobile-app/sapot-mobile-app/features/shared/hooks/index.ts b/mobile-app/sapot-mobile-app/features/shared/hooks/index.ts index 2392fadc..333041d6 100644 --- a/mobile-app/sapot-mobile-app/features/shared/hooks/index.ts +++ b/mobile-app/sapot-mobile-app/features/shared/hooks/index.ts @@ -8,6 +8,7 @@ export * from "./use-connection-service"; export * from "./use-dialog-visibility"; export * from "./use-discovery-service"; export * from "./use-foreground-sync"; +export * from "./use-gsm-service"; export * from "./use-health-poll"; export * from "./use-loading-overlay"; export * from "./use-main-container"; @@ -24,4 +25,3 @@ export * from "./use-user-profile"; export * from "./use-user-search"; export * from "./use-user-store"; export * from "./use-zeroconf-published"; - diff --git a/mobile-app/sapot-mobile-app/features/shared/hooks/use-gsm-health.ts b/mobile-app/sapot-mobile-app/features/shared/hooks/use-gsm-health.ts index d310ded3..0534fb85 100644 --- a/mobile-app/sapot-mobile-app/features/shared/hooks/use-gsm-health.ts +++ b/mobile-app/sapot-mobile-app/features/shared/hooks/use-gsm-health.ts @@ -1,11 +1,12 @@ -import { getGsmHealth } from "@/features/shared/core/api/gsm.api"; import { hookLog } from "@/features/shared/core/utils/logger"; import { useEffect, useState } from "react"; +import { useGsmService } from "./use-gsm-service"; hookLog.debug("[use-gsm-health] module loaded"); const GSM_POLL_INTERVAL_MS = 30_000; export function useGsmHealth(): { gsmReady: boolean; loading: boolean } { + const gsmService = useGsmService(); const [gsmReady, setGsmReady] = useState(false); const [loading, setLoading] = useState(true); @@ -14,7 +15,7 @@ export function useGsmHealth(): { gsmReady: boolean; loading: boolean } { const check = (isInitial: boolean) => { if (isInitial) setLoading(true); - getGsmHealth() + gsmService.getHealth() .then((res) => { if (!cancelled) setGsmReady(res.gsm_ready === true); }) .catch(() => { if (!cancelled) setGsmReady(false); }) .finally(() => { if (isInitial && !cancelled) setLoading(false); }); @@ -27,7 +28,7 @@ export function useGsmHealth(): { gsmReady: boolean; loading: boolean } { cancelled = true; clearInterval(id); }; - }, []); + }, [gsmService]); return { gsmReady, loading }; } diff --git a/mobile-app/sapot-mobile-app/features/shared/hooks/use-gsm-service.ts b/mobile-app/sapot-mobile-app/features/shared/hooks/use-gsm-service.ts new file mode 100644 index 00000000..d5ece30b --- /dev/null +++ b/mobile-app/sapot-mobile-app/features/shared/hooks/use-gsm-service.ts @@ -0,0 +1,5 @@ +import { useMainContainer } from "./use-main-container"; + +export function useGsmService() { + return useMainContainer().gsmService; +} diff --git a/mobile-app/sapot-mobile-app/features/shared/main-container.ts b/mobile-app/sapot-mobile-app/features/shared/main-container.ts index a0b1fc4e..aee7c726 100644 --- a/mobile-app/sapot-mobile-app/features/shared/main-container.ts +++ b/mobile-app/sapot-mobile-app/features/shared/main-container.ts @@ -14,6 +14,7 @@ import { CleanUpService, ConnectionService, DiscoveryService, + GsmService, NotificationService, SignalingService, WebrtcSessionManager, @@ -79,6 +80,7 @@ export class MainContainer { readonly zeroconfAdapter: ZeroconfAdapter; readonly networkConfig: NetworkConfig; readonly discoveryService: DiscoveryService; + readonly gsmService: GsmService; readonly tcpServerAdapter: TcpServerAdapter; readonly webrtcSessionManager: WebrtcSessionManager; readonly signalingService: SignalingService; @@ -126,6 +128,7 @@ export class MainContainer { this.appModeStore = appModeStore; this.networkConfig = new NetworkConfig(); + this.gsmService = new GsmService(); this.localEncryptionService = new LocalEncryptionService({ getPassword: () => _pendingRawPassword, diff --git a/server/app/api/gsm.py b/server/app/api/gsm.py index 902bb71f..e630c72a 100644 --- a/server/app/api/gsm.py +++ b/server/app/api/gsm.py @@ -4,6 +4,7 @@ from uuid import UUID, uuid4, uuid5 from fastapi import Depends, HTTPException, Query, Request from fastapi.routing import APIRouter +from fastapi.responses import JSONResponse import time from pydantic import BaseModel @@ -22,8 +23,15 @@ import httpx import json +# The gateway allows 60 seconds before write, a 5-second serial write, and a +# fresh 60-second confirmation window. The proxy adds 10 seconds for HTTP overhead. +GSM_GATEWAY_WORST_CASE_SECONDS = 60.0 + 5.0 + 60.0 +GSM_PROXY_READ_TIMEOUT_SECONDS = GSM_GATEWAY_WORST_CASE_SECONDS + 10.0 +GSM_GATEWAY_MAX_ADMITTED_REQUESTS = 21 +GSM_PROXY_MAX_CONNECTIONS = GSM_GATEWAY_MAX_ADMITTED_REQUESTS + 1 +GSM_PROXY_POOL_TIMEOUT_SECONDS = 1.0 + # Module-level client reuses TCP connections to localhost:8001 across requests. -# The 120s timeout matches the SMS send worst-case; health checks are much faster. _gsm_http_client: httpx.AsyncClient | None = None logger = logging.getLogger("app") @@ -37,8 +45,16 @@ def _get_gsm_client() -> httpx.AsyncClient: if _gsm_http_client is None or _gsm_http_client.is_closed: _gsm_http_client = httpx.AsyncClient( base_url="http://localhost:8001", - timeout=120.0, - limits=httpx.Limits(max_connections=4, max_keepalive_connections=2), + timeout=httpx.Timeout( + connect=5.0, + read=GSM_PROXY_READ_TIMEOUT_SECONDS, + write=5.0, + pool=GSM_PROXY_POOL_TIMEOUT_SECONDS, + ), + limits=httpx.Limits( + max_connections=GSM_PROXY_MAX_CONNECTIONS, + max_keepalive_connections=4, + ), ) return _gsm_http_client @@ -52,7 +68,7 @@ def _gsm_log_extra(current_user: User | None, path: str) -> dict: } -async def _get_gsm_health(path: str, current_user: User) -> dict: +async def _get_gsm_health(path: str, current_user: User) -> dict | JSONResponse: try: response = await _get_gsm_client().get(path) except httpx.RequestError as exc: @@ -63,7 +79,10 @@ async def _get_gsm_health(path: str, current_user: User) -> dict: ) raise HTTPException(status_code=503, detail="GSM gateway is unavailable") from exc - return response.json() + payload = response.json() + if response.status_code >= 400: + return JSONResponse(status_code=response.status_code, content=payload) + return payload class InboundSMSPayload(BaseModel): @@ -72,6 +91,46 @@ class InboundSMSPayload(BaseModel): body: str +class GsmFailureDetail(BaseModel): + message: str + reason: str + msg_id: str | None = None + + +class GsmFailureResponse(BaseModel): + detail: GsmFailureDetail + + +class GsmHealthResponse(BaseModel): + status: str + gsm_ready: bool + connected: bool + detail: str + + +class GsmHealthUnavailableResponse(BaseModel): + detail: str + + +GSM_SEND_ERROR_RESPONSES = { + 502: { + "model": GsmFailureResponse, + "description": "The modem rejected the SMS or delivery confirmation timed out.", + }, + 503: { + "model": GsmFailureResponse | GsmHealthUnavailableResponse, + "description": "The GSM gateway is unavailable, stopping, or at queue capacity.", + }, +} + +GSM_HEALTH_ERROR_RESPONSES = { + 503: { + "model": GsmHealthResponse | GsmHealthUnavailableResponse, + "description": "The GSM gateway is unavailable or reports degraded health.", + } +} + + def _gsm_secret_ok(request: Request) -> bool: return request.headers.get("X-GSM-Secret") == GSM_SECRET @@ -202,14 +261,14 @@ async def inbound_sms( return {"ok": True, "message_id": str(msg.id)} -@router.get("/health") +@router.get("/health", responses=GSM_HEALTH_ERROR_RESPONSES) async def gsm_health( current_user : Annotated[User, Depends(get_current_user)], ): return await _get_gsm_health("/health", current_user) -@router.get("/health/detailed") +@router.get("/health/detailed", responses=GSM_HEALTH_ERROR_RESPONSES) async def gsm_health_detailed( current_user : Annotated[User, Depends(get_current_user_admin)], ): @@ -236,7 +295,7 @@ async def gsm_messages( response = await client.get("/sms/messages", params=params) return response.json() -@router.post("/sms/send") +@router.post("/sms/send", responses=GSM_SEND_ERROR_RESPONSES) async def send_sms( current_user : Annotated[User, Depends(get_current_user)], user_id: UUID, @@ -260,13 +319,25 @@ async def send_sms( async def sendToModule(phone_number: str, message: str): client = _get_gsm_client() - response = await client.post( - "/sms/send", - json={"number": phone_number, "body": f"FROM {phone_number}: " + message}, - ) - return response.json() - -@router.post("/request") + try: + response = await client.post( + "/sms/send", + json={"number": phone_number, "body": f"FROM {phone_number}: " + message}, + headers={"X-GSM-Secret": GSM_SECRET}, + ) + except httpx.RequestError as exc: + raise HTTPException(status_code=503, detail={ + "message": "GSM gateway is unavailable", + "reason": "GATEWAY_UNAVAILABLE", + }) from exc + + payload = response.json() + if response.status_code >= 400: + detail = payload.get("detail", payload) if isinstance(payload, dict) else payload + raise HTTPException(status_code=response.status_code, detail=detail) + return payload + +@router.post("/request", responses=GSM_SEND_ERROR_RESPONSES) async def request_phone_verification( data: RequestPhoneVerification, request: Request, @@ -402,7 +473,7 @@ def verify_phone_code( # RESEND CODE # ============================================================================= -@router.post("/resend") +@router.post("/resend", responses=GSM_SEND_ERROR_RESPONSES) async def resend_phone_code( current_user : Annotated[User, Depends(get_current_user)], session: SessionDep @@ -574,7 +645,7 @@ def sms_conversation_id(user_id_a: str, user_id_b: str) -> str: -@router.post("/contact-unknown-user") +@router.post("/contact-unknown-user", responses=GSM_SEND_ERROR_RESPONSES) async def contact_unknown_user( current_user : Annotated[User, Depends(get_current_user)], target_phone_number: Annotated[str, Query(pattern=r"^\+639\d{9}$")], diff --git a/server/app/api/testing.py b/server/app/api/testing.py index 84b9d968..c9813419 100644 --- a/server/app/api/testing.py +++ b/server/app/api/testing.py @@ -39,6 +39,7 @@ "qa_large", "qa_banned", "qa_locked", + "qa_phone_verified", "qa_gps", "qa_map_user", "qa_map_user_2", diff --git a/server/app/db_operations/qa_scenarios.py b/server/app/db_operations/qa_scenarios.py index 09f1efdd..5a81a25d 100644 --- a/server/app/db_operations/qa_scenarios.py +++ b/server/app/db_operations/qa_scenarios.py @@ -28,6 +28,7 @@ from app.models.location import UserLocation from app.models.login_attempt import LoginAttempt from app.models.message import Message, MessageType +from app.models.phone_verification import PhoneVerified from app.models.rescuer import Rescuer from app.models.users import User @@ -423,6 +424,15 @@ def build_locked_out(session: Session) -> dict: return {"user": user.username, "locked_until": locked_until.isoformat()} +def build_verified_phone(session: Session) -> dict: + user = get_or_create_user(session, "qa_phone_verified", phone_number="+639300000751") + verified = session.exec(select(PhoneVerified).where(PhoneVerified.user_id == user.id)).first() + if not verified: + session.add(PhoneVerified(user_id=user.id)) + session.commit() + return {"user": user.username, "phone_verified": True} + + _ANNOUNCEMENT_PRIORITIES = (PriorityType.low, PriorityType.normal, PriorityType.high) _ANNOUNCEMENT_AUDIENCES = (AudienceType.user, AudienceType.rescuer, AudienceType.admin) @@ -592,6 +602,10 @@ class Scenario(NamedTuple): "qa_locked with a LoginAttempt row at attempt_count=5, locked_until +6h.", build_locked_out, ), + "verified-phone": Scenario( + "qa_phone_verified with a verified Philippine phone number.", + build_verified_phone, + ), "announcements": Scenario( "Active + expired announcements across all 3 priorities x 3 audiences (18 rows).", build_announcements, diff --git a/server/app/tests/test_gsm_health.py b/server/app/tests/test_gsm_health.py index 6e05b23a..45d6cf11 100644 --- a/server/app/tests/test_gsm_health.py +++ b/server/app/tests/test_gsm_health.py @@ -12,6 +12,19 @@ async def get(self, path: str): raise httpx.ConnectError("All connection attempts failed") +class DegradedGsmClient: + async def get(self, path: str): + return httpx.Response( + 503, + json={ + "status": "degraded", + "gsm_ready": False, + "connected": True, + "detail": "network unavailable", + }, + ) + + def test_gsm_health_reports_unavailable_gateway(client, monkeypatch, caplog): monkeypatch.setattr(gsm, "_get_gsm_client", lambda: UnavailableGsmClient()) monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: None) @@ -26,3 +39,18 @@ def test_gsm_health_reports_unavailable_gateway(client, monkeypatch, caplog): assert log_record.user_id == "ANONYMOUS" assert log_record.action == "gsm_health_unavailable" assert log_record.metadata_json == {"path": "/health"} + + +def test_gsm_health_preserves_degraded_gateway_status(client, monkeypatch): + monkeypatch.setattr(gsm, "_get_gsm_client", lambda: DegradedGsmClient()) + monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: None) + + response = client.get("/gsm/health") + + assert response.status_code == 503 + assert response.json() == { + "status": "degraded", + "gsm_ready": False, + "connected": True, + "detail": "network unavailable", + } diff --git a/server/app/tests/test_gsm_proxy.py b/server/app/tests/test_gsm_proxy.py new file mode 100644 index 00000000..65738769 --- /dev/null +++ b/server/app/tests/test_gsm_proxy.py @@ -0,0 +1,247 @@ +import asyncio + +import httpx + +from app.api import gsm +from app.db_operations.token import get_current_user +from app.main import app +from app.models.phone_verification import PhoneVerification, now_ms +from app.models.users import User +from sqlmodel import select + + +QUEUE_FULL_RESPONSE = { + "detail": { + "message": "Outbound SMS queue is full", + "reason": "QUEUE_FULL", + "msg_id": "sms-log-id", + } +} + +SERVICE_STOPPING_RESPONSE = { + "detail": { + "message": "SMS service is stopping", + "reason": "SERVICE_STOPPING", + "msg_id": "sms-log-id", + } +} + + +class FakeGsmResponse: + def __init__(self, status_code: int, payload: dict): + self.status_code = status_code + self._payload = payload + + @property + def is_error(self) -> bool: + return self.status_code >= 400 + + def json(self) -> dict: + return self._payload + + +class SaturatedGsmClient: + async def post(self, path: str, json: dict, **kwargs): + return FakeGsmResponse(503, QUEUE_FULL_RESPONSE) + + +class StoppingGsmClient: + async def post(self, path: str, json: dict, **kwargs): + return FakeGsmResponse(503, SERVICE_STOPPING_RESPONSE) + + +class UnavailableGsmClient: + async def post(self, path: str, json: dict, **kwargs): + raise httpx.ConnectError("All connection attempts failed") + + +class ModemNotReadyGsmClient: + async def post(self, path: str, json: dict, **kwargs): + return FakeGsmResponse(503, {"detail": "GSM modem not ready"}) + + +class PoolExhaustedGsmClient: + async def post(self, path: str, json: dict, **kwargs): + raise httpx.PoolTimeout("GSM proxy connection pool is full") + + +def _authenticated_user(session): + return session.exec(select(User)).first() + + +def test_proxy_capacity_and_timeouts_cover_gateway_contract(monkeypatch): + captured = {} + + class CapturingClient: + is_closed = False + + def __init__(self, **kwargs): + captured.update(kwargs) + + monkeypatch.setattr(gsm, "_gsm_http_client", None) + monkeypatch.setattr(gsm.httpx, "AsyncClient", CapturingClient) + + gsm._get_gsm_client() + + timeout = captured["timeout"] + limits = captured["limits"] + assert timeout.read == gsm.GSM_PROXY_READ_TIMEOUT_SECONDS + assert timeout.read > gsm.GSM_GATEWAY_WORST_CASE_SECONDS + assert timeout.pool == gsm.GSM_PROXY_POOL_TIMEOUT_SECONDS + assert limits.max_connections == gsm.GSM_PROXY_MAX_CONNECTIONS + assert limits.max_connections > gsm.GSM_GATEWAY_MAX_ADMITTED_REQUESTS + + +def test_send_to_module_authenticates_with_shared_secret(monkeypatch): + captured = {} + + class CapturingClient: + async def post(self, path: str, **kwargs): + captured["path"] = path + captured.update(kwargs) + return FakeGsmResponse(200, {"ok": True}) + + monkeypatch.setattr(gsm, "_get_gsm_client", lambda: CapturingClient()) + + result = asyncio.run(gsm.sendToModule("+639171234567", "message")) + + assert result == {"ok": True} + assert captured["path"] == "/sms/send" + assert captured["headers"] == {"X-GSM-Secret": gsm.GSM_SECRET} + + +def test_send_sms_preserves_queue_full_status(client, session, monkeypatch): + current_user = _authenticated_user(session) + target = session.exec(select(User).where(User.id != current_user.id)).first() + monkeypatch.setattr(gsm, "_get_gsm_client", lambda: SaturatedGsmClient()) + monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) + + response = client.post( + "/gsm/sms/send", + params={"user_id": str(target.id), "message": "Help is on the way"}, + ) + + assert response.status_code == 503 + assert response.json() == QUEUE_FULL_RESPONSE + + +def test_send_sms_preserves_service_stopping_status(client, session, monkeypatch): + current_user = _authenticated_user(session) + target = session.exec(select(User).where(User.id != current_user.id)).first() + monkeypatch.setattr(gsm, "_get_gsm_client", lambda: StoppingGsmClient()) + monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) + + response = client.post( + "/gsm/sms/send", + params={"user_id": str(target.id), "message": "Help is on the way"}, + ) + + assert response.status_code == 503 + assert response.json() == SERVICE_STOPPING_RESPONSE + + +def test_phone_verification_preserves_queue_full_status(client, session, monkeypatch): + current_user = _authenticated_user(session) + monkeypatch.setattr(gsm, "_get_gsm_client", lambda: SaturatedGsmClient()) + monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) + + response = client.post( + "/gsm/request", + json={"phone_number": current_user.phone_number}, + ) + + assert response.status_code == 503 + assert response.json() == QUEUE_FULL_RESPONSE + + +def test_phone_verification_resend_preserves_queue_full_status( + client, session, monkeypatch +): + current_user = _authenticated_user(session) + session.add( + PhoneVerification( + user_id=current_user.id, + phone_number=current_user.phone_number, + verification_code="123456", + expires_at=now_ms() + 300_000, + ) + ) + session.commit() + monkeypatch.setattr(gsm, "_get_gsm_client", lambda: SaturatedGsmClient()) + monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) + + response = client.post("/gsm/resend") + + assert response.status_code == 503 + assert response.json() == QUEUE_FULL_RESPONSE + + +def test_contact_unknown_user_preserves_queue_full_status(client, session, monkeypatch): + current_user = _authenticated_user(session) + monkeypatch.setattr(gsm, "_get_gsm_client", lambda: SaturatedGsmClient()) + monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) + + response = client.post( + "/gsm/contact-unknown-user", + params={"target_phone_number": "+639991234567"}, + ) + + assert response.status_code == 503 + assert response.json() == QUEUE_FULL_RESPONSE + + +def test_send_sms_reports_unavailable_gateway(client, session, monkeypatch): + current_user = _authenticated_user(session) + target = session.exec(select(User).where(User.id != current_user.id)).first() + monkeypatch.setattr(gsm, "_get_gsm_client", lambda: UnavailableGsmClient()) + monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) + + response = client.post( + "/gsm/sms/send", + params={"user_id": str(target.id), "message": "Help is on the way"}, + ) + + assert response.status_code == 503 + assert response.json() == { + "detail": { + "message": "GSM gateway is unavailable", + "reason": "GATEWAY_UNAVAILABLE", + } + } + + +def test_send_sms_preserves_modem_not_ready_status(client, session, monkeypatch): + current_user = _authenticated_user(session) + target = session.exec(select(User).where(User.id != current_user.id)).first() + monkeypatch.setattr(gsm, "_get_gsm_client", lambda: ModemNotReadyGsmClient()) + monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) + + response = client.post( + "/gsm/sms/send", + params={"user_id": str(target.id), "message": "Help is on the way"}, + ) + + assert response.status_code == 503 + assert response.json() == {"detail": "GSM modem not ready"} + + +def test_send_sms_rejects_proxy_pool_exhaustion_without_gateway_send( + client, session, monkeypatch +): + current_user = _authenticated_user(session) + target = session.exec(select(User).where(User.id != current_user.id)).first() + monkeypatch.setattr(gsm, "_get_gsm_client", lambda: PoolExhaustedGsmClient()) + monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) + + response = client.post( + "/gsm/sms/send", + params={"user_id": str(target.id), "message": "Help is on the way"}, + ) + + assert response.status_code == 503 + assert response.json() == { + "detail": { + "message": "GSM gateway is unavailable", + "reason": "GATEWAY_UNAVAILABLE", + } + } diff --git a/server/app/tests/test_qa_scenarios.py b/server/app/tests/test_qa_scenarios.py index 3bbda6d1..ba4f4305 100644 --- a/server/app/tests/test_qa_scenarios.py +++ b/server/app/tests/test_qa_scenarios.py @@ -14,6 +14,7 @@ from app.models.location import UserLocation from app.models.login_attempt import LoginAttempt from app.models.message import Message +from app.models.phone_verification import PhoneVerified from app.models.rescuer import Rescuer from app.models.users import User @@ -79,6 +80,16 @@ def test_build_locked_out_sets_attempt_count_and_lock(session: Session): assert attempt.locked_until.replace(tzinfo=None) > datetime.now(timezone.utc).replace(tzinfo=None) +def test_build_verified_phone_creates_verified_user(session: Session): + result = qa_scenarios.build_verified_phone(session) + + user = session.exec(select(User).where(User.username == "qa_phone_verified")).first() + assert user is not None + assert user.phone_number == "+639300000751" + assert session.exec(select(PhoneVerified).where(PhoneVerified.user_id == user.id)).first() + assert result == {"user": "qa_phone_verified", "phone_verified": True} + + def test_build_announcements_covers_priority_and_audience_matrix(session: Session): qa_scenarios.build_announcements(session) diff --git a/server/app/tests/test_testing_endpoints.py b/server/app/tests/test_testing_endpoints.py index e2e93cf6..73ebe352 100644 --- a/server/app/tests/test_testing_endpoints.py +++ b/server/app/tests/test_testing_endpoints.py @@ -43,6 +43,16 @@ def test_seed_gps_roles_scenario_creates_multi_role_fixtures(client): assert body["result"]["rescuers"] == ["qa_map_rescuer", "qa_map_rescuer_2"] +def test_seed_verified_phone_scenario_creates_login_fixture(client): + response = client.post("/testing/seed/verified-phone", headers=QA_HEADERS) + assert response.status_code == 200 + assert response.json()["result"] == {"user": "qa_phone_verified", "phone_verified": True} + + login = client.post("/testing/login-as/qa_phone_verified", headers=QA_HEADERS) + assert login.status_code == 200 + assert login.json()["username"] == "qa_phone_verified" + + def test_login_as_qa_map_rescuer_mints_usable_tokens(client): client.post("/testing/seed/gps-roles", headers=QA_HEADERS) diff --git a/server/nginx.conf b/server/nginx.conf index 371b702d..b43c2227 100644 --- a/server/nginx.conf +++ b/server/nginx.conf @@ -68,7 +68,7 @@ server { proxy_pass http://127.0.0.1:8000; proxy_set_header Host $host; - proxy_read_timeout 135s; + proxy_read_timeout 155s; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; # proxy_set_header X-Forwarded-For $remote_addr; From a76a4a7b60caba59e5b9a7fcacd0cb1a89a92fff Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Sat, 15 Aug 2026 20:36:46 +0800 Subject: [PATCH 08/12] fix(gsm-compose): route gateway traffic on Docker network (#372) --- admin-frontend/sapot-admin/Dockerfile | 11 ++++++++++- docker-compose.yml | 7 ++++++- docs/deployment/environment-config.md | 2 ++ docs/getting-started/docker-setup.md | 4 ++++ server/.env.example | 1 + server/app/api/gsm.py | 5 +++-- server/app/tests/test_gsm_proxy.py | 21 ++++++++++++++++++++- 7 files changed, 46 insertions(+), 5 deletions(-) diff --git a/admin-frontend/sapot-admin/Dockerfile b/admin-frontend/sapot-admin/Dockerfile index 982865fb..c881b681 100644 --- a/admin-frontend/sapot-admin/Dockerfile +++ b/admin-frontend/sapot-admin/Dockerfile @@ -1,13 +1,22 @@ # syntax=docker/dockerfile:1 -FROM node:22-slim AS builder +FROM node:22-slim AS base RUN corepack enable WORKDIR /app + +FROM base AS dependencies + COPY package.json pnpm-lock.yaml ./ RUN pnpm install --frozen-lockfile +FROM dependencies AS development +ENV NODE_ENV=development +COPY . . + +FROM dependencies AS builder + COPY . . RUN pnpm build diff --git a/docker-compose.yml b/docker-compose.yml index b4142ae1..834dd983 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,8 @@ services: restart: unless-stopped env_file: - ./server/.env + environment: + GSM_GATEWAY_URL: http://gsm-fastapi:8001 depends_on: db: condition: service_healthy @@ -78,6 +80,7 @@ services: admin: build: context: ./admin-frontend/sapot-admin + target: development command: pnpm dev restart: unless-stopped env_file: @@ -125,7 +128,9 @@ services: - ./GSM-module/GSM-fastapi/.env environment: HOST: 0.0.0.0 - SAPOT_API_URL: https://nginx + # The callback stays on Docker's private network. Going through nginx + # would require the gateway image to trust the development TLS CA. + SAPOT_API_URL: http://api:8000 # No /dev/ttyACM0 device passthrough here — the modem isn't present on # most dev machines, and Compose has no "optional device" syntax, so # declaring it here would abort the whole `docker compose up` (nginx/ diff --git a/docs/deployment/environment-config.md b/docs/deployment/environment-config.md index c1c826a6..61521a69 100644 --- a/docs/deployment/environment-config.md +++ b/docs/deployment/environment-config.md @@ -16,6 +16,7 @@ All SAPOT components are configured via environment variables. This document lis | `REDIS_URL` | `redis://localhost:6379` | Set if Redis is on a non-default host/port | | `SERVER_ED25519_SEED` | `None` (server key signing disabled if unset) | Set to enable server-signed peer keys | | `GSM_SECRET` | None — required, raises `RuntimeError` at import if unset | **MUST** be set — shared secret for GSM module webhooks | +| `GSM_GATEWAY_URL` | `http://localhost:8001` | Base URL of the deployed GSM FastAPI gateway. Set `http://gsm-fastapi:8001` in Docker Compose. | See [SECURITY.md](../../SECURITY.md) for why `DATABASE_URL`, `JWT_SECRET_KEY`, `CORS_ALLOWED_ORIGINS`, and `GSM_SECRET` are required. @@ -37,6 +38,7 @@ ENVIRONMENT=production REDIS_URL=redis://127.0.0.1:6379/0 SERVER_ED25519_SEED= GSM_SECRET= +GSM_GATEWAY_URL=http://127.0.0.1:8001 ``` --- diff --git a/docs/getting-started/docker-setup.md b/docs/getting-started/docker-setup.md index 0c092eae..bd68e6d4 100644 --- a/docs/getting-started/docker-setup.md +++ b/docs/getting-started/docker-setup.md @@ -59,6 +59,10 @@ cp GSM-module/GSM-fastapi/.env.example GSM-module/GSM-fastapi/.env `gsm-fastapi`'s `GSM_SECRET` must match `server/.env`'s `GSM_SECRET` — they authenticate the webhook calls between the two services (see [environment-config.md](../deployment/environment-config.md)). + +Compose sets the server's `GSM_GATEWAY_URL` to `http://gsm-fastapi:8001`, which resolves through the +internal Docker network. Do not replace it with `localhost`: inside the `api` container, that address +refers to the API container rather than the separate GSM gateway container. The `gsm-fastapi` container passes through the GSM modem at `/dev/ttyACM0`, but only when `docker-compose.gsm-hardware.yml` is explicitly merged in (Compose has no "optional device" syntax, so this stays out of the base `docker-compose.yml`/`docker-compose.override.yml` — otherwise the diff --git a/server/.env.example b/server/.env.example index 5746eeca..ffeaf79d 100644 --- a/server/.env.example +++ b/server/.env.example @@ -11,6 +11,7 @@ TLS_KEY=~/server.key # so `cp .env.example .env` works out of the box for docs/getting-started/docker-setup.md. # Running bare-metal instead (docs/getting-started/server-setup.md)? Change both to 127.0.0.1/localhost. REDIS_URL=redis://redis:6379 +GSM_GATEWAY_URL=http://gsm-fastapi:8001 DATABASE_URL=mysql+pymysql://sapot:sapot@db:3306/sapot_dev diff --git a/server/app/api/gsm.py b/server/app/api/gsm.py index e630c72a..207cba9f 100644 --- a/server/app/api/gsm.py +++ b/server/app/api/gsm.py @@ -31,20 +31,21 @@ GSM_PROXY_MAX_CONNECTIONS = GSM_GATEWAY_MAX_ADMITTED_REQUESTS + 1 GSM_PROXY_POOL_TIMEOUT_SECONDS = 1.0 -# Module-level client reuses TCP connections to localhost:8001 across requests. +# Module-level client reuses TCP connections to the configured GSM gateway. _gsm_http_client: httpx.AsyncClient | None = None logger = logging.getLogger("app") GSM_SECRET = os.environ.get("GSM_SECRET") if not GSM_SECRET: raise RuntimeError("GSM_SECRET environment variable is not set") +GSM_GATEWAY_URL = os.environ.get("GSM_GATEWAY_URL", "http://localhost:8001").rstrip("/") def _get_gsm_client() -> httpx.AsyncClient: global _gsm_http_client if _gsm_http_client is None or _gsm_http_client.is_closed: _gsm_http_client = httpx.AsyncClient( - base_url="http://localhost:8001", + base_url=GSM_GATEWAY_URL, timeout=httpx.Timeout( connect=5.0, read=GSM_PROXY_READ_TIMEOUT_SECONDS, diff --git a/server/app/tests/test_gsm_proxy.py b/server/app/tests/test_gsm_proxy.py index 65738769..81eaea62 100644 --- a/server/app/tests/test_gsm_proxy.py +++ b/server/app/tests/test_gsm_proxy.py @@ -69,7 +69,7 @@ def _authenticated_user(session): return session.exec(select(User)).first() -def test_proxy_capacity_and_timeouts_cover_gateway_contract(monkeypatch): +def test_proxy_capacity_timeouts_and_gateway_url_cover_gateway_contract(monkeypatch): captured = {} class CapturingClient: @@ -90,6 +90,25 @@ def __init__(self, **kwargs): assert timeout.pool == gsm.GSM_PROXY_POOL_TIMEOUT_SECONDS assert limits.max_connections == gsm.GSM_PROXY_MAX_CONNECTIONS assert limits.max_connections > gsm.GSM_GATEWAY_MAX_ADMITTED_REQUESTS + assert captured["base_url"] == gsm.GSM_GATEWAY_URL + + +def test_proxy_uses_configured_gateway_url(monkeypatch): + captured = {} + + class CapturingClient: + is_closed = False + + def __init__(self, **kwargs): + captured.update(kwargs) + + monkeypatch.setattr(gsm, "_gsm_http_client", None) + monkeypatch.setattr(gsm, "GSM_GATEWAY_URL", "http://gsm-fastapi:8001") + monkeypatch.setattr(gsm.httpx, "AsyncClient", CapturingClient) + + gsm._get_gsm_client() + + assert captured["base_url"] == "http://gsm-fastapi:8001" def test_send_to_module_authenticates_with_shared_secret(monkeypatch): From 6a61a57d7d9966be36f8514b7cd3a2156d8b9ffe Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Sat, 15 Aug 2026 20:41:05 +0800 Subject: [PATCH 09/12] feat(gsm-emulator): add virtual modem development stack (#373) * fix(gsm-compose): route gateway traffic on Docker network * feat(gsm-emulator): add virtual modem development stack * fix(docs): format localhost urls as code to pass link check --- GSM-module/GSM-fastapi/Dockerfile | 13 +- GSM-module/GSM-fastapi/mock_modem.py | 362 ++++++++++++++++++ GSM-module/GSM-fastapi/run-with-mock-modem.sh | 31 ++ .../GSM-fastapi/tests/test_mock_modem.py | 188 +++++++++ docker-compose.gsm-emulator.yml | 15 + docker-compose.gsm-hardware.yml | 7 +- docker/up.sh | 24 +- docs/features/sms-gateway/testing.md | 33 +- docs/getting-started/docker-setup.md | 25 ++ docs/getting-started/gsm-module-setup.md | 93 +++++ 10 files changed, 786 insertions(+), 5 deletions(-) create mode 100644 GSM-module/GSM-fastapi/mock_modem.py create mode 100755 GSM-module/GSM-fastapi/run-with-mock-modem.sh create mode 100644 GSM-module/GSM-fastapi/tests/test_mock_modem.py create mode 100644 docker-compose.gsm-emulator.yml diff --git a/GSM-module/GSM-fastapi/Dockerfile b/GSM-module/GSM-fastapi/Dockerfile index ed4d9fab..a62e1bcb 100644 --- a/GSM-module/GSM-fastapi/Dockerfile +++ b/GSM-module/GSM-fastapi/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -FROM python:3.13-slim +FROM python:3.13-slim AS base RUN pip install --no-cache-dir uv @@ -12,6 +12,17 @@ ENV PATH="/opt/venv/bin:$PATH" COPY . . +FROM base AS emulator + +EXPOSE 8001 8002 + +CMD ["python", "main.py"] + +FROM base AS production + +COPY api.py app_version.py config.py database.py main.py protocol.py serial_worker.py sms_handler.py ./ +COPY models ./models + EXPOSE 8001 CMD ["python", "main.py"] diff --git a/GSM-module/GSM-fastapi/mock_modem.py b/GSM-module/GSM-fastapi/mock_modem.py new file mode 100644 index 00000000..04b34938 --- /dev/null +++ b/GSM-module/GSM-fastapi/mock_modem.py @@ -0,0 +1,362 @@ +"""PTY-backed virtual GSM modem and browser phone for local development.""" + +import argparse +import errno +import json +import os +import re +import select +import sys +import threading +import time +from collections import defaultdict +from datetime import datetime, timezone +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Optional +from urllib.parse import parse_qs, urlparse + +if os.name == "posix": + import pty + import tty + + +DISCONNECTED_POLL_SECONDS = 0.1 +ATTACH_SETTLE_SECONDS = 0.05 +E164_PATTERN = re.compile(r"^\+[0-9]{7,15}$") +INBOUND_BODY_MAX_BYTES = 127 # GSM_BUF is 128 bytes, including its NUL terminator. +OUTBOUND_RESULT_MODES = {"success", "NO_PROMPT", "TIMEOUT"} + + +def valid_phone_number(number: object) -> bool: + return isinstance(number, str) and bool(E164_PATTERN.fullmatch(number)) + + +def normalize_body(body: object, *, inbound: bool = False) -> Optional[str]: + if not isinstance(body, str): + return None + normalized = body.replace("\r", " ").replace("\n", " ") + if not normalized.strip(): + return None + if inbound: + normalized = normalized.replace("|", "/") + if len(normalized.encode("utf-8")) > INBOUND_BODY_MAX_BYTES: + return None + return normalized + + +def parse_send_sms(line: str) -> Optional[tuple[str, str]]: + """Return the destination and body for a valid outbound SMS frame.""" + parts = line.split("|", 2) + if len(parts) != 3 or parts[0] != "SEND_SMS" or not valid_phone_number(parts[1]): + return None + body = normalize_body(parts[2]) + if body is None: + return None + return parts[1], body + + +class VirtualModem: + """Thread-safe modem state shared by the PTY loop and browser server.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._master_fd: Optional[int] = None + self._attached = False + self._sim_present = True + self._network_connected = True + self._outbound_result_mode = "success" + self._messages: dict[str, list[dict[str, Any]]] = defaultdict(list) + self._next_message_id = 1 + + def attach(self, master_fd: int) -> None: + with self._lock: + self._master_fd = master_fd + self._attached = True + + def detach(self) -> None: + with self._lock: + self._attached = False + self._master_fd = None + + def status(self) -> dict[str, Any]: + with self._lock: + usable = self._sim_present and self._network_connected + return { + "connected": self._attached, + "sim_present": self._sim_present, + "network_connected": self._network_connected, + "gsm_ready": self._attached and usable, + "outbound_result_mode": self._outbound_result_mode, + } + + def _add_message(self, number: str, direction: str, body: str, status: str) -> dict[str, Any]: + message = { + "id": self._next_message_id, + "phone_number": number, + "direction": direction, + "body": body, + "timestamp": datetime.now(timezone.utc).isoformat(), + "status": status, + } + self._next_message_id += 1 + self._messages[number].append(message) + return message + + def messages(self, number: str) -> list[dict[str, Any]]: + with self._lock: + return list(self._messages.get(number, ())) + + def reset(self) -> None: + with self._lock: + self._messages.clear() + self._next_message_id = 1 + + def receive_outbound(self, number: str, body: str) -> tuple[bool, str]: + with self._lock: + if not self._sim_present: + return False, "SIM_MISSING" + if not self._network_connected: + return False, "NETWORK_LOST" + if self._outbound_result_mode != "success": + return False, self._outbound_result_mode + self._add_message(number, "received", body, "delivered") + return True, "" + + def inject_inbound(self, number: object, body: object) -> tuple[Optional[dict[str, Any]], Optional[str]]: + if not valid_phone_number(number): + return None, "phone_number must be E.164 format e.g. +639171234567" + normalized = normalize_body(body, inbound=True) + if normalized is None: + return None, f"body must be non-empty and at most {INBOUND_BODY_MAX_BYTES} UTF-8 bytes" + with self._lock: + if not self._sim_present: + return None, "SIM_MISSING" + if not self._network_connected: + return None, "NETWORK_LOST" + if not self._attached or self._master_fd is None: + return None, "MODEM_DISCONNECTED" + message = self._add_message(number, "sent", normalized, "sent") + master_fd = self._master_fd + if not _write_frames(master_fd, [f"SMS_RECEIVED|{number}|{normalized}\n".encode("utf-8")]): + with self._lock: + self._attached = False + self._master_fd = None + return None, "MODEM_DISCONNECTED" + return message, None + + def update(self, changes: dict[str, Any]) -> tuple[Optional[dict[str, Any]], Optional[str], list[bytes]]: + allowed = {"sim_present", "network_connected", "outbound_result_mode"} + if not changes or not set(changes).issubset(allowed): + return None, "provide sim_present, network_connected, or outbound_result_mode", [] + frames: list[bytes] = [] + with self._lock: + if "sim_present" in changes and not isinstance(changes["sim_present"], bool): + return None, "sim_present must be a boolean", [] + if "network_connected" in changes and not isinstance(changes["network_connected"], bool): + return None, "network_connected must be a boolean", [] + if "outbound_result_mode" in changes and changes["outbound_result_mode"] not in OUTBOUND_RESULT_MODES: + return None, "outbound_result_mode must be success, NO_PROMPT, or TIMEOUT", [] + before_usable = self._sim_present and self._network_connected + self._sim_present = changes.get("sim_present", self._sim_present) + self._network_connected = changes.get("network_connected", self._network_connected) + self._outbound_result_mode = changes.get("outbound_result_mode", self._outbound_result_mode) + after_usable = self._sim_present and self._network_connected + if self._attached: + if not self._sim_present: + frames.append(b"SIM_MISSING\n") + elif not self._network_connected: + frames.append(b"NETWORK_LOST\n") + elif not before_usable and after_usable: + frames.extend([b"GSM_READY\n", b"NETWORK_OK\n"]) + return self.status(), None, frames + + def readiness_frames(self) -> list[bytes]: + with self._lock: + if not self._sim_present: + return [b"SIM_MISSING\n"] + if not self._network_connected: + return [b"GSM_READY\n", b"NETWORK_LOST\n"] + return [b"GSM_READY\n", b"NETWORK_OK\n"] + + def master_fd(self) -> Optional[int]: + with self._lock: + return self._master_fd + + +def _is_disconnected(error: OSError) -> bool: + return error.errno == errno.EIO + + +def _write_frames(master_fd: int, frames: list[bytes]) -> bool: + try: + for frame in frames: + remaining = memoryview(frame) + while remaining: + written = os.write(master_fd, remaining) + remaining = remaining[written:] + except OSError as error: + if _is_disconnected(error): + return False + raise + return True + + +def _announce_readiness(modem: VirtualModem, master_fd: int) -> bool: + time.sleep(ATTACH_SETTLE_SECONDS) + return _write_frames(master_fd, modem.readiness_frames()) + + +PHONE_UI = """SAPOT Virtual Phone

SAPOT Virtual Phone

""" + + +def make_http_handler(modem: VirtualModem): + class Handler(BaseHTTPRequestHandler): + def _send_json(self, status: int, value: Any) -> None: + data = json.dumps(value).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _payload(self) -> Optional[dict[str, Any]]: + try: + length = int(self.headers.get("Content-Length", "0")) + value = json.loads(self.rfile.read(length).decode("utf-8")) + return value if isinstance(value, dict) else None + except (UnicodeDecodeError, ValueError, json.JSONDecodeError): + return None + + def do_GET(self) -> None: + parsed = urlparse(self.path) + if parsed.path == "/": + data = PHONE_UI.encode("utf-8") + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + elif parsed.path == "/api/messages": + number = parse_qs(parsed.query).get("phone", [None])[0] + if not valid_phone_number(number): + self._send_json(HTTPStatus.BAD_REQUEST, {"detail": "phone must be E.164 format"}) + else: + self._send_json(HTTPStatus.OK, {"phone_number": number, "messages": modem.messages(number)}) + elif parsed.path == "/api/modem": + self._send_json(HTTPStatus.OK, modem.status()) + else: + self._send_json(HTTPStatus.NOT_FOUND, {"detail": "not found"}) + + def do_POST(self) -> None: + if self.path == "/api/messages": + payload = self._payload() + if payload is None: + self._send_json(HTTPStatus.BAD_REQUEST, {"detail": "body must be a JSON object"}) + return + message, error = modem.inject_inbound(payload.get("phone_number"), payload.get("body")) + self._send_json(HTTPStatus.CREATED if message else HTTPStatus.SERVICE_UNAVAILABLE if error in {"SIM_MISSING", "NETWORK_LOST", "MODEM_DISCONNECTED"} else HTTPStatus.BAD_REQUEST, {"message": message} if message else {"detail": error}) + elif self.path == "/api/reset": + modem.reset() + self._send_json(HTTPStatus.OK, {"ok": True}) + else: + self._send_json(HTTPStatus.NOT_FOUND, {"detail": "not found"}) + + def do_PUT(self) -> None: + if self.path != "/api/modem": + self._send_json(HTTPStatus.NOT_FOUND, {"detail": "not found"}) + return + payload = self._payload() + if payload is None: + self._send_json(HTTPStatus.BAD_REQUEST, {"detail": "body must be a JSON object"}) + return + status, error, frames = modem.update(payload) + if error: + self._send_json(HTTPStatus.BAD_REQUEST, {"detail": error}) + return + fd = modem.master_fd() + if fd is not None and frames and not _write_frames(fd, frames): + modem.detach() + self._send_json(HTTPStatus.OK, status) + + def log_message(self, _format: str, *_args: object) -> None: + return + return Handler + + +def start_http_server(modem: VirtualModem, host: str, port: int) -> ThreadingHTTPServer: + server = ThreadingHTTPServer((host, port), make_http_handler(modem)) + threading.Thread(target=server.serve_forever, name="virtual-phone-http", daemon=True).start() + return server + + +def run(port_file: Optional[str] = None, web_host: str = "127.0.0.1", web_port: int = 8002) -> None: + if os.name != "posix": + raise RuntimeError("mock_modem.py requires POSIX PTY support") + modem = VirtualModem() + http_server = start_http_server(modem, web_host, web_port) + master_fd, slave_fd = pty.openpty() + slave_path = os.ttyname(slave_fd) + tty.setraw(slave_fd) + os.close(slave_fd) + if port_file: + with open(port_file, "w", encoding="utf-8") as file: + file.write(slave_path) + print(f"Virtual modem port: {slave_path}", flush=True) + print(f"Run the gateway with: SERIAL_PORT={slave_path} python main.py", flush=True) + attached = False + buffer = "" + try: + while True: + if not attached: + if _announce_readiness(modem, master_fd): + modem.attach(master_fd) + attached = True + else: + time.sleep(DISCONNECTED_POLL_SECONDS) + continue + readable, _, _ = select.select([master_fd], [], [], DISCONNECTED_POLL_SECONDS) + if not readable: + continue + try: + chunk = os.read(master_fd, 4096) + except OSError as error: + if not _is_disconnected(error): + raise + modem.detach(); attached = False; buffer = ""; continue + if not chunk: + modem.detach(); attached = False; buffer = ""; continue + buffer += chunk.decode("utf-8", errors="replace") + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + command = parse_send_sms(line.removesuffix("\r")) + if command is None: + print(f"Ignored modem command: {line!r}", flush=True) + continue + number, body = command + print(f"SMS request to {number}: {body}", flush=True) + delivered, reason = modem.receive_outbound(number, body) + if reason == "TIMEOUT": + continue + frame = f"SMS_SENT|{number}\n" if delivered else f"SMS_FAILED|{number}|{reason}\n" + if not _write_frames(master_fd, [frame.encode()]): + modem.detach(); attached = False; buffer = ""; break + finally: + modem.detach() + http_server.shutdown() + http_server.server_close() + os.close(master_fd) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--port-file", help="write the generated PTY path here before accepting a serial client") + parser.add_argument("--web-host", default="127.0.0.1", help="virtual-phone HTTP bind address") + parser.add_argument("--web-port", type=int, default=8002, help="virtual-phone HTTP port") + arguments = parser.parse_args() + try: + run(arguments.port_file, arguments.web_host, arguments.web_port) + except KeyboardInterrupt: + sys.exit(0) diff --git a/GSM-module/GSM-fastapi/run-with-mock-modem.sh b/GSM-module/GSM-fastapi/run-with-mock-modem.sh new file mode 100755 index 00000000..6b2c3515 --- /dev/null +++ b/GSM-module/GSM-fastapi/run-with-mock-modem.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -eu + +port_file="$(mktemp)" +cleanup() { + if [ -n "${gateway_pid:-}" ]; then + kill "$gateway_pid" 2>/dev/null || true + fi + if [ -n "${modem_pid:-}" ]; then + kill "$modem_pid" 2>/dev/null || true + fi + rm -f "$port_file" +} +trap cleanup EXIT INT TERM + +python -u mock_modem.py --port-file "$port_file" \ + --web-host "${VIRTUAL_PHONE_HOST:-127.0.0.1}" \ + --web-port "${VIRTUAL_PHONE_PORT:-8002}" & +modem_pid=$! + +while [ ! -s "$port_file" ]; do + if ! kill -0 "$modem_pid" 2>/dev/null; then + wait "$modem_pid" + fi + sleep 0.1 +done + +export SERIAL_PORT="$(cat "$port_file")" +python main.py & +gateway_pid=$! +wait "$gateway_pid" diff --git a/GSM-module/GSM-fastapi/tests/test_mock_modem.py b/GSM-module/GSM-fastapi/tests/test_mock_modem.py new file mode 100644 index 00000000..9f484a33 --- /dev/null +++ b/GSM-module/GSM-fastapi/tests/test_mock_modem.py @@ -0,0 +1,188 @@ +import json +import os +import re +import select +import signal +import subprocess +import sys +import time +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +import pytest +import serial + +from mock_modem import ( + INBOUND_BODY_MAX_BYTES, + VirtualModem, + normalize_body, + parse_send_sms, + start_http_server, + valid_phone_number, +) + + +requires_pty = pytest.mark.skipif( + os.name != "posix", reason="PTY modem integration tests require POSIX" +) + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +class OutputReader: + def __init__(self, stream): + self.stream = stream + self.buffer = b"" + + def readline(self, timeout: float = 2.0) -> str: + deadline = time.monotonic() + timeout + while b"\n" not in self.buffer: + remaining = deadline - time.monotonic() + assert remaining > 0, "timed out waiting for emulator output" + ready, _, _ = select.select([self.stream], [], [], remaining) + assert ready, "timed out waiting for emulator output" + self.buffer += os.read(self.stream.fileno(), 4096) + line, self.buffer = self.buffer.split(b"\n", 1) + return line.decode("utf-8") + + +class ModemProcess: + def __init__(self, port_file=None): + self.port_file = port_file + + def __enter__(self): + command = [sys.executable, "-u", "mock_modem.py"] + if self.port_file: + command.extend(["--port-file", str(self.port_file)]) + self.process = subprocess.Popen( + command, + cwd=PROJECT_ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.output = OutputReader(self.process.stdout) + first_line = self.output.readline() + self.path = re.fullmatch(r"Virtual modem port: (/dev/pts/\d+)", first_line).group(1) + assert self.output.readline() == ( + f"Run the gateway with: SERIAL_PORT={self.path} python main.py" + ) + return self + + def readline(self, timeout: float = 2.0) -> str: + return self.output.readline(timeout) + + def __exit__(self, *_): + self.process.send_signal(signal.SIGINT) + self.process.wait(timeout=2) + assert self.process.returncode == 0 + self.process.stdout.close() + self.process.stderr.close() + + +def _open_modem(path: str) -> serial.Serial: + port = serial.Serial(path, 9600, timeout=1) + assert port.readline() == b"GSM_READY\n" + assert port.readline() == b"NETWORK_OK\n" + return port + + +def test_parse_send_sms_preserves_pipes(): + assert parse_send_sms("SEND_SMS|+639171234567|one|two|three") == ( + "+639171234567", "one|two|three" + ) + + +def test_phone_validation_and_inbound_normalization_match_firmware_frames(): + assert valid_phone_number("+639171234567") + assert not valid_phone_number("09171234567") + assert normalize_body("one|two\nthree", inbound=True) == "one/two three" + assert normalize_body("x" * (INBOUND_BODY_MAX_BYTES + 1), inbound=True) is None + + +def test_virtual_modem_stores_successful_messages_and_reset_clears_them(): + modem = VirtualModem() + assert modem.receive_outbound("+639171234567", "hello") == (True, "") + assert modem.messages("+639171234567")[0]["direction"] == "received" + modem.reset() + assert modem.messages("+639171234567") == [] + + +def test_virtual_modem_state_transitions_emit_gateway_events(): + modem = VirtualModem() + modem.attach(99) + _, error, frames = modem.update({"network_connected": False}) + assert error is None + assert frames == [b"NETWORK_LOST\n"] + _, error, frames = modem.update({"network_connected": True}) + assert error is None + assert frames == [b"GSM_READY\n", b"NETWORK_OK\n"] + _, error, frames = modem.update({"sim_present": False}) + assert error is None + assert frames == [b"SIM_MISSING\n"] + + +def test_virtual_phone_http_reports_validation_errors_and_modem_state(): + modem = VirtualModem() + server = start_http_server(modem, "127.0.0.1", 0) + base_url = f"http://127.0.0.1:{server.server_port}" + try: + with urlopen(f"{base_url}/api/modem") as response: + assert json.load(response)["gsm_ready"] is False + request = Request( + f"{base_url}/api/messages", + data=b'{"phone_number":"not-a-number","body":"hi"}', + method="POST", + headers={"Content-Type": "application/json"}, + ) + with pytest.raises(HTTPError) as error: + urlopen(request) + assert error.value.code == 400 + finally: + server.shutdown() + server.server_close() + + +@pytest.mark.parametrize("line", ["OTHER|+63|body", "SEND_SMS|+63", "SEND_SMS||body"]) +def test_parse_send_sms_rejects_malformed_commands(line): + assert parse_send_sms(line) is None + + +@requires_pty +def test_emulator_writes_port_file_before_printing_path(tmp_path): + port_file = tmp_path / "modem-port" + with ModemProcess(port_file) as modem: + assert port_file.read_text() == modem.path + + +@requires_pty +def test_emulator_confirms_fragmented_and_batched_commands(): + with ModemProcess() as modem: + with _open_modem(modem.path) as port: + port.write(b"SEND_SMS|+639171234567|fragment") + port.write(b"ed body\nSEND_SMS|+639188888888|second|body\n") + port.flush() + + assert port.readline() == b"SMS_SENT|+639171234567\n" + assert port.readline() == b"SMS_SENT|+639188888888\n" + assert modem.readline() == "SMS request to +639171234567: fragmented body" + assert modem.readline() == "SMS request to +639188888888: second|body" + + +@requires_pty +def test_emulator_reannounces_after_reconnect_and_ignores_malformed_input(): + with ModemProcess() as modem: + with _open_modem(modem.path) as port: + port.write(b"NOT_A_COMMAND\n") + port.flush() + assert modem.readline() == "Ignored modem command: 'NOT_A_COMMAND'" + assert port.read(1) == b"" + + port.write(b"SEND_SMS|+639171234567|still works\n") + port.flush() + assert port.readline() == b"SMS_SENT|+639171234567\n" + assert modem.readline() == "SMS request to +639171234567: still works" + + time.sleep(0.25) + with _open_modem(modem.path) as port: + assert port.read(1) == b"" diff --git a/docker-compose.gsm-emulator.yml b/docker-compose.gsm-emulator.yml new file mode 100644 index 00000000..13e2b7f3 --- /dev/null +++ b/docker-compose.gsm-emulator.yml @@ -0,0 +1,15 @@ +# Opt-in local development overlay. The emulator and gateway must run in the +# same container because a PTY path belongs to one Linux device namespace. +# +# Usage: docker/up.sh -f docker-compose.yml -f docker-compose.gsm-emulator.yml up --build -d +services: + gsm-fastapi: + build: + context: ./GSM-module/GSM-fastapi + target: emulator + command: ["./run-with-mock-modem.sh"] + environment: + VIRTUAL_PHONE_HOST: 0.0.0.0 + VIRTUAL_PHONE_PORT: 8002 + ports: + - "127.0.0.1:${VIRTUAL_PHONE_PORT:-8002}:8002" diff --git a/docker-compose.gsm-hardware.yml b/docker-compose.gsm-hardware.yml index 509fb526..121ecf1d 100644 --- a/docker-compose.gsm-hardware.yml +++ b/docker-compose.gsm-hardware.yml @@ -1,10 +1,13 @@ # Opt-in overlay: passes the GSM modem through to gsm-fastapi. Not # auto-loaded (unlike docker-compose.override.yml) — only merge this in on -# a machine that actually has the modem attached at /dev/ttyACM0, otherwise +# a machine that actually has the modem attached, otherwise # `docker compose up` aborts and leaves nginx/admin stuck in "Created". # # Usage: docker/up.sh -f docker-compose.yml -f docker-compose.gsm-hardware.yml up -d +# docker/up.sh loads GSM-module/GSM-fastapi/.env for SERIAL_PORT substitution. services: gsm-fastapi: + environment: + SERIAL_PORT: ${SERIAL_PORT:-/dev/ttyACM0} devices: - - "/dev/ttyACM0:/dev/ttyACM0" + - "${SERIAL_PORT:-/dev/ttyACM0}:${SERIAL_PORT:-/dev/ttyACM0}" diff --git a/docker/up.sh b/docker/up.sh index 098faf3f..38e04b4d 100755 --- a/docker/up.sh +++ b/docker/up.sh @@ -16,7 +16,9 @@ set -eu # case unless you export CERT_SAN in the shell first. # # Passes --env-file server/.env explicitly: docker-compose.yml's -# ${MYSQL_*} substitutions are read from this file. Passing --env-file at +# ${MYSQL_*} substitutions are read from this file. When the optional GSM +# hardware overlay is selected, its SERIAL_PORT substitution is read from the +# gateway's .env too, so the device mapping matches the serial worker. Passing --env-file at # all disables Compose's default auto-load of a root-level .env, so we # also pass repo-root .env (port overrides, see .env.example) when present # — --env-file can be repeated, later ones win on overlapping keys, and @@ -45,4 +47,24 @@ if [ -f .env ]; then ENV_FILE_ARGS="--env-file .env $ENV_FILE_ARGS" fi +case " $* " in + *" docker-compose.gsm-hardware.yml "*) + if [ ! -f GSM-module/GSM-fastapi/.env ]; then + echo "docker/up.sh: GSM-module/GSM-fastapi/.env is required for the hardware overlay" >&2 + exit 1 + fi + serial_port="$(sed -n 's/^[[:space:]]*SERIAL_PORT[[:space:]]*=[[:space:]]*//p' GSM-module/GSM-fastapi/.env | tail -n 1)" + serial_port="${serial_port#\"}" + serial_port="${serial_port#\'}" + case "$serial_port" in + /dev/pts/*) + echo "docker/up.sh: SERIAL_PORT=$serial_port is a host PTY and cannot be passed through with the hardware overlay" >&2 + echo "docker/up.sh: use docker-compose.gsm-emulator.yml for PTY testing, or set SERIAL_PORT to a host /dev/ttyACM* or /dev/ttyUSB* device" >&2 + exit 1 + ;; + esac + ENV_FILE_ARGS="$ENV_FILE_ARGS --env-file GSM-module/GSM-fastapi/.env" + ;; +esac + exec docker compose $ENV_FILE_ARGS "$@" diff --git a/docs/features/sms-gateway/testing.md b/docs/features/sms-gateway/testing.md index 256d0109..90ee350a 100644 --- a/docs/features/sms-gateway/testing.md +++ b/docs/features/sms-gateway/testing.md @@ -46,6 +46,7 @@ pnpm run testAll | `tests/test_database_reconciliation.py` | Idempotent startup recovery of orphaned pending log rows | | `tests/test_incoming_sms.py` | Sender rejection reason codes and inbound log status updates | | `tests/test_lifespan.py` | Reconciliation ordering before serial worker startup | +| `tests/test_mock_modem.py` | Virtual-phone validation, firmware-compatible normalization, modem state transitions, HTTP responses, PTY framing, reconnects, and subprocess cleanup | | `server/app/tests/test_gsm_proxy.py` | Main-server shared-secret header, status preservation, and timeout headroom for chat, verification, resend, and first-contact requests | | `mobile-app/sapot-mobile-app/features/shared/core/errors/__tests__/gsm-error.test.ts` | Typed `QUEUE_FULL` parsing and user-visible error messages | | `mobile-app/sapot-mobile-app/features/chat/components/__tests__/message-list.test.tsx` | Manual resend rejection and `not_sent` restoration | @@ -83,7 +84,37 @@ The saturation test starts 21 blocking send requests, representing 20 waiting re This covers the thread-pool exhaustion described by issue #252. A test with only a rejecting fake would verify the response shape but would not prove that the rejection handler can still obtain a worker thread. -## Manual modem smoke test +## Software-only PTY smoke test + +Use this Linux host workflow to validate the real `SerialWorker` and outbound FastAPI path without +an Arduino or carrier account. It still needs development `DB_PATH` and `GSM_SECRET` values because +the emulator replaces only the serial device. + +1. In one terminal, run `python mock_modem.py` from `GSM-module/GSM-fastapi/` and copy its printed `/dev/pts/` path. The virtual phone is available at `http://127.0.0.1:8002`. +2. In another terminal, start the gateway with `SERIAL_PORT=/dev/pts/ python main.py`. +3. Confirm `curl http://127.0.0.1:8001/health` reports `connected: true` and `gsm_ready: true`. +4. Send an authenticated request: + + ```bash + curl -X POST http://127.0.0.1:8001/sms/send \ + -H 'Content-Type: application/json' \ + -H 'X-GSM-Secret: ' \ + -d '{"number":"+639171234567","body":"SAPOT PTY smoke test"}' + ``` + +5. Confirm the API reports success and the selected virtual-phone inbox shows the message from SAPOT Gateway. +6. Reply from that inbox and confirm the gateway processes it through the normal inbound session and callback path. +7. Set the virtual-phone network or SIM control to unavailable, confirm the gateway health degrades, then restore it and confirm it becomes ready again. +8. Restart only the gateway, using the same PTY path, and confirm it becomes ready again. + +The emulator can also return `NO_PROMPT` or withhold a confirmation (`TIMEOUT`) from its browser controls. +It cannot validate USB access, real SIM state, signal, carrier acceptance, or physical-phone delivery. + +For Compose-based testing, start the stack with +`docker-compose.gsm-emulator.yml`. The overlay runs the emulator inside the gateway container because +a host-created PTY is not visible to that container. + +## Real-modem smoke test Run this only on a host with the configured Arduino and SIM: diff --git a/docs/getting-started/docker-setup.md b/docs/getting-started/docker-setup.md index bd68e6d4..6df78cf3 100644 --- a/docs/getting-started/docker-setup.md +++ b/docs/getting-started/docker-setup.md @@ -76,6 +76,20 @@ whole `docker compose up` would abort on any machine without the GSM modem attac Without the GSM modem, just run the normal `./docker/up.sh up --build -d` below — `gsm-fastapi` still starts, it just won't have serial access. +To exercise outbound SMS flow in Docker without hardware, merge the PTY emulator overlay instead. +It starts the emulator in the `gsm-fastapi` container, so its generated device path is visible to the +gateway process. Do not set `SERIAL_PORT` to a host `/dev/pts/` path: containers have separate PTY +namespaces. + +```bash +./docker/up.sh -f docker-compose.yml -f docker-compose.gsm-emulator.yml up --build -d +docker compose logs -f gsm-fastapi +``` + +The gateway logs the generated port and becomes ready after the emulator handshake. The emulator +prints each valid outbound destination and body in the same service logs. Do not merge the emulator +overlay with `docker-compose.gsm-hardware.yml`; use the hardware overlay for real modem testing. + See the repo-root `SECURITY.md` for why `DATABASE_URL`, `JWT_SECRET_KEY`, `CORS_ALLOWED_ORIGINS`, and `SERVER_ED25519_SEED` are required at import time. `server/.env.example` supplies safe defaults only for local service addresses; it never supplies usable secrets. ## Run @@ -169,6 +183,17 @@ The new stack runs under a different project name (derived from the repo root di docker compose up -d db redis api certgen nginx # pulls in admin + tileserver, skips gsm-fastapi ``` +**`gsm-fastapi` logs `Cannot open /dev/ttyACM0: No such file or directory`.** The Arduino may be connected to the host, but the running container was created without the hardware overlay, so Docker did not expose the serial device inside it. Confirm the host sees the device, then recreate only the gateway with the overlay: +```bash +ls -l /dev/ttyACM* /dev/ttyUSB* +./docker/up.sh -f docker-compose.yml -f docker-compose.gsm-hardware.yml up -d --force-recreate gsm-fastapi +``` +If the first command reports a port other than `/dev/ttyACM0`, update `SERIAL_PORT` in +`GSM-module/GSM-fastapi/.env`. The `docker/up.sh` wrapper reads that file when the hardware overlay is +selected, so Compose maps the same device path into the gateway container. +This overlay accepts host serial devices such as `/dev/ttyACM0` and `/dev/ttyUSB0`. It cannot pass a +host `/dev/pts/` pseudo-terminal into Docker. Use `docker-compose.gsm-emulator.yml` for PTY testing. + **`https://localhost/admin` works but `http://localhost:3000` returns 404.** Expected. The admin app sets `basePath: "/admin"` in `next.config.ts`, so its published port serves the dashboard at `http://localhost:3000/admin`, not at the root path. **`nginx` logs `host not found in upstream "api"` even though `api` is running.** The `nginx` container was created against a stale image/config and never recreated (Compose reuses an existing container if it thinks nothing relevant changed). Force it: diff --git a/docs/getting-started/gsm-module-setup.md b/docs/getting-started/gsm-module-setup.md index 5141c188..4fe07d15 100644 --- a/docs/getting-started/gsm-module-setup.md +++ b/docs/getting-started/gsm-module-setup.md @@ -18,6 +18,11 @@ Docker, in which case follow this doc. For deploying it as a systemd service, se - A GSM modem attached via USB serial (default expected at `/dev/ttyACM0`, matching `SERIAL_PORT`'s default in `config.py`) - Access to the same MariaDB instance the server uses (`DB_PATH`) +Physical modem hardware is required to deliver messages through a carrier. For Linux-only local +outbound-flow testing, `mock_modem.py` can replace the serial hardware with a virtual port. It does +not replace the database or shared-secret configuration: `.env` must still provide `DB_PATH` and +`GSM_SECRET`. + ## Install ```bash @@ -68,6 +73,94 @@ the main SAPOT server on `8000`, and it matches the server's `_gsm_http_client`, `http://localhost:8001` (`server/app/api/gsm.py`). Setting `PORT` yourself changes nothing except the port printed in the startup log. See `GSM-module/CLAUDE.md`'s "Common Pitfalls". +## Test two-way SMS flow without a modem + +The virtual modem exercises the unchanged serial worker and outbound API flow without an Arduino, +SIM card, or carrier connection. It uses a POSIX pseudo-terminal (PTY), so this workflow is intended +for Linux host development, not Windows. For Docker Compose, use the +[`docker-compose.gsm-emulator.yml`](../../docker-compose.gsm-emulator.yml) overlay instead of passing +a host PTY into the container. + +In one terminal, start the emulator and copy the printed port path: + +```bash +cd GSM-module/GSM-fastapi +python mock_modem.py +``` + +```text +Virtual modem port: /dev/pts/3 +Run the gateway with: SERIAL_PORT=/dev/pts/3 python main.py +``` + +Before starting the gateway, put that exact path in `GSM-module/GSM-fastapi/.env`. Keep the required +`DB_PATH` and `GSM_SECRET` values there too. + +```dotenv +# Development-only PTY created by mock_modem.py. Replace this value each time +# the emulator is restarted because its /dev/pts path can change. +SERIAL_PORT=/dev/pts/3 +``` + +Start the gateway in a second terminal: + +```bash +cd GSM-module/GSM-fastapi +python main.py +``` + +Open `http://127.0.0.1:8002` to use the virtual phone. Enter any E.164 phone number, such as +`+639171234567`, then send through the normal SAPOT API. The accepted message appears as an incoming +message from **SAPOT Gateway**. Reply in the browser to inject `SMS_RECEIVED` into the unchanged serial +worker, so account checks, `[target]` sessions, message logging, and callbacks still run in the gateway. + +The controls intentionally model the Arduino-facing boundary: + +| Control | Effect | +| --- | --- | +| SIM | Removing it emits `SIM_MISSING`; restoring a working modem emits `GSM_READY` then `NETWORK_OK`. | +| Network | Losing it emits `NETWORK_LOST`; restoring it emits `GSM_READY` then `NETWORK_OK`. | +| Outbound result | `success` accepts and displays a message, `NO_PROMPT` returns `SMS_FAILED`, and `TIMEOUT` leaves the gateway send waiting for its normal timeout. | + +Browser replies are disabled while the SIM or network is unavailable. The emulator replaces inbound +pipe characters with `/`, replaces newlines with spaces, and caps replies at the firmware's 127-byte +inbound buffer limit. It keeps messages only in memory, so restarting it clears every virtual inbox. + +For Docker Compose, run: + +```bash +./docker/up.sh -f docker-compose.yml -f docker-compose.gsm-emulator.yml up --build -d +``` + +The overlay publishes the virtual phone only at `127.0.0.1:${VIRTUAL_PHONE_PORT:-8002}` and runs it in +the same container as the gateway because PTY paths do not cross container boundaries. Do not put the +host's `/dev/pts/` path in `.env` for this workflow: `run-with-mock-modem.sh` creates the PTY inside +the container and overrides `SERIAL_PORT` for the gateway process. You can restart only the gateway and +reuse the same PTY path while the emulator continues running. + +The normal gateway image is a production target that excludes the virtual modem, virtual phone, and +their startup script. The emulator overlay explicitly selects a separate emulator target, so it cannot +be activated by the standard production image or command. + +For a physical modem in Docker, set the host device path in `GSM-module/GSM-fastapi/.env` before using +the hardware overlay. The `docker/up.sh` wrapper reads this value and uses it both as the gateway's +`SERIAL_PORT` and as the Docker device mapping. + +```dotenv +# Physical Arduino/GSM modem attached to the Docker host. +SERIAL_PORT=/dev/ttyACM0 +``` + +```bash +./docker/up.sh -f docker-compose.yml -f docker-compose.gsm-hardware.yml up --build -d +``` + +Use a host device such as `/dev/ttyACM0` or `/dev/ttyUSB0` here. A `/dev/pts/` path is valid only +for direct-host testing; Docker cannot pass it through as a hardware device. + +An emulator success means that the simulated modem accepted the request at the Arduino protocol boundary. +It does not mean a carrier accepted the SMS or that a physical phone received it. + ## Verify ```bash From b567ef7d56282b5e770954d74eeaf041af57ce2f Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:26:44 +0800 Subject: [PATCH 10/12] fix(server-activity): ignore writes for deleted users (#374) --- server/app/db_operations/activity.py | 3 +++ server/app/tests/test_activity.py | 33 +++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/server/app/db_operations/activity.py b/server/app/db_operations/activity.py index 4f11be6c..4e2de955 100644 --- a/server/app/db_operations/activity.py +++ b/server/app/db_operations/activity.py @@ -4,6 +4,7 @@ from uuid import UUID from fastapi import HTTPException, Request import jwt +from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select from app.db_operations.token import ALGORITHM, SECRET_KEY from app.models.activity import UserActivity @@ -48,6 +49,8 @@ def _write_user_activity_sync(user_id: UUID, ip: str, user_agent: str) -> None: activity.ip_address = ip session.add(activity) session.commit() + except IntegrityError: + pass except Exception as e: print(f"[activity] write failed: {e}") diff --git a/server/app/tests/test_activity.py b/server/app/tests/test_activity.py index febee30b..efa8fc92 100644 --- a/server/app/tests/test_activity.py +++ b/server/app/tests/test_activity.py @@ -1,7 +1,12 @@ """Regression coverage for concurrent user-presence updates.""" from uuid import uuid4 -from app.db_operations.activity import _get_user_activity_for_update +from sqlalchemy.exc import IntegrityError + +from app.db_operations.activity import ( + _get_user_activity_for_update, + _write_user_activity_sync, +) class _Result: @@ -24,3 +29,29 @@ def test_presence_lookup_locks_the_user_activity_row(): assert _get_user_activity_for_update(session, uuid4()) is None assert session.statement._for_update_arg is not None + + +def test_activity_write_ignores_missing_user_foreign_key(monkeypatch, capsys): + class FailingSession: + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + def exec(self, _): + return _Result() + + def add(self, _): + pass + + def commit(self): + raise IntegrityError("INSERT", {}, Exception("foreign key constraint")) + + monkeypatch.setattr( + "app.db_operations.activity.Session", lambda _: FailingSession() + ) + + _write_user_activity_sync(uuid4(), "127.0.0.1", "pytest") + + assert capsys.readouterr().out == "" From f10c0748a6816df0d4f1a6a66a6a5cfb34194471 Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Sat, 15 Aug 2026 23:47:24 +0800 Subject: [PATCH 11/12] fix(server-gsm): require verified phone number for outgoing SMS (#375) * fix(server-gsm): require verified phone for SMS sends * docs(gsm): document verified SMS sender policy --- docs/api/gsm-sms.md | 17 ++++++- docs/api/openapi/gsm-sms.yaml | 12 +++++ docs/features/sms-gateway/design.md | 2 +- docs/features/sms-gateway/requirements.md | 3 +- mobile-app/sapot-mobile-app/docs/API.md | 13 ++++- server/app/api/gsm.py | 32 +++++++++++- server/app/tests/test_gsm_proxy.py | 61 ++++++++++++++++++++--- 7 files changed, 125 insertions(+), 15 deletions(-) diff --git a/docs/api/gsm-sms.md b/docs/api/gsm-sms.md index 3d7d942d..abd96933 100644 --- a/docs/api/gsm-sms.md +++ b/docs/api/gsm-sms.md @@ -15,7 +15,7 @@ The GSM endpoints proxy SMS operations from the SAPOT server to the GSM module ( | GET | `/gsm/health` | JWT Bearer | Check GSM module availability. | | GET | `/gsm/health/detailed` | JWT Bearer | Detailed GSM module health/diagnostics. | | GET | `/gsm/sms/messages` | JWT Bearer | List recent SMS messages seen by the module. | -| POST | `/gsm/sms/send` | JWT Bearer | Send an SMS to a phone number. | +| POST | `/gsm/sms/send` | JWT Bearer + verified phone | Send an SMS to a phone number. | | POST | `/gsm/request` | None | Initiate an SMS OTP for a phone number (registration/verification). | | POST | `/gsm/verify` | None | Verify an SMS OTP code. | | POST | `/gsm/resend` | None | Resend an SMS OTP. | @@ -25,7 +25,7 @@ The GSM endpoints proxy SMS operations from the SAPOT server to the GSM module ( | GET | `/gsm/mock/health` | JWT Bearer | Mock variant of `/gsm/health`. | | GET | `/gsm/mock/health/detailed` | JWT Bearer | Mock variant of `/gsm/health/detailed`. | | GET | `/gsm/mock/sms/messages` | JWT Bearer | Mock variant of `/gsm/sms/messages`. | -| POST | `/gsm/mock/sms/send` | JWT Bearer | Mock variant of `/gsm/sms/send`. | +| POST | `/gsm/mock/sms/send` | JWT Bearer + verified phone | Mock variant of `/gsm/sms/send`. | | POST | `/gsm/mock/request` | None | Mock variant of `/gsm/request`. | | POST | `/gsm/mock/verify` | None | Mock variant of `/gsm/verify`. | | POST | `/gsm/mock/resend` | None | Mock variant of `/gsm/resend`. | @@ -43,6 +43,19 @@ Webhook endpoint the GSM hardware gateway calls when it receives an SMS. Protect The GSM module's own standalone hardware-facing API is documented in [`docs/deployment/gsm-module.md`](../deployment/gsm-module.md). Its `POST /sms/send` route requires the same `X-GSM-Secret` shared secret that protects the main server's inbound webhook. +## Outbound sender eligibility + +`POST /gsm/sms/send` and its mock variant require a `PhoneVerified` record for the authenticated account. The server checks this before looking up the recipient or contacting the GSM gateway. An unverified account receives HTTP 403: + +```json +{ + "detail": { + "reason": "PHONE_VERIFICATION_REQUIRED", + "message": "Verify your phone number before sending SMS." + } +} +``` + ## Gateway failure contract The main server preserves synchronous gateway failures for `/gsm/sms/send`, `/gsm/request`, `/gsm/resend`, and `/gsm/contact-unknown-user`. Queue saturation returns HTTP 503: diff --git a/docs/api/openapi/gsm-sms.yaml b/docs/api/openapi/gsm-sms.yaml index 9656fe9b..e22a57d1 100644 --- a/docs/api/openapi/gsm-sms.yaml +++ b/docs/api/openapi/gsm-sms.yaml @@ -366,6 +366,12 @@ paths: schema: {} '404': description: Not Found + '403': + description: The sending account does not have a verified phone number. + content: + application/json: + schema: + $ref: '#/components/schemas/GsmFailureResponse' '422': description: Validation Error content: @@ -592,6 +598,12 @@ paths: - $ref: '#/components/schemas/GsmFailureResponse' - $ref: '#/components/schemas/GsmHealthUnavailableResponse' title: Response 503 Send Sms Gsm Sms Send Post + '403': + description: The sending account does not have a verified phone number. + content: + application/json: + schema: + $ref: '#/components/schemas/GsmFailureResponse' '422': description: Validation Error content: diff --git a/docs/features/sms-gateway/design.md b/docs/features/sms-gateway/design.md index 3db8906e..20bb0222 100644 --- a/docs/features/sms-gateway/design.md +++ b/docs/features/sms-gateway/design.md @@ -33,7 +33,7 @@ sequenceDiagram GSM->>Main: POST /gsm/inbound with X-GSM-Secret ``` -The main server authenticates the user-facing `/gsm/sms/send` route with a JSON Web Token (JWT). It authenticates its direct gateway call with the same shared `GSM_SECRET` used for callbacks. The GSM service validates `X-GSM-Secret` before logging or queueing a send, which prevents another container on the internal network from occupying the serial modem. +The main server authenticates the user-facing `/gsm/sms/send` route with a JSON Web Token (JWT) and requires the sender to have a verified phone number. It rejects an unverified sender before contacting the gateway. The main server authenticates its direct gateway call with the same shared `GSM_SECRET` used for callbacks. The GSM service validates `X-GSM-Secret` before logging or queueing a send, which prevents another container on the internal network from occupying the serial modem. ## How does outbound admission work? diff --git a/docs/features/sms-gateway/requirements.md b/docs/features/sms-gateway/requirements.md index 61bdd05f..3aaabb60 100644 --- a/docs/features/sms-gateway/requirements.md +++ b/docs/features/sms-gateway/requirements.md @@ -119,7 +119,8 @@ Only one request may await a modem confirmation. A confirmation received before ### FR-SG-08: Main server integration -- The user-facing `/gsm/sms/send` route remains on the main server and requires its normal JWT authentication. +- The user-facing `/gsm/sms/send` route remains on the main server and requires its normal JWT authentication and a verified phone number for the sending account. +- An authenticated account without a verified phone number receives HTTP 403 with `reason: "PHONE_VERIFICATION_REQUIRED"` before the main server calls the GSM service. - The main server calls the direct gateway at `http://localhost:8001/sms/send` with `X-GSM-Secret`. - The direct gateway is a trusted local service and must not be exposed to untrusted networks. - The main server must preserve gateway HTTP 502 and 503 failures for user-facing send, verification, resend, and first-contact requests. diff --git a/mobile-app/sapot-mobile-app/docs/API.md b/mobile-app/sapot-mobile-app/docs/API.md index 79d59109..2a9b88a6 100644 --- a/mobile-app/sapot-mobile-app/docs/API.md +++ b/mobile-app/sapot-mobile-app/docs/API.md @@ -564,7 +564,8 @@ Failure is non-fatal — the client logs and retries on next login. --- ### `POST /user-utils/search-user` — Search Users -**Auth:** Required +**Auth:** Required; the authenticated account must have a verified phone number + **Query params:** `identifier_string=&limit=&offset=` **Response `200`:** @@ -864,6 +865,16 @@ independently — surface them separately rather than collapsing to one "offline { "msg_id": "string", "ok": "boolean", "to": "string" } ``` +**Response `403` when the sender's phone number is not verified:** +```json +{ + "detail": { + "reason": "PHONE_VERIFICATION_REQUIRED", + "message": "Verify your phone number before sending SMS." + } +} +``` + **Response `503` when the outbound queue is full:** ```json { diff --git a/server/app/api/gsm.py b/server/app/api/gsm.py index 207cba9f..892933e7 100644 --- a/server/app/api/gsm.py +++ b/server/app/api/gsm.py @@ -124,6 +124,18 @@ class GsmHealthUnavailableResponse(BaseModel): }, } +GSM_PHONE_VERIFICATION_ERROR_RESPONSES = { + 403: { + "model": GsmFailureResponse, + "description": "The sending account does not have a verified phone number.", + }, +} + +GSM_SMS_SEND_ERROR_RESPONSES = { + **GSM_SEND_ERROR_RESPONSES, + **GSM_PHONE_VERIFICATION_ERROR_RESPONSES, +} + GSM_HEALTH_ERROR_RESPONSES = { 503: { "model": GsmHealthResponse | GsmHealthUnavailableResponse, @@ -296,7 +308,19 @@ async def gsm_messages( response = await client.get("/sms/messages", params=params) return response.json() -@router.post("/sms/send", responses=GSM_SEND_ERROR_RESPONSES) + +def _require_verified_phone(user: User) -> None: + if not user.phone_is_verified: + raise HTTPException( + status_code=403, + detail={ + "reason": "PHONE_VERIFICATION_REQUIRED", + "message": "Verify your phone number before sending SMS.", + }, + ) + + +@router.post("/sms/send", responses=GSM_SMS_SEND_ERROR_RESPONSES) async def send_sms( current_user : Annotated[User, Depends(get_current_user)], user_id: UUID, @@ -307,6 +331,8 @@ async def send_sms( if current_user.banned: raise HTTPException(403) + _require_verified_phone(current_user) + target = session.get(User, user_id) if not target: @@ -770,7 +796,7 @@ async def MOCK_gsm_messages( """Admin only""" return "Admin only!" -@router.post("/mock/sms/send") +@router.post("/mock/sms/send", responses=GSM_PHONE_VERIFICATION_ERROR_RESPONSES) async def MOCK_send_sms( current_user : Annotated[User, Depends(get_current_user)], user_id: UUID, @@ -781,6 +807,8 @@ async def MOCK_send_sms( if current_user.banned: raise HTTPException(403) + _require_verified_phone(current_user) + target = session.get(User, user_id) if not target: diff --git a/server/app/tests/test_gsm_proxy.py b/server/app/tests/test_gsm_proxy.py index 81eaea62..caddce40 100644 --- a/server/app/tests/test_gsm_proxy.py +++ b/server/app/tests/test_gsm_proxy.py @@ -5,7 +5,7 @@ from app.api import gsm from app.db_operations.token import get_current_user from app.main import app -from app.models.phone_verification import PhoneVerification, now_ms +from app.models.phone_verification import PhoneVerification, PhoneVerified, now_ms from app.models.users import User from sqlmodel import select @@ -65,8 +65,13 @@ async def post(self, path: str, json: dict, **kwargs): raise httpx.PoolTimeout("GSM proxy connection pool is full") -def _authenticated_user(session): - return session.exec(select(User)).first() +def _authenticated_user(session, *, phone_verified=False): + user = session.exec(select(User)).first() + if phone_verified: + session.add(PhoneVerified(user_id=user.id)) + session.commit() + session.refresh(user) + return user def test_proxy_capacity_timeouts_and_gateway_url_cover_gateway_contract(monkeypatch): @@ -130,7 +135,7 @@ async def post(self, path: str, **kwargs): def test_send_sms_preserves_queue_full_status(client, session, monkeypatch): - current_user = _authenticated_user(session) + current_user = _authenticated_user(session, phone_verified=True) target = session.exec(select(User).where(User.id != current_user.id)).first() monkeypatch.setattr(gsm, "_get_gsm_client", lambda: SaturatedGsmClient()) monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) @@ -145,7 +150,7 @@ def test_send_sms_preserves_queue_full_status(client, session, monkeypatch): def test_send_sms_preserves_service_stopping_status(client, session, monkeypatch): - current_user = _authenticated_user(session) + current_user = _authenticated_user(session, phone_verified=True) target = session.exec(select(User).where(User.id != current_user.id)).first() monkeypatch.setattr(gsm, "_get_gsm_client", lambda: StoppingGsmClient()) monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) @@ -210,7 +215,7 @@ def test_contact_unknown_user_preserves_queue_full_status(client, session, monke def test_send_sms_reports_unavailable_gateway(client, session, monkeypatch): - current_user = _authenticated_user(session) + current_user = _authenticated_user(session, phone_verified=True) target = session.exec(select(User).where(User.id != current_user.id)).first() monkeypatch.setattr(gsm, "_get_gsm_client", lambda: UnavailableGsmClient()) monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) @@ -230,7 +235,7 @@ def test_send_sms_reports_unavailable_gateway(client, session, monkeypatch): def test_send_sms_preserves_modem_not_ready_status(client, session, monkeypatch): - current_user = _authenticated_user(session) + current_user = _authenticated_user(session, phone_verified=True) target = session.exec(select(User).where(User.id != current_user.id)).first() monkeypatch.setattr(gsm, "_get_gsm_client", lambda: ModemNotReadyGsmClient()) monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) @@ -247,7 +252,7 @@ def test_send_sms_preserves_modem_not_ready_status(client, session, monkeypatch) def test_send_sms_rejects_proxy_pool_exhaustion_without_gateway_send( client, session, monkeypatch ): - current_user = _authenticated_user(session) + current_user = _authenticated_user(session, phone_verified=True) target = session.exec(select(User).where(User.id != current_user.id)).first() monkeypatch.setattr(gsm, "_get_gsm_client", lambda: PoolExhaustedGsmClient()) monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) @@ -264,3 +269,43 @@ def test_send_sms_rejects_proxy_pool_exhaustion_without_gateway_send( "reason": "GATEWAY_UNAVAILABLE", } } + + +def test_send_sms_rejects_unverified_sender_without_gateway_send( + client, session, monkeypatch +): + current_user = _authenticated_user(session) + target = session.exec(select(User).where(User.id != current_user.id)).first() + + def fail_if_gateway_client_is_requested(): + raise AssertionError("Unverified sender reached the GSM gateway") + + monkeypatch.setattr(gsm, "_get_gsm_client", fail_if_gateway_client_is_requested) + monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) + + response = client.post( + "/gsm/sms/send", + params={"user_id": str(target.id), "message": "Help is on the way"}, + ) + + assert response.status_code == 403 + assert response.json() == { + "detail": { + "reason": "PHONE_VERIFICATION_REQUIRED", + "message": "Verify your phone number before sending SMS.", + } + } + + +def test_mock_send_sms_rejects_unverified_sender(client, session, monkeypatch): + current_user = _authenticated_user(session) + target = session.exec(select(User).where(User.id != current_user.id)).first() + monkeypatch.setitem(app.dependency_overrides, get_current_user, lambda: current_user) + + response = client.post( + "/gsm/mock/sms/send", + params={"user_id": str(target.id), "message": "Help is on the way"}, + ) + + assert response.status_code == 403 + assert response.json()["detail"]["reason"] == "PHONE_VERIFICATION_REQUIRED" From 3986d4a56ec4b123e3650b84fca483de6846f2b7 Mon Sep 17 00:00:00 2001 From: devAMT <162143021+Adamskiee@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:49:18 +0800 Subject: [PATCH 12/12] fix(server-api): harden error handling logs and make role writes atomic (#377) * fix(server-api): harden error handling logs * fix(server-api): make role writes atomic --- docs/api/conventions.md | 1 + server/app/api/admin.py | 157 ++++++++++++------ server/app/api/peer_connection.py | 76 ++++++--- server/app/api/update_info.py | 9 +- server/app/db_operations/GPS_manager.py | 17 +- server/app/db_operations/auth.py | 38 +++-- .../app/db_operations/connection_manager.py | 37 ++++- server/app/db_operations/token.py | 13 +- server/app/structured_logging.py | 16 ++ server/app/tests/test_admin_me.py | 96 ++++++++++- server/app/tests/test_gps.py | 40 ++++- server/app/tests/test_websocket_pool.py | 69 ++++++++ 12 files changed, 474 insertions(+), 95 deletions(-) create mode 100644 server/app/structured_logging.py diff --git a/docs/api/conventions.md b/docs/api/conventions.md index 62bdedc8..e7c65a38 100644 --- a/docs/api/conventions.md +++ b/docs/api/conventions.md @@ -95,6 +95,7 @@ set (every decorated route in `server/app/api/`): | `POST /auth/token` | 5/minute | | `POST /auth/` | 3/minute | | `POST /auth/refresh` | 10/minute | +| `POST /api/admin/refresh` | 10/minute | | `POST /auth/reauthenticate` | 5/minute | | `POST /auth/change-password` | 3/minute | | `POST /auth/forgot-password/otp/send` | 3/minute | diff --git a/server/app/api/admin.py b/server/app/api/admin.py index 9c532a89..ea07b266 100644 --- a/server/app/api/admin.py +++ b/server/app/api/admin.py @@ -1,6 +1,7 @@ from app.models.announcement import Announcement, PriorityType, AnnouncementStatusType, AudienceType from app.models.users import User from datetime import datetime, timedelta, timezone +import logging import os from uuid import UUID, uuid4 from math import ceil @@ -46,6 +47,8 @@ refresh_token, ) from app.models.rescuer import Rescuer +from app.structured_logging import log_context +from app.limiter import limiter from app.models.token import Token from app.models.users import ( User, @@ -59,6 +62,7 @@ router = APIRouter( prefix="/api/admin", tags=["admin"], responses={404: {"description": "Not Found"}} ) +logger = logging.getLogger("app") class AdminLoginResponse(BaseModel): @@ -125,6 +129,7 @@ async def login_for_access_token( @router.post("/refresh") +@limiter.limit("10/minute") async def refresh_access_token( request: Request, response: Response, session: SessionDep ): @@ -156,8 +161,12 @@ async def refresh_access_token( # max_age=604800, # 7 days # ) return {"status": "refreshed", "refresh_token": new_access_token.refresh_token, "access_token": new_access_token.access_token} - except: - raise HTTPException(status_code=401) + except Exception as exc: + logger.info( + "Admin refresh token validation failed", + extra=log_context(None, "admin_refresh_failed"), + ) + raise HTTPException(status_code=401) from exc @router.post("/logout") @@ -244,6 +253,11 @@ def perform_ping_probe(): # Success if return code is 0 ping_history.append(result.returncode == 0) except Exception: + logger.warning( + "Network ping probe failed", + exc_info=True, + extra=log_context(None, "network_ping_probe_failed"), + ) ping_history.append(False) @@ -343,7 +357,7 @@ def get_network_details(): struct.pack("256s", iface[:15].encode("utf-8")), )[20:24] ) - except Exception: + except OSError: # Likely no IPv4 assigned to this interface ip_addr = "N/A" @@ -461,24 +475,26 @@ def get_admin_users( } -def makeAdmin(user: User, session: SessionDep): +def makeAdmin(user: User, session: SessionDep, commit: bool = True): if not user.id: raise HTTPException(500) admin = Admin(user_id=user.id) session.add(admin) - session.commit() - session.refresh(admin) + if commit: + session.commit() + session.refresh(admin) -def makeRescuer(user: User, session: SessionDep): +def makeRescuer(user: User, session: SessionDep, commit: bool = True): if not user.id: raise HTTPException(500) rescuer = Rescuer( user_id=user.id, ) session.add(rescuer) - session.commit() - session.refresh(rescuer) + if commit: + session.commit() + session.refresh(rescuer) @router.post("/create/user/rescuer") @@ -499,8 +515,14 @@ def create_rescuer( session.commit() return {"status": "ok"} except IntegrityError as _: + session.rollback() raise HTTPException(403, "user is already a rescuer") - except Exception as _: + except Exception: + session.rollback() + logger.exception( + "Failed to grant rescuer role", + extra=log_context(current_user.id, "admin_rescuer_grant_failed", user_id), + ) raise HTTPException(500) @@ -517,13 +539,18 @@ def create_admin( session.commit() return {"status": "ok"} except IntegrityError as _: + session.rollback() raise HTTPException(403, "user is already an admin") - except Exception as _: + except Exception: session.rollback() + logger.exception( + "Failed to grant admin role", + extra=log_context(current_user.id, "admin_role_grant_failed", user_id), + ) raise HTTPException(500) -def removeAdmin(user: User, session: SessionDep): +def removeAdmin(user: User, session: SessionDep, commit: bool = True): if not user.id: raise HTTPException(500) if not user.id: @@ -534,14 +561,15 @@ def removeAdmin(user: User, session: SessionDep): # 2. If it exists, delete it if admin: session.delete(admin) - session.commit() + if commit: + session.commit() return {"message": "Admin deleted successfully"} # 3. Handle the case where it doesn't exist raise HTTPException(404, "Admin not found") -def removeRescuer(user: User, session: SessionDep): +def removeRescuer(user: User, session: SessionDep, commit: bool = True): if not user.id: raise HTTPException(500) statement = select(Rescuer).where(Rescuer.user_id == user.id) @@ -550,7 +578,8 @@ def removeRescuer(user: User, session: SessionDep): # 2. If it exists, delete it if rescuer: session.delete(rescuer) - session.commit() + if commit: + session.commit() return {"message": "Rescuer deleted successfully"} # 3. Handle the case where it doesn't exist @@ -567,8 +596,15 @@ def remove_admin( try: removeAdmin(user, session) return {"status": "ok"} - except Exception as _: + except HTTPException: + session.rollback() + raise + except Exception: session.rollback() + logger.exception( + "Failed to remove admin role", + extra=log_context(current_user.id, "admin_role_removal_failed", user_id), + ) raise HTTPException(500) @@ -582,8 +618,15 @@ def remove_rescuer( try: removeRescuer(user, session) return {"status": "ok"} - except Exception as _: + except HTTPException: session.rollback() + raise + except Exception: + session.rollback() + logger.exception( + "Failed to remove rescuer role", + extra=log_context(current_user.id, "admin_rescuer_removal_failed", user_id), + ) raise HTTPException(500) @@ -594,23 +637,30 @@ def create_user( session: SessionDep, ): try: - user = db_create_user(userData, session) + user = db_create_user(userData, session, commit=False) if not user.id: raise HTTPException(500) if userData.is_rescuer: - makeRescuer(user, session) + makeRescuer(user, session, commit=False) if userData.is_admin: - makeAdmin(user, session) + makeAdmin(user, session, commit=False) + + session.commit() + session.refresh(user) return user except HTTPException as e: session.rollback() raise e - except Exception as e: + except Exception: session.rollback() + logger.exception( + "Failed to create user through admin console", + extra=log_context(current_user.id, "admin_user_creation_failed"), + ) raise HTTPException(500) @@ -625,41 +675,38 @@ def edit_user( if not user or not user.id: raise HTTPException(404, "user not found") - print("here") update_user_info( - user, UserUpdate(**userData.model_dump(exclude_unset=True)), session + user, UserUpdate(**userData.model_dump(exclude_unset=True)), session, commit=False ) - print("here after") updated_user = get_user_by_ID(session, userData.id) if not updated_user: - print("raise 500") raise HTTPException(500) - print("here after 500") - try: - if userData.is_rescuer: - makeRescuer(updated_user, session) - else: - removeRescuer(updated_user, session) - except: - pass + if userData.is_rescuer is not None: + if userData.is_rescuer and not updated_user.rescuer: + makeRescuer(updated_user, session, commit=False) + elif not userData.is_rescuer and updated_user.rescuer: + removeRescuer(updated_user, session, commit=False) - try: - if userData.is_admin: - makeAdmin(updated_user, session) - else: - removeAdmin(updated_user, session) - except: - pass + if userData.is_admin is not None: + if userData.is_admin and not updated_user.admin: + makeAdmin(updated_user, session, commit=False) + elif not userData.is_admin and updated_user.admin: + removeAdmin(updated_user, session, commit=False) + + session.commit() return {"status": "ok"} except HTTPException as e: session.rollback() raise e - except Exception as e: + except Exception: session.rollback() - print("E", e) + logger.exception( + "Failed to update user", + extra=log_context(current_user.id, "admin_user_update_failed", userData.id), + ) raise HTTPException(500) @@ -677,7 +724,12 @@ def delete_user( session.commit() except HTTPException as e: raise e - except Exception as e: + except Exception: + session.rollback() + logger.exception( + "Failed to delete user", + extra=log_context(_.id, "admin_user_deletion_failed", user_id), + ) raise HTTPException(500) return {"status": "ok"} @@ -715,8 +767,12 @@ def ban_user( session.refresh(ban) except HTTPException as e: raise e - except Exception as e: - print(e) + except Exception: + session.rollback() + logger.exception( + "Failed to ban user", + extra=log_context(_.id, "admin_user_ban_failed", user_id), + ) raise HTTPException(500) return {"status": "ok"} @@ -734,14 +790,21 @@ def unban_user( now = datetime.now(timezone.utc).replace(tzinfo=None) nowreal = datetime.now(timezone.utc) statement = ( - update(BannedUser).where(BannedUser.until > now).values(until=nowreal) + update(BannedUser) + .where(BannedUser.user_id == user_id, BannedUser.until > now) + .values(until=nowreal) ) session.exec(statement) session.commit() except HTTPException as e: raise e - except Exception as e: + except Exception: + session.rollback() + logger.exception( + "Failed to unban user", + extra=log_context(_.id, "admin_user_unban_failed", user_id), + ) raise HTTPException(500) return {"status": "ok"} diff --git a/server/app/api/peer_connection.py b/server/app/api/peer_connection.py index b2036d9e..5c90fdd8 100644 --- a/server/app/api/peer_connection.py +++ b/server/app/api/peer_connection.py @@ -19,20 +19,25 @@ from app.models.queued import Queue from app.models.signalling import SignalMessage from fastapi import Query, WebSocketDisconnect +from pydantic import ValidationError from app.db_operations.websockets import authenticate_websocket, relay_message, relay_public_message, validate_message_sender, validate_sender, relay_signal, receive_signal_message, WebSocketAuthError from app.db_operations.connection_manager import manager from app.db_operations.activity import set_user_status from app.models.websocketComms import MessageData, PublicMessageData +from app.structured_logging import log_context -logger = logging.getLogger(__name__) +logger = logging.getLogger("app") def _set_status_bg(user_id: UUID, status: str) -> None: try: with Session(engine) as session: set_user_status(session, user_id, status) - except Exception as e: - print(f"[activity] status update failed for {user_id}: {e}") + except Exception: + logger.exception( + "Activity status update failed", + extra=log_context(user_id, "websocket_activity_status_update_failed", metadata={"status": status}), + ) router = APIRouter( prefix='/ws', @@ -129,8 +134,11 @@ def get_queued_messages(user_id: UUID, session: SessionDep, limit: int = 100): try: statement = select(Queue).where(Queue.to == user_id).limit(limit) return session.exec(statement).all() - except Exception as e: - print("EX", e) + except Exception: + logger.exception( + "Failed to fetch queued messages", + extra=log_context(user_id, "websocket_queue_fetch_failed"), + ) return None @@ -149,10 +157,10 @@ def deep_parse_dict(data): if (data.startswith('{') and data.endswith('}')) or (data.startswith('[') and data.endswith(']')): try: data = json.loads(data) - except Exception: + except json.JSONDecodeError: try: data = ast.literal_eval(data) - except Exception: + except (ValueError, SyntaxError): return data else: return data @@ -179,15 +187,23 @@ async def main_web_socket(token: str, websocket: WebSocket, target_id: UUID|None try: user_id = await authenticate_websocket(websocket, token) except WebSocketAuthError: - logger.warning("WebSocket auth rejected: invalid or expired token client=%s", websocket.client) + logger.warning( + "WebSocket auth rejected: invalid or expired token client=%s", + websocket.client, + extra=log_context(None, "websocket_auth_rejected"), + ) return await manager.connect(UUID(user_id), websocket) asyncio.get_event_loop().run_in_executor(None, _set_status_bg, UUID(user_id), "Active") try: await manager.broadcast({"type": "status-update", "user_id": user_id, 'status': "online"}) - except: - pass + except Exception: + logger.warning( + "Failed to broadcast WebSocket online status", + exc_info=True, + extra=log_context(user_id, "websocket_online_status_broadcast_failed"), + ) try: with Session(engine) as session: @@ -211,10 +227,16 @@ async def main_web_socket(token: str, websocket: WebSocket, target_id: UUID|None if message.payload_type == 'seen': session.delete(message) session.commit() - except Exception as e: - print(f"[drain] failed to deliver queued message {message.id}: {e}") - except Exception as e: - print(f"[drain] failed to fetch queued messages for {user_id}: {e}") + except Exception: + logger.exception( + "Failed to deliver queued WebSocket message", + extra=log_context(user_id, "websocket_queue_delivery_failed", message.id), + ) + except Exception: + logger.exception( + "Failed to drain queued WebSocket messages", + extra=log_context(user_id, "websocket_queue_drain_failed"), + ) try: while True: @@ -227,15 +249,23 @@ async def main_web_socket(token: str, websocket: WebSocket, target_id: UUID|None if raw_type == "public-chat": try: payload = PublicMessageData.model_validate(raw_payload) - except Exception: + except ValidationError: + logger.debug( + "Invalid public WebSocket payload", + extra=log_context(user_id, "websocket_invalid_payload", metadata={"type": raw_type}), + ) payload = raw_payload else: try: payload = MessageData.model_validate(raw_payload) - except Exception: + except ValidationError: try: - payload = SignalMessage(**raw_payload) - except Exception: + payload = SignalMessage.model_validate(raw_payload) + except ValidationError: + logger.debug( + "Invalid WebSocket payload", + extra=log_context(user_id, "websocket_invalid_payload", metadata={"type": raw_type}), + ) payload = raw_payload if isinstance(payload, dict) and payload.get("type") == "ping": await manager.send_personal_message(UUID(user_id), {"type": "pong"}) @@ -261,7 +291,11 @@ async def main_web_socket(token: str, websocket: WebSocket, target_id: UUID|None except WebSocketDisconnect: try: await manager.broadcast({"type": "status-update","user_id": user_id, 'status': "offline"}) - except: - pass + except Exception: + logger.warning( + "Failed to broadcast WebSocket offline status", + exc_info=True, + extra=log_context(user_id, "websocket_offline_status_broadcast_failed"), + ) asyncio.get_event_loop().run_in_executor(None, _set_status_bg, UUID(user_id), "Inactive") - await manager.disconnect(UUID(user_id)) + await manager.disconnect(UUID(user_id), websocket=websocket) diff --git a/server/app/api/update_info.py b/server/app/api/update_info.py index eba8d4a9..5b319823 100644 --- a/server/app/api/update_info.py +++ b/server/app/api/update_info.py @@ -1,4 +1,5 @@ from typing import Annotated +import logging from sqlalchemy import or_ from fastapi import Depends, HTTPException from fastapi.routing import APIRouter @@ -9,6 +10,7 @@ from app.models.users import UserUpdate, User from app.db_operations.auth import update_user_info from app.db_operations.auth import SessionDep +from app.structured_logging import log_context from sqlalchemy.exc import IntegrityError from fastapi import status @@ -20,6 +22,7 @@ 404: {'description': 'Not Found'} } ) +logger = logging.getLogger("app") @router.post("/", status_code=status.HTTP_200_OK) @@ -73,8 +76,12 @@ def update_user( raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=detail) - except Exception as e: + except Exception: session.rollback() + logger.exception( + "Failed to update profile", + extra=log_context(current_user.id, "profile_update_failed", current_user.id), + ) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="An unexpected error occurred" diff --git a/server/app/db_operations/GPS_manager.py b/server/app/db_operations/GPS_manager.py index 59ef327e..4a1065bd 100644 --- a/server/app/db_operations/GPS_manager.py +++ b/server/app/db_operations/GPS_manager.py @@ -1,6 +1,10 @@ import json +import logging from fastapi import WebSocket from typing import Dict +from app.structured_logging import log_context + +logger = logging.getLogger("app") class GPSManager: def __init__(self): @@ -17,11 +21,18 @@ def disconnect_monitor(self, user_id: str): async def broadcast_to_rescuers(self, message: dict): """Send the GPS packet to every connected Rescuer.""" - for user_id, connection in self.active_monitors.items(): + stale_monitors: list[tuple[str, WebSocket]] = [] + for user_id, connection in list(self.active_monitors.items()): try: await connection.send_json(message) except Exception: - # If a connection is dead, we'll clean it up later or on next fail - pass + logger.info( + "GPS broadcast delivery failed for disconnected monitor", + extra=log_context(user_id, "gps_monitor_disconnected"), + ) + stale_monitors.append((user_id, connection)) + for user_id, connection in stale_monitors: + if self.active_monitors.get(user_id) is connection: + self.disconnect_monitor(user_id) gps_manager = GPSManager() diff --git a/server/app/db_operations/auth.py b/server/app/db_operations/auth.py index e627ff10..ecddd9f3 100644 --- a/server/app/db_operations/auth.py +++ b/server/app/db_operations/auth.py @@ -1,16 +1,19 @@ import os +import logging from datetime import datetime, timezone from typing import Annotated, Dict from uuid import UUID from fastapi import Depends, HTTPException, Request from pwdlib import PasswordHash -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, SQLAlchemyError from pwdlib.hashers.argon2 import Argon2Hasher from sqlmodel import SQLModel, Session, create_engine, select, or_ from app.models.users import User, UserCreate from app.models.users import UserUpdate, UserPasswordUpdate +from app.structured_logging import log_context +logger = logging.getLogger("app") # Reduced from recommended() defaults (time_cost=2, memory_cost=65536) @@ -55,12 +58,17 @@ def verify_password(plain_password : str, hashed__password : str): return password_hash.verify(plain_password, hashed__password) -def db_create_user(user: UserCreate, session: SessionDep): +def db_create_user(user: UserCreate, session: SessionDep, commit: bool = True): try: user_in_db = get_user_by_ID(session, user.id) if user.id else None except HTTPException: user_in_db = None - except: + except SQLAlchemyError: + session.rollback() + logger.exception( + "Failed to look up existing user during user creation", + extra=log_context(None, "user_creation_lookup_failed"), + ) raise HTTPException(500, "Internal server error.") errors: Dict[str, str] = {} @@ -97,11 +105,15 @@ def db_create_user(user: UserCreate, session: SessionDep): session.add(db_user) try: - session.commit() + if commit: + session.commit() + else: + session.flush() except IntegrityError: session.rollback() raise HTTPException(status_code=400, detail={"username": "Username or contact already registered"}) - session.refresh(db_user) + if commit: + session.refresh(db_user) return db_user elif user_in_db and user_in_db.guest: # modify existing user @@ -121,8 +133,11 @@ def db_create_user(user: UserCreate, session: SessionDep): session.add(user_in_db) # delete guest record session.delete(user_in_db.guest) - session.commit() - session.refresh(user_in_db) + if commit: + session.commit() + session.refresh(user_in_db) + else: + session.flush() return user_in_db # TODO: all guest accounts are disabled from getting a token in any way shape or form @@ -192,15 +207,18 @@ def authenticate_user( return user -def update_user_info(user: User, new_user_data : UserUpdate, session : SessionDep): +def update_user_info( + user: User, new_user_data: UserUpdate, session: SessionDep, commit: bool = True +): new_user_dump = new_user_data.model_dump(exclude_unset=True) for field, value in new_user_dump.items(): setattr(user, field, value) session.add(user) - session.commit() - session.refresh(user) + if commit: + session.commit() + session.refresh(user) PASSWORD_MIN_LENGTH = 8 diff --git a/server/app/db_operations/connection_manager.py b/server/app/db_operations/connection_manager.py index 55802969..5e43e7b3 100644 --- a/server/app/db_operations/connection_manager.py +++ b/server/app/db_operations/connection_manager.py @@ -5,10 +5,11 @@ from uuid import UUID, uuid4 from fastapi import WebSocket from typing import Dict, Optional +from app.structured_logging import log_context import redis.asyncio as aioredis -logger = logging.getLogger(__name__) +logger = logging.getLogger("app") _PRESENCE_KEY = "ws:online_users" # Sorted set; score = expiry epoch (float) _BROADCAST_CHANNEL = "ws:broadcast" @@ -74,7 +75,9 @@ async def connect(self, user_id: UUID, websocket: WebSocket) -> None: self._heartbeat_loop(user_id), name=f"ws-hb-{user_id}" ) - async def disconnect(self, user_id: UUID) -> None: + async def disconnect(self, user_id: UUID, websocket: WebSocket | None = None) -> None: + if websocket is not None and self._local.get(user_id) is not websocket: + return self._local.pop(user_id, None) for tasks in (self._sub_tasks, self._hb_tasks): task = tasks.pop(user_id, None) @@ -93,7 +96,11 @@ async def send_personal_message(self, target_id: UUID, message: dict) -> None: await ws.send_json(message) return except Exception: - await self.disconnect(target_id) + logger.info( + "WebSocket delivery failed for disconnected user", + extra=log_context(target_id, "websocket_user_disconnected"), + ) + await self.disconnect(target_id, websocket=ws) # Cross-worker: publish so the holding worker delivers it if self._redis: await self._redis.publish( @@ -150,8 +157,11 @@ async def _user_sub_loop( try: data = json.loads(raw["data"]) await websocket.send_json(data) - except Exception as exc: - logger.debug("[ws-sub] delivery failed user=%s: %s", user_id, exc) + except Exception: + logger.info( + "WebSocket subscription delivery failed for disconnected user", + extra=log_context(user_id, "websocket_subscription_disconnected"), + ) break except asyncio.CancelledError: pass @@ -170,15 +180,24 @@ async def _broadcast_loop(self, pubsub: aioredis.client.PubSub) -> None: continue message = envelope["msg"] except Exception: + logger.warning( + "Discarded malformed WebSocket broadcast", + exc_info=True, + extra=log_context(None, "websocket_broadcast_malformed"), + ) continue - stale: list[UUID] = [] + stale: list[tuple[UUID, WebSocket]] = [] for uid, ws in list(self._local.items()): try: await ws.send_json(message) except Exception: - stale.append(uid) - for uid in stale: - await self.disconnect(uid) + logger.info( + "WebSocket broadcast delivery failed for disconnected user", + extra=log_context(uid, "websocket_broadcast_disconnected"), + ) + stale.append((uid, ws)) + for uid, websocket in stale: + await self.disconnect(uid, websocket=websocket) except asyncio.CancelledError: pass finally: diff --git a/server/app/db_operations/token.py b/server/app/db_operations/token.py index b2aab2be..594b40a6 100644 --- a/server/app/db_operations/token.py +++ b/server/app/db_operations/token.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os import threading +import logging import redis as _redis_module from uuid import UUID, uuid4 from pydantic import BaseModel @@ -20,6 +21,9 @@ from app.models.users import UserCreate from app.models.jti import BlacklistedToken from app.db_operations.auth import get_user, get_user_by_ID +from app.structured_logging import log_context + +logger = logging.getLogger("app") _REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379") @@ -27,7 +31,12 @@ _redis: _redis_module.Redis = _redis_module.from_url(_REDIS_URL, decode_responses=True) _redis.ping() _REDIS_AVAILABLE = True -except Exception: +except _redis_module.RedisError: + logger.warning( + "Redis blacklist cache is unavailable; using database checks", + exc_info=True, + extra=log_context(None, "token_blacklist_cache_unavailable"), + ) _redis = None # type: ignore[assignment] _REDIS_AVAILABLE = False @@ -90,7 +99,7 @@ def get_user_id_from_header(request: Request): token = auth_header.split(" ")[1] payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) return payload.get("sub") # or payload.get("user_id") - except Exception: + except (IndexError, PyJWTError): return None def verify_token(token: str): diff --git a/server/app/structured_logging.py b/server/app/structured_logging.py new file mode 100644 index 00000000..ac470f8f --- /dev/null +++ b/server/app/structured_logging.py @@ -0,0 +1,16 @@ +from typing import Any +from uuid import UUID + + +def log_context( + user_id: UUID | str | None, + action: str, + entity_id: UUID | str | None = None, + metadata: dict[str, Any] | None = None, +) -> dict[str, Any]: + return { + "user_id": str(user_id) if user_id else "ANONYMOUS", + "action": action, + "entity_id": str(entity_id) if entity_id else None, + "metadata_json": metadata or {}, + } diff --git a/server/app/tests/test_admin_me.py b/server/app/tests/test_admin_me.py index 27e4b162..432f7295 100644 --- a/server/app/tests/test_admin_me.py +++ b/server/app/tests/test_admin_me.py @@ -1,8 +1,14 @@ from fastapi.testclient import TestClient +from fastapi import HTTPException +from datetime import datetime, timedelta, timezone +import pytest from sqlmodel import Session, select +from app.api import admin from app.models.admin import Admin -from app.models.users import User +from app.models.banned_user import BannedUser +from app.models.rescuer import Rescuer +from app.models.users import User, UserCreateThroughAdmin, UserUpdateThroughAdmin def _login_as_admin(client: TestClient, session: Session, username: str, password: str) -> str: @@ -74,3 +80,91 @@ def test_logout_without_refresh_token_cookie_returns_401_not_500(client: TestCli ) assert response.status_code == 401 + + +def test_admin_edit_rolls_back_profile_when_role_change_fails(session: Session, monkeypatch): + user = session.exec(select(User).where(User.username == "test")).one() + user_id = user.id + original_username = user.username + update = UserUpdateThroughAdmin( + id=user_id, + username="updated-admin-user", + is_admin=True, + is_rescuer=False, + ) + + def fail_role_grant(*args, **kwargs): + raise RuntimeError("role write failed") + + monkeypatch.setattr(admin, "makeAdmin", fail_role_grant) + + with pytest.raises(HTTPException) as exc_info: + admin.edit_user(user, update, session) + + assert exc_info.value.status_code == 500 + session.expire_all() + persisted_user = session.exec(select(User).where(User.id == user_id)).one() + assert persisted_user.username == original_username + + +def test_admin_edit_preserves_roles_when_role_fields_are_omitted(session: Session): + user = session.exec(select(User).where(User.username == "test")).one() + session.add_all([Admin(user_id=user.id), Rescuer(user_id=user.id)]) + session.commit() + session.expire_all() + user = session.exec(select(User).where(User.username == "test")).one() + + result = admin.edit_user( + user, + UserUpdateThroughAdmin(id=user.id, username="renamed-admin-user"), + session, + ) + + assert result == {"status": "ok"} + session.expire_all() + persisted_user = session.exec(select(User).where(User.id == user.id)).one() + assert persisted_user.admin is not None + assert persisted_user.rescuer is not None + + +def test_admin_create_rolls_back_user_and_role_when_role_grant_fails(session: Session, monkeypatch): + actor = session.exec(select(User).where(User.username == "test")).one() + user_data = UserCreateThroughAdmin( + username="atomic-admin-user", + first_name="Atomic", + last_name="Admin", + password="StrongPass123", + is_rescuer=True, + is_admin=True, + ) + + def fail_admin_grant(*args, **kwargs): + raise RuntimeError("role write failed") + + monkeypatch.setattr(admin, "makeAdmin", fail_admin_grant) + + with pytest.raises(HTTPException) as exc_info: + admin.create_user(actor, user_data, session) + + assert exc_info.value.status_code == 500 + assert session.exec(select(User).where(User.username == "atomic-admin-user")).first() is None + + +def test_unban_only_updates_requested_user(session: Session): + actor = session.exec(select(User).where(User.username == "test")).one() + target = session.exec(select(User).where(User.username == "Tony Stark")).one() + other = session.exec(select(User).where(User.username == "Steve Rogers")).one() + expires_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=1) + session.add_all([ + BannedUser(user_id=target.id, until=expires_at), + BannedUser(user_id=other.id, until=expires_at), + ]) + session.commit() + + admin.unban_user(actor, target.id, session) + + session.expire_all() + target_ban = session.exec(select(BannedUser).where(BannedUser.user_id == target.id)).one() + other_ban = session.exec(select(BannedUser).where(BannedUser.user_id == other.id)).one() + assert target_ban.until < expires_at + assert other_ban.until == expires_at diff --git a/server/app/tests/test_gps.py b/server/app/tests/test_gps.py index 3288afd8..6e7bd37c 100644 --- a/server/app/tests/test_gps.py +++ b/server/app/tests/test_gps.py @@ -1,4 +1,6 @@ import pytest +import asyncio +import logging from datetime import datetime, timedelta, timezone from fastapi.testclient import TestClient from fastapi import APIRouter, WebSocket, WebSocketDisconnect, Depends @@ -10,6 +12,43 @@ import json from app.tests.test_db_utils import get_auth_headers +from app.db_operations.GPS_manager import GPSManager + + +class FailingWebSocket: + async def send_json(self, message: dict) -> None: + raise RuntimeError("connection closed") + + +def test_gps_broadcast_logs_and_removes_failed_monitor(caplog): + manager = GPSManager() + manager.active_monitors["rescuer-1"] = FailingWebSocket() + caplog.set_level(logging.INFO, logger="app") + + asyncio.run(manager.broadcast_to_rescuers({"lat": 1, "lng": 2})) + + assert manager.active_monitors == {} + record = next(record for record in caplog.records if "GPS broadcast delivery failed" in record.message) + assert record.user_id == "rescuer-1" + assert record.action == "gps_monitor_disconnected" + assert record.entity_id is None + assert record.metadata_json == {} + + +def test_gps_broadcast_keeps_monitor_that_reconnects_during_delivery(): + manager = GPSManager() + replacement = object() + + class ReconnectingWebSocket: + async def send_json(self, message: dict) -> None: + manager.active_monitors["rescuer-1"] = replacement + raise RuntimeError("connection closed") + + manager.active_monitors["rescuer-1"] = ReconnectingWebSocket() + + asyncio.run(manager.broadcast_to_rescuers({"lat": 1, "lng": 2})) + + assert manager.active_monitors["rescuer-1"] is replacement def test_stream_gps_location_success(client: TestClient): @@ -184,4 +223,3 @@ def test_gps_broadcast_to_rescuer(client, session, test_user_instance, test_resc # After exiting the block, the rescuer is disconnected # and the 'raise' inside your endpoint is handled by the TestClient - diff --git a/server/app/tests/test_websocket_pool.py b/server/app/tests/test_websocket_pool.py index 39489a14..ff2f04f4 100644 --- a/server/app/tests/test_websocket_pool.py +++ b/server/app/tests/test_websocket_pool.py @@ -10,7 +10,10 @@ The fix: open short-lived sessions per DB operation so an idle peer holds zero connections. """ +import asyncio +import logging import time +from uuid import uuid4 import pytest from sqlmodel import Session, SQLModel, create_engine @@ -21,6 +24,72 @@ from app.main import app from app.db_operations.auth import get_session from app.db_operations.token import create_access_token +from app.db_operations.connection_manager import ConnectionManager + + +class FailingWebSocket: + async def send_json(self, message: dict) -> None: + raise RuntimeError("connection closed") + + +def test_personal_delivery_failure_is_logged_and_disconnects(caplog): + manager = ConnectionManager() + user_id = uuid4() + manager._local[user_id] = FailingWebSocket() + caplog.set_level(logging.INFO, logger="app") + + asyncio.run(manager.send_personal_message(user_id, {"type": "ping"})) + + assert user_id not in manager._local + record = next(record for record in caplog.records if "WebSocket delivery failed" in record.message) + assert record.user_id == str(user_id) + assert record.action == "websocket_user_disconnected" + assert record.entity_id is None + assert record.metadata_json == {} + + +def test_personal_delivery_failure_keeps_reconnected_socket(): + manager = ConnectionManager() + user_id = uuid4() + replacement = object() + + class ReconnectingWebSocket: + async def send_json(self, message: dict) -> None: + manager._local[user_id] = replacement + raise RuntimeError("connection closed") + + manager._local[user_id] = ReconnectingWebSocket() + + asyncio.run(manager.send_personal_message(user_id, {"type": "ping"})) + + assert manager._local[user_id] is replacement + + +def test_broadcast_delivery_failure_keeps_reconnected_socket(): + manager = ConnectionManager() + user_id = uuid4() + replacement = object() + + class ReconnectingWebSocket: + async def send_json(self, message: dict) -> None: + manager._local[user_id] = replacement + raise RuntimeError("connection closed") + + class OneMessagePubSub: + async def listen(self): + yield {"type": "message", "data": '{"_wid": "other", "msg": {"type": "ping"}}'} + + async def unsubscribe(self, channel: str) -> None: + return None + + async def aclose(self) -> None: + return None + + manager._local[user_id] = ReconnectingWebSocket() + + asyncio.run(manager._broadcast_loop(OneMessagePubSub())) + + assert manager._local[user_id] is replacement @pytest.fixture(name="pool_engine")