From 930792aba87d140cd7adec740ee97966aa31aa62 Mon Sep 17 00:00:00 2001 From: Adamskiee Date: Sat, 15 Aug 2026 20:20:59 +0800 Subject: [PATCH 1/3] fix(gsm-compose): route gateway traffic on Docker network --- admin-frontend/sapot-admin/Dockerfile | 11 ++++++++++- docker-compose.yml | 7 ++++++- docs/deployment/environment-config.md | 2 ++ docs/getting-started/docker-setup.md | 4 ++++ server/.env.example | 1 + server/app/api/gsm.py | 5 +++-- server/app/tests/test_gsm_proxy.py | 21 ++++++++++++++++++++- 7 files changed, 46 insertions(+), 5 deletions(-) diff --git a/admin-frontend/sapot-admin/Dockerfile b/admin-frontend/sapot-admin/Dockerfile index 982865fb..c881b681 100644 --- a/admin-frontend/sapot-admin/Dockerfile +++ b/admin-frontend/sapot-admin/Dockerfile @@ -1,13 +1,22 @@ # syntax=docker/dockerfile:1 -FROM node:22-slim AS builder +FROM node:22-slim AS base RUN corepack enable WORKDIR /app + +FROM base AS dependencies + COPY package.json pnpm-lock.yaml ./ RUN pnpm install --frozen-lockfile +FROM dependencies AS development +ENV NODE_ENV=development +COPY . . + +FROM dependencies AS builder + COPY . . RUN pnpm build diff --git a/docker-compose.yml b/docker-compose.yml index b4142ae1..834dd983 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,8 @@ services: restart: unless-stopped env_file: - ./server/.env + environment: + GSM_GATEWAY_URL: http://gsm-fastapi:8001 depends_on: db: condition: service_healthy @@ -78,6 +80,7 @@ services: admin: build: context: ./admin-frontend/sapot-admin + target: development command: pnpm dev restart: unless-stopped env_file: @@ -125,7 +128,9 @@ services: - ./GSM-module/GSM-fastapi/.env environment: HOST: 0.0.0.0 - SAPOT_API_URL: https://nginx + # The callback stays on Docker's private network. Going through nginx + # would require the gateway image to trust the development TLS CA. + SAPOT_API_URL: http://api:8000 # No /dev/ttyACM0 device passthrough here — the modem isn't present on # most dev machines, and Compose has no "optional device" syntax, so # declaring it here would abort the whole `docker compose up` (nginx/ diff --git a/docs/deployment/environment-config.md b/docs/deployment/environment-config.md index c1c826a6..61521a69 100644 --- a/docs/deployment/environment-config.md +++ b/docs/deployment/environment-config.md @@ -16,6 +16,7 @@ All SAPOT components are configured via environment variables. This document lis | `REDIS_URL` | `redis://localhost:6379` | Set if Redis is on a non-default host/port | | `SERVER_ED25519_SEED` | `None` (server key signing disabled if unset) | Set to enable server-signed peer keys | | `GSM_SECRET` | None — required, raises `RuntimeError` at import if unset | **MUST** be set — shared secret for GSM module webhooks | +| `GSM_GATEWAY_URL` | `http://localhost:8001` | Base URL of the deployed GSM FastAPI gateway. Set `http://gsm-fastapi:8001` in Docker Compose. | See [SECURITY.md](../../SECURITY.md) for why `DATABASE_URL`, `JWT_SECRET_KEY`, `CORS_ALLOWED_ORIGINS`, and `GSM_SECRET` are required. @@ -37,6 +38,7 @@ ENVIRONMENT=production REDIS_URL=redis://127.0.0.1:6379/0 SERVER_ED25519_SEED= GSM_SECRET= +GSM_GATEWAY_URL=http://127.0.0.1:8001 ``` --- diff --git a/docs/getting-started/docker-setup.md b/docs/getting-started/docker-setup.md index 0c092eae..bd68e6d4 100644 --- a/docs/getting-started/docker-setup.md +++ b/docs/getting-started/docker-setup.md @@ -59,6 +59,10 @@ cp GSM-module/GSM-fastapi/.env.example GSM-module/GSM-fastapi/.env `gsm-fastapi`'s `GSM_SECRET` must match `server/.env`'s `GSM_SECRET` — they authenticate the webhook calls between the two services (see [environment-config.md](../deployment/environment-config.md)). + +Compose sets the server's `GSM_GATEWAY_URL` to `http://gsm-fastapi:8001`, which resolves through the +internal Docker network. Do not replace it with `localhost`: inside the `api` container, that address +refers to the API container rather than the separate GSM gateway container. The `gsm-fastapi` container passes through the GSM modem at `/dev/ttyACM0`, but only when `docker-compose.gsm-hardware.yml` is explicitly merged in (Compose has no "optional device" syntax, so this stays out of the base `docker-compose.yml`/`docker-compose.override.yml` — otherwise the diff --git a/server/.env.example b/server/.env.example index 5746eeca..ffeaf79d 100644 --- a/server/.env.example +++ b/server/.env.example @@ -11,6 +11,7 @@ TLS_KEY=~/server.key # so `cp .env.example .env` works out of the box for docs/getting-started/docker-setup.md. # Running bare-metal instead (docs/getting-started/server-setup.md)? Change both to 127.0.0.1/localhost. REDIS_URL=redis://redis:6379 +GSM_GATEWAY_URL=http://gsm-fastapi:8001 DATABASE_URL=mysql+pymysql://sapot:sapot@db:3306/sapot_dev diff --git a/server/app/api/gsm.py b/server/app/api/gsm.py index e630c72a..207cba9f 100644 --- a/server/app/api/gsm.py +++ b/server/app/api/gsm.py @@ -31,20 +31,21 @@ GSM_PROXY_MAX_CONNECTIONS = GSM_GATEWAY_MAX_ADMITTED_REQUESTS + 1 GSM_PROXY_POOL_TIMEOUT_SECONDS = 1.0 -# Module-level client reuses TCP connections to localhost:8001 across requests. +# Module-level client reuses TCP connections to the configured GSM gateway. _gsm_http_client: httpx.AsyncClient | None = None logger = logging.getLogger("app") GSM_SECRET = os.environ.get("GSM_SECRET") if not GSM_SECRET: raise RuntimeError("GSM_SECRET environment variable is not set") +GSM_GATEWAY_URL = os.environ.get("GSM_GATEWAY_URL", "http://localhost:8001").rstrip("/") def _get_gsm_client() -> httpx.AsyncClient: global _gsm_http_client if _gsm_http_client is None or _gsm_http_client.is_closed: _gsm_http_client = httpx.AsyncClient( - base_url="http://localhost:8001", + base_url=GSM_GATEWAY_URL, timeout=httpx.Timeout( connect=5.0, read=GSM_PROXY_READ_TIMEOUT_SECONDS, diff --git a/server/app/tests/test_gsm_proxy.py b/server/app/tests/test_gsm_proxy.py index 65738769..81eaea62 100644 --- a/server/app/tests/test_gsm_proxy.py +++ b/server/app/tests/test_gsm_proxy.py @@ -69,7 +69,7 @@ def _authenticated_user(session): return session.exec(select(User)).first() -def test_proxy_capacity_and_timeouts_cover_gateway_contract(monkeypatch): +def test_proxy_capacity_timeouts_and_gateway_url_cover_gateway_contract(monkeypatch): captured = {} class CapturingClient: @@ -90,6 +90,25 @@ def __init__(self, **kwargs): assert timeout.pool == gsm.GSM_PROXY_POOL_TIMEOUT_SECONDS assert limits.max_connections == gsm.GSM_PROXY_MAX_CONNECTIONS assert limits.max_connections > gsm.GSM_GATEWAY_MAX_ADMITTED_REQUESTS + assert captured["base_url"] == gsm.GSM_GATEWAY_URL + + +def test_proxy_uses_configured_gateway_url(monkeypatch): + captured = {} + + class CapturingClient: + is_closed = False + + def __init__(self, **kwargs): + captured.update(kwargs) + + monkeypatch.setattr(gsm, "_gsm_http_client", None) + monkeypatch.setattr(gsm, "GSM_GATEWAY_URL", "http://gsm-fastapi:8001") + monkeypatch.setattr(gsm.httpx, "AsyncClient", CapturingClient) + + gsm._get_gsm_client() + + assert captured["base_url"] == "http://gsm-fastapi:8001" def test_send_to_module_authenticates_with_shared_secret(monkeypatch): From 285367e650c367e44ee9b992b921e3bceff1af0d Mon Sep 17 00:00:00 2001 From: Adamskiee Date: Sat, 15 Aug 2026 20:23:08 +0800 Subject: [PATCH 2/3] feat(gsm-emulator): add virtual modem development stack --- GSM-module/GSM-fastapi/Dockerfile | 13 +- GSM-module/GSM-fastapi/mock_modem.py | 362 ++++++++++++++++++ GSM-module/GSM-fastapi/run-with-mock-modem.sh | 31 ++ .../GSM-fastapi/tests/test_mock_modem.py | 188 +++++++++ docker-compose.gsm-emulator.yml | 15 + docker-compose.gsm-hardware.yml | 7 +- docker/up.sh | 24 +- docs/features/sms-gateway/testing.md | 33 +- docs/getting-started/docker-setup.md | 26 +- docs/getting-started/gsm-module-setup.md | 93 +++++ 10 files changed, 786 insertions(+), 6 deletions(-) create mode 100644 GSM-module/GSM-fastapi/mock_modem.py create mode 100755 GSM-module/GSM-fastapi/run-with-mock-modem.sh create mode 100644 GSM-module/GSM-fastapi/tests/test_mock_modem.py create mode 100644 docker-compose.gsm-emulator.yml diff --git a/GSM-module/GSM-fastapi/Dockerfile b/GSM-module/GSM-fastapi/Dockerfile index ed4d9fab..a62e1bcb 100644 --- a/GSM-module/GSM-fastapi/Dockerfile +++ b/GSM-module/GSM-fastapi/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -FROM python:3.13-slim +FROM python:3.13-slim AS base RUN pip install --no-cache-dir uv @@ -12,6 +12,17 @@ ENV PATH="/opt/venv/bin:$PATH" COPY . . +FROM base AS emulator + +EXPOSE 8001 8002 + +CMD ["python", "main.py"] + +FROM base AS production + +COPY api.py app_version.py config.py database.py main.py protocol.py serial_worker.py sms_handler.py ./ +COPY models ./models + EXPOSE 8001 CMD ["python", "main.py"] diff --git a/GSM-module/GSM-fastapi/mock_modem.py b/GSM-module/GSM-fastapi/mock_modem.py new file mode 100644 index 00000000..04b34938 --- /dev/null +++ b/GSM-module/GSM-fastapi/mock_modem.py @@ -0,0 +1,362 @@ +"""PTY-backed virtual GSM modem and browser phone for local development.""" + +import argparse +import errno +import json +import os +import re +import select +import sys +import threading +import time +from collections import defaultdict +from datetime import datetime, timezone +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Optional +from urllib.parse import parse_qs, urlparse + +if os.name == "posix": + import pty + import tty + + +DISCONNECTED_POLL_SECONDS = 0.1 +ATTACH_SETTLE_SECONDS = 0.05 +E164_PATTERN = re.compile(r"^\+[0-9]{7,15}$") +INBOUND_BODY_MAX_BYTES = 127 # GSM_BUF is 128 bytes, including its NUL terminator. +OUTBOUND_RESULT_MODES = {"success", "NO_PROMPT", "TIMEOUT"} + + +def valid_phone_number(number: object) -> bool: + return isinstance(number, str) and bool(E164_PATTERN.fullmatch(number)) + + +def normalize_body(body: object, *, inbound: bool = False) -> Optional[str]: + if not isinstance(body, str): + return None + normalized = body.replace("\r", " ").replace("\n", " ") + if not normalized.strip(): + return None + if inbound: + normalized = normalized.replace("|", "/") + if len(normalized.encode("utf-8")) > INBOUND_BODY_MAX_BYTES: + return None + return normalized + + +def parse_send_sms(line: str) -> Optional[tuple[str, str]]: + """Return the destination and body for a valid outbound SMS frame.""" + parts = line.split("|", 2) + if len(parts) != 3 or parts[0] != "SEND_SMS" or not valid_phone_number(parts[1]): + return None + body = normalize_body(parts[2]) + if body is None: + return None + return parts[1], body + + +class VirtualModem: + """Thread-safe modem state shared by the PTY loop and browser server.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._master_fd: Optional[int] = None + self._attached = False + self._sim_present = True + self._network_connected = True + self._outbound_result_mode = "success" + self._messages: dict[str, list[dict[str, Any]]] = defaultdict(list) + self._next_message_id = 1 + + def attach(self, master_fd: int) -> None: + with self._lock: + self._master_fd = master_fd + self._attached = True + + def detach(self) -> None: + with self._lock: + self._attached = False + self._master_fd = None + + def status(self) -> dict[str, Any]: + with self._lock: + usable = self._sim_present and self._network_connected + return { + "connected": self._attached, + "sim_present": self._sim_present, + "network_connected": self._network_connected, + "gsm_ready": self._attached and usable, + "outbound_result_mode": self._outbound_result_mode, + } + + def _add_message(self, number: str, direction: str, body: str, status: str) -> dict[str, Any]: + message = { + "id": self._next_message_id, + "phone_number": number, + "direction": direction, + "body": body, + "timestamp": datetime.now(timezone.utc).isoformat(), + "status": status, + } + self._next_message_id += 1 + self._messages[number].append(message) + return message + + def messages(self, number: str) -> list[dict[str, Any]]: + with self._lock: + return list(self._messages.get(number, ())) + + def reset(self) -> None: + with self._lock: + self._messages.clear() + self._next_message_id = 1 + + def receive_outbound(self, number: str, body: str) -> tuple[bool, str]: + with self._lock: + if not self._sim_present: + return False, "SIM_MISSING" + if not self._network_connected: + return False, "NETWORK_LOST" + if self._outbound_result_mode != "success": + return False, self._outbound_result_mode + self._add_message(number, "received", body, "delivered") + return True, "" + + def inject_inbound(self, number: object, body: object) -> tuple[Optional[dict[str, Any]], Optional[str]]: + if not valid_phone_number(number): + return None, "phone_number must be E.164 format e.g. +639171234567" + normalized = normalize_body(body, inbound=True) + if normalized is None: + return None, f"body must be non-empty and at most {INBOUND_BODY_MAX_BYTES} UTF-8 bytes" + with self._lock: + if not self._sim_present: + return None, "SIM_MISSING" + if not self._network_connected: + return None, "NETWORK_LOST" + if not self._attached or self._master_fd is None: + return None, "MODEM_DISCONNECTED" + message = self._add_message(number, "sent", normalized, "sent") + master_fd = self._master_fd + if not _write_frames(master_fd, [f"SMS_RECEIVED|{number}|{normalized}\n".encode("utf-8")]): + with self._lock: + self._attached = False + self._master_fd = None + return None, "MODEM_DISCONNECTED" + return message, None + + def update(self, changes: dict[str, Any]) -> tuple[Optional[dict[str, Any]], Optional[str], list[bytes]]: + allowed = {"sim_present", "network_connected", "outbound_result_mode"} + if not changes or not set(changes).issubset(allowed): + return None, "provide sim_present, network_connected, or outbound_result_mode", [] + frames: list[bytes] = [] + with self._lock: + if "sim_present" in changes and not isinstance(changes["sim_present"], bool): + return None, "sim_present must be a boolean", [] + if "network_connected" in changes and not isinstance(changes["network_connected"], bool): + return None, "network_connected must be a boolean", [] + if "outbound_result_mode" in changes and changes["outbound_result_mode"] not in OUTBOUND_RESULT_MODES: + return None, "outbound_result_mode must be success, NO_PROMPT, or TIMEOUT", [] + before_usable = self._sim_present and self._network_connected + self._sim_present = changes.get("sim_present", self._sim_present) + self._network_connected = changes.get("network_connected", self._network_connected) + self._outbound_result_mode = changes.get("outbound_result_mode", self._outbound_result_mode) + after_usable = self._sim_present and self._network_connected + if self._attached: + if not self._sim_present: + frames.append(b"SIM_MISSING\n") + elif not self._network_connected: + frames.append(b"NETWORK_LOST\n") + elif not before_usable and after_usable: + frames.extend([b"GSM_READY\n", b"NETWORK_OK\n"]) + return self.status(), None, frames + + def readiness_frames(self) -> list[bytes]: + with self._lock: + if not self._sim_present: + return [b"SIM_MISSING\n"] + if not self._network_connected: + return [b"GSM_READY\n", b"NETWORK_LOST\n"] + return [b"GSM_READY\n", b"NETWORK_OK\n"] + + def master_fd(self) -> Optional[int]: + with self._lock: + return self._master_fd + + +def _is_disconnected(error: OSError) -> bool: + return error.errno == errno.EIO + + +def _write_frames(master_fd: int, frames: list[bytes]) -> bool: + try: + for frame in frames: + remaining = memoryview(frame) + while remaining: + written = os.write(master_fd, remaining) + remaining = remaining[written:] + except OSError as error: + if _is_disconnected(error): + return False + raise + return True + + +def _announce_readiness(modem: VirtualModem, master_fd: int) -> bool: + time.sleep(ATTACH_SETTLE_SECONDS) + return _write_frames(master_fd, modem.readiness_frames()) + + +PHONE_UI = """SAPOT Virtual Phone

SAPOT Virtual Phone

""" + + +def make_http_handler(modem: VirtualModem): + class Handler(BaseHTTPRequestHandler): + def _send_json(self, status: int, value: Any) -> None: + data = json.dumps(value).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _payload(self) -> Optional[dict[str, Any]]: + try: + length = int(self.headers.get("Content-Length", "0")) + value = json.loads(self.rfile.read(length).decode("utf-8")) + return value if isinstance(value, dict) else None + except (UnicodeDecodeError, ValueError, json.JSONDecodeError): + return None + + def do_GET(self) -> None: + parsed = urlparse(self.path) + if parsed.path == "/": + data = PHONE_UI.encode("utf-8") + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + elif parsed.path == "/api/messages": + number = parse_qs(parsed.query).get("phone", [None])[0] + if not valid_phone_number(number): + self._send_json(HTTPStatus.BAD_REQUEST, {"detail": "phone must be E.164 format"}) + else: + self._send_json(HTTPStatus.OK, {"phone_number": number, "messages": modem.messages(number)}) + elif parsed.path == "/api/modem": + self._send_json(HTTPStatus.OK, modem.status()) + else: + self._send_json(HTTPStatus.NOT_FOUND, {"detail": "not found"}) + + def do_POST(self) -> None: + if self.path == "/api/messages": + payload = self._payload() + if payload is None: + self._send_json(HTTPStatus.BAD_REQUEST, {"detail": "body must be a JSON object"}) + return + message, error = modem.inject_inbound(payload.get("phone_number"), payload.get("body")) + self._send_json(HTTPStatus.CREATED if message else HTTPStatus.SERVICE_UNAVAILABLE if error in {"SIM_MISSING", "NETWORK_LOST", "MODEM_DISCONNECTED"} else HTTPStatus.BAD_REQUEST, {"message": message} if message else {"detail": error}) + elif self.path == "/api/reset": + modem.reset() + self._send_json(HTTPStatus.OK, {"ok": True}) + else: + self._send_json(HTTPStatus.NOT_FOUND, {"detail": "not found"}) + + def do_PUT(self) -> None: + if self.path != "/api/modem": + self._send_json(HTTPStatus.NOT_FOUND, {"detail": "not found"}) + return + payload = self._payload() + if payload is None: + self._send_json(HTTPStatus.BAD_REQUEST, {"detail": "body must be a JSON object"}) + return + status, error, frames = modem.update(payload) + if error: + self._send_json(HTTPStatus.BAD_REQUEST, {"detail": error}) + return + fd = modem.master_fd() + if fd is not None and frames and not _write_frames(fd, frames): + modem.detach() + self._send_json(HTTPStatus.OK, status) + + def log_message(self, _format: str, *_args: object) -> None: + return + return Handler + + +def start_http_server(modem: VirtualModem, host: str, port: int) -> ThreadingHTTPServer: + server = ThreadingHTTPServer((host, port), make_http_handler(modem)) + threading.Thread(target=server.serve_forever, name="virtual-phone-http", daemon=True).start() + return server + + +def run(port_file: Optional[str] = None, web_host: str = "127.0.0.1", web_port: int = 8002) -> None: + if os.name != "posix": + raise RuntimeError("mock_modem.py requires POSIX PTY support") + modem = VirtualModem() + http_server = start_http_server(modem, web_host, web_port) + master_fd, slave_fd = pty.openpty() + slave_path = os.ttyname(slave_fd) + tty.setraw(slave_fd) + os.close(slave_fd) + if port_file: + with open(port_file, "w", encoding="utf-8") as file: + file.write(slave_path) + print(f"Virtual modem port: {slave_path}", flush=True) + print(f"Run the gateway with: SERIAL_PORT={slave_path} python main.py", flush=True) + attached = False + buffer = "" + try: + while True: + if not attached: + if _announce_readiness(modem, master_fd): + modem.attach(master_fd) + attached = True + else: + time.sleep(DISCONNECTED_POLL_SECONDS) + continue + readable, _, _ = select.select([master_fd], [], [], DISCONNECTED_POLL_SECONDS) + if not readable: + continue + try: + chunk = os.read(master_fd, 4096) + except OSError as error: + if not _is_disconnected(error): + raise + modem.detach(); attached = False; buffer = ""; continue + if not chunk: + modem.detach(); attached = False; buffer = ""; continue + buffer += chunk.decode("utf-8", errors="replace") + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + command = parse_send_sms(line.removesuffix("\r")) + if command is None: + print(f"Ignored modem command: {line!r}", flush=True) + continue + number, body = command + print(f"SMS request to {number}: {body}", flush=True) + delivered, reason = modem.receive_outbound(number, body) + if reason == "TIMEOUT": + continue + frame = f"SMS_SENT|{number}\n" if delivered else f"SMS_FAILED|{number}|{reason}\n" + if not _write_frames(master_fd, [frame.encode()]): + modem.detach(); attached = False; buffer = ""; break + finally: + modem.detach() + http_server.shutdown() + http_server.server_close() + os.close(master_fd) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--port-file", help="write the generated PTY path here before accepting a serial client") + parser.add_argument("--web-host", default="127.0.0.1", help="virtual-phone HTTP bind address") + parser.add_argument("--web-port", type=int, default=8002, help="virtual-phone HTTP port") + arguments = parser.parse_args() + try: + run(arguments.port_file, arguments.web_host, arguments.web_port) + except KeyboardInterrupt: + sys.exit(0) diff --git a/GSM-module/GSM-fastapi/run-with-mock-modem.sh b/GSM-module/GSM-fastapi/run-with-mock-modem.sh new file mode 100755 index 00000000..6b2c3515 --- /dev/null +++ b/GSM-module/GSM-fastapi/run-with-mock-modem.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -eu + +port_file="$(mktemp)" +cleanup() { + if [ -n "${gateway_pid:-}" ]; then + kill "$gateway_pid" 2>/dev/null || true + fi + if [ -n "${modem_pid:-}" ]; then + kill "$modem_pid" 2>/dev/null || true + fi + rm -f "$port_file" +} +trap cleanup EXIT INT TERM + +python -u mock_modem.py --port-file "$port_file" \ + --web-host "${VIRTUAL_PHONE_HOST:-127.0.0.1}" \ + --web-port "${VIRTUAL_PHONE_PORT:-8002}" & +modem_pid=$! + +while [ ! -s "$port_file" ]; do + if ! kill -0 "$modem_pid" 2>/dev/null; then + wait "$modem_pid" + fi + sleep 0.1 +done + +export SERIAL_PORT="$(cat "$port_file")" +python main.py & +gateway_pid=$! +wait "$gateway_pid" diff --git a/GSM-module/GSM-fastapi/tests/test_mock_modem.py b/GSM-module/GSM-fastapi/tests/test_mock_modem.py new file mode 100644 index 00000000..9f484a33 --- /dev/null +++ b/GSM-module/GSM-fastapi/tests/test_mock_modem.py @@ -0,0 +1,188 @@ +import json +import os +import re +import select +import signal +import subprocess +import sys +import time +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +import pytest +import serial + +from mock_modem import ( + INBOUND_BODY_MAX_BYTES, + VirtualModem, + normalize_body, + parse_send_sms, + start_http_server, + valid_phone_number, +) + + +requires_pty = pytest.mark.skipif( + os.name != "posix", reason="PTY modem integration tests require POSIX" +) + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +class OutputReader: + def __init__(self, stream): + self.stream = stream + self.buffer = b"" + + def readline(self, timeout: float = 2.0) -> str: + deadline = time.monotonic() + timeout + while b"\n" not in self.buffer: + remaining = deadline - time.monotonic() + assert remaining > 0, "timed out waiting for emulator output" + ready, _, _ = select.select([self.stream], [], [], remaining) + assert ready, "timed out waiting for emulator output" + self.buffer += os.read(self.stream.fileno(), 4096) + line, self.buffer = self.buffer.split(b"\n", 1) + return line.decode("utf-8") + + +class ModemProcess: + def __init__(self, port_file=None): + self.port_file = port_file + + def __enter__(self): + command = [sys.executable, "-u", "mock_modem.py"] + if self.port_file: + command.extend(["--port-file", str(self.port_file)]) + self.process = subprocess.Popen( + command, + cwd=PROJECT_ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.output = OutputReader(self.process.stdout) + first_line = self.output.readline() + self.path = re.fullmatch(r"Virtual modem port: (/dev/pts/\d+)", first_line).group(1) + assert self.output.readline() == ( + f"Run the gateway with: SERIAL_PORT={self.path} python main.py" + ) + return self + + def readline(self, timeout: float = 2.0) -> str: + return self.output.readline(timeout) + + def __exit__(self, *_): + self.process.send_signal(signal.SIGINT) + self.process.wait(timeout=2) + assert self.process.returncode == 0 + self.process.stdout.close() + self.process.stderr.close() + + +def _open_modem(path: str) -> serial.Serial: + port = serial.Serial(path, 9600, timeout=1) + assert port.readline() == b"GSM_READY\n" + assert port.readline() == b"NETWORK_OK\n" + return port + + +def test_parse_send_sms_preserves_pipes(): + assert parse_send_sms("SEND_SMS|+639171234567|one|two|three") == ( + "+639171234567", "one|two|three" + ) + + +def test_phone_validation_and_inbound_normalization_match_firmware_frames(): + assert valid_phone_number("+639171234567") + assert not valid_phone_number("09171234567") + assert normalize_body("one|two\nthree", inbound=True) == "one/two three" + assert normalize_body("x" * (INBOUND_BODY_MAX_BYTES + 1), inbound=True) is None + + +def test_virtual_modem_stores_successful_messages_and_reset_clears_them(): + modem = VirtualModem() + assert modem.receive_outbound("+639171234567", "hello") == (True, "") + assert modem.messages("+639171234567")[0]["direction"] == "received" + modem.reset() + assert modem.messages("+639171234567") == [] + + +def test_virtual_modem_state_transitions_emit_gateway_events(): + modem = VirtualModem() + modem.attach(99) + _, error, frames = modem.update({"network_connected": False}) + assert error is None + assert frames == [b"NETWORK_LOST\n"] + _, error, frames = modem.update({"network_connected": True}) + assert error is None + assert frames == [b"GSM_READY\n", b"NETWORK_OK\n"] + _, error, frames = modem.update({"sim_present": False}) + assert error is None + assert frames == [b"SIM_MISSING\n"] + + +def test_virtual_phone_http_reports_validation_errors_and_modem_state(): + modem = VirtualModem() + server = start_http_server(modem, "127.0.0.1", 0) + base_url = f"http://127.0.0.1:{server.server_port}" + try: + with urlopen(f"{base_url}/api/modem") as response: + assert json.load(response)["gsm_ready"] is False + request = Request( + f"{base_url}/api/messages", + data=b'{"phone_number":"not-a-number","body":"hi"}', + method="POST", + headers={"Content-Type": "application/json"}, + ) + with pytest.raises(HTTPError) as error: + urlopen(request) + assert error.value.code == 400 + finally: + server.shutdown() + server.server_close() + + +@pytest.mark.parametrize("line", ["OTHER|+63|body", "SEND_SMS|+63", "SEND_SMS||body"]) +def test_parse_send_sms_rejects_malformed_commands(line): + assert parse_send_sms(line) is None + + +@requires_pty +def test_emulator_writes_port_file_before_printing_path(tmp_path): + port_file = tmp_path / "modem-port" + with ModemProcess(port_file) as modem: + assert port_file.read_text() == modem.path + + +@requires_pty +def test_emulator_confirms_fragmented_and_batched_commands(): + with ModemProcess() as modem: + with _open_modem(modem.path) as port: + port.write(b"SEND_SMS|+639171234567|fragment") + port.write(b"ed body\nSEND_SMS|+639188888888|second|body\n") + port.flush() + + assert port.readline() == b"SMS_SENT|+639171234567\n" + assert port.readline() == b"SMS_SENT|+639188888888\n" + assert modem.readline() == "SMS request to +639171234567: fragmented body" + assert modem.readline() == "SMS request to +639188888888: second|body" + + +@requires_pty +def test_emulator_reannounces_after_reconnect_and_ignores_malformed_input(): + with ModemProcess() as modem: + with _open_modem(modem.path) as port: + port.write(b"NOT_A_COMMAND\n") + port.flush() + assert modem.readline() == "Ignored modem command: 'NOT_A_COMMAND'" + assert port.read(1) == b"" + + port.write(b"SEND_SMS|+639171234567|still works\n") + port.flush() + assert port.readline() == b"SMS_SENT|+639171234567\n" + assert modem.readline() == "SMS request to +639171234567: still works" + + time.sleep(0.25) + with _open_modem(modem.path) as port: + assert port.read(1) == b"" diff --git a/docker-compose.gsm-emulator.yml b/docker-compose.gsm-emulator.yml new file mode 100644 index 00000000..13e2b7f3 --- /dev/null +++ b/docker-compose.gsm-emulator.yml @@ -0,0 +1,15 @@ +# Opt-in local development overlay. The emulator and gateway must run in the +# same container because a PTY path belongs to one Linux device namespace. +# +# Usage: docker/up.sh -f docker-compose.yml -f docker-compose.gsm-emulator.yml up --build -d +services: + gsm-fastapi: + build: + context: ./GSM-module/GSM-fastapi + target: emulator + command: ["./run-with-mock-modem.sh"] + environment: + VIRTUAL_PHONE_HOST: 0.0.0.0 + VIRTUAL_PHONE_PORT: 8002 + ports: + - "127.0.0.1:${VIRTUAL_PHONE_PORT:-8002}:8002" diff --git a/docker-compose.gsm-hardware.yml b/docker-compose.gsm-hardware.yml index 509fb526..121ecf1d 100644 --- a/docker-compose.gsm-hardware.yml +++ b/docker-compose.gsm-hardware.yml @@ -1,10 +1,13 @@ # Opt-in overlay: passes the GSM modem through to gsm-fastapi. Not # auto-loaded (unlike docker-compose.override.yml) — only merge this in on -# a machine that actually has the modem attached at /dev/ttyACM0, otherwise +# a machine that actually has the modem attached, otherwise # `docker compose up` aborts and leaves nginx/admin stuck in "Created". # # Usage: docker/up.sh -f docker-compose.yml -f docker-compose.gsm-hardware.yml up -d +# docker/up.sh loads GSM-module/GSM-fastapi/.env for SERIAL_PORT substitution. services: gsm-fastapi: + environment: + SERIAL_PORT: ${SERIAL_PORT:-/dev/ttyACM0} devices: - - "/dev/ttyACM0:/dev/ttyACM0" + - "${SERIAL_PORT:-/dev/ttyACM0}:${SERIAL_PORT:-/dev/ttyACM0}" diff --git a/docker/up.sh b/docker/up.sh index 098faf3f..38e04b4d 100755 --- a/docker/up.sh +++ b/docker/up.sh @@ -16,7 +16,9 @@ set -eu # case unless you export CERT_SAN in the shell first. # # Passes --env-file server/.env explicitly: docker-compose.yml's -# ${MYSQL_*} substitutions are read from this file. Passing --env-file at +# ${MYSQL_*} substitutions are read from this file. When the optional GSM +# hardware overlay is selected, its SERIAL_PORT substitution is read from the +# gateway's .env too, so the device mapping matches the serial worker. Passing --env-file at # all disables Compose's default auto-load of a root-level .env, so we # also pass repo-root .env (port overrides, see .env.example) when present # — --env-file can be repeated, later ones win on overlapping keys, and @@ -45,4 +47,24 @@ if [ -f .env ]; then ENV_FILE_ARGS="--env-file .env $ENV_FILE_ARGS" fi +case " $* " in + *" docker-compose.gsm-hardware.yml "*) + if [ ! -f GSM-module/GSM-fastapi/.env ]; then + echo "docker/up.sh: GSM-module/GSM-fastapi/.env is required for the hardware overlay" >&2 + exit 1 + fi + serial_port="$(sed -n 's/^[[:space:]]*SERIAL_PORT[[:space:]]*=[[:space:]]*//p' GSM-module/GSM-fastapi/.env | tail -n 1)" + serial_port="${serial_port#\"}" + serial_port="${serial_port#\'}" + case "$serial_port" in + /dev/pts/*) + echo "docker/up.sh: SERIAL_PORT=$serial_port is a host PTY and cannot be passed through with the hardware overlay" >&2 + echo "docker/up.sh: use docker-compose.gsm-emulator.yml for PTY testing, or set SERIAL_PORT to a host /dev/ttyACM* or /dev/ttyUSB* device" >&2 + exit 1 + ;; + esac + ENV_FILE_ARGS="$ENV_FILE_ARGS --env-file GSM-module/GSM-fastapi/.env" + ;; +esac + exec docker compose $ENV_FILE_ARGS "$@" diff --git a/docs/features/sms-gateway/testing.md b/docs/features/sms-gateway/testing.md index 256d0109..537b002d 100644 --- a/docs/features/sms-gateway/testing.md +++ b/docs/features/sms-gateway/testing.md @@ -46,6 +46,7 @@ pnpm run testAll | `tests/test_database_reconciliation.py` | Idempotent startup recovery of orphaned pending log rows | | `tests/test_incoming_sms.py` | Sender rejection reason codes and inbound log status updates | | `tests/test_lifespan.py` | Reconciliation ordering before serial worker startup | +| `tests/test_mock_modem.py` | Virtual-phone validation, firmware-compatible normalization, modem state transitions, HTTP responses, PTY framing, reconnects, and subprocess cleanup | | `server/app/tests/test_gsm_proxy.py` | Main-server shared-secret header, status preservation, and timeout headroom for chat, verification, resend, and first-contact requests | | `mobile-app/sapot-mobile-app/features/shared/core/errors/__tests__/gsm-error.test.ts` | Typed `QUEUE_FULL` parsing and user-visible error messages | | `mobile-app/sapot-mobile-app/features/chat/components/__tests__/message-list.test.tsx` | Manual resend rejection and `not_sent` restoration | @@ -83,7 +84,37 @@ The saturation test starts 21 blocking send requests, representing 20 waiting re This covers the thread-pool exhaustion described by issue #252. A test with only a rejecting fake would verify the response shape but would not prove that the rejection handler can still obtain a worker thread. -## Manual modem smoke test +## Software-only PTY smoke test + +Use this Linux host workflow to validate the real `SerialWorker` and outbound FastAPI path without +an Arduino or carrier account. It still needs development `DB_PATH` and `GSM_SECRET` values because +the emulator replaces only the serial device. + +1. In one terminal, run `python mock_modem.py` from `GSM-module/GSM-fastapi/` and copy its printed `/dev/pts/` path. The virtual phone is available at . +2. In another terminal, start the gateway with `SERIAL_PORT=/dev/pts/ python main.py`. +3. Confirm `curl http://127.0.0.1:8001/health` reports `connected: true` and `gsm_ready: true`. +4. Send an authenticated request: + + ```bash + curl -X POST http://127.0.0.1:8001/sms/send \ + -H 'Content-Type: application/json' \ + -H 'X-GSM-Secret: ' \ + -d '{"number":"+639171234567","body":"SAPOT PTY smoke test"}' + ``` + +5. Confirm the API reports success and the selected virtual-phone inbox shows the message from SAPOT Gateway. +6. Reply from that inbox and confirm the gateway processes it through the normal inbound session and callback path. +7. Set the virtual-phone network or SIM control to unavailable, confirm the gateway health degrades, then restore it and confirm it becomes ready again. +8. Restart only the gateway, using the same PTY path, and confirm it becomes ready again. + +The emulator can also return `NO_PROMPT` or withhold a confirmation (`TIMEOUT`) from its browser controls. +It cannot validate USB access, real SIM state, signal, carrier acceptance, or physical-phone delivery. + +For Compose-based testing, start the stack with +`docker-compose.gsm-emulator.yml`. The overlay runs the emulator inside the gateway container because +a host-created PTY is not visible to that container. + +## Real-modem smoke test Run this only on a host with the configured Arduino and SIM: diff --git a/docs/getting-started/docker-setup.md b/docs/getting-started/docker-setup.md index bd68e6d4..276c49e7 100644 --- a/docs/getting-started/docker-setup.md +++ b/docs/getting-started/docker-setup.md @@ -59,7 +59,6 @@ cp GSM-module/GSM-fastapi/.env.example GSM-module/GSM-fastapi/.env `gsm-fastapi`'s `GSM_SECRET` must match `server/.env`'s `GSM_SECRET` — they authenticate the webhook calls between the two services (see [environment-config.md](../deployment/environment-config.md)). - Compose sets the server's `GSM_GATEWAY_URL` to `http://gsm-fastapi:8001`, which resolves through the internal Docker network. Do not replace it with `localhost`: inside the `api` container, that address refers to the API container rather than the separate GSM gateway container. @@ -76,6 +75,20 @@ whole `docker compose up` would abort on any machine without the GSM modem attac Without the GSM modem, just run the normal `./docker/up.sh up --build -d` below — `gsm-fastapi` still starts, it just won't have serial access. +To exercise outbound SMS flow in Docker without hardware, merge the PTY emulator overlay instead. +It starts the emulator in the `gsm-fastapi` container, so its generated device path is visible to the +gateway process. Do not set `SERIAL_PORT` to a host `/dev/pts/` path: containers have separate PTY +namespaces. + +```bash +./docker/up.sh -f docker-compose.yml -f docker-compose.gsm-emulator.yml up --build -d +docker compose logs -f gsm-fastapi +``` + +The gateway logs the generated port and becomes ready after the emulator handshake. The emulator +prints each valid outbound destination and body in the same service logs. Do not merge the emulator +overlay with `docker-compose.gsm-hardware.yml`; use the hardware overlay for real modem testing. + See the repo-root `SECURITY.md` for why `DATABASE_URL`, `JWT_SECRET_KEY`, `CORS_ALLOWED_ORIGINS`, and `SERVER_ED25519_SEED` are required at import time. `server/.env.example` supplies safe defaults only for local service addresses; it never supplies usable secrets. ## Run @@ -169,6 +182,17 @@ The new stack runs under a different project name (derived from the repo root di docker compose up -d db redis api certgen nginx # pulls in admin + tileserver, skips gsm-fastapi ``` +**`gsm-fastapi` logs `Cannot open /dev/ttyACM0: No such file or directory`.** The Arduino may be connected to the host, but the running container was created without the hardware overlay, so Docker did not expose the serial device inside it. Confirm the host sees the device, then recreate only the gateway with the overlay: +```bash +ls -l /dev/ttyACM* /dev/ttyUSB* +./docker/up.sh -f docker-compose.yml -f docker-compose.gsm-hardware.yml up -d --force-recreate gsm-fastapi +``` +If the first command reports a port other than `/dev/ttyACM0`, update `SERIAL_PORT` in +`GSM-module/GSM-fastapi/.env`. The `docker/up.sh` wrapper reads that file when the hardware overlay is +selected, so Compose maps the same device path into the gateway container. +This overlay accepts host serial devices such as `/dev/ttyACM0` and `/dev/ttyUSB0`. It cannot pass a +host `/dev/pts/` pseudo-terminal into Docker. Use `docker-compose.gsm-emulator.yml` for PTY testing. + **`https://localhost/admin` works but `http://localhost:3000` returns 404.** Expected. The admin app sets `basePath: "/admin"` in `next.config.ts`, so its published port serves the dashboard at `http://localhost:3000/admin`, not at the root path. **`nginx` logs `host not found in upstream "api"` even though `api` is running.** The `nginx` container was created against a stale image/config and never recreated (Compose reuses an existing container if it thinks nothing relevant changed). Force it: diff --git a/docs/getting-started/gsm-module-setup.md b/docs/getting-started/gsm-module-setup.md index 5141c188..09a376c9 100644 --- a/docs/getting-started/gsm-module-setup.md +++ b/docs/getting-started/gsm-module-setup.md @@ -18,6 +18,11 @@ Docker, in which case follow this doc. For deploying it as a systemd service, se - A GSM modem attached via USB serial (default expected at `/dev/ttyACM0`, matching `SERIAL_PORT`'s default in `config.py`) - Access to the same MariaDB instance the server uses (`DB_PATH`) +Physical modem hardware is required to deliver messages through a carrier. For Linux-only local +outbound-flow testing, `mock_modem.py` can replace the serial hardware with a virtual port. It does +not replace the database or shared-secret configuration: `.env` must still provide `DB_PATH` and +`GSM_SECRET`. + ## Install ```bash @@ -68,6 +73,94 @@ the main SAPOT server on `8000`, and it matches the server's `_gsm_http_client`, `http://localhost:8001` (`server/app/api/gsm.py`). Setting `PORT` yourself changes nothing except the port printed in the startup log. See `GSM-module/CLAUDE.md`'s "Common Pitfalls". +## Test two-way SMS flow without a modem + +The virtual modem exercises the unchanged serial worker and outbound API flow without an Arduino, +SIM card, or carrier connection. It uses a POSIX pseudo-terminal (PTY), so this workflow is intended +for Linux host development, not Windows. For Docker Compose, use the +[`docker-compose.gsm-emulator.yml`](../../docker-compose.gsm-emulator.yml) overlay instead of passing +a host PTY into the container. + +In one terminal, start the emulator and copy the printed port path: + +```bash +cd GSM-module/GSM-fastapi +python mock_modem.py +``` + +```text +Virtual modem port: /dev/pts/3 +Run the gateway with: SERIAL_PORT=/dev/pts/3 python main.py +``` + +Before starting the gateway, put that exact path in `GSM-module/GSM-fastapi/.env`. Keep the required +`DB_PATH` and `GSM_SECRET` values there too. + +```dotenv +# Development-only PTY created by mock_modem.py. Replace this value each time +# the emulator is restarted because its /dev/pts path can change. +SERIAL_PORT=/dev/pts/3 +``` + +Start the gateway in a second terminal: + +```bash +cd GSM-module/GSM-fastapi +python main.py +``` + +Open to use the virtual phone. Enter any E.164 phone number, such as +`+639171234567`, then send through the normal SAPOT API. The accepted message appears as an incoming +message from **SAPOT Gateway**. Reply in the browser to inject `SMS_RECEIVED` into the unchanged serial +worker, so account checks, `[target]` sessions, message logging, and callbacks still run in the gateway. + +The controls intentionally model the Arduino-facing boundary: + +| Control | Effect | +| --- | --- | +| SIM | Removing it emits `SIM_MISSING`; restoring a working modem emits `GSM_READY` then `NETWORK_OK`. | +| Network | Losing it emits `NETWORK_LOST`; restoring it emits `GSM_READY` then `NETWORK_OK`. | +| Outbound result | `success` accepts and displays a message, `NO_PROMPT` returns `SMS_FAILED`, and `TIMEOUT` leaves the gateway send waiting for its normal timeout. | + +Browser replies are disabled while the SIM or network is unavailable. The emulator replaces inbound +pipe characters with `/`, replaces newlines with spaces, and caps replies at the firmware's 127-byte +inbound buffer limit. It keeps messages only in memory, so restarting it clears every virtual inbox. + +For Docker Compose, run: + +```bash +./docker/up.sh -f docker-compose.yml -f docker-compose.gsm-emulator.yml up --build -d +``` + +The overlay publishes the virtual phone only at `127.0.0.1:${VIRTUAL_PHONE_PORT:-8002}` and runs it in +the same container as the gateway because PTY paths do not cross container boundaries. Do not put the +host's `/dev/pts/` path in `.env` for this workflow: `run-with-mock-modem.sh` creates the PTY inside +the container and overrides `SERIAL_PORT` for the gateway process. You can restart only the gateway and +reuse the same PTY path while the emulator continues running. + +The normal gateway image is a production target that excludes the virtual modem, virtual phone, and +their startup script. The emulator overlay explicitly selects a separate emulator target, so it cannot +be activated by the standard production image or command. + +For a physical modem in Docker, set the host device path in `GSM-module/GSM-fastapi/.env` before using +the hardware overlay. The `docker/up.sh` wrapper reads this value and uses it both as the gateway's +`SERIAL_PORT` and as the Docker device mapping. + +```dotenv +# Physical Arduino/GSM modem attached to the Docker host. +SERIAL_PORT=/dev/ttyACM0 +``` + +```bash +./docker/up.sh -f docker-compose.yml -f docker-compose.gsm-hardware.yml up --build -d +``` + +Use a host device such as `/dev/ttyACM0` or `/dev/ttyUSB0` here. A `/dev/pts/` path is valid only +for direct-host testing; Docker cannot pass it through as a hardware device. + +An emulator success means that the simulated modem accepted the request at the Arduino protocol boundary. +It does not mean a carrier accepted the SMS or that a physical phone received it. + ## Verify ```bash From b4b1e216ddae91d42891023cc75add9e170d0702 Mon Sep 17 00:00:00 2001 From: Adamskiee Date: Sat, 15 Aug 2026 20:33:36 +0800 Subject: [PATCH 3/3] fix(docs): format localhost urls as code to pass link check --- docs/features/sms-gateway/testing.md | 2 +- docs/getting-started/gsm-module-setup.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/sms-gateway/testing.md b/docs/features/sms-gateway/testing.md index 537b002d..90ee350a 100644 --- a/docs/features/sms-gateway/testing.md +++ b/docs/features/sms-gateway/testing.md @@ -90,7 +90,7 @@ Use this Linux host workflow to validate the real `SerialWorker` and outbound Fa an Arduino or carrier account. It still needs development `DB_PATH` and `GSM_SECRET` values because the emulator replaces only the serial device. -1. In one terminal, run `python mock_modem.py` from `GSM-module/GSM-fastapi/` and copy its printed `/dev/pts/` path. The virtual phone is available at . +1. In one terminal, run `python mock_modem.py` from `GSM-module/GSM-fastapi/` and copy its printed `/dev/pts/` path. The virtual phone is available at `http://127.0.0.1:8002`. 2. In another terminal, start the gateway with `SERIAL_PORT=/dev/pts/ python main.py`. 3. Confirm `curl http://127.0.0.1:8001/health` reports `connected: true` and `gsm_ready: true`. 4. Send an authenticated request: diff --git a/docs/getting-started/gsm-module-setup.md b/docs/getting-started/gsm-module-setup.md index 09a376c9..4fe07d15 100644 --- a/docs/getting-started/gsm-module-setup.md +++ b/docs/getting-started/gsm-module-setup.md @@ -109,7 +109,7 @@ cd GSM-module/GSM-fastapi python main.py ``` -Open to use the virtual phone. Enter any E.164 phone number, such as +Open `http://127.0.0.1:8002` to use the virtual phone. Enter any E.164 phone number, such as `+639171234567`, then send through the normal SAPOT API. The accepted message appears as an incoming message from **SAPOT Gateway**. Reply in the browser to inject `SMS_RECEIVED` into the unchanged serial worker, so account checks, `[target]` sessions, message logging, and callbacks still run in the gateway.