diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 0000000..9248cb3 --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,11 @@ +#!/bin/sh +# Pre-push hook — push protection for secrets (local side of check:secrets). +# Install once per clone: git config core.hooksPath .githooks +# Blocks the push when gitleaks finds a leak. +set -eu +if ! command -v gitleaks >/dev/null 2>&1; then + echo "pre-push: missing tool gitleaks (see .github/workflows/verify.yml pinned install)" + exit 2 +fi +echo "pre-push: running gitleaks (history)…" +gitleaks detect --source . --config .gitleaks.toml --verbose diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..89b09fd --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,21 @@ + +## What / why + +- What changed: +- Why: + +## Verification (Delivery+Verification) + +Commands actually run + outcomes (include baseline failures separately): + +- [ ] `python -m pytest tests/ -v` +- [ ] frontend `npm run lint` / `npm run build` (if touched) +- [ ] `git diff --check` +- [ ] `gitleaks detect --source . --config .gitleaks.toml --verbose` +- [ ] Docs updated in same PR (if behavior changed) + +## Trailer (required — Closes/Refs where supported) + + + +Refs # - diff --git a/.github/workflows/secrets.yml b/.github/workflows/secrets.yml new file mode 100644 index 0000000..13c5298 --- /dev/null +++ b/.github/workflows/secrets.yml @@ -0,0 +1,50 @@ +# secrets — history-aware secrets scan for `main`. +# Local parity: `poe check:secrets` (gitleaks). +# Triggers: PRs and pushes targeting `main` only, plus nightly and manual. +# No `|| true`, no `continue-on-error`. Does not touch release/ccpe-v1.0. +name: secrets + +on: + pull_request: + branches: [main] + push: + branches: [main] + schedule: + - cron: "17 2 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: secrets-${{ github.ref }} + cancel-in-progress: true + +jobs: + secrets-history: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout (full history for history-aware scan) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install gitleaks (pinned) + run: | + set -eu + GITLEAKS_VERSION="8.30.1" + curl -sSL "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" -o /tmp/gitleaks.tar.gz + tar -xzf /tmp/gitleaks.tar.gz -C /tmp + sudo mv /tmp/gitleaks /usr/local/bin/gitleaks + gitleaks version + + - name: Gitleaks history scan + run: | + set -eu + gitleaks detect --source . --config .gitleaks.toml --verbose + + - name: Gitleaks working-tree scan + run: | + set -eu + gitleaks detect --source . --config .gitleaks.toml --no-git --verbose diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..7a4af51 --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,131 @@ +# verify — required verification gate for `main`. +# Local parity: `poe verify` / `make verify` run the same standard checks. +# Triggers: PRs and pushes targeting `main` only, plus manual. +# No `|| true`, no `continue-on-error`. Does not touch release/ccpe-v1.0. +# Standard tools only: pytest, git, grep, gitleaks, python stdlib, npm. +name: verify + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: verify-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout (full history for history-aware scan) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install Python deps + run: | + set -eu + python -m pip install -r requirements.txt + + - name: Backend suite (pytest) + run: | + set -eu + python -m pytest tests/ -v + + - name: Whitespace hygiene (git diff --check) + run: | + set -eu + git diff --check + + - name: Stub hygiene (grep, no placeholders) + run: | + set -eu + # Fail on bare stubs; scoped TODO(ref) passes, binaries skipped. + if git grep -I -n -E -e 'TODO[^\(]|FIXME|XXX|HACK|lorem|NotImplementedError' --and --not -e 'TODO\[' -- . ':!frontend/node_modules' ':!paper' ':!frontend/package-lock.json'; then + echo "stub hygiene FAIL — remove bare stubs above" + exit 1 + fi + echo "stub hygiene ok" + + - name: Install gitleaks (pinned) + run: | + set -eu + GITLEAKS_VERSION="8.30.1" + curl -sSL "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" -o /tmp/gitleaks.tar.gz + tar -xzf /tmp/gitleaks.tar.gz -C /tmp + sudo mv /tmp/gitleaks /usr/local/bin/gitleaks + gitleaks version + + - name: Gitleaks history scan + run: | + set -eu + gitleaks detect --source . --config .gitleaks.toml --verbose + + - name: Gitleaks working-tree scan + run: | + set -eu + gitleaks detect --source . --config .gitleaks.toml --no-git --verbose + + - name: Schema compat + restore proof (stdlib only, temp copies) + run: | + set -eu + python3 - <<'PY' + import sqlite3, tempfile + from contextlib import closing + from pathlib import Path + from storage import ensure_schema + with closing(sqlite3.connect(":memory:")) as c: + ensure_schema(c) + tables = [r[0] for r in c.execute("SELECT name FROM sqlite_master WHERE type='table'")] + assert "aws_connections" in tables, "missing aws_connections" + print(f"fresh schema ok: {len(tables)} tables") + with tempfile.TemporaryDirectory() as d: + src = Path("data/cloud_optimizer.db") + dst = Path(d) / "restore.db" + if src.exists(): + with closing(sqlite3.connect(f"{src.as_posix()}?mode=ro", uri=True)) as s: + with closing(sqlite3.connect(dst.as_posix())) as t: + s.backup(t) + assert t.execute("PRAGMA integrity_check").fetchone()[0] == "ok" + print("restore round-trip ok") + else: + print("restore skipped: no tracked demo DB") + PY + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: "npm" + cache-dependency-path: frontend/package-lock.json + + - name: Install frontend deps (committed lockfile) + working-directory: frontend + run: | + set -eu + npm ci + + - name: Frontend lint + working-directory: frontend + run: | + set -eu + npm run lint + + - name: Frontend build + working-directory: frontend + run: | + set -eu + npm run build diff --git a/.gitignore b/.gitignore index de12bb1..441f16b 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,14 @@ backups/ # Per-user runtime settings, including temporary and corrupt recovery copies backend_api/runtime_settings.json* + +# Paper + support — never on main (paper lives on release/paper branch) +paper/ +support_info/ +*.Zone.Identifier +# Local runtime artifacts +backend_api/runtime_settings.json +backend_api/runtime_settings.json.corrupt +data/*.backup +data/*.db-shm +data/*.db-wal diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..5a6cc68 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,32 @@ +# Gitleaks config — Smart Cloud Optimizer (main). +# Full-history + working-tree scan runs via `poe check:secrets` / `make check-secrets` +# and .github/workflows/secrets.yml + verify.yml (pinned gitleaks). +# Fail-closed: extends default rules; allowlists cover only synthetic demo +# placeholders and AWS documentation example keys, plus build-artifact paths. +# Real leaks fail the scan. Do not add real-key patterns here. + +title = "smart-cloud-optimizer" + +[extend] +useDefault = true + +[allowlist] +description = "Synthetic demo placeholders and AWS docs example keys — never real credentials" +regexTarget = "match" +regexes = [ + '''(?i)SYNTHETIC-001''', + '''aws-SYNTHETIC-001''', + '''SYNTHETIC-001''', + '''AKIAIOSFODNN7EXAMPLE''', + '''wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY''', +] +paths = [ + '''frontend/\.next/.*''', + '''\.next/.*''', + '''frontend/node_modules/.*''', + '''frontend/out/.*''', + '''\.pytest_cache/.*''', + '''\.pytest_tmp/.*''', + '''htmlcov/.*''', + '''.*\.pyc$''', +] diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..356bcc2 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,8 @@ +# CODEOWNERS — application PRs target `main`; `release/ccpe-v1.0` untouched. +# Fallback first: everything needs the owner when no scoped rule matches. + +* @G1DO +/backend_api/ @G1DO +/frontend/ @G1DO +/storage/ @G1DO +/docs/ @G1DO diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..89dcc01 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,38 @@ +# Contributing + +Application changes target **`main`** only. The default `release/ccpe-v1.0` +branch is a separate paper artifact; select `main` explicitly when opening +an application PR. This repo is a single-user localhost demo — see +[threat model](docs/security/threat-model.md) before connecting real data. + +## PRs: title, body, trailers + +- Title: `: ` (e.g. `docs: clarify sync recovery`). + Keep it specific; no `WIP` in the title at merge time. +- Body must state: what changed, why, and verification actually run + (commands + outcomes, including baseline failures). Template: + [.github/pull_request_template.md](.github/pull_request_template.md). +- Every PR body ends with a trailer line. Use `Closes #` when the PR + resolves the issue, `Refs #` when it only relates: + `Closes #12` / `Refs #12`. No issue → `Refs N/A - ` + (e.g. small doc fix). GitHub preserves the PR body on squash-merge, + so the trailer is the audit link. Next 2 PRs must carry trailers; + baseline at time of writing: 0 `Closes`/`Refs` trailers in 112 commits. + +## Before opening + +- Start with [README setup](README.md#run--docker), then + `README.md` Verify for commands. Verbs: `pyproject.toml [tool.poe.tasks]`, `Makefile`. +- Run the relevant checks and report results in the PR: + `pytest`, frontend `lint`/`build` when touched, `git diff --check`, + `gitleaks detect --source . --config .gitleaks.toml --verbose`. + Same checks run in CI on `main` (see `.github/workflows/`). +- Update affected technical docs in the same PR as code. Docs are minimal: + root `README.md` + `docs/architecture/*.mmd` + `data-model.md` + `threat-model.md` + 1 ADR. + `operations/runbook.md` absent pre-prod. +- Never add credentials, personal data, account identifiers, or runtime + database changes. Keep `.env` and snapshots out of Git. Redact logs + before sharing. + +For vulnerability reporting use [SECURITY.md](SECURITY.md) — private route +only, never public issues/PRs. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d4df3ef --- /dev/null +++ b/Makefile @@ -0,0 +1,79 @@ +# Makefile — same verbs as [tool.poe.tasks] (short shell steps, real tools). +# No glue scripts; missing tool fails closed (exit 2), never silent pass. +# Prototype note: test + verify are the only required gates for +# prototype-stage changes; the other verbs run on demand or in review. +# Usage: make verify | make test | make check-secrets (or make 'check:secrets'). + +.PHONY: help dev test migrate build test-contract test\:contract \ + check-sast check\:sast check-secrets check\:secrets check-audit check\:audit \ + check-hygiene check\:hygiene diagrams verify + +help: + @echo "verbs: dev test migrate build test:contract check:sast check:secrets" + @echo " check:audit check:hygiene diagrams verify (prototype: test + verify only)" + +dev: + @python3 -c "import uvicorn" 2>/dev/null || { echo "dev: missing tool uvicorn (pip install -r requirements.txt)"; exit 2; } + python3 -m uvicorn backend_api.main:app --reload --host 127.0.0.1 --port 8000 + +test: + @python3 -c "import pytest" 2>/dev/null || { echo "test: missing tool pytest (pip install -r requirements.txt)"; exit 2; } + python3 -m pytest tests/ -v + +migrate: + python3 -c "import sqlite3; from storage import ensure_schema; c=sqlite3.connect(':memory:'); ensure_schema(c); c.close(); print('migrate ok')" + +build: + @command -v npm >/dev/null 2>&1 || { echo "build: missing tool npm (install Node 22+)"; exit 2; } + python3 -m compileall -q backend_api cloud_optimizer aws_collector storage data_generation ml_engine optimizer ai_module dashboard + npm --prefix frontend run build + +test-contract: + @test -f tests/test_frontend_backend_contract.py || { echo "test:contract omitted: no contract suite in tree"; exit 0; } + @python3 -c "import pytest" 2>/dev/null || { echo "test:contract: missing tool pytest (pip install -r requirements.txt)"; exit 2; } + python3 -m pytest tests/test_frontend_backend_contract.py -v + +test\:contract: test-contract + +check-sast: + @python3 -c "import bandit" 2>/dev/null || { echo "check:sast: missing tool bandit (pip install bandit)"; exit 2; } + python3 -m bandit -r backend_api storage -q + +check\:sast: check-sast + +check-secrets: + @command -v gitleaks >/dev/null 2>&1 || { echo "check:secrets: missing tool gitleaks (see .github/workflows/verify.yml pinned install)"; exit 2; } + gitleaks detect --source . --config .gitleaks.toml --verbose + gitleaks detect --source . --config .gitleaks.toml --no-git --verbose + +check\:secrets: check-secrets + +check-audit: + python3 -m pip check + @if command -v pip-audit >/dev/null 2>&1; then pip-audit; else echo "check:audit note: pip-audit not installed — pip check passed (omitted with reason)"; fi + +check\:audit: check-audit + +check-hygiene: + @command -v git >/dev/null 2>&1 || { echo "check:hygiene: missing tool git"; exit 2; } + git diff --check + @if git grep -I -n -E -e 'TODO[^(]|FIXME|XXX|HACK|lorem|NotImplementedError' --and --not -e 'TODO\[' -- . ':!frontend/node_modules' ':!paper' ':!frontend/package-lock.json'; then echo "stub hygiene FAIL"; exit 1; fi + +check\:hygiene: check-hygiene + +diagrams: + @files=$$(find . -name '*.mmd' -not -path './node_modules/*' -not -path './frontend/node_modules/*' -not -path './frontend/.next/*' -not -path './.git/*' -print); \ + if [ -z "$$files" ]; then echo "diagrams omitted: no .mmd sources in tree (add docs//.mmd to enable)"; exit 0; fi; \ + command -v mmdc >/dev/null 2>&1 || { echo "diagrams: missing tool mermaid-cli (npm i -g @mermaid-js/mermaid-cli)"; exit 2; }; \ + for f in $$files; do mmdc -i "$$f" -o "$${f%.mmd}.svg"; done + +verify: + @python3 -c "import pytest" 2>/dev/null || { echo "verify: missing tool pytest (pip install -r requirements.txt)"; exit 2; } + @command -v npm >/dev/null 2>&1 || { echo "verify: missing tool npm (install Node 22+)"; exit 2; } + @command -v git >/dev/null 2>&1 || { echo "verify: missing tool git"; exit 2; } + @command -v gitleaks >/dev/null 2>&1 || { echo "verify: missing tool gitleaks"; exit 2; } + python3 -m pytest tests/ -v + git diff --check + gitleaks detect --source . --config .gitleaks.toml --verbose + npm --prefix frontend run lint + npm --prefix frontend run build diff --git a/README.md b/README.md index 7d727e2..8f3475f 100644 --- a/README.md +++ b/README.md @@ -1,113 +1,151 @@ # Smart Cloud Optimizer (OptiCloud) -An AWS cost-analysis and recommendation application: a Next.js dashboard calls a -FastAPI backend for collection, forecasting, anomaly detection, and optimization. -It recommends changes; it does not apply them to AWS. +Next.js UI + FastAPI backend for AWS cost collection, forecasting, anomaly detection, and optimization. Recommends changes; never applies them. -**Use this build on a single-user local machine with synthetic data.** The HTTP -API trusts client-supplied identities and stored AWS keys are plaintext. Read the -[security boundaries](docs/security/README.md) before connecting an account. +App work targets **`main`**. Default `release/ccpe-v1.0` is a separate paper artifact: +`git clone --branch main https://github.com/G1DO/smart-cloud-optimizer.git` -Application development targets **`main`**. GitHub's default -`release/ccpe-v1.0` is a separate paper software artifact; clone `main` explicitly: - -```bash -git clone --branch main https://github.com/G1DO/smart-cloud-optimizer.git -cd smart-cloud-optimizer -``` - -Start with the [technical documentation](docs/README.md), -[development guide](docs/development/README.md), or [project report (PDF)](docs/project-report.pdf). +> Single-user localhost + synthetic data only. +> HTTP trusts client `user_id`; AWS keys stored plaintext in SQLite. +> Read `docs/security/threat-model.md` before connecting a real account. ## Prerequisites -Python 3.12+, Node.js 20.9+, npm, and Git for manual setup; Docker with Compose -for the container setup. Python dependencies are in [requirements.txt](requirements.txt), -and frontend dependencies/scripts in [package.json](frontend/package.json) and -its committed lockfile. The Python image uses 3.12; the frontend uses Next.js 16, -React 19, and TypeScript. - -## Quick start +Python 3.12+, Node 20.9+, npm, Git. Docker + Compose for containers. +Python pins: `requirements.txt`. Frontend pins: `frontend/package.json` + `frontend/package-lock.json`. +Templates: `.env.example`, `.env.docker.example`, `frontend/.env.local.example`. -### Option A — Docker (simplest) - -Runs the stack with the demo SQLite DB baked into the backend image. Published -ports bind to `127.0.0.1`. Exploring stored demo data needs no external credentials; -Gemini-backed recommendations require an API key. +## Run — Docker ```bash docker compose up --build +# docker compose --profile full up --build # adds legacy Streamlit on :8501 ``` -| Service | URL | -| --- | --- | -| Frontend | http://localhost:3000 | -| Backend API docs | http://localhost:8000/docs | -| Backend health | http://localhost:8000/health | - -Add the optional legacy Streamlit dashboard (the `full` profile reuses the backend image): +Frontend http://localhost:3000 · API docs http://localhost:8000/docs · liveness http://localhost:8000/health · readiness http://localhost:8000/ready. +DB in named volume `optimizer-data`; `down -v` destroys it. `NEXT_PUBLIC_API_BASE_URL` is a frontend build arg, must be host-reachable (`http://localhost:8000`). ```bash -docker compose --profile full up --build # adds Streamlit on http://localhost:8501 +cp .env.docker.example .env # only for Gemini / real AWS ``` -To set env vars (e.g. `GOOGLE_API_KEY`), copy the template and edit it: +## Run — manual ```bash -cp .env.docker.example .env +python3 -m venv venv && source venv/bin/activate +pip install -r requirements.txt +cp .env.example .env # optional +uvicorn backend_api.main:app --reload --host 127.0.0.1 --port 8000 +cd frontend && npm ci && cp .env.local.example .env.local && npm run dev -- --hostname 127.0.0.1 ``` -Notes: -- The DB lives in a **named volume** `optimizer-data` (not a host bind mount). The destructive `docker compose down -v` removes the database volume; the next start seeds it again from the image. Runtime settings have a separate lifetime described in [local operations](docs/operations/local-runtime.md#runtime-settings-recovery). -- `NEXT_PUBLIC_API_BASE_URL` is a frontend **build** arg (Next.js inlines it into the browser bundle), so it must be host-reachable (`http://localhost:8000`) — never the in-network service name `backend`. - -### Option B — Manual (backend + frontend) +## First use -Commands below are for **macOS/Linux**. On **Windows PowerShell**, swap `source venv/bin/activate` for `.\venv\Scripts\Activate.ps1` and `cp` for `Copy-Item`. +Open http://localhost:3000 → Try Demo Mode (`aws-SYNTHETIC-001`). New users start empty. +Connections: Test and Save are separate — save with supplied account ID does not re-verify keys; sync can still fail. Sync clears prior account data then collects newest-month-first incrementally. Forecasts via UI/API. Optimizer: run on disposable DB copy, see `optimizer/engine.py`. -**1. Backend** (from the repo root): +## Verify ```bash -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -cp .env.example .env # optional: only needed for AI / real AWS -uvicorn backend_api.main:app --reload --host 127.0.0.1 --port 8000 +python3 -m pip check +python3 -m pytest tests/ -v +git diff --check +gitleaks detect --source . --config .gitleaks.toml --verbose +cd frontend && npm ci && npm run lint && npm run build ``` -Backend health: http://127.0.0.1:8000/health · API docs: http://127.0.0.1:8000/docs +`/health` = liveness cheap. `/ready` = readiness with DB check (503 when down). +OpenAPI without server: `python3 -c "from backend_api.main import app; import json; print(json.dumps(app.openapi(),indent=2))"`. +Same checks run in CI `verify` job, required for merge. Verbs: `pyproject.toml [tool.poe.tasks]`, `Makefile`. + +## Security + +- No tokens: login verifies PBKDF2 hash, issues nothing; all data routes trust `user_id` query — localhost only. +- AWS keys plaintext in `storage/db.py`; API exposes last 4 only; save ≠ verified. +- Tracked `data/cloud_optimizer.db` contains real PII + account ID/ARN + synthetic demo — add nothing real, distribute no rebuilt images with real data. +- AI: prompt >4000 → 400, Gemini fail → 502, optional `ONBOARDING_API_TOKEN`; advice unvalidated. +- Browser identity `localStorage` only; settings `backend_api/runtime_settings.json` unwired to engines, lost on recreate. Reports: `SECURITY.md` private only. + +## Architecture + +```mermaid +flowchart LR + Browser["Browser"] + NextUI["Next.js UI
(port 3000)"] + + subgraph BE["FastAPI backend (port 8000)"] + API["FastAPI routers
(trust-based user_id)"] + Storage["storage facade
(shared data API)"] + Optimizer["optimizer
(PuLP right-sizing + rules)"] + ML["ml_engine
(forecasts / anomalies)"] + AImod["ai_module
(Gemini advisor)"] + Collector["aws_collector
(boto3 pipeline)"] + end + + DB[("SQLite DB
data/cloud_optimizer.db")] + Synthetic["data_generation.synthetic
(demo seed)"] + AWS["AWS
(Cost Explorer / CloudWatch via boto3)"] + Gemini["Google Gemini
(AI recommendations)"] + + Browser --> NextUI + NextUI -->|"HTTP / JSON"| API + API --> Storage + API --> Optimizer + API --> ML + API --> AImod + API --> Collector + Optimizer --> Storage + ML --> Storage + AImod --> Storage + Collector --> Storage + Storage --> DB + API -->|"route-specific SQL"| DB + + Collector -. "real mode" .-> AWS + AImod -. "AI calls" .-> Gemini + Synthetic -. "demo mode" .-> Storage + + classDef ext stroke-dasharray:5 5,stroke:#888,color:#444; + class AWS,Gemini,Synthetic ext; +``` -**2. Frontend** (second terminal): +Code is canonical. Diagrams: Mermaid `.mmd` validated with mermaid-cli, Excalidraw never canonical. -```bash -cd frontend -npm ci -cp .env.local.example .env.local -npm run dev -- --hostname 127.0.0.1 # http://localhost:3000 -``` +| View | Source | +|---|---| +| Context | `docs/architecture/c1-system-context.mmd` | +| Containers | `docs/architecture/c2-containers.mmd` | +| Components | `docs/architecture/c3-components.mmd` | +| Sequence | `docs/architecture/sequence.mmd` | +| State | `docs/architecture/state.mmd` | +| Deployment | `docs/architecture/deployment.mmd` | +| Data model | `docs/architecture/data-model.md` — generator from `storage/db.py`, additive `ensure_schema` | -> Run Python commands from the repo root with the venv active — `cloud_optimizer` and the other application packages resolve from there. +`storage/` is preferred facade but forecast/recommendation/connection routes also issue direct SQL. Identity: `usr-…` on register, web save switches to `aws-`; legacy Streamlit stays on `usr-…`. -## First use +Active env: `GOOGLE_API_KEY`, `GOOGLE_MODEL`, `ONBOARDING_API_TOKEN`, `NEXT_PUBLIC_API_BASE_URL`, `AWS_DEFAULT_REGION`, `DEMO_MODE`. See `cloud_optimizer/config.py`, `aws_collector/config.py`, `docker-compose.yml`. + +## Operations + +Single-user localhost. No prod, no SLO, no on-call. `docs/operations/runbook.md` absent until prod. -Open http://localhost:3000 and choose **Try Demo Mode** to view the existing -`aws-SYNTHETIC-001` workspace. New registrations begin with empty dashboards. -**Account Settings → Connections** can test/save AWS keys and start a sync; -[testing, saving, and workspace selection are distinct](docs/architecture/README.md#connection-identity-and-verification). -Collection replaces prior account data incrementally, so read the -[data pipeline](docs/architecture/data-pipeline.md) before syncing. +| Symptom | Check | +|---|---| +| Empty dashboard | Workspace selected? Demo data only in `aws-SYNTHETIC-001` | +| Sync 409 | Stale `in_progress` — stop writers, set `failed` via `storage.update_aws_connection_status`, restart, retry | +| AI 502 / settings no-op | Key set? Logs? Settings are prefs, not engine inputs | -Forecasting runs from the Forecasts page or HTTP API; the ML CLI is a placeholder. -Use the [optimizer guide](docs/architecture/engines/optimizer.md#usage) for a -command-line experiment on a disposable database copy. +Preserve: SQLite backup API to temp dir outside repo, `integrity_check` ok, keep WAL/SHM with DB, never commit snapshots. Restore: stop writers, preserve current + sidecars, restore verified snapshot, verify workspace/dates. `down -v` destroys compose volume — preserve first. -## Develop and maintain +Engines: `optimizer/engine.py` (PuLP + rules, replaces per-user recs), `ml_engine/forecaster.py` (30d default, 1–180d, null-on-fail), `ai_module/recommender.py` (questionnaire → Gemini, router does not persist). Collection: `aws_collector/runner.py` newest-first. -- [Development checks](docs/development/README.md): Python tests, frontend lint/build, and contract verification. -- [Architecture](docs/architecture/README.md): components, identity, persistence, and engine boundaries. -- [API contracts](docs/api/http.md) and [configuration](docs/reference/configuration.md): generated reference and active inputs. -- [Local operations](docs/operations/local-runtime.md): diagnostics, data preservation, and recovery. -- [Security](docs/security/README.md) and [vulnerability reporting](SECURITY.md): current trust limits and reporting route. +## Index -Keep runtime data, credentials, and `.env` files out of commits. Environment -examples and dependency manifests/lockfiles are tracked for reproducible setup. +| Need | Go here, not a copy | +|---|---| +| Run/verify | This file; `pyproject.toml`, `Makefile`, `.github/workflows/verify.yml` | +| HTTP | `backend_api/main.py`, `backend_api/routers/`, live `/docs` | +| Storage/schema | `storage/db.py`, `storage/__init__.py`, `docs/architecture/data-model.md` | +| Decisions | `docs/decisions/` | +| Security | `docs/security/threat-model.md`, `SECURITY.md`, `CODEOWNERS` | +| Changes | `CONTRIBUTING.md`, `.github/pull_request_template.md` | diff --git a/SECURITY.md b/SECURITY.md index c49d629..e0f9f43 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,17 +1,37 @@ # Vulnerability reporting -This application is a local research/demo build. Its -[current security boundaries](docs/security/README.md) include trust-based HTTP -identity and plaintext credential storage; it is not suitable for public or -multi-tenant deployment. +Scope: application on **`main`** (single-user localhost demo with synthetic +data). This build trusts client-supplied identities and stores AWS keys in +plaintext; see [threat model](docs/security/threat-model.md). Do not use it multi-tenant or +public. -GitHub private vulnerability reporting is not currently enabled for this -repository, and no dedicated private contact is published. Request a private -reporting channel from the [repository maintainer](https://github.com/G1DO). -If a public issue is the only available contact route, include only the request -for a private channel, without vulnerability details, credentials, personal data, -or an exploit. Share the affected revision, impact, and minimal reproduction -privately once a channel is established. +## Private contact -Changes to the private reporting route should update this file. The repository -does not currently declare a security support window or response-time commitment. +- Preferred: GitHub private vulnerability reporting for this repository + (Security tab → Report a vulnerability). Reports stay private until triaged. +- If private reporting is unavailable in your view, request a private channel + from the [repository maintainer](https://github.com/G1DO) with only the + affected revision and a request for a channel — no details yet. +- Do not open a public issue, discussion, or PR with vulnerability details, + credentials, personal data, account identifiers, prompts, or exploits. + Public contact may only request the private channel. + +Include once a private channel is established: affected revision/commit, +impact, minimal reproduction, and redacted logs (redact identifiers, +credentials, prompts, and personal data; never dump credential columns). + +## Response commitments + +- Acknowledge receipt within **2 days**. +- Provide an initial assessment / next steps within **7 days**. +- Review security posture (dependencies, fixture PII, scan and schema-proof + status) at least **monthly**; critical localhost-demo fixes target the next + monthly review or sooner when feasible. +- Supported window: `main` only while this file names the private route above. + The default `release/ccpe-v1.0` branch is a separate paper artifact and is + not in scope for application fixes. + +Changes to the private route, acknowledgment time, or review cadence update +this file. Recheck the [threat model](docs/security/threat-model.md) whenever +identity, connection storage, browser state, external prompts, network +exposure, or persistence changes. diff --git a/backend_api/main.py b/backend_api/main.py index fdd1127..6113e18 100644 --- a/backend_api/main.py +++ b/backend_api/main.py @@ -1,7 +1,16 @@ +import logging +import uuid from contextlib import asynccontextmanager -from fastapi import FastAPI +from cloud_optimizer.config import setup_logging +from cloud_optimizer.logging_json import set_trace_id + +setup_logging(service="backend_api") +logger = logging.getLogger(__name__) + +from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse from backend_api.routers.costs import router as costs_router from backend_api.routers.dashboard import router as dashboard_router @@ -15,6 +24,27 @@ from storage.db import get_connection, ensure_schema, ensure_user +def _trace_id_from_request(request: Request) -> str: + """W3C traceparent trace-id where propagated, else request id, else new id. + + Starter §Service standards Logging: trace/span IDs (W3C traceparent + where propagated, else env correlation IDs). + """ + traceparent = request.headers.get("traceparent", "") + parts = traceparent.split("-") + # W3C: version-traceid-spanid-flags; trace-id is 32 lowercase hex chars. + if len(parts) == 4 and len(parts[1]) == 32: + try: + int(parts[1], 16) + return parts[1] + except ValueError: + pass + incoming = request.headers.get("x-request-id", "").strip() + if incoming: + return incoming[:64] + return uuid.uuid4().hex + + @asynccontextmanager async def lifespan(app: FastAPI): conn = get_connection() @@ -44,6 +74,20 @@ async def lifespan(app: FastAPI): allow_headers=["*"], ) + +@app.middleware("http") +async def trace_middleware(request: Request, call_next): + trace_id = _trace_id_from_request(request) + set_trace_id(trace_id) + logger.info(f"{request.method} {request.url.path}") + try: + response = await call_next(request) + finally: + # Reset to env default so the contextvar never leaks across requests. + set_trace_id("") + response.headers["X-Request-ID"] = trace_id + return response + app.include_router(dashboard_router) app.include_router(forecast_router) app.include_router(costs_router) @@ -61,4 +105,26 @@ def root(): @app.get("/health") def health(): - return {"ok": True} \ No newline at end of file + """Liveness: cheap, no dependency checks. When to restart.""" + return {"ok": True, "status": "ok"} + + +@app.get("/ready") +def ready(): + """Readiness: with dependency checks. When to accept traffic.""" + checks: dict[str, str] = {} + try: + conn = get_connection() + try: + conn.execute("SELECT 1").fetchone() + checks["database"] = "ok" + finally: + conn.close() + except Exception as exc: # noqa: BLE001 — readiness must fail closed + # Full detail goes to JSON logs (redacted); clients get a generic fail. + logger.error(f"readiness database check failed: {exc}") + checks["database"] = "fail" + return JSONResponse( + status_code=503, content={"ready": False, "checks": checks} + ) + return {"ready": True, "checks": checks} \ No newline at end of file diff --git a/cloud_optimizer/config.py b/cloud_optimizer/config.py index 610448a..16e20dd 100644 --- a/cloud_optimizer/config.py +++ b/cloud_optimizer/config.py @@ -8,6 +8,7 @@ """ import logging import os +import sys from pathlib import Path from typing import NamedTuple @@ -125,14 +126,39 @@ class InstanceSpec(NamedTuple): } -def setup_logging(level: str = "INFO") -> None: - """Configure project-wide logging. +def _resolve_level(level: str | None = None) -> int: + """Resolve a level name to a logging level, defaulting to env/INFO.""" + name = (level or os.getenv("LOG_LEVEL", "INFO")).upper() + return getattr(logging, name, logging.INFO) + + +def setup_logging(level: str | None = None, service: str | None = None) -> None: + """Configure project-wide JSON single-line logging to stdout. + + Starter §Service standards Logging: single-line JSON to stdout with + ``timestamp``/``level``/``service``/``msg``/``trace``, secret/PII + redaction, no secrets in logs. Args: level: Logging level name (DEBUG, INFO, WARNING, ERROR, CRITICAL). + Defaults to ``LOG_LEVEL`` env or ``INFO``. + service: Service name for the ``service`` field. Defaults to + ``SERVICE_NAME`` env or ``"api"``. """ - logging.basicConfig( - level=getattr(logging, level.upper()), - format="%(asctime)s | %(name)s | %(levelname)s | %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) + from cloud_optimizer.logging_json import JsonFormatter + + svc = service or os.getenv("SERVICE_NAME", "api") + lvl = _resolve_level(level) + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonFormatter(service=svc)) + root = logging.getLogger() + # Drop the legacy non-JSON basicConfig handlers so output is JSON-only. + for h in list(root.handlers): + root.removeHandler(h) + root.addHandler(handler) + root.setLevel(lvl) + # Keep uvicorn/fastapi loggers on the same JSON handler. + for name in ("uvicorn", "uvicorn.error", "uvicorn.access", "fastapi"): + lg = logging.getLogger(name) + lg.handlers = [] + lg.propagate = True diff --git a/cloud_optimizer/logging_json.py b/cloud_optimizer/logging_json.py new file mode 100644 index 0000000..178edff --- /dev/null +++ b/cloud_optimizer/logging_json.py @@ -0,0 +1,149 @@ +"""logging_json.py — single-line JSON formatter with redaction + trace. + +Starter §Service standards Logging (substantive only): single-line JSON to +stdout with timestamp/level/service/msg/trace, secret/PII redaction tested, +no secrets in logs. Trace follows W3C traceparent where propagated, else +env correlation IDs. +""" +from __future__ import annotations + +import contextvars +import json +import logging +import os +import re +from datetime import datetime, timezone + +REDACTED = "[REDACTED]" + +# Context-local trace/span id set per-request by backend_api middleware. +_trace_ctx: contextvars.ContextVar[str] = contextvars.ContextVar( + "trace_id", default="" +) + + +def set_trace_id(value: str) -> None: + """Set the current request trace id (internal use by middleware/tests).""" + _trace_ctx.set(value or "") + + +def get_trace_id() -> str: + """Return W3C trace id if set, else env correlation id, else '-'.""" + current = _trace_ctx.get() + if current: + return current + for env_key in ("TRACE_ID", "CORRELATION_ID", "REQUEST_ID"): + val = os.getenv(env_key, "") + if val: + return val + return "-" + + +# Keys whose values must never appear in logs (case-insensitive). +SENSITIVE_KEYS = { + "password", + "passwd", + "pwd", + "secret", + "client_secret", + "api_key", + "apikey", + "google_api_key", + "openai_api_key", + "access_token", + "auth_token", + "id_token", + "refresh_token", + "session_token", + "aws_secret_access_key", + "aws_access_key_id", + "aws_session_token", + "authorization", + "x-api-token", + "onboarding_api_token", + "private_key", + "email", +} + +# key[=:]value and "key": "value" shapes (covers query strings, headers, JSON). +_KEY_ALT = ( + r"password|passwd|pwd|secret|client_secret|api[_-]?key|access[_-]?token|" + r"auth[_-]?token|id[_-]?token|refresh[_-]?token|session[_-]?token|" + r"aws_secret_access_key|aws_session_token|aws_access_key_id|" + r"authorization|x[_-]?api[_-]?token|onboarding_api_token|private_key" +) +_RE_KV = re.compile( + rf"(?i)(({_KEY_ALT})\s*[:=]\s*)([\"']?)([^\s\"',;}}&]+)([\"']?)" +) +_RE_JSON_KV = re.compile( + rf"(?i)(\"(?:{_KEY_ALT})\"\s*:\s*\")(.*?)(\")" +) +_RE_BEARER = re.compile(r"(?i)\b(Bearer\s+)[A-Za-z0-9\-._~+/=]+") +_RE_AKIA = re.compile(r"\bAKIA[0-9A-Z]{16}\b") +_RE_AWS_SECRET = re.compile(r"\b[A-Za-z0-9/+=]{40}\b") +_RE_PRIVATE_KEY = re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----") +_RE_EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") + + +def redact_str(text: str) -> str: + """Redact secrets/PII from a free-text log message.""" + if not isinstance(text, str): + text = str(text) + redacted = _RE_JSON_KV.sub(rf"\1{REDACTED}\3", text) + redacted = _RE_KV.sub(rf"\1\3{REDACTED}\5", redacted) + redacted = _RE_BEARER.sub(rf"\1{REDACTED}", redacted) + redacted = _RE_AKIA.sub(REDACTED, redacted) + redacted = _RE_PRIVATE_KEY.sub(REDACTED, redacted) + # Only redact long base64-ish blobs when they appear next to a sensitive + # key context is hard; keep the generic rule narrow: redact 40-char + # AWS-secret-shaped tokens only if the message already mentions secrets. + if re.search(rf"(?i)({_KEY_ALT})", redacted): + redacted = _RE_AWS_SECRET.sub(REDACTED, redacted) + redacted = _RE_EMAIL.sub(REDACTED, redacted) + return redacted + + +def redact_obj(obj): + """Recursively redact dict/list values with sensitive keys.""" + if isinstance(obj, dict): + out = {} + for k, v in obj.items(): + if isinstance(k, str) and k.lower() in SENSITIVE_KEYS: + out[k] = REDACTED + else: + out[k] = redact_obj(v) + return out + if isinstance(obj, (list, tuple)): + return [redact_obj(v) for v in obj] + if isinstance(obj, str): + return redact_str(obj) + return obj + + +class JsonFormatter(logging.Formatter): + """Single-line JSON formatter: timestamp/level/service/msg/trace.""" + + def __init__(self, service: str = "api"): + super().__init__() + self.service = service or os.getenv("SERVICE_NAME", "api") + + def format(self, record: logging.LogRecord) -> str: + msg = record.getMessage() + msg = redact_str(msg) + payload: dict = { + "timestamp": datetime.now(timezone.utc) + .isoformat() + .replace("+00:00", "Z"), + "level": record.levelname, + "service": os.getenv("SERVICE_NAME", self.service), + "msg": msg, + "trace": get_trace_id(), + "logger": record.name, + } + if record.exc_info and record.exc_info[0] is not None: + exc_text = self.formatException(record.exc_info) + payload["error"] = redact_str(exc_text) + # Redact any structured extras without breaking the required keys. + if hasattr(record, "extra") and isinstance(record.extra, dict): + payload["extra"] = redact_obj(record.extra) + return json.dumps(payload, ensure_ascii=False) diff --git a/docker-compose.yml b/docker-compose.yml index 493ea51..c3becd6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ # Smart Cloud Optimizer — local stack. # Published ports bind to host loopback; container listeners remain reachable -# inside optimizer-net. See docs/operations/local-runtime.md. +# inside optimizer-net. See README.md Operations. # Core: docker compose up --build (backend + frontend) # Full: docker compose --profile full up --build (adds Streamlit) # diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index ce1c1d0..0000000 --- a/docs/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Technical documentation - -These guides describe the implemented application on **`main`**. -Start with the [README](../README.md#quick-start) for installation and first use, -or the [development guide](development/README.md) for checks and contribution -conventions. - -| Responsibility | Start here | -| --- | --- | -| Architecture | [System boundaries and source map](architecture/README.md), [data pipeline](architecture/data-pipeline.md), [data model](architecture/data-model.md) | -| Engine behavior | [Forecasting](architecture/engines/forecasting.md), [optimizer](architecture/engines/optimizer.md), [AI advisor](architecture/engines/ai-advisor.md) | -| Development | [Checks and conventions](development/README.md), [legacy Streamlit UI](development/streamlit.md) | -| API / contracts | [Generated HTTP reference and compatibility](api/http.md), [storage contracts and signatures](api/storage.md) | -| Operations | [Local diagnostics, persistence, and recovery](operations/local-runtime.md) | -| Security | [Trust boundaries and data handling](security/README.md), [vulnerability reporting](../SECURITY.md) | -| Reference | [Environment variables and active configuration](reference/configuration.md) | - -## Project report - -[Project report (PDF)](project-report.pdf) is exported from the latest corrected -graduation report. It records the project at the time it was written; use the -guides above for current application behavior. diff --git a/docs/api/http.md b/docs/api/http.md deleted file mode 100644 index 54745f3..0000000 --- a/docs/api/http.md +++ /dev/null @@ -1,48 +0,0 @@ -# HTTP contracts - -The [FastAPI app](../../backend_api/main.py) and its -[routers](../../backend_api/routers/) own endpoint and Pydantic model definitions. -With the backend running, use http://localhost:8000/docs for interactive reference -and http://localhost:8000/openapi.json for the machine-readable schema. - -Generate the same schema without starting the server or running its database -initialization, from the repository root with Python dependencies installed: - -```bash -python - <<'PYTHON' -import json -from backend_api.main import app -print(json.dumps(app.openapi(), indent=2)) -PYTHON -``` - -Do not maintain a handwritten endpoint/schema inventory alongside this output. -Generated schemas describe input/output shapes; the important behavioral -contracts are linked below. - -## Behavioral contracts - -- [Identity and connections](../architecture/README.md#connection-identity-and-verification): registration identity, AWS workspace selection, and separate test/save behavior. -- [Security boundaries](../security/README.md): client-supplied user IDs, optional onboarding token, and plaintext credentials. -- [Data pipeline](../architecture/data-pipeline.md): sync is asynchronous and replaces prior account data incrementally. -- [Forecasting](../architecture/engines/forecasting.md): history requirements, caching, and comparison versus offline accuracy evaluation. -- [Optimization](../architecture/engines/optimizer.md): replacement writes, budget scope, and recommendation limits. -- [AI advisor](../architecture/engines/ai-advisor.md): failure responses, prompt guard, and persistence differences. -- [Runtime settings](../architecture/README.md#runtime-settings): saved preferences are not active engine inputs. - -`GET /health` returns a static `{"ok": true}` response. It establishes HTTP -liveness, not database integrity or AWS/Gemini readiness. Use the -[operations guide](../operations/local-runtime.md) for diagnosis. - -## Compatibility and verification - -The browser UI and API are developed together on `main`. Routes use `/api/` -without a version segment; there is no published external-client compatibility -or deprecation policy. Do not infer a stable third-party contract from OpenAPI's -default version field. - -When changing request/response behavior, update the affected frontend consumers, -[contract tests](../../tests/test_frontend_backend_contract.py), and behavioral -docs in the same PR. Describe consumer impact and any migration needed for saved -state; exercise the affected page with the backend running. The -[development guide](../development/README.md) owns check commands. diff --git a/docs/api/storage.md b/docs/api/storage.md deleted file mode 100644 index 2480b57..0000000 --- a/docs/api/storage.md +++ /dev/null @@ -1,125 +0,0 @@ -# Storage API Reference - -The `storage` package exports the shared SQLite interface from -[storage/__init__.py](../../storage/__init__.py). Implementations, accepted row fields, -and SQL live in [storage/db.py](../../storage/db.py). Some HTTP routes also issue -SQL directly; see [Architecture](../architecture/README.md). - -## Current signatures - -Generate the public function reference from the checked-out code: - -```bash -python - <<'PYTHON' -import inspect -import storage - -for name in storage.__all__: - value = getattr(storage, name) - if inspect.isfunction(value): - print(f"{name}{inspect.signature(value)}") -PYTHON -``` - -For a function's field requirements and behavior, use, for example, -`python -m pydoc storage.insert_ec2_instances`. Get exact table keys and types -from the [generated schema](../architecture/data-model.md). - -Filters are function-specific: inventory reads accept only `conn` and `user_id`; -cost queries use `start_date`/`end_date`; EC2/RDS metrics use `start`/`end`. -Other metric getters accept a resource filter but no time range. -ElastiCache uses `cache_cluster_id`, ECS uses `service_name`, and Lambda metrics -use `date`. Pricing reads accept `service` and `instance_type`, without a user ID. -Do not assume every getter accepts arbitrary keyword filters. - -## Transaction contract - -- Data `insert_*` functions return an inserted-row count and do not commit. - Data `get_*` functions return lists of dictionaries and do not commit. -- Commit after batching data inserts; roll back on failure and close the - connection when finished. SQLite's connection context manager does not close it. -- Schema, user, authentication/profile, and AWS connection mutation helpers - manage their own commits. These calls are not part of a caller-controlled - batch of data inserts. -- `ensure_schema()` creates missing objects and applies supported additive - credential migrations. `create_schema()` drops and recreates all tables; - use it only for disposable fixtures. -- `clear_user_data()` commits deletion of a user's data/results while retaining - `users`, `aws_connections`, and global `instance_pricing`. - -`get_connection(db_path=None)` defaults to the configured demo DB, enables WAL -and foreign keys, and sets a 10-second busy timeout. Many inventory/metric/cost -inserts use `INSERT OR REPLACE`; result inserts have their own semantics in the -source. Sync and optimization can delete previous results; see -[Data pipeline](../architecture/data-pipeline.md) and [Optimizer](../architecture/engines/optimizer.md#usage). - -## Disposable example - -This example creates a temporary database, inserts cost data, reads it back, -and closes the connection without touching the tracked database: - -```python -from contextlib import closing -from pathlib import Path -from tempfile import TemporaryDirectory -from storage import ( - get_connection, ensure_schema, ensure_user, - insert_daily_costs, get_daily_costs, -) - -with TemporaryDirectory() as directory: - with closing(get_connection(Path(directory) / "example.db")) as conn: - ensure_schema(conn) - user_id = ensure_user(conn, "EXAMPLE-001") - insert_daily_costs(conn, user_id, [ - {"date": "2024-01-15", "total_cost": 125.50}, - ]) - conn.commit() - print(get_daily_costs( - conn, user_id, start_date="2024-01-01", end_date="2024-01-31", - )) -``` - -## Identity and credentials - -`register_user()` creates a `usr-` identity; `ensure_user()` creates an -`aws-` workspace. Connection rows link the caller-supplied owner to -an AWS account; the web and Streamlit flows choose different owners. See -[Connection identity and verification](../architecture/README.md#connection-identity-and-verification). -`get_aws_connections()` returns stored credentials to Python callers; HTTP -routes must remove secrets before returning responses. User-scoped queries -filter the supplied ID but do not authenticate it. See the -[security notes](../security/README.md). - -`INSTANCE_SPECS` and `SERVICE_NAME_MAP` are also exported by `storage`; their -canonical definitions are in [cloud_optimizer/config.py](../../cloud_optimizer/config.py). - -## Loading data for analysis - -[`ml_engine.data_prep`](../../ml_engine/data_prep.py) converts storage results into -DataFrames. For example, read cost data without running schema initialization or -writing the committed fixture: - -```python -import sqlite3 -from contextlib import closing -from pathlib import Path -from ml_engine.data_prep import load_cost_data - -path = Path("data/cloud_optimizer.db").resolve() -with closing(sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)) as conn: - conn.row_factory = sqlite3.Row - frame = load_cost_data(conn, user_id="aws-SYNTHETIC-001") -print(frame[["date", "total_cost"]].head()) -``` - -A schema column does not prove a metric was observed. Check the -[collectors](../../aws_collector/collectors/) and mappings: the current EC2 and RDS -collectors do not fetch memory utilization. Numeric coercion may turn missing or -invalid inputs into zero; other fields remain NULL. Interpret availability before -using these values for sizing or model training. - -Service totals are not resource-level costs. Joining `service_costs` to inventory -by `user_id` alone multiplies rows and does not attribute spend to resources. -See [forecasting evaluation](../architecture/engines/forecasting.md#reproducible-evaluation) -for training/evaluation boundaries. diff --git a/docs/architecture/README.md b/docs/architecture/README.md deleted file mode 100644 index 28d8d27..0000000 --- a/docs/architecture/README.md +++ /dev/null @@ -1,151 +0,0 @@ -# Architecture - -The application on `main` uses a TypeScript/Next.js browser UI and a Python -FastAPI backend. The optional Streamlit UI calls the Python engines directly. - -```mermaid -flowchart LR - Browser["Browser"] - NextUI["Next.js UI
(port 3000)"] - - subgraph BE["FastAPI backend (port 8000)"] - API["FastAPI routers
(trust-based user_id)"] - Storage["storage facade
(shared data API)"] - Optimizer["optimizer
(PuLP right-sizing + rules)"] - ML["ml_engine
(forecasts / anomalies)"] - AImod["ai_module
(Gemini advisor)"] - Collector["aws_collector
(boto3 pipeline)"] - end - - DB[("SQLite DB
data/cloud_optimizer.db")] - Synthetic["data_generation.synthetic
(demo seed)"] - AWS["AWS
(Cost Explorer / CloudWatch via boto3)"] - Gemini["Google Gemini
(AI recommendations)"] - - Browser --> NextUI - NextUI -->|"HTTP / JSON"| API - API --> Storage - API --> Optimizer - API --> ML - API --> AImod - API --> Collector - Optimizer --> Storage - ML --> Storage - AImod --> Storage - Collector --> Storage - Storage --> DB - API -->|"route-specific SQL"| DB - - Collector -. "real mode" .-> AWS - AImod -. "AI calls" .-> Gemini - Synthetic -. "demo mode" .-> Storage - - classDef ext stroke-dasharray:5 5,stroke:#888,color:#444; - class AWS,Gemini,Synthetic ext; -``` - -## Boundaries - -| Component | Responsibility | -| --- | --- | -| [`frontend/`](../../frontend/) | Next.js routes, client state, Plotly charts, HTTP calls to the backend | -| [`backend_api/`](../../backend_api/) | FastAPI routes for auth, connections/sync, costs, dashboard, forecasting, recommendations, settings, and AI onboarding | -| [`cloud_optimizer/`](../../cloud_optimizer/) | Shared Python configuration, paths, root `.env` loading | -| [`storage/`](../../storage/) | SQLite schema, migrations, authentication helpers, connection and data APIs | -| [`aws_collector/`](../../aws_collector/) | boto3 clients and per-service collectors for live AWS data | -| [`data_generation/`](../../data_generation/) | Synthetic fixtures and a CLI that seeds SQLite | -| [`ml_engine/`](../../ml_engine/) | Time-series preparation, forecasting, anomaly detection, evaluation | -| [`optimizer/`](../../optimizer/) | Inventory/metrics/pricing-based LP and rule recommendations | -| [`ai_module/`](../../ai_module/) | Questionnaire-based recommendations using Gemini | -| [`dashboard/`](../../dashboard/) | Optional legacy Streamlit application | - -Use the running backend's generated http://localhost:8000/docs for HTTP contracts. -See [HTTP contracts](../api/http.md) and -[Configuration](../reference/configuration.md) for generated reference and settings. - -## Persistence and computation - -Most Python components access `data/cloud_optimizer.db` through `storage`. -The forecast router currently opens SQLite directly; recommendations and -connections also execute SQL on storage-created connections. The storage facade -is the preferred shared interface, but it is not an enforced access boundary. -Per-user runtime settings are stored separately; see [Runtime settings](#runtime-settings). - -`ensure_schema()` creates missing tables/indexes and adds missing AWS credential -columns for older databases. `create_schema()` is destructive. See -[Storage API](../api/storage.md) for transaction ownership and -[Data schemas](data-model.md) for table details. - -The optimizer reads observed inventory, metrics, and pricing; it does not consume -ML forecast output. Optimization replaces stored recommendations for the selected -user. Forecasting and Gemini recommendations are separate computations. See -[optimizer constraints and usage](engines/optimizer.md) before interpreting savings. - -## Data sources - -The demo workspace uses existing synthetic data in SQLite. The current tracked -generator creates fixtures for tests and demos without downloading external -research datasets. It does not reproduce the historical committed fixture -byte-for-byte. - -The Next.js connection form stores access keys in SQLite; see -[connection identity and verification](#connection-identity-and-verification) -for the separate test, save, and workspace-selection behavior. The collector -supports both key-based connections and legacy IAM-role connections. Sync runs -in a background thread, clears the selected account's previous data, then -collects month ranges newest first. See [Data pipeline](data-pipeline.md) for -write behavior. - -## Connection identity and verification - -Registration creates a `usr-…` identity. In the primary web flow, -[the connection router](../../backend_api/routers/connections.py) saves a connection -under `aws-`, the same workspace ID used for collected data. After -save, [the frontend session helper](../../frontend/app/lib/session.ts) switches the -stored active identity to that workspace. Web connections are not attached to -the registered `usr-…` identity. The legacy -[Streamlit settings page](../../dashboard/settings.py) instead attaches role-based -connections to its authenticated owner and selects an AWS workspace separately. - -**Test connection** calls STS to resolve the supplied keys' account. Testing and -saving are separate operations: save resolves the account through STS only when -no account ID is supplied. With a supplied account ID, save validates its format -and stores the connection without rechecking the keys against that account. -A saved connection is therefore not proof of verified AWS access or ownership. -Sync creates an AWS session from the stored credentials and can fail later. - -## Trust boundary - -The API accepts client-supplied identity. Its `user_id` filters are not server-side -authorization. See [security boundaries](../security/README.md) for password, -credential, browser-session, and external-service limits. The -[operations guide](../operations/local-runtime.md) covers local exposure, -diagnostics, and recovery. - -## Runtime settings - -[backend_api/routers/settings.py](../../backend_api/routers/settings.py) stores -per-user preferences in `backend_api/runtime_settings.json`. The settings API -validates and saves them, but engines and the collector do not read that file: -saving a budget, collection interval, retention period, or model preference does -not change their current behavior. Request parameters and Python configuration -remain the active inputs. - -The file is excluded from Git and the image build context. Compose persists only -`/app/data`, so settings are lost on container recreation. See -[runtime settings recovery](../operations/local-runtime.md#runtime-settings-recovery) -for file lifetime, locking limits, and corrupt-file handling. - -## Engine entry points - -| Subsystem | Current behavior and source | -| --- | --- | -| Collection | [Pipeline](data-pipeline.md); [`CollectorRunner`](../../aws_collector/runner.py) and [service collectors](../../aws_collector/collectors/) | -| Forecasting | [Models and evaluation](engines/forecasting.md); [`forecaster.py`](../../ml_engine/forecaster.py), [`data_prep.py`](../../ml_engine/data_prep.py), [`evaluator.py`](../../ml_engine/evaluator.py) | -| Optimization | [LP/rule contracts and usage](engines/optimizer.md); [`engine.py`](../../optimizer/engine.py) | -| AI advisor | [Questionnaire and Gemini contract](engines/ai-advisor.md); [`recommender.py`](../../ai_module/recommender.py) | -| Legacy UI | [Streamlit startup](../development/streamlit.md); [`dashboard/app.py`](../../dashboard/app.py) | - -The [data model](data-model.md) explains relationships; the -[storage contract](../api/storage.md) explains transaction ownership and generated -signatures. Exact function and table inventories come from those sources. diff --git a/docs/architecture/c1-system-context.mmd b/docs/architecture/c1-system-context.mmd new file mode 100644 index 0000000..d8ebe30 --- /dev/null +++ b/docs/architecture/c1-system-context.mmd @@ -0,0 +1,18 @@ +flowchart TB + User["Person
Single local user
Browser operator"] + Browser["Browser
localStorage holds user_id
No server session"] + + subgraph System["Smart Cloud Optimizer
Local single-user system"] + App["Next.js UI + FastAPI API + SQLite
Demo data: aws-SYNTHETIC-001"] + end + + AWS["AWS
Cost Explorer + CloudWatch
via boto3"] + Gemini["Google Gemini
AI recommendations
gemini-2.5-flash"] + + User -->|"Uses
localhost:3000"| Browser + Browser -->|"HTTP JSON, trust-based user_id
CORS: 127.0.0.1:3000, localhost:3000"| App + App -.->|"Real mode: STS + collectors
Stored plaintext keys"| AWS + App -.->|"Advisor prompts
GOOGLE_API_KEY, 4000-char limit"| Gemini + + classDef ext stroke-dasharray:5 5,stroke:#888,color:#444; + class AWS,Gemini ext; diff --git a/docs/architecture/c2-containers.mmd b/docs/architecture/c2-containers.mmd new file mode 100644 index 0000000..dbf485a --- /dev/null +++ b/docs/architecture/c2-containers.mmd @@ -0,0 +1,45 @@ +flowchart LR + Browser["Browser
localStorage user_id
No Next.js middleware guard"] + + subgraph FE["Frontend container
optimizer-frontend :3000"] + NextUI["Next.js UI
Routes, Plotly charts
NEXT_PUBLIC_API_BASE_URL
http://localhost:8000 build-time"] + end + + subgraph BE["Backend container
optimizer-backend :8000
uvicorn backend_api.main:app"] + API["FastAPI routers
trust-based user_id query param
traceparent + X-Request-ID"] + Storage["storage facade
shared data API
get_connection + ensure_schema"] + Optimizer["optimizer
PuLP right-sizing + rules
Replaces recommendations"] + ML["ml_engine
forecaster + data_prep
anomaly + evaluator"] + AImod["ai_module
Gemini advisor
Questionnaire prompt"] + Collector["aws_collector
CollectorRunner + collectors
Background thread sync"] + end + + DB[("SQLite
data/cloud_optimizer.db
WAL, 30 tables")] + Synthetic["data_generation.synthetic
Seed CLI, demo fixtures"] + Streamlit["Streamlit UI :8501
profile full only
Calls Python directly"] + AWS["AWS
Cost Explorer + CloudWatch
boto3"] + Gemini["Google Gemini
gemini-2.5-flash"] + + Browser -->|"HTTP JSON
browser-side"| NextUI + NextUI -->|"HTTP JSON
in-network via host URL"| API + API --> Storage + API --> Optimizer + API --> ML + API --> AImod + API --> Collector + Streamlit --> Storage + Streamlit --> Optimizer + Streamlit --> ML + Streamlit --> AImod + Optimizer --> Storage + ML --> Storage + AImod --> Storage + Collector --> Storage + Storage --> DB + API -.->|"Route-specific direct SQL
forecast, recommendations, connections"| DB + Synthetic -.->|"Demo seed
INSERT OR REPLACE"| Storage + Collector -.->|"Real mode
Stored keys or role"| AWS + AImod -.->|"Advisor calls
GOOGLE_API_KEY"| Gemini + + classDef ext stroke-dasharray:5 5,stroke:#888,color:#444; + class AWS,Gemini,Synthetic ext; diff --git a/docs/architecture/c3-components.mmd b/docs/architecture/c3-components.mmd new file mode 100644 index 0000000..7904bad --- /dev/null +++ b/docs/architecture/c3-components.mmd @@ -0,0 +1,53 @@ +flowchart TB + subgraph API["FastAPI app backend_api/main.py
lifespan: ensure_schema + ensure_user SYNTHETIC-001
CORS localhost:3000 + 127.0.0.1:3000, allow_credentials False
GET /health liveness, GET /ready DB check"] + Auth["auth router
/api/auth login signup demo logout
PBKDF2 verify, no token issued
Per-IP throttle 10 per 300s"] + Conn["connections router
/api/connections test save list delete sync
STS resolve, _SYNCING guard
Thread + CollectorRunner.from_connection"] + Costs["costs router
/api/costs preset range
get_daily_costs + get_service_costs"] + Dash["dashboard router
/api/dashboard/home
daily + service + recs + anomalies
30-day rollup"] + Forecast["forecast router
/api/forecast + /compare
Direct SQLite + ml_engine
forecaster, data_prep, evaluator"] + Recs["recommendations router
/api/recommendations GET + POST generate
Direct SQL read, optimizer.engine.optimize"] + Settings["settings router
/api/settings GET PUT POST reset
runtime_settings.json file
Engines do not read file"] + Onboard["ai-onboarding router
/api/ai-onboarding questions
build-prompt generate
4000-char guard, 502 on Gemini fail
X-API-Token when set"] + end + + subgraph Store["storage facade storage/db.py
Preferred shared interface, not enforced"] + ConnMgmt["Connection helpers
get_connection WAL + busy_timeout
ensure_schema additive only
create_schema destructive fixtures only
register authenticate get_user
add get delete connections
update status + clear_user_data"] + Writes["Write API insert_*
daily service region costs
ec2 rds elasticache ecs lambda
ebs s3 dynamodb nat elb metrics
instance_pricing ai_recs forecasts
recommendations anomalies
No commit inside, caller commits"] + Reads["Read API get_*
daily service costs
metrics by user resource time
recommendations forecasts anomalies
_query_metrics ORDER BY time"] + end + + subgraph Engines["Compute engines"] + Opt["optimizer/engine.py
Inventory + metrics + pricing
LP + rules, replaces user recs"] + MLEng["ml_engine
forecaster data_prep anomaly evaluator
Reads history via storage or SQL"] + AIMod["ai_module/recommender.py
Questionnaire to Gemini prompt
Writes ai_recommendations"] + Coll["aws_collector
CollectorRunner 12 months newest-first
Per-service collectors
clear_user_data then incremental commit"] + end + + DB[("SQLite data/cloud_optimizer.db
users aws_connections
costs inventory metrics pricing results")] + + Auth --> ConnMgmt + Conn --> ConnMgmt + Conn --> Coll + Costs --> Reads + Dash --> Reads + Forecast --> Reads + Forecast -.->|"Direct SQL on storage connection"| DB + Recs --> Reads + Recs --> Opt + Recs -.->|"Direct SQL read"| DB + Conn -.->|"Inline UPDATE access_verified
Direct SELECT by id in sync thread"| DB + Settings -.->|"File only
backend_api/runtime_settings.json"| Settings + Onboard --> AIMod + Opt --> Writes + Opt --> Reads + MLEng --> Reads + MLEng --> Writes + AIMod --> Writes + Coll --> Writes + Coll --> ConnMgmt + ConnMgmt --> DB + Writes --> DB + Reads --> DB + + classDef ext stroke-dasharray:5 5,stroke:#888,color:#444; + class Settings ext; diff --git a/docs/architecture/data-model.md b/docs/architecture/data-model.md index 6b14828..f08f775 100644 --- a/docs/architecture/data-model.md +++ b/docs/architecture/data-model.md @@ -41,15 +41,28 @@ It does not reconcile arbitrary older table definitions. User IDs represent both registered identities (`usr-…`) and AWS workspaces (`aws-`). Web connections belong to the AWS workspace; legacy Streamlit connections belong to the registered owner. Collection writes data -into the AWS workspace. See -[Connection identity and verification](README.md#connection-identity-and-verification). -Demo data uses `aws-SYNTHETIC-001`. Filtering these IDs is not HTTP authorization. +into the AWS workspace. Demo data uses `aws-SYNTHETIC-001`. Filtering these IDs is not HTTP authorization. Passwords use PBKDF2-HMAC-SHA256. Connection access keys and session tokens are stored in plaintext, and storage reads expose those columns to Python callers. -See the [security limitations](../security/README.md). +See `docs/security/threat-model.md`. -For insert/query behavior and commit ownership, see [Storage API](../api/storage.md). -For data replacement during sync or seeding, see [Data pipeline](data-pipeline.md). -Runtime settings are a separate JSON file, described in -[Architecture](README.md#runtime-settings). +## Schema compatibility check + +`ensure_schema()` is additive only: it creates missing tables/indexes and adds +the supported AWS credential columns +(`aws_access_key_id`, `aws_secret_access_key`, `aws_session_token`, +`auth_type`) to older databases. It never drops data or reconciles arbitrary +older table definitions. `create_schema()` drops and recreates all tables and +belongs only in disposable fixtures. + +Recorded proof (2026-10-20, `main`): +fresh `:memory:` init holds 30 tables; migration adds the 4 supported +credential columns and preserves both users; temp-copy backup/restore +round-trip `integrity_check` ok. Re-run `poe check:schema` / CI `verify` +after any schema change. + +Re-run the script after any schema or migration change. + +For insert/query behavior see `storage/db.py` and `storage/__init__.py`. +Runtime settings are a separate JSON file `backend_api/runtime_settings.json`, lost on container recreation. diff --git a/docs/architecture/data-pipeline.md b/docs/architecture/data-pipeline.md deleted file mode 100644 index 8ea5ba5..0000000 --- a/docs/architecture/data-pipeline.md +++ /dev/null @@ -1,63 +0,0 @@ -# Data Pipeline - -## Live AWS collection - -The Next.js **Account Settings → Connections** flow saves access keys; testing -and account resolution are described in -[Connection identity and verification](README.md#connection-identity-and-verification). -Sync starts a background -`CollectorRunner.from_connection()`, using stored keys or legacy role -credentials. The standalone `python -m aws_collector.main` entry point uses -boto3's configured credentials. - -The runner clears previous data for the selected account before collecting -12 months of history. It processes month ranges **newest first**, collecting -costs, service inventory/metrics, and pricing, with incremental commits. A -failure can leave partial data; sync is not an atomic replacement. - -Per-service collectors live in [aws_collector/collectors/](../../aws_collector/collectors). -Their writes use `storage.insert_*()`. Many data tables use -`INSERT OR REPLACE` on composite primary keys to update matching observations. -Global instance pricing has no `user_id`; account data is keyed by user. -See [Storage API](../api/storage.md) for transaction ownership and -[Data schemas](data-model.md) for keys. - -Collectors log individual API failures where they can continue. Consult sync -status and backend logs if data is incomplete. See [local recovery](../operations/local-runtime.md#interrupted-sync-recovery) -for interrupted workers and partial collection. Runtime AWS credentials and -account data must stay out of commits; see the -[security notes](../security/README.md). - -## Synthetic data - -[data_generation/synthetic.py](../../data_generation/synthetic.py) is tracked and -used by tests and the demo seeder. Its functions generate inventory, metrics, -costs, and pricing; the CLI maps generator columns into storage records and -commits the results. Current fixtures are generated in code without downloading -external research datasets. - -```bash -python -m data_generation.synthetic --help -``` - -The CLI writes to the configured `data/cloud_optimizer.db`; it has no -`--db-path` option. Seeding overwrites matching target-user rows and global -instance-pricing rows. It leaves unmatched rows and other users' data in place, -so running it on an existing database is not a clean reset. Use a disposable -checkout/database for experiments. - -Dates are relative to the execution date. The same seed and day count reproduce -random values for that date; a later execution shifts timestamps. The committed -historical demo fixture is not a byte-for-byte output contract for this generator. - -## Consumers - -The TypeScript UI reads JSON from FastAPI. The API routes call storage helpers -and engines; some routes also use direct SQL as described in -[Architecture](README.md). The legacy Streamlit UI calls Python modules -directly. - -Forecasting reads historical series. Optimization reads inventory, observed -metrics, and prices and replaces stored recommendations. Its write and budget -semantics are documented in [Optimizer](engines/optimizer.md#usage). These consumers -share the same database schema regardless of how observations were collected. diff --git a/docs/architecture/deployment.mmd b/docs/architecture/deployment.mmd new file mode 100644 index 0000000..038f353 --- /dev/null +++ b/docs/architecture/deployment.mmd @@ -0,0 +1,29 @@ +flowchart TB + Dev["Developer laptop
Docker Compose
smart-cloud-optimizer project"] + Browser["Browser
http://localhost:3000
http://127.0.0.1:3000"] + + subgraph Net["Bridge network optimizer-net
In-network DNS: backend, frontend"] + subgraph BackendC["backend service
image optimizer-backend
python:3.12-slim + uvicorn :8000"] + BE["FastAPI backend_api.main:app
--host 0.0.0.0 --port 8000
lifespan ensure_schema
Health GET /health"] + end + subgraph FrontendC["frontend service
image optimizer-frontend
Next.js :3000"] + FE["Next.js UI
NEXT_PUBLIC_API_BASE_URL build arg
http://localhost:8000 host-reachable
depends_on backend healthy"] + end + subgraph StreamlitC["streamlit service
profile full only
Reuses optimizer-backend image"] + ST["Streamlit dashboard/app.py :8501
--server.address 0.0.0.0
depends_on backend healthy"] + end + end + + Vol[("Named volume optimizer-data
Mounted at /app/data
Seeded from image, appuser uid 10001
SQLite WAL writable
down -v resets demo DB")] + EnvFile[".env optional
DEMO_MODE GOOGLE_API_KEY MODEL
AWS_REGION ACCOUNT_ID
ONBOARDING_API_TOKEN
Base AWS keys for CLI role path"] + SettingsNote["runtime_settings.json
Not in image, not in volume
Lost on container recreate"] + + Browser -->|"127.0.0.1:3000 loopback"| FE + Browser -->|"127.0.0.1:8000 loopback
Docs /docs, health /health"| BE + Browser -->|"127.0.0.1:8501 loopback
full profile only"| ST + FE -.->|"Browser-side fetch only
Never backend service name"| BE + BE -->|"Read write
data/cloud_optimizer.db"| Vol + ST -->|"Shares same DB file"| Vol + EnvFile -.->|"env_file + environment override"| BE + EnvFile -.->|"env_file + environment override"| ST + BE -.->|"Writes, excluded from Git + image"| SettingsNote diff --git a/docs/architecture/engines/ai-advisor.md b/docs/architecture/engines/ai-advisor.md deleted file mode 100644 index 83d941c..0000000 --- a/docs/architecture/engines/ai-advisor.md +++ /dev/null @@ -1,68 +0,0 @@ -# AI Module - -The AI module builds a questionnaire prompt and calls Gemini for architecture -advice. This is separate from the inventory-based [cost optimizer](optimizer.md). -The configured model is `GOOGLE_MODEL`; see [Configuration](../../reference/configuration.md). -A valid `GOOGLE_API_KEY` is needed even for the synthetic demo workspace. - -## Entry points and persistence - -| Entry point | Behavior | -| --- | --- | -| Next.js guided questions → [FastAPI onboarding router](../../../backend_api/routers/on_boarding.py) | Fetches questions, builds a prompt, and returns generated recommendations and raw text. The router does not persist them to SQLite. | -| Legacy [Streamlit Home](../../../dashboard/home.py) | Offers the questionnaire during cold start and stores generated recommendations for the selected user. | -| Standalone [ai_module/ui.py](../../../ai_module/ui.py) | Writes recommendations to the synthetic user; running it can modify the tracked demo DB. Use a disposable checkout/database for experiments. | -| Python callers | Decide whether to store the returned result through `storage.insert_ai_recommendations()`. | - -HTTP request/response schemas are generated at http://localhost:8000/docs and -http://localhost:8000/openapi.json when the backend runs. The generate endpoint -caps prompts at 4000 characters (HTTP 400), returns HTTP 502 for in-band Gemini -errors, and optionally requires `X-API-Token` when `ONBOARDING_API_TOKEN` is set. -The current Next.js client does not send that header. These guards do not provide -session authentication or validate the factual correctness of AI advice. - -## Python contract - -- [guided_questions.py](../../../ai_module/guided_questions.py) owns the question IDs, - text, and options. Use `get_guided_questions()` instead of maintaining another - questionnaire definition. -- [prompt_builder.py](../../../ai_module/prompt_builder.py) maps answers to a prompt - requesting `recommended_setup`, `estimated_cost`, and `explanation`. -- [recommender.py](../../../ai_module/recommender.py) creates `google.genai.Client`, - calls `client.models.generate_content()` with the configured model and - temperature `0.3`, extracts the text between the first `{` and last `}`, and - parses JSON. -- `get_ai_recommendations(prompt)` returns `(parsed_result, raw_response_text)`. - Missing credentials, API failures, empty responses, and JSON failures return - `({"error": message}, "")`. Parsing does not enforce the requested output - fields or prove that the recommendation or price is correct. - -A Python caller can construct the prompt without making an external request: - -```python -from ai_module import get_guided_questions, build_prompt - -questions = get_guided_questions() -answers = { - question["id"]: question["options"][0] if question["options"] else "" - for question in questions -} -prompt = build_prompt(answers) -print(prompt) -``` - -Calling `get_ai_recommendations(prompt)` then uses the configured Gemini service. -Check the error result before consuming fields. For persistence, -`recommended_setup` is JSON-serialized into a TEXT column; include the input -profile, prompt, estimated cost, model identifier, and raw response according to -`insert_ai_recommendations()` in [storage/db.py](../../../storage/db.py). -The caller commits and closes the connection; see [Storage API](../../api/storage.md). - -## Verification - -```bash -python -m pytest tests/test_ai_module.py tests/test_home.py tests/test_frontend_backend_contract.py -v -``` - -These tests use mocks and local fixtures. They verify integration/error behavior, -not live Gemini availability, generated advice, or forecast/optimizer accuracy. diff --git a/docs/architecture/engines/forecasting.md b/docs/architecture/engines/forecasting.md deleted file mode 100644 index d8c31f3..0000000 --- a/docs/architecture/engines/forecasting.md +++ /dev/null @@ -1,48 +0,0 @@ -# Forecasting Models - -## Current application behavior - -[ml_engine/forecaster.py](../../../ml_engine/forecaster.py) implements Naive, -Seasonal Naive, ETS, Prophet, and SARIMAX. The primary Next.js UI calls -[backend_api/routers/forecast.py](../../../backend_api/routers/forecast.py): - -- `GET /api/forecast` defaults to Prophet and a 30-day horizon. It accepts - horizons of 1–180 days and requires at least 30 normalized daily observations. -- The router loads the latest 365 stored daily observations. - It applies anomaly filtering, retaining the original series if filtering - leaves fewer than 30 observations. -- `GET /api/forecast/compare` runs the five models and compares forecast spend - with the historical daily average. `vs_historical_pct` is not prediction - accuracy, MAPE, or walk-forward validation. A failed model produces null - forecast values in the comparison response. -- Responses are cached in process by user, horizon, series fingerprint, and - model where applicable. These HTTP routes do not save forecasts to SQLite. - The legacy Streamlit page runs models directly and offers CSV export; it - also does not save forecasts to SQLite. - -There is no automatic data-age-based model selection in the HTTP router. -Use the running backend's generated http://localhost:8000/docs for the current -HTTP contract. The CLI remains a placeholder; use the UI or Python API. - -## Reproducible evaluation - -[ml_engine/evaluator.py](../../../ml_engine/evaluator.py) provides `time_series_cv()` -and `compare_models()` for expanding-window validation against held-out values. -They report MAPE, RMSE, and MAE; MAPE excludes zero actuals. These routines are -separate from the HTTP model-comparison endpoint. - -For a local evaluation, load the selected workspace through -`ml_engine.data_prep.load_cost_data()`, choose explicit initial/horizon/step -parameters, and call `compare_models()` with the forecasters under test. Close -the database connection after loading. The synthetic demo workspace is -`aws-SYNTHETIC-001`. Preserve the data snapshot, parameters, code revision, -dependency versions, and output when using results to make a model decision. - -Anomaly detection in [ml_engine/anomaly.py](../../../ml_engine/anomaly.py) combines -rolling Z-score and IQR flags by default. For offline evaluation, document any -filtering and fit preprocessing on training data so future observations do not -influence a fold's training set. - -```bash -python -m pytest tests/test_ml_utils.py tests/test_frontend_backend_contract.py -v -``` diff --git a/docs/architecture/engines/optimizer.md b/docs/architecture/engines/optimizer.md deleted file mode 100644 index 2e959f6..0000000 --- a/docs/architecture/engines/optimizer.md +++ /dev/null @@ -1,232 +0,0 @@ -# Optimizer Module - -## Overview - -The optimizer analyzes AWS resource inventory and metrics to produce cost-saving recommendations. It uses two complementary approaches: a **Linear Programming (LP) solver** for compute right-sizing and a **rule engine** for everything else. - -It reads observed metrics and pricing; ML forecasts are not inputs to the current -optimizer. Recommendation generation writes SQLite and does not apply AWS changes. - -``` - ┌─────────────────────┐ - │ engine.optimize() │ single entry point - └──────────┬──────────┘ - │ - ┌────────────────┼────────────────┐ - ▼ ▼ - ┌─────────────────┐ ┌─────────────────┐ - │ compute_lp.py │ │ rules.py │ - │ LP solver │ │ threshold │ - │ │ │ checks │ - │ EC2 rightsize │ │ EC2 pricing │ - │ RDS rightsize │ │ RDS pricing │ - └─────────────────┘ │ Lambda memory │ - │ EBS volumes │ - │ S3 tiering │ - │ DynamoDB mode │ - │ NAT endpoints │ - │ ELB idle │ - └─────────────────┘ -``` - -**Why two systems?** The problems are structurally different: - -- **Right-sizing** = "which of these 20 instance types is cheapest while still handling my load?" — a menu of alternatives with quantifiable specs. Perfect for LP. -- **Everything else** = "is this thing wasting money? yes/no" — threshold decisions with no alternatives to choose between. Rules are simpler and more transparent. - -## LP Solver (`compute_lp.py`) - -### Formulation - -Binary LP that assigns each instance the cheapest candidate type from the pricing catalog. - -``` -Variables: x[instance, candidate] ∈ {0, 1} - -Objective: minimize Σ cost(candidate) × x[instance, candidate] - -Constraints: - 1. Assignment: Σ x[i, c] = 1 each instance gets exactly one type - 2. CPU: Σ vcpus(c) × x[i,c] ≥ min_vcpus[i] - 3. Memory: Σ memory(c) × x[i,c] ≥ min_memory[i] (when data available) - 4. Budget: Σ total cost ≤ budget_cap (optional) -``` - -### How requirements are computed - -``` - P95 CPU utilization (from metrics) - │ - ▼ - actual_vcpus_needed = (p95_cpu / 100) × current_vcpus - │ - ▼ - min_vcpus = actual_vcpus_needed × headroom (default 1.3) -``` - -Example: instance with 4 vCPUs running at P95 CPU = 40% -- Actual need: 0.40 × 4 = 1.6 vCPUs -- With 30% headroom: 1.6 × 1.3 = 2.08 vCPUs -- A candidate must have at least 2.08 vCPUs, so a 2-vCPU instance is insufficient. - -### EC2 vs RDS differences - -| | EC2 | RDS | -|---|---|---| -| CPU constraint | P95 from metrics | P95 from metrics | -| Memory constraint | P95 when available | Skipped (the implemented RDS collector does not fetch memory) | -| Multi-AZ | N/A | Doubles candidate cost | -| Pricing source | `instance_pricing` table (service=EC2) | `instance_pricing` table (service=RDS) | - -### Edge cases - -- **No metrics** → instance skipped (can't compute requirements) -- **Stopped instances** → skipped (not running, no need to optimize) -- **LP infeasible** → logged warning, returns empty (e.g., instance needs more vCPUs than any candidate offers) -- **Optimal = current type** → no recommendation (already optimal) -- **Optimal costs more** → no recommendation (only recommend savings) - -## Rule Engine (`rules.py`) - -### Rule summary - -| # | Function | Service | Trigger condition | Recommendation | -|---|----------|---------|-------------------|----------------| -| 1 | `check_ec2_pricing` | EC2 | on-demand + currently running + metric timestamps span 60+ days + cheaper RI pricing | Switch to reserved-1yr | -| 2 | `check_rds_pricing` | RDS | on-demand + metric timestamps span 60+ days + cheaper RI pricing | Switch to reserved-1yr | -| 3 | `check_lambda_memory` | Lambda | avg memory < 50% of allocation | Downsize to next lower tier | -| 4 | `check_ebs_volumes` | EBS | gp2 type, OR unattached, OR idle >90% | Upgrade gp2→gp3, delete unused | -| 5 | `check_s3_buckets` | S3 | STANDARD class + <100 daily requests | Switch to INTELLIGENT_TIERING | -| 6 | `check_dynamodb_tables` | DynamoDB | PROVISIONED + both RCU/WCU utilization <50% | Switch to ON_DEMAND | -| 7 | `check_nat_gateways` | VPC | monthly cost ≥$30 | Add VPC gateway endpoints | -| 8 | `check_elb_idle` | ELB | 0 targets AND traffic check does not establish ≥100 average daily requests (see limits below) | Delete load balancer | - -### Key thresholds and constants - -``` -EBS: gp2 = $0.10/GB/mo gp3 = $0.08/GB/mo (20% savings) -S3: STANDARD = $0.023/GB/mo IT monitoring = $0.0025/1000 objects -DynamoDB: RCU = $0.09/mo WCU = $0.47/mo (provisioned) -NAT: $0.045/hr + $0.045/GB processed -Lambda: tiers = [128, 256, 512, 1024, 2048, 3008] MB -``` - -### Checks and limits - -- **Lambda**: skips downsizing when `avg + 2×std > new_tier`. This is a statistical - estimate from observed memory usage, not a guarantee against out-of-memory failures. -- **EBS idle**: requires at least 168 metric rows (assumed hourly) before flagging; - row count alone does not establish continuous coverage. -- **ELB**: requires zero targets, but checks traffic only with at least 168 metric - rows and a `request_count` column. Missing or shorter history leaves the traffic - flag false and can still produce a high-confidence deletion recommendation. - With sufficient history, the rule skips deletion at ≥100 average daily requests. - Independently check workload use and telemetry before acting on a deletion recommendation. -- **DynamoDB**: requires BOTH read AND write utilization below threshold -- **EC2/RDS pricing**: requires a 60-day span between the earliest and latest metric - timestamps; it does not verify uninterrupted operation or future commitment needs. - -## Orchestrator (`engine.py`) - -### Flow - -``` -optimize(conn, user_id) - │ - ├── 1. DELETE old recommendations for user_id - │ - ├── 2. Run LP solver - │ ├── optimize_ec2() - │ └── optimize_rds() - │ - ├── 3. Run rule checks - │ ├── check_ec2_pricing() - │ ├── check_rds_pricing() - │ ├── check_lambda_memory() - │ ├── check_ebs_volumes() - │ ├── check_s3_buckets() - │ ├── check_dynamodb_tables() - │ ├── check_nat_gateways() - │ └── check_elb_idle() - │ - ├── 4. Deduplicate by (resource_id, recommendation_type) - │ └── keep highest monthly_savings - │ - └── 5. INSERT into recommendations table + COMMIT -``` - -### Parameters - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `budget_cap` | `config.DEFAULT_BUDGET_CAP` (5000.0) | Cap applied separately to each EC2/RDS LP, not one combined account budget | -| `services` | all 8 services | Select checks; see `ALL_SERVICES` in [engine.py](../../../optimizer/engine.py) | - -Every run deletes **all** previous recommendations for the selected user before -generating replacements, including when `services` selects only some checks. -An infeasible LP contributes no sizing recommendations; independent rules can -still produce results. Rule recommendations are outside the LP budget constraint. - -### Deduplication - -If the LP and a rule both produce a recommendation for the same `(resource_id, recommendation_type)`, only the one with higher `monthly_savings` is kept. This prevents duplicate entries in the DB. - -Different action types for one resource remain separate. Their savings may -overlap, so the CLI's summed savings are not a validated combined execution plan. - -## Usage - -Run from the repository root with dependencies installed. Stop writers to the -source database before copying it; if it is live, use SQLite's backup API to -include committed WAL contents. Keep the committed fixture unchanged: - -```bash -optimizer_tmp=$(mktemp -d) -cp data/cloud_optimizer.db "$optimizer_tmp/cloud_optimizer.db" -python -m optimizer --user-id aws-SYNTHETIC-001 \ - --db-path "$optimizer_tmp/cloud_optimizer.db" -``` - -The disposable copy receives the recommendations. Inspect it before removing -the temporary directory. Use `python -m optimizer --help` for service and budget -arguments. Library callers can pass a connection from -`storage.get_connection(copy_path)` to `optimizer.optimize()`; close it afterward. - -The LP code explicitly uses PuLP's `PULP_CBC_CMD`. Verify that solver in the -active Python environment: - -```bash -python -c "from pulp import PULP_CBC_CMD; print(PULP_CBC_CMD().available())" -``` - -An available executable path is expected. Installing a system `cbc` executable -alone does not change the solver selected by this code. - -### Each recommendation dict contains - -```python -{ - "service": "EC2", - "resource_id": "i-abc123", - "recommendation_type": "rightsize", # or pricing_plan_switch, delete_unused, etc. - "current_config": "c5.xlarge, on-demand", - "recommended_config": "c5.large, on-demand", - "current_monthly_cost": 124.00, - "estimated_monthly_cost": 62.00, - "monthly_savings": 62.00, - "savings_percent": 50.0, - "confidence": "high", # high, medium, or low - "reasoning": "P95 CPU requires 1.8 vCPUs ..." -} -``` - -## Design rationale - -| Decision | Choice | Why | -|----------|--------|-----| -| LP for compute, rules for rest | Approach C | LP needs a menu of alternatives with specs — only compute has this. Rules handle threshold checks. | -| P95 metric (not avg or max) | P95 | Avg hides spikes. Max is one-off outliers. P95 balances real workload. | -| 1.3x headroom | Default | 30% buffer for unexpected traffic. Configurable per call. | -| Clear + replace recs | DELETE before INSERT | Fresh results each run. No stale recs from previous configs. | -| RDS: CPU only | No memory constraint | Original rationale: `FreeableMemory` is bytes, not utilization %. The current collector does not fetch this metric. | -| Confidence levels | high/medium/low | Rule-specific heuristic labels; see [`compute_lp.py`](../../../optimizer/compute_lp.py) and [`rules.py`](../../../optimizer/rules.py). They are not calibrated probabilities or proof an action is safe. | diff --git a/docs/architecture/sequence.mmd b/docs/architecture/sequence.mmd new file mode 100644 index 0000000..f1d9bad --- /dev/null +++ b/docs/architecture/sequence.mmd @@ -0,0 +1,39 @@ +flowchart TB + Browser["Browser
localStorage user_id
Defaults to aws-SYNTHETIC-001"] + Next["Next.js :3000
Account Settings, dashboard
recommendations, forecast pages"] + API["FastAPI :8000
Routers validate + scope by user_id
Trust-based, no server auth"] + Storage["storage facade
get_connection ensure_schema
insert_* get_* no-commit"] + Collector["CollectorRunner thread
from_connection keys or role
12 months newest-first"] + DB[("SQLite
data/cloud_optimizer.db")] + AWS["AWS via boto3
STS Cost Explorer CloudWatch"] + Gemini["Gemini
Advisor completions"] + + Browser -->|"1 HTTP GET with user_id
Page load or refresh"| Next + Next -->|"2 HTTP JSON user_id query
NEXT_PUBLIC_API_BASE_URL"| API + API -->|"3 ensure_schema
Open WAL connection"| Storage + Storage -->|"4 SELECT scoped by user_id"| DB + DB -->|"5 Rows"| Storage + Storage -->|"6 Dicts"| API + API -->|"7 JSON costs recs forecast"| Next + Next -->|"8 Render Plotly + tables"| Browser + + Browser -->|"A Save keys in Account Settings"| Next + Next -->|"B POST /api/connections
Keys + region + optional account_id"| API + API -->|"C Validate AKIA-ASIA + format
STS resolve when account missing
ensure_user + add_aws_connection"| Storage + Storage -->|"D INSERT aws_connections
Plaintext keys, UNIQUE user account"| DB + + Browser -->|"E POST sync"| Next + Next -->|"F POST /api/connections/id/sync"| API + API -->|"G 409 when _SYNCING or in_progress
Else mark in_progress + 202
Spawn daemon thread"| Collector + Collector -->|"H Fresh thread connection
clear_user_data + collectors
Costs EC2 RDS Lambda S3 DynamoDB
ElastiCache ECS NAT ELB pricing
Commit per month"| Storage + Storage -->|"I INSERT OR REPLACE
Partial data when failure"| DB + Collector -->|"J success or failed + message
last_sync_at now"| DB + Collector -.->|"K Live read when real mode"| AWS + Next -->|"L Poll GET /api/connections
sync_status badge"| API + + Browser -->|"M Build advisor prompt"| Next + Next -->|"N POST /api/ai-onboarding/generate
4000-char guard"| API + API -->|"O Prompt to Gemini
502 when key missing or fail"| Gemini + Gemini -->|"P Recommendation text"| API + API -->|"Q INSERT ai_recommendations"| Storage + Storage -->|"R Commit"| DB diff --git a/docs/architecture/state.mmd b/docs/architecture/state.mmd new file mode 100644 index 0000000..438a28c --- /dev/null +++ b/docs/architecture/state.mmd @@ -0,0 +1,31 @@ +flowchart LR + subgraph SyncStatus["aws_connections.sync_status
CHECK never success failed in_progress
Default never"] + Never["never
Saved, not yet synced
Fresh row default"] + InProg["in_progress
Thread live
_SYNCING holds id"] + Success["success
12-month run committed
last_sync_at set"] + Failed["failed
Exception truncated 500 chars
error_message set, partial data kept"] + Never -->|"POST sync
202 + thread start"| InProg + Success -->|"POST sync again
Full clear + re-collect"| InProg + Failed -->|"POST sync retry
Full clear + re-collect"| InProg + InProg -->|"Runner completes"| Success + InProg -->|"Exception in thread"| Failed + InProg -.->|"Second POST sync
409 sync already in progress"| InProg + end + + subgraph ConnLife["Connection row lifecycle
user_id aws-account_id UNIQUE"] + Test["Test keys
POST /test STS GetCallerIdentity
ok True sets access_verified 1
ok False returns 200 reason"] + Save["Save
POST / save, format checks
STS resolve when account missing
409 on duplicate account"] + List["List
GET user_id scoped
Secrets omitted, key last4 only"] + Delete["Delete
DELETE id + user_id
404 when not owned"] + Test --> Save + Save --> List + List --> Delete + Save --> Never + List --> Never + end + + subgraph Identity["Identity workspace switch"] + Reg["usr identity
POST /api/auth/signup
PBKDF2 stored, no token"] + Ws["aws-account workspace
ensure_user aws-account_id
Web save switches to it
Collector writes into it"] + Reg -->|"Save connection
Owner becomes aws-account"| Ws + end diff --git a/docs/decisions/0001-starter-base-with-extra-docs.md b/docs/decisions/0001-starter-base-with-extra-docs.md new file mode 100644 index 0000000..a52f08e --- /dev/null +++ b/docs/decisions/0001-starter-base-with-extra-docs.md @@ -0,0 +1,35 @@ +# ADR 0001: Ultra-minimal docs — root README + 9 required files + +- Status: Accepted +- Date: 2026-10-20 +- Branch: `main` +- Supersedes: Starter-base draft with engines/pipeline/local-runtime companions + +## Context + +`main` is a substantive single-user localhost system (persistent SQLite, Auth/sensitive firing, no prod). Starter requires for substantive: `README.md`, 6x `.mmd` + `data-model.md`, `threat-model.md` (Auth fires), verbs, `verify.yml`. Operations `runbook/slo/recovery` absent until prod. Template wants ~6 pages heuristic, merge over add, absent over stub. + +Previous detail pages (`engines/`, `data-pipeline.md`, `local-runtime.md`, `api/`, `reference/configuration.md`, `development/`, `security/README.md`, `arch/README.md`, `report.pdf`) duplicated code facts and inflated review burden. Root README can carry run/verify/ops as pointers with code canonical. + +## Decision + +Keep 10 versioned docs only: + +- `README.md` — pointer + run + verify + index, code links only, no pastes +- `docs/architecture/c1-system-context.mmd`, `c2-containers.mmd`, `c3-components.mmd`, `sequence.mmd`, `state.mmd`, `deployment.mmd` +- `docs/architecture/data-model.md` — generator from `storage/db.py`, compat proof +- `docs/security/threat-model.md` — Auth fires +- This ADR + +Delete: `docs/README.md`, `arch/README.md`, `api/http.md`, `api/storage.md`, `data-pipeline.md`, `engines/*`, `development/*`, `operations/local-runtime.md`, `runbook.md` (N/A pre-prod), `reference/configuration.md`, `security/README.md`, `project-report.pdf` (release asset). Fold essential facts into root README; code is canonical for schemas, contracts, config, identity, sync order, LP formulation. + +## Consequences + +- One home per fact: intent in root, diagrams in `.mmd`, schemas in `storage/db.py`, decisions here. +- Fewer files to rot; docs PRs touch root or 1 `.mmd`. +- Risk: root grows; mitigate by links not copies, 150-line cap. Engine LP detail lives in `optimizer/engine.py`, sync order in `aws_collector/runner.py`, identity in `backend_api/routers/connections.py`. + +## Verification + +- `git ls-files docs/` = 9 files above. +- `git diff --check` + `gitleaks detect` green; `poe verify` / CI `verify` green. diff --git a/docs/development/README.md b/docs/development/README.md deleted file mode 100644 index 22b0101..0000000 --- a/docs/development/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Development Guide - -Use the [README](../../README.md#quick-start) for setup. Run Python commands from -the repository root with the virtual environment active. - -Application PRs target **`main`**; the default `release/ccpe-v1.0` branch contains -the separate paper artifact. Keep changes focused, update affected technical -documentation, and describe what changed, why, and the verification actually run. -Report pre-existing failures separately from regressions and respect any target -branch checks or review requirements. Use [SECURITY.md](../../SECURITY.md) for -vulnerability reporting. - -## Checks - -```bash -python -m pip check -python -m pytest tests/ -v -python -m compileall -q backend_api cloud_optimizer aws_collector storage \ - data_generation ml_engine optimizer ai_module dashboard -git diff --check -``` - -The pytest suite covers storage/authentication, AWS configuration and connections, -synthetic generation, forecasting, optimization, AI helpers, legacy dashboard -behavior, and frontend/backend contracts. Discover the current test inventory -with `python -m pytest --collect-only -q`. Tests use local fixtures and mocked -services; they require no live AWS or AI credentials. - -For the TypeScript frontend, use the committed lockfile: - -```bash -cd frontend -npm ci -npm run lint -npm run build -``` - -The scripts are defined in [package.json](../../frontend/package.json). For changes -to request/response behavior, also exercise the affected page with the backend -running. FastAPI generates the current endpoint and model reference at -http://localhost:8000/docs and http://localhost:8000/openapi.json. - -There is currently no repository CI or documentation validation tool configured. -Use the [schema generator](../architecture/data-model.md) and -[storage signature lookup](../api/storage.md#current-signatures) when changing those -contracts; [HTTP reference generation](../api/http.md) also works without starting -the server or touching the tracked database. -Check changed Markdown links and anchors, preview rendering, and verify commands -against the implementation. Report commands and outcomes in the PR, including -any baseline failure; keep existing tests and validation intact. - -## Project patterns - -- Follow the surrounding Python or TypeScript style. Python uses type hints and - module loggers; lazy imports also occur for optional or expensive integrations. -- Shared paths, `.env` loading, and constants live in - [cloud_optimizer/config.py](../../cloud_optimizer/config.py). AWS session/client - construction lives in [aws_collector/config.py](../../aws_collector/config.py). -- Prefer the `storage` facade for database access. Some API routes currently use - SQL directly; see [Architecture](../architecture/README.md). Data keyed by `user_id` - does not provide server-side authorization. -- Use `ensure_schema()` for additive initialization. `create_schema()` drops and - recreates tables and belongs only in disposable fixtures. See the - [storage contract](../api/storage.md#transaction-contract) for commit ownership. -- Keep tests independent of external services. Use pytest temporary paths for - writes and mocks for AWS/AI calls. Do not regenerate the tracked database as - part of routine testing. -- Read [optimizer usage](../architecture/engines/optimizer.md#usage) before running a write operation; - both optimization and collection can replace existing stored results. - -Python dependencies live in [requirements.txt](../../requirements.txt); -[pyproject.toml](../../pyproject.toml) configures pytest and coverage. Update the -numeric/ML pins together when changing that stack, then verify imports and the -relevant forecasting tests. diff --git a/docs/development/streamlit.md b/docs/development/streamlit.md deleted file mode 100644 index 3623f15..0000000 --- a/docs/development/streamlit.md +++ /dev/null @@ -1,75 +0,0 @@ -# Legacy Streamlit Startup - -The primary TypeScript/FastAPI application starts with the -[README quick start](../../README.md#quick-start). This guide covers the optional -legacy Python UI in `dashboard/`. - -## Prerequisites - -- Python 3.12+ -- pip - -## 1. Create virtual environment & install dependencies - -```bash -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -``` - -## 2. Environment variables (optional) - -The project runs in **demo mode** by default (pre-loaded synthetic data, no AWS credentials needed). - -For real collection, configure boto3 credentials for the principal that assumes -the connection's role; see [Configuration](../reference/configuration.md#environment-variables). -Set the default boto3 region before running: - -```bash -export AWS_DEFAULT_REGION=us-east-1 -``` - -For AI recommendations, separately set `GOOGLE_API_KEY` in the environment or -root `.env`. Gemini requires this key even when using synthetic data. - -## 3. Launch the dashboard - -```bash -python -m streamlit run dashboard/app.py --server.address=127.0.0.1 -``` - -Opens at `http://localhost:8501` with a **login screen**: - -- **Login** -- Sign in with email and password -- **Register** -- Create a new account (password hashed with PBKDF2-HMAC-SHA256) -- **Try Demo Mode** -- Explore with pre-loaded synthetic data (no account required) - -After authentication, the sidebar shows 5 pages: Home, Costs, Forecasts, Recommendations, Settings. An **account switcher** in the sidebar lets you select which AWS account to view. - -## 4. Connecting an AWS account - -After registering, go to **Settings** and use the "Add AWS Account" form: - -1. Enter your AWS Account ID and IAM Role ARN -2. Click "Test Connection" to verify access -3. Click "Add Account" to save - -The caller needs `sts:AssumeRole`, and the role needs permissions for the AWS -services being collected. This form tests role access with STS. The primary -Next.js UI instead accepts access keys; see [Configuration](../reference/configuration.md). - -## 5. CLI tools - -Use the [optimizer example](../architecture/engines/optimizer.md#usage) to generate recommendations in a -disposable database copy. Optimization replaces existing recommendations. - -Run forecasting from the Forecasts page. The ML CLI is a placeholder that prints -a notice and exits with status 1 when a user ID is supplied. - -## 6. Run tests - -```bash -python -m pytest tests/ -v -``` - -See [Development](README.md) for the full application checks. diff --git a/docs/operations/local-runtime.md b/docs/operations/local-runtime.md deleted file mode 100644 index 0b326a1..0000000 --- a/docs/operations/local-runtime.md +++ /dev/null @@ -1,129 +0,0 @@ -# Local operations and recovery - -This guide covers the single-user development/demo stack. Start it with the -[README](../../README.md#quick-start). Host ports bind to `127.0.0.1`; container -listeners and the Compose network remain accessible within the stack. This does -not repair the [application's trust boundaries](../security/README.md). - -## Diagnose the running stack - -```bash -docker compose --profile full ps -docker compose logs --tail=100 backend frontend -# If the optional legacy UI is running: -docker compose logs --tail=100 streamlit -``` - -Open http://127.0.0.1:3000, http://127.0.0.1:8000/health, and, for the `full` -profile, http://127.0.0.1:8501. `/health` is a static liveness response: it does not -verify database contents, AWS permissions, or Gemini availability. Check the -specific failed page/request and backend logs. Redact account identifiers, -credentials, prompts, and personal data before sharing diagnostics. - -| Symptom | Checks and next action | -| --- | --- | -| Browser cannot reach API | Check backend health and the frontend's build-time `NEXT_PUBLIC_API_BASE_URL`; it must be browser-reachable. Rebuild the frontend after changing it. CORS permits the two local port-3000 origins. | -| Empty account dashboard | Confirm the selected workspace, stored dates, and sync state. Registration alone does not populate data; demo data belongs to `aws-SYNTHETIC-001`. | -| Incomplete AWS data | Inspect sync status and collector logs for service-level failures. Successful completion does not prove every AWS read succeeded. Review [collection/write behavior](../architecture/data-pipeline.md) before retrying. | -| Sync returns 409 after a restart | An interrupted daemon thread can leave persisted `in_progress`; use the stopped-worker procedure below. | -| AI returns 502 | Check the API key and upstream error in backend logs. Synthetic data does not remove the Gemini key requirement. Enabling the optional token gate also requires a client that sends the token. | -| Settings saved but behavior unchanged | Settings are stored preferences; engines and collection do not consume them. See [active configuration](../reference/configuration.md). | - -## State and lifetime - -| State | Location | Lifetime | -| --- | --- | --- | -| Manual-run database | `data/cloud_optimizer.db` | Local tracked fixture, modified by application writes; keep runtime changes out of Git | -| Compose database | `/app/data/cloud_optimizer.db` on `optimizer-data` | Seeded from the image when the volume is empty; survives container recreation | -| Web runtime settings | `backend_api/runtime_settings.json` (`/app/backend_api/` in the container) | Separate from SQLite; lost on container recreation in the supplied Compose setup | -| Browser identity | Browser localStorage | Client state, not an authenticated server session | - -A rebuild does not replace an existing database volume with the image's fixture. -`docker compose down` retains the volume. **`docker compose down -v` destroys the -Compose database volume**; use it only for an intentional disposable-demo reset -after preserving anything needed. The next start seeds the volume from the image. -The synthetic generator updates matching rows and is not a clean database reset. - -## Preserve and restore state - -Before collection or other replacement writes, preserve data you need. For a -manual-run database, use SQLite's backup API so committed WAL contents are -included. This example creates a new private directory outside the repository: - -```bash -backup_dir=$(mktemp -d) -python - "$backup_dir/cloud_optimizer.db" <<'PYTHON' -import sqlite3 -import sys -from contextlib import closing -from pathlib import Path - -source = Path("data/cloud_optimizer.db").resolve() -with closing(sqlite3.connect(source.as_uri() + "?mode=ro", uri=True)) as src: - with closing(sqlite3.connect(sys.argv[1])) as dst: - src.backup(dst) - assert dst.execute("PRAGMA integrity_check").fetchone()[0] == "ok" -print(sys.argv[1]) -PYTHON -``` - -For Compose, execute the same backup operation against `/app/data/cloud_optimizer.db` -inside the backend container, write the snapshot to a temporary container path, -and copy that snapshot out with `docker compose cp` before recreating the container. -Copy runtime settings separately if needed. Store snapshots privately: the database -can contain plaintext credentials and account data. An integrity check proves -SQLite structural consistency, not collection completeness or correct values. - -To restore, stop every database writer (backend, optional Streamlit, and standalone -collectors/engines), preserve the current database and any WAL/SHM sidecars as a -separate recovery copy, then restore the verified snapshot to the configured path. -Do not pair a restored snapshot with old WAL/SHM files. In Compose retain writable -ownership for UID 10001 on `/app/data`. Restart and verify the intended workspace, -connection state, and data dates through the application. Do not commit snapshots. - -## Interrupted sync recovery - -Sync runs in a daemon thread in the backend process. It clears previous account -data and commits incrementally. Restarting the backend can stop that thread while -leaving `sync_status = 'in_progress'`, which blocks a new sync. There is no durable -queue or automatic resume; retry starts a fresh replacement collection. - -1. Confirm the old worker is no longer running; stop the backend and all other - writers. Do not clear the status of an active sync. -2. Preserve state as above. Identify the exact connection and workspace from - `aws_connections`, selecting only `id`, `user_id`, `aws_account_id`, and - `sync_status`; do not dump its credential columns. -3. On that database, call the existing storage helper with the confirmed ID: - - ```python - from contextlib import closing - from storage import get_connection, update_aws_connection_status - - # Use the verified database path and connection ID from the preceding step. - with closing(get_connection(database_path)) as conn: - update_aws_connection_status( - conn, connection_id, "failed", "Interrupted sync; worker stopped", - ) - ``` - - The helper commits and updates `last_sync_at` to the repair time; that timestamp - is not evidence of a successful collection. -4. Restart the backend, confirm the failed status, and decide whether to retry - through Account Settings. Inspect backend logs and data coverage afterward; - changing the status alone does not recover deleted or partial data. - -## Runtime settings recovery - -The [settings router](../../backend_api/routers/settings.py) replaces the JSON -file atomically and serializes access within one process. It does not coordinate -multiple backend processes. Malformed JSON is renamed to -`runtime_settings.json.corrupt`; reads then use defaults until valid settings are -saved. A syntactically valid file with invalid setting values can still fail model -validation. - -Stop the backend, preserve the current file and any corrupt copy outside Git, -then repair or restore a valid user-to-settings mapping. Validate values against -`RuntimeSettings` in the router, restart, and read the affected user's settings. -Repeated malformed reads can replace the previous `.corrupt` copy, so preserve it -before trying recovery. The database volume does not preserve this file; copy it -out before container recreation and restore it into the replacement container. diff --git a/docs/project-report.pdf b/docs/project-report.pdf deleted file mode 100644 index 563c222..0000000 Binary files a/docs/project-report.pdf and /dev/null differ diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md deleted file mode 100644 index 321455d..0000000 --- a/docs/reference/configuration.md +++ /dev/null @@ -1,166 +0,0 @@ -# Configuration - -Environment variables, constants, and operational modes. The separate -[web runtime settings](../architecture/README.md#runtime-settings) are saved preferences; -they do not currently configure the engines or collector. - ---- - -## Two Config Files - -The project has two separate config files with no overlap: - -| File | Scope | What it controls | -| --- | --- | --- | -| `cloud_optimizer/config.py` | Project-wide | Paths, constants, env vars, ML settings, logging | -| `aws_collector/config.py` | AWS only | boto3 session, service clients, region discovery | - ---- - -## Environment Variables - -| Variable | Default | Description | -| --- | --- | --- | -| `DEMO_MODE` | `true` | Legacy Streamlit status label; does not select data or control collection | -| `AWS_REGION` | `us-east-1` | Shared display value in legacy Streamlit Settings; does not configure boto3 sessions | -| `AWS_ACCOUNT_ID` | `SYNTHETIC-001` | Legacy constant; the collector gets the active account from STS | -| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_SESSION_TOKEN` | (unset) | boto3 credentials for CLI collection or the principal assuming a legacy role; web connections use stored keys | -| `AWS_DEFAULT_REGION` | (unset locally; `us-east-1` in Compose) | Region for default boto3 sessions; web connections pass their own region | -| `OPENAI_API_KEY` | (empty) | OpenAI API key (legacy, unused) | -| `OPENAI_MODEL` | `gpt-4o-mini` | OpenAI model (legacy, unused) | -| `GOOGLE_API_KEY` | (empty) | Google API key for AI recommendations (Gemini) | -| `GOOGLE_MODEL` | `gemini-2.5-flash` | Google Gemini model for AI module | -| `ONBOARDING_API_TOKEN` | (empty) | Optional `X-API-Token` gate on `POST /api/ai-onboarding/generate` | -| `NEXT_PUBLIC_API_BASE_URL` | `http://127.0.0.1:8000` | Frontend backend URL, embedded at build time | - -Set backend variables in the repository-root `.env` file or export them before running. -`cloud_optimizer/config.py` resolves that file and the database path relative to -the repository root, independently of the current working directory. Existing -environment variables take precedence over `.env` values. - -The frontend uses `frontend/.env.local`; Compose builds it with -`NEXT_PUBLIC_API_BASE_URL=http://localhost:8000` in -[docker-compose.yml](../../docker-compose.yml). This URL must be reachable from the -browser, and changing it requires rebuilding the frontend. Templates: -[local backend](../../.env.example), [Docker](../../.env.docker.example), and -[frontend](../../frontend/.env.local.example). The Next.js UI currently sends no -`X-API-Token`; enabling the optional onboarding gate requires clients to supply it. - ---- - -## Modes - -### Demo Mode (click "Try Demo Mode" on login screen) - -- No AWS credentials needed -- Database ships with pre-loaded synthetic data -- The TypeScript UI selects the `aws-SYNTHETIC-001` workspace -- Gemini-backed features still require a valid API key - -### Real Mode (register + connect AWS account in Settings) - -- The Next.js form accepts access keys (and an optional session token). The - backend stores them server-side and syncs using `AWSConfig.from_keys()`. - Testing is separate from saving; see - [Connection identity and verification](../architecture/README.md#connection-identity-and-verification). -- Legacy Streamlit connections use IAM role ARNs. The collector also supports - these stored connections through `AWSConfig.from_role()`. -- STS identity resolution alone does not establish permission to collect data. - The service reads are defined by [AWSConfig](../../aws_collector/config.py) and - the [collectors](../../aws_collector/collectors); the repository does not ship a - complete IAM policy. Check backend logs for denied collection calls. - -### How mode switching works - -Demo and connected-account data share SQLite. The web UI selects a workspace; -`DEMO_MODE=false` is not required to connect AWS through FastAPI. HTTP endpoints -trust the supplied user ID, so the client login screen does not enforce data -authorization. See the [security notes](../security/README.md). - ---- - -## Constants - -### Paths - -| Constant | Value | Description | -| --- | --- | --- | -| `PROJECT_ROOT` | repo root | Base directory | -| `DATA_DIR` | `data/` | Parent data directory | -| `DB_PATH` | `data/cloud_optimizer.db` | SQLite database path | - -### Collection - -| Constant | Value | Description | -| --- | --- | --- | -| `DEFAULT_COLLECTION_MONTHS` | `12` | Declared default; the runner, CLI, and sync route currently specify 12 independently | -| `DEFAULT_SYNTHETIC_DAYS` | `365` | Days of sample data to generate | -| `API_TIMEOUT` | `30` | Displayed in legacy Settings; not passed to boto3 clients | -| `MAX_RETRIES` | `3` | Displayed in legacy Settings; not passed to boto3 clients | -| `CHUNK_SIZE` | `100` | Declared but unused by collection/storage | - -The collector does not apply these timeout/retry constants; its clients use -boto3's resolved configuration. Changing these constants does not change AWS -request timeouts, retries, or batching. - -### ML - -| Constant | Value | Description | -| --- | --- | --- | -| `FORECAST_HORIZON_DAYS` | `30` | Displayed in legacy Settings; HTTP requests and model methods define their own horizons | -| `MIN_TRAINING_DAYS` | `30` | Legacy Streamlit forecast gate; the HTTP router independently requires 30 observations | -| `COLD_START_DAYS` | `30` | Days of cost data needed for the legacy Streamlit Home dashboard | -| `SEASONALITY_PERIOD` | `7` | Displayed in legacy Settings; forecasters define their own seasonality | - -The Streamlit Home page shows its questionnaire or saved AI recommendations while -fewer than 30 days of cost data are available. It shows collection progress after -the first day of data; accounts with no data do not see an empty progress bar. - -### Optimization - -| Constant | Value | Description | -| --- | --- | --- | -| `DEFAULT_BUDGET_CAP` | `5000.0` | Monthly LP budget, applied separately to EC2 and RDS; see [optimizer](../architecture/engines/optimizer.md) | -| `SPOT_RELIABILITY` | `False` | Displayed in legacy Settings; not consumed by the optimizer | - -### Supported Services - -See the current constants in [cloud_optimizer/config.py](../../cloud_optimizer/config.py). -The optimizer's own service-filter names are defined by `ALL_SERVICES` in -[optimizer/engine.py](../../optimizer/engine.py). - ---- - -## AWS Collector Config - -`aws_collector/config.py` constructs session-specific `AWSConfig` instances. -`get_config()` and `init_config()` also expose a shared instance for default -callers; `from_keys()` and `from_role()` construct separate connection instances. - -```text -AWSConfig - ├── session boto3.Session - ├── ce Cost Explorer client - ├── ec2 EC2 client (default region) - ├── cloudwatch CloudWatch client (default region) - ├── pricing Pricing client (always us-east-1) - ├── s3 S3 client - ├── account_id From STS - └── regions From ec2.describe_regions() -``` - -Regional clients are created on-demand via `get_ec2_client(region)`, `get_rds_client(region)`, `get_lambda_client(region)`, `get_cloudwatch_client(region)`. - ---- - -## Logging - -Configured via `setup_logging()` in `cloud_optimizer/config.py`: - -```text -Format: %(asctime)s | %(name)s | %(levelname)s | %(message)s -Example: 2025-06-15 14:30:00 | aws_collector.cost_collector | INFO | Fetched daily costs for 2025-05 -``` - -Modules generally use `logger = logging.getLogger(__name__)`; CLI entry points -also print summaries directly. diff --git a/docs/security/README.md b/docs/security/README.md deleted file mode 100644 index 10c6927..0000000 --- a/docs/security/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Security boundaries - -These are implemented limitations, not production security guarantees. The app is intended for **single-user local use with synthetic data**. Loopback port binding limits network reachability but does not authenticate clients or isolate users. - -- **Auth is trust-based — no tokens or sessions.** Login/sign-up verify a PBKDF2-hashed password but issue **no token or session cookie**. Every data and settings endpoint trusts a `user_id` query parameter with **no server-side authorization check**, so any client could read another user's data by supplying their `user_id`. The login route has only an in-process, per-IP throttle (10 failures / 300s → HTTP 429). This is **not safe for multi-tenant or public deployment** — a real deployment must add token/session auth and derive `user_id` server-side, and configure CORS for the chosen credential transport. -- **AWS credentials are stored in plaintext.** The connection form sends AWS access keys to the backend, which persists them **in the SQLite DB in plaintext**. Connection responses omit secret fields and expose only the last 4 chars of the access key id; sync reads credentials server-side. Saving does not always verify AWS access; see [connection identity and verification](../architecture/README.md#connection-identity-and-verification). Do not point this at a shared or public host with real credentials. -- **The committed demo DB contains real data.** `data/cloud_optimizer.db` ships with the synthetic demo user **plus** a real personal email (PII) and a real AWS account id / IAM role ARN in plaintext. Do not add more real account data to the committed DB. -- **AI onboarding guards.** `POST /api/ai-onboarding/generate` rejects prompts longer than 4000 characters (HTTP 400) and returns HTTP 502 when the upstream Gemini call fails (e.g. `GOOGLE_API_KEY` unset). When `ONBOARDING_API_TOKEN` is set, callers must send a matching `X-API-Token` header; it is off by default so the demo works with no token. -- **Settings are file-backed, not in the DB.** Per-user runtime settings persist to `backend_api/runtime_settings.json`. Reads are always allowed; writes/resets are read-only for demo users and for users without a connected AWS account. Settings are not wired into the engines or collector, and Compose does not persist this file across container recreation; see [runtime settings](../architecture/README.md#runtime-settings). -- **Frontend auth is client-only.** Session identity lives entirely in browser `localStorage` — there is no Next.js middleware or server-side route protection, so any `/dashboard` route is reachable and falls back to the synthetic demo user. - -## External services and data handling - -AWS sync sends stored credentials to AWS through boto3 and writes account data -locally. The AI advisor sends the questionnaire-derived prompt to Gemini; avoid -including secrets or sensitive account details. Generated advice is not validated -for correctness or safety; review any proposed AWS change independently. - -[Data pipeline](../architecture/data-pipeline.md) describes replacement and partial -sync behavior. [Local operations](../operations/local-runtime.md) covers state -preservation; backups inherit the database's sensitive contents and must stay out -of Git. The build context excludes `.env` and runtime settings but intentionally -includes the committed demo database, so do not distribute locally rebuilt images -that contain real account data. - -## Reporting and review triggers - -Use [SECURITY.md](../../SECURITY.md) for vulnerability reporting. Recheck this -page whenever identity, connection storage, browser state, external prompts, -network exposure, or persistence changes. diff --git a/docs/security/threat-model.md b/docs/security/threat-model.md new file mode 100644 index 0000000..e0a34fb --- /dev/null +++ b/docs/security/threat-model.md @@ -0,0 +1,123 @@ +# Threat model — Smart Cloud Optimizer (`main`) + +Scope: single-user localhost demo with synthetic data. This is a tailoring +record for the implemented trust limits, not a production security claim. +See [security boundaries](README.md) and [vulnerability reporting](../../SECURITY.md). + +## Assets + +- AWS access keys / session tokens submitted via the connection form and + persisted in SQLite (`aws_connections`). +- Account identifiers: AWS account id, IAM role ARN, `user_id` values + (`usr-…`, `aws-`). +- Personal data in the committed fixture: `data/cloud_optimizer.db` ships a + real personal email, a real AWS account id, and a real IAM role ARN + alongside the synthetic `aws-SYNTHETIC-001` workspace. +- Password verifiers: PBKDF2-HMAC-SHA256 `salt:hash` in `users.password_hash`. +- Cost / inventory / metric history, optimizer and forecast output. +- Gemini prompts and generated advice; `.env` values (`GOOGLE_API_KEY`, + `ONBOARDING_API_TOKEN`). + +## Trust boundaries + +- Browser `localStorage` holds session identity with no server session; + any `/dashboard` route is reachable and falls back to the synthetic demo user. +- Every data and settings HTTP endpoint trusts a client-supplied `user_id` + query parameter with no server-side authorization check + (`backend_api/routers/auth.py` issues no token; `dashboard.py`, `costs.py`, + `connections.py`, `forecast.py`, `recommendations.py`, `settings.py` + filter by the supplied id only). +- SQLite file `data/cloud_optimizer.db` is the trust store; `storage/db.py` + is the preferred accessor but some routes issue direct SQL. +- Loopback binding (`127.0.0.1`) limits network reachability but does not + authenticate callers. +- External: AWS via boto3 with stored credentials; Gemini via questionnaire + prompt. Generated advice is unvalidated. +- Build / runtime: `.env` and `backend_api/runtime_settings.json` are excluded + from Git and the image context, but the committed demo DB is intentionally + included in the backend image. Backups inherit DB contents and must stay out + of Git. + +## Threats (in scope for this deployment model) + +- Any local client can read or mutate another user's rows by supplying their + `user_id` (cross-workspace read/write, connection delete, sync trigger). +- Stolen DB file, backup, or image layer exposes plaintext AWS keys (when + key-based connections exist), account ids, role ARNs, emails, and password + hashes (offline brute force is the only barrier; PBKDF2 slows but does not + prevent it). +- Committed PII / account identifiers in `data/cloud_optimizer.db` leak via + clone, fork, image, or log redaction failure. +- Credential capture via logs, diagnostics, or AI prompts (questionnaire text + sent to Gemini; backend JSON logs redact secrets/PII but callers can still + paste them into prompts or public issues). +- Unverified save: `POST /api/connections` with a supplied `aws_account_id` + stores without re-verifying keys against that account; `GET /test` and save + are separate operations. +- Login enumeration / brute force is only blunted by an in-process per-IP + throttle (10 failures / 300 s → 429); no distributed or durable lockout. +- Sync race / stale `in_progress` blocks retry; interrupted daemon thread + leaves persisted status without automatic resume. + +Out of scope (explicit non-goals): multi-tenant isolation, public hosting, +encryption at rest, token/session auth, server-side authorization, durable +rate limiting, and automatic secret rotation. Do not deploy this build +multi-user or off-localhost without fixing those first. + +## Controls (implemented) + +- Passwords: PBKDF2-HMAC-SHA256, 260k iterations, 16-byte salt, constant-time + compare; generic auth errors to avoid enumeration. +- Throttle: in-process per-IP 10/300 s on `/api/auth/*`. +- Input validation before AWS calls: `AKIA|ASIA` + 16 chars, 12-digit account + id; STS-derived account id when none is supplied. +- Secret minimization in API: `ConnectionOut` omits secret, token, + `external_id`, `iam_role_arn`; only last 4 of the access key id is exposed. + Sync reads credentials server-side from the stored row. +- AI guard: prompt > 4000 chars → 400; Gemini failure → 502; optional + `ONBOARDING_API_TOKEN` (`X-API-Token`) gate, off by default for demo. +- Transport scope: Compose and manual runs bind to `127.0.0.1`; CORS allows + only the two local port-3000 origins. +- Logging: single-line JSON to stdout with `timestamp`/`level`/`service`/ + `msg`/`trace` (W3C `traceparent` where propagated); secrets/PII redaction + (`[REDACTED]`) covered by `tests/test_service_standards.py`. +- Hygiene: `.gitignore` excludes `.env`, runtime settings, `data/real/`, + backups, WAL/SHM sidecars; `git diff --check`, `compileall`, frontend lint, + and `check:secrets` (working tree + full history) run in CI with push + protection via the pre-push hook. + +## Accepted risks (tailoring) + +- Auth remains trust-based (`user_id` query param, no token). Accepted only + for single-user localhost with synthetic data. +- AWS keys persist in plaintext in SQLite when key-based connections are + used (`storage/db.py` `aws_access_key_id` / `aws_secret_access_key` / + `aws_session_token`, with inline `SECURITY` comments). Accepted only for + the localhost demo with no machine-side key store; must be encrypted before + any shared or public use. +- Committed `data/cloud_optimizer.db` contains real PII / account identifiers + in plaintext. Accepted as a known fixture limitation; do not add more real + account data, do not distribute rebuilt images containing real data, and + keep backups and diagnostics redacted and out of Git. + +## Sensitive handling rules + +- Never commit `.env`, `runtime_settings.json`, backups, WAL/SHM files, or + real AWS data under `data/real/`. +- Never paste secrets, account ids, emails, prompts, or exploit details into + public issues; use the private route in `SECURITY.md` (ack in 2 days, + status in 7 days, monthly review). +- When sharing diagnostics, redact identifiers, credentials, prompts, and + personal data; query `aws_connections` by id columns only, never dump + credential columns. +- Schema changes stay additive: `ensure_schema()` creates missing objects and + adds supported credential columns; `create_schema()` (drop/recreate) belongs + only in disposable fixtures. See `docs/architecture/data-model.md` and + `storage/db.py`. + +## Review triggers + +Recheck this model whenever identity, connection storage, browser state, +external prompts, network exposure, persistence, or the committed fixture +changes. Secret scan (`check:secrets`), schema compat, and restore proof must +stay green; see `README.md` Verify and `pyproject.toml [tool.poe.tasks]`. diff --git a/ml_engine/__main__.py b/ml_engine/__main__.py index d2a058c..79bb6fd 100644 --- a/ml_engine/__main__.py +++ b/ml_engine/__main__.py @@ -5,11 +5,10 @@ """ import argparse import sys -from pathlib import Path -# TODO: Import forecaster when ready -# from storage import db -# from ml_engine import forecaster +# NOTE(ml-cli): this entry point intentionally only prints the handoff notice +# and exits 1. Real forecasting entry points are the dashboard Forecasts page +# and ml_engine.forecaster (BaseForecaster.fit/predict) via data_prep loaders. def main(): diff --git a/pyproject.toml b/pyproject.toml index 3bf6508..6acb59e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,3 +9,57 @@ omit = ["tests/*"] [tool.coverage.report] show_missing = true fail_under = 0 + +# Task verbs (Starter §Task verbs, §CI Same checks locally). +# Short shell steps calling standard tools directly, no glue scripts. +# Omitted suites stay omitted with reason, never stubbed/skipped; +# a missing tool fails closed (exit 2 with install message), never passes. +# Prototype note: test + verify are the only required gates for +# prototype-stage changes; the other verbs run on demand or in review. +[tool.poe.tasks.dev] +help = "Backend dev server (uvicorn, loopback only)" +shell = """python -c "import uvicorn" 2>/dev/null || { echo "dev: missing tool uvicorn (pip install -r requirements.txt)"; exit 2; }; python -m uvicorn backend_api.main:app --reload --host 127.0.0.1 --port 8000""" + +[tool.poe.tasks.test] +help = "Backend suite (pytest; missing tool fails closed)" +shell = """python -c "import pytest" 2>/dev/null || { echo "test: missing tool pytest (pip install -r requirements.txt)"; exit 2; }; python -m pytest tests/ -v""" + +[tool.poe.tasks.migrate] +help = "Schema compat gate: ensure_schema on disposable DB (real DB migrates on backend startup)" +shell = "python -c \"import sqlite3; from storage import ensure_schema; c=sqlite3.connect(':memory:'); ensure_schema(c); c.close(); print('migrate ok')\"" + +[tool.poe.tasks.build] +help = "Backend compileall + frontend production build (missing npm fails closed)" +shell = """command -v npm >/dev/null 2>&1 || { echo "build: missing tool npm (install Node 22+)"; exit 2; }; python -m compileall -q backend_api cloud_optimizer aws_collector storage data_generation ml_engine optimizer ai_module dashboard && npm --prefix frontend run build""" + +[tool.poe.tasks."test:contract"] +help = "Frontend/backend contract tests (pytest; omitted with reason if suite absent)" +shell = """test -f tests/test_frontend_backend_contract.py || { echo "test:contract omitted: no contract suite in tree"; exit 0; }; python -c "import pytest" 2>/dev/null || { echo "test:contract: missing tool pytest (pip install -r requirements.txt)"; exit 2; }; python -m pytest tests/test_frontend_backend_contract.py -v""" + +[tool.poe.tasks."check:sast"] +help = "SAST on trust boundary (bandit backend_api+storage; missing tool fails closed)" +shell = """python -c "import bandit" 2>/dev/null || { echo "check:sast: missing tool bandit (pip install bandit)"; exit 2; }; python -m bandit -r backend_api storage -q""" + +[tool.poe.tasks."check:secrets"] +help = "Secrets scan: gitleaks history + working tree (fail-closed)" +shell = """command -v gitleaks >/dev/null 2>&1 || { echo "check:secrets: missing tool gitleaks (see .github/workflows/verify.yml pinned install)"; exit 2; }; gitleaks detect --source . --config .gitleaks.toml --verbose && gitleaks detect --source . --config .gitleaks.toml --no-git --verbose""" + +[tool.poe.tasks."check:audit"] +help = "Dependency audit: pip check authoritative, pip-audit when installed" +shell = """python -m pip check || exit 1; if command -v pip-audit >/dev/null 2>&1; then pip-audit; else echo "check:audit note: pip-audit not installed — pip check passed (omitted with reason)"; fi""" + +[tool.poe.tasks."check:hygiene"] +help = "Hygiene: git diff --check + grep for bare stubs" +shell = """command -v git >/dev/null 2>&1 || { echo "check:hygiene: missing tool git"; exit 2; }; git diff --check && ! git grep -I -n -E -e 'TODO[^\\(]|FIXME|XXX|HACK|lorem|NotImplementedError' --and --not -e 'TODO\\[' -- . ':!frontend/node_modules' ':!paper' ':!frontend/package-lock.json'""" + +[tool.poe.tasks."check:schema"] +help = "Schema compat + restore proof on temp copies (stdlib only)" +shell = """python3 -c "import sqlite3; from contextlib import closing; from storage import ensure_schema; c=closing(sqlite3.connect(':memory:')).__enter__(); ensure_schema(c); tables=[r[0] for r in c.execute(\"SELECT name FROM sqlite_master WHERE type='table'\")]; assert 'aws_connections' in tables; print(f'schema ok: {len(tables)} tables'); c.close()\"""" + +[tool.poe.tasks.diagrams] +help = "Render .mmd sources with mermaid-cli (omitted with reason when no .mmd exists)" +shell = """files=$(find . -name '*.mmd' -not -path './node_modules/*' -not -path './frontend/node_modules/*' -not -path './frontend/.next/*' -not -path './.git/*' -print); if [ -z "$files" ]; then echo "diagrams omitted: no .mmd sources in tree (add docs//.mmd to enable)"; exit 0; fi; command -v mmdc >/dev/null 2>&1 || { echo "diagrams: missing tool mermaid-cli (npm i -g @mermaid-js/mermaid-cli)"; exit 2; }; for f in $files; do mmdc -i "$f" -o "${f%.mmd}.svg"; done""" + +[tool.poe.tasks.verify] +help = "Required local gate, same checks as CI verify" +shell = """python -c "import pytest" 2>/dev/null || { echo "verify: missing tool pytest (pip install -r requirements.txt)"; exit 2; }; command -v npm >/dev/null 2>&1 || { echo "verify: missing tool npm (install Node 22+)"; exit 2; }; command -v git >/dev/null 2>&1 || { echo "verify: missing tool git"; exit 2; }; command -v gitleaks >/dev/null 2>&1 || { echo "verify: missing tool gitleaks"; exit 2; }; python -m pytest tests/ -v && git diff --check && gitleaks detect --source . --config .gitleaks.toml --verbose && npm --prefix frontend run lint && npm --prefix frontend run build""" diff --git a/tests/test_service_standards.py b/tests/test_service_standards.py new file mode 100644 index 0000000..5f4213c --- /dev/null +++ b/tests/test_service_standards.py @@ -0,0 +1,157 @@ +"""Service standards: health split + JSON logging with redaction (Starter §Service standards).""" +from __future__ import annotations + +import io +import json +import logging + +import backend_api.main as main +from cloud_optimizer import config +from cloud_optimizer.logging_json import ( + JsonFormatter, + get_trace_id, + redact_obj, + redact_str, + set_trace_id, +) +from fastapi.testclient import TestClient + + +def test_health_is_cheap_liveness_no_db(monkeypatch): + def _fail(*a, **k): + raise AssertionError("liveness must not touch dependencies") + + monkeypatch.setattr(main, "get_connection", _fail) + client = TestClient(main.app, raise_server_exceptions=False) + resp = client.get("/health") + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["status"] == "ok" + + +def test_ready_ok_with_db(monkeypatch, tmp_path): + import sqlite3 + + db_file = tmp_path / "ready.db" + real_connect = sqlite3.connect + + def fake_get_connection(*a, **k): + conn = real_connect(str(db_file)) + conn.execute("CREATE TABLE IF NOT EXISTS t (id INTEGER)") + return conn + + monkeypatch.setattr(main, "get_connection", fake_get_connection) + client = TestClient(main.app, raise_server_exceptions=False) + resp = client.get("/ready") + assert resp.status_code == 200 + body = resp.json() + assert body["ready"] is True + assert body["checks"]["database"] == "ok" + + +def test_ready_fails_closed_503(monkeypatch): + def _boom(*a, **k): + raise RuntimeError("db down") + + monkeypatch.setattr(main, "get_connection", _boom) + client = TestClient(main.app, raise_server_exceptions=False) + resp = client.get("/ready") + assert resp.status_code == 503 + body = resp.json() + assert body["ready"] is False + assert body["checks"]["database"] == "fail" + # No leak of internal error text to clients. + assert "db down" not in json.dumps(body) + + +def test_traceparent_propagated(monkeypatch, tmp_path): + import sqlite3 + + trace_id = "0af7651916cd43dd8448eb211c80319c" + traceparent = f"00-{trace_id}-b7ad6b7169203331-01" + + def fake_get_connection(*a, **k): + conn = sqlite3.connect(":memory:") + return conn + + monkeypatch.setattr(main, "get_connection", fake_get_connection) + client = TestClient(main.app, raise_server_exceptions=False) + resp = client.get("/ready", headers={"traceparent": traceparent}) + assert resp.status_code == 200 + assert resp.headers["X-Request-ID"] == trace_id + + +def test_json_single_line_shape(): + set_trace_id("abc123") + try: + fmt = JsonFormatter(service="backend_api") + record = logging.LogRecord( + name="test", level=logging.INFO, pathname=__file__, + lineno=1, msg="hello %s", args=("world",), exc_info=None, + ) + line = fmt.format(record) + finally: + set_trace_id("") + assert "\n" not in line + payload = json.loads(line) # must be single-line JSON + assert set(["timestamp", "level", "service", "msg", "trace"]) <= set(payload) + assert payload["level"] == "INFO" + assert payload["service"] == "backend_api" + assert payload["msg"] == "hello world" + assert payload["trace"] == "abc123" + assert "timestamp" in payload and payload["timestamp"].endswith("Z") + + +def test_redaction_secrets_and_pii(): + cases = [ + "password=supersecret123", + '{"password": "supersecret123"}', + "api_key=AKIAIOSFODNN7EXAMPLE extra", + "AKIAIOSFODNN7EXAMPLE", + "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload", + "Bearer mytoken123", + "aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "user mariam@example.com logged in", + "-----BEGIN RSA PRIVATE KEY----- MIIE", + ] + for raw in cases: + redacted = redact_str(raw) + assert "supersecret123" not in redacted, raw + assert "AKIAIOSFODNN7EXAMPLE" not in redacted, raw + assert "mariam@example.com" not in redacted, raw + assert "wJalrXUtnFEMI" not in redacted, raw + assert "[REDACTED]" in redacted, raw + # Dict values with sensitive keys. + obj = redact_obj({"password": "x", "nested": {"api_key": "y"}, "ok": "keep"}) + assert obj == { + "password": "[REDACTED]", + "nested": {"api_key": "[REDACTED]"}, + "ok": "keep", + } + + +def test_setup_logging_emits_json_stdout(capsys): + config.setup_logging(level="INFO", service="svc-test") + try: + logger = logging.getLogger("svc-test-logger") + logger.info("password=hunter2 hello user bob@example.com") + out = capsys.readouterr().out + finally: + # Restore default-ish state for other tests; next setup_logging resets. + pass + assert out.strip(), "expected JSON log on stdout" + for line in out.strip().splitlines(): + payload = json.loads(line) + assert {"timestamp", "level", "service", "msg", "trace"} <= set(payload) + joined = "\n".join(out.strip().splitlines()) + assert "hunter2" not in joined + assert "bob@example.com" not in joined + + +def test_trace_falls_back_to_env(monkeypatch): + set_trace_id("") + monkeypatch.setenv("TRACE_ID", "env-trace-1") + assert get_trace_id() == "env-trace-1" + monkeypatch.delenv("TRACE_ID") + assert get_trace_id() == "-"