From 1db1e3c0c0c8a95cc17e2bad933898c5f7d7628c Mon Sep 17 00:00:00 2001 From: Ibrahim Date: Tue, 20 Oct 2026 06:34:03 +0300 Subject: [PATCH 1/5] chore(ci): add verify gate, CODEOWNERS, and hygiene/diagram sources for main Add .github/workflows/verify.yml invoking poe verify (test, migrate, build, check:hygiene, diagrams) with full-history gitleaks scan and a single aggregating gate job as the sole required check. Nightly schedule covers main. Add .github/CODEOWNERS (real owner @G1DO + wildcard fallback) and .gitleaks.toml allowlisting synthetic SYNTHETIC-001 placeholders so the history scan stays green. Include versioned .mmd diagram sources, decisions index, and poe task verbs. Wire ml_engine CLI to real forecaster to clear stub-marker hygiene check. Refs: main audit (no .github, no CODEOWNERS, unprotected main) --- .github/CODEOWNERS | 21 ++ .github/workflows/verify.yml | 116 +++++++++++ .gitleaks.toml | 17 ++ README.md | 2 +- docs/architecture/README.md | 21 ++ docs/architecture/c1-system-context.mmd | 12 ++ docs/architecture/c2-containers.mmd | 30 +++ docs/architecture/c3-components.mmd | 62 ++++++ docs/architecture/deployment.mmd | 28 +++ docs/architecture/er-diagram.mmd | 256 ++++++++++++++++++++++++ docs/architecture/sequence.mmd | 39 ++++ docs/architecture/state.mmd | 25 +++ docs/decisions/README.md | 24 +++ ml_engine/__main__.py | 61 +++++- pyproject.toml | 55 +++++ 15 files changed, 759 insertions(+), 10 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/verify.yml create mode 100644 .gitleaks.toml create mode 100644 docs/architecture/c1-system-context.mmd create mode 100644 docs/architecture/c2-containers.mmd create mode 100644 docs/architecture/c3-components.mmd create mode 100644 docs/architecture/deployment.mmd create mode 100644 docs/architecture/er-diagram.mmd create mode 100644 docs/architecture/sequence.mmd create mode 100644 docs/architecture/state.mmd create mode 100644 docs/decisions/README.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..2cc9702 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,21 @@ +# CODEOWNERS — main only. +# Every path has an owner (no unowned files). Real owner + wildcard fallback. +# Default branch remains release/ccpe-v1.0; this file governs `main`. + +# Fallback: everything owned by the maintainer. +* @G1DO + +# Explicit ownership (same maintainer, explicit for audit clarity). +/docs/ @G1DO +/.github/ @G1DO +/backend_api/ @G1DO +/frontend/ @G1DO +/storage/ @G1DO +/ml_engine/ @G1DO +/optimizer/ @G1DO +/aws_collector/ @G1DO +/cloud_optimizer/ @G1DO +/ai_module/ @G1DO +/dashboard/ @G1DO +/data_generation/ @G1DO +/tests/ @G1DO diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..2704010 --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,116 @@ +# verify — sole CI gate for `main` (Starter §CI Gate job). +# Local parity: `python -m poe verify` runs test → migrate → build → +# check:hygiene → diagrams (see pyproject.toml [tool.poe.tasks.verify]). +# CI mirrors the same task verbs; `gate` is the single aggregating check +# required in branch protection. No `|| true`, no `continue-on-error`. +# Triggers: PRs and pushes targeting `main` only, plus nightly on `main` +# and manual dispatch. Does not touch release/ccpe-v1.0. +name: verify + +on: + pull_request: + branches: [main] + push: + branches: [main] + schedule: + # Nightly on main (02:17 UTC). Active once this workflow exists on the + # default branch; until then PR/push triggers still cover `main`. + - cron: "17 2 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: verify-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Full local gate: test, migrate, build, hygiene, diagrams. + # Hygiene covers `git diff --check`, compileall, frontend lint, stub-marker + # grep. Diagrams validates versioned `.mmd` sources via mmdc and rejects + # pasted ```mermaid blocks (see poe tasks check:hygiene, diagrams). + verify: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout (full history for diff-check) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python (match Dockerfile 3.12) + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Set up Node (frontend lint + mmdc) + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Install Python deps + task runner + run: | + set -eu + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install poethepoet + + - name: Install frontend deps (required by check:hygiene) + run: | + set -eu + npm ci --prefix frontend + + - name: Install mermaid-cli (diagrams validation) + run: | + set -eu + npm install -g @mermaid-js/mermaid-cli + mmdc --version + + - name: Run verify (test → migrate → build → check:hygiene → diagrams) + run: | + set -eu + python -m poe verify + + # Full-history secrets scan (gitleaks). Uses .gitleaks.toml allowlist for + # synthetic SYNTHETIC-001 placeholders only; real leaks fail closed. + secrets-history: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout (full history) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install gitleaks (pinned) + run: | + set -eu + GITLEAKS_VERSION="8.30.1" + curl -sSL "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" -o /tmp/gitleaks.tar.gz + tar -xzf /tmp/gitleaks.tar.gz -C /tmp + sudo mv /tmp/gitleaks /usr/local/bin/gitleaks + gitleaks version + + - name: Gitleaks full-history scan + run: | + set -eu + gitleaks detect --source . --config .gitleaks.toml --verbose + + # Single aggregating gate — the SOLE required status check in branch + # protection for `main`. Fails unless every required job succeeded. + gate: + runs-on: ubuntu-latest + needs: [verify, secrets-history] + if: always() + steps: + - name: Aggregate required checks + run: | + set -eu + echo "verify=${{ needs.verify.result }} secrets-history=${{ needs.secrets-history.result }}" + test "${{ needs.verify.result }}" = "success" + test "${{ needs.secrets-history.result }}" = "success" + echo "gate ok" diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..825035f --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,17 @@ +# Gitleaks config — Smart Cloud Optimizer (main). +# Full-history secrets scan runs in CI (verify.yml → secrets-history). +# Synthetic demo placeholders (SYNTHETIC-001 family) are not secrets and are +# allowlisted here so the history scan stays green without weakening rules. +# Fail-closed: no allowlisted real-key patterns, no generic skips. + +title = "smart-cloud-optimizer" + +[allowlist] +description = "Synthetic demo placeholders — never real credentials" +regexTarget = "match" +regexes = [ + '''(?i)SYNTHETIC-001''', + '''(?i)synthetic-001''', + '''aws-SYNTHETIC-001''', + '''SYNTHETIC-001''', +] diff --git a/README.md b/README.md index 7d727e2..e557cfe 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ command-line experiment on a disposable database copy. ## Develop and maintain - [Development checks](docs/development/README.md): Python tests, frontend lint/build, and contract verification. -- [Architecture](docs/architecture/README.md): components, identity, persistence, and engine boundaries. +- [Architecture](docs/architecture/README.md): components, identity, persistence, and engine boundaries — versioned `.mmd` diagrams (C1/C2/C3, sequence, state, deployment, ER) are indexed there. - [API contracts](docs/api/http.md) and [configuration](docs/reference/configuration.md): generated reference and active inputs. - [Local operations](docs/operations/local-runtime.md): diagnostics, data preservation, and recovery. - [Security](docs/security/README.md) and [vulnerability reporting](SECURITY.md): current trust limits and reporting route. diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 28d8d27..cdc1827 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -3,6 +3,27 @@ The application on `main` uses a TypeScript/Next.js browser UI and a Python FastAPI backend. The optional Streamlit UI calls the Python engines directly. +## Diagram sources (versioned `.mmd` files) + +Change diagrams and text together: every behavior change in the same commit +must update the affected diagram file(s) below plus this index. + +| Level | Source file | Renders | +| --- | --- | --- | +| C1 system context | [c1-system-context.mmd](c1-system-context.mmd) | Users, the optimizer system, AWS, Gemini | +| C2 containers | [c2-containers.mmd](c2-containers.mmd) | Next.js (:3000), FastAPI (:8000), Streamlit (:8501), SQLite, externals | +| C3 components | [c3-components.mmd](c3-components.mmd) | FastAPI routers, engines, storage facade, SQLite | +| Sequence | [sequence.mmd](sequence.mmd) | Connection save/sync, forecast request | +| State | [state.mmd](state.mmd) | Connection/sync and recommendation lifecycle | +| Deployment | [deployment.mmd](deployment.mmd) | Compose services, network, named volume, ports | +| Data model | [data-model.md](data-model.md) | How to regenerate the authoritative DDL from `storage/db.py` | +| ER | [er-diagram.mmd](er-diagram.mmd) | 30 tables, keys + notable columns (DDL is authority; inventory↔metrics links are logical) | + +Cross-cutting choices: [decisions](../decisions/README.md) (currently N/A with +reason; add dated notes there for new hard-to-reverse decisions). + +## Overview + ```mermaid flowchart LR Browser["Browser"] diff --git a/docs/architecture/c1-system-context.mmd b/docs/architecture/c1-system-context.mmd new file mode 100644 index 0000000..5453748 --- /dev/null +++ b/docs/architecture/c1-system-context.mmd @@ -0,0 +1,12 @@ +%% C1 — System context (Smart Cloud Optimizer on main). +%% Actors and external systems only; no internal containers. +%% Sources: docs/architecture/README.md, docker-compose.yml, backend_api/main.py +flowchart LR + user["User
(browser)"] + system["Smart Cloud Optimizer
(Next.js UI + FastAPI backend + SQLite)"] + aws["AWS
(Cost Explorer / CloudWatch / service APIs via boto3)"] + gemini["Google Gemini
(AI recommendations via GOOGLE_MODEL)"] + + user -->|"uses (HTTP, Next.js :3000 / FastAPI :8000)"| system + system -->|"collects costs / inventory / metrics
(live sync, stored keys or role creds)"| aws + system -->|"questionnaire prompt -> advice JSON
(requires GOOGLE_API_KEY)"| gemini diff --git a/docs/architecture/c2-containers.mmd b/docs/architecture/c2-containers.mmd new file mode 100644 index 0000000..d6c6dcb --- /dev/null +++ b/docs/architecture/c2-containers.mmd @@ -0,0 +1,30 @@ +%% C2 — Containers (deployable units on main). +%% Sources: docs/architecture/README.md, docker-compose.yml, Dockerfile, +%% backend_api/main.py, dashboard/app.py, storage/db.py +flowchart LR + browser["Browser
(user agent)"] + + subgraph frontend["Frontend container (:3000)"] + nextui["Next.js UI
(routes, Plotly charts, session helper)"] + end + + subgraph backend["Backend container (:8000)"] + fastapi["FastAPI app
(routers + engines + storage facade)"] + end + + subgraph legacy["Legacy container (:8501, profile full, optional)"] + streamlit["Streamlit UI
(dashboard/app.py, direct Python calls)"] + end + + db[("SQLite DB
(data/cloud_optimizer.db
named volume optimizer-data)")] + aws["AWS
(Cost Explorer / CloudWatch via boto3)"] + gemini["Google Gemini
(AI recommendations)"] + + browser -->|"HTTP (pages + API calls)"| nextui + nextui -->|"HTTP / JSON
(NEXT_PUBLIC_API_BASE_URL=http://localhost:8000)"| fastapi + browser -->|"HTTP (optional legacy UI)"| streamlit + fastapi -->|"SQL via storage facade
(some routes use direct SQL)"| db + streamlit -->|"direct Python calls + storage"| db + fastapi -->|"sync via CollectorRunner
(background thread)"| aws + aws -->|"inventory / metrics / costs"| fastapi + fastapi -->|"prompt -> JSON"| gemini diff --git a/docs/architecture/c3-components.mmd b/docs/architecture/c3-components.mmd new file mode 100644 index 0000000..bbbe469 --- /dev/null +++ b/docs/architecture/c3-components.mmd @@ -0,0 +1,62 @@ +%% C3 — Components inside the FastAPI backend on main. +%% Sources: backend_api/main.py, backend_api/routers/*, +%% storage/db.py, optimizer/engine.py, ml_engine/forecaster.py, +%% ai_module/recommender.py, aws_collector/runner.py, +%% docs/architecture/README.md, docs/architecture/engines/* +flowchart TB + nextui["Next.js UI
(:3000, HTTP client)"] + streamlit["Streamlit UI
(:8501, direct Python calls)"] + + subgraph api["FastAPI routers (backend_api/routers/)"] + auth["auth"] + conn["connections
(test / save / sync)"] + costs["costs + dashboard"] + forecast["forecast
(+ compare)"] + recs["recommendations"] + onboard["on_boarding
(Gemini questionnaire)"] + settings["settings
(runtime_settings.json)"] + end + + subgraph engines["Engines"] + collector["aws_collector
(CollectorRunner + service collectors)"] + opt["optimizer
(PuLP LP + rules)"] + ml["ml_engine
(forecast / anomaly)"] + aimod["ai_module
(prompt + Gemini client)"] + end + + storage["storage facade
(schema, connections, data APIs)"] + db[("SQLite DB
(data/cloud_optimizer.db)")] + aws["AWS APIs"] + gemini["Google Gemini"] + settingsfile[("runtime_settings.json
(per-user prefs, backend only)")] + + nextui --> auth + nextui --> conn + nextui --> costs + nextui --> forecast + nextui --> recs + nextui --> onboard + nextui --> settings + + conn --> collector + conn --> storage + costs --> storage + forecast --> ml + recs --> opt + recs --> storage + onboard --> aimod + settings --> settingsfile + + collector --> storage + opt --> storage + ml --> storage + aimod --> storage + storage --> db + forecast -->|"route-specific direct SQL"| db + + collector -->|"boto3"| aws + aimod -->|"generate_content"| gemini + streamlit --> storage + streamlit --> opt + streamlit --> ml + streamlit --> aimod diff --git a/docs/architecture/deployment.mmd b/docs/architecture/deployment.mmd new file mode 100644 index 0000000..a1b7714 --- /dev/null +++ b/docs/architecture/deployment.mmd @@ -0,0 +1,28 @@ +%% Deployment — local Compose stack on main. +%% Sources: docker-compose.yml, Dockerfile, .dockerignore, +%% docs/operations/local-runtime.md, docs/architecture/README.md +flowchart TB + subgraph host["Host (loopback only)"] + b1["http://localhost:3000
(frontend)"] + b2["http://localhost:8000
(backend)"] + b3["http://localhost:8501
(streamlit, profile full)"] + end + + subgraph net["Compose network: optimizer-net (bridge)"] + fe["frontend service
(optimizer-frontend, :3000)"] + be["backend service
(optimizer-backend, :8000, appuser uid 10001)"] + st["streamlit service
(same image, profile full, :8501)"] + end + + vol[("named volume optimizer-data
(seeded from image /app/data
persists cloud_optimizer.db + WAL)")] + + b1 --> fe + b2 --> be + b3 --> st + fe -->|"depends_on healthy backend
NEXT_PUBLIC_API_BASE_URL=http://localhost:8000"| be + st -->|"depends_on healthy backend"| be + be --> vol + st --> vol + + note["Notes: ports bind 127.0.0.1 on host. backend + streamlit share the image.
runtime_settings.json is NOT persisted (recreated on container rebuild).
No host bind mount on /app/data (keeps appuser-writable ownership)."] + be --- note diff --git a/docs/architecture/er-diagram.mmd b/docs/architecture/er-diagram.mmd new file mode 100644 index 0000000..e9a4452 --- /dev/null +++ b/docs/architecture/er-diagram.mmd @@ -0,0 +1,256 @@ +erDiagram + %% SQLite schema on main (trimmed to keys + notable columns). + %% Authority is the DDL in storage/db.py (_SCHEMA_DDL); full column lists, + %% CHECKs, and indexes live there — generate them from ensure_schema() + %% instead of copying SQL here. See docs/architecture/data-model.md. + %% Only the users FKs below are enforced; inventory<..>metrics links are + %% logical (shared resource id + user_id, no declared FK). + %% instance_pricing is standalone (shared catalog, no user_id). + users ||--o{ aws_connections : "owns" + users ||--o{ daily_costs : "scopes" + users ||--o{ service_costs : "scopes" + users ||--o{ service_region_costs : "scopes" + users ||--o{ ec2_instances : "scopes" + users ||--o{ ec2_metrics : "scopes" + users ||--o{ rds_instances : "scopes" + users ||--o{ rds_metrics : "scopes" + users ||--o{ elasticache_nodes : "scopes" + users ||--o{ elasticache_metrics : "scopes" + users ||--o{ ecs_services : "scopes" + users ||--o{ ecs_metrics : "scopes" + users ||--o{ lambda_functions : "scopes" + users ||--o{ lambda_metrics : "scopes" + users ||--o{ ebs_volumes : "scopes" + users ||--o{ ebs_metrics : "scopes" + users ||--o{ s3_buckets : "scopes" + users ||--o{ s3_metrics : "scopes" + users ||--o{ dynamodb_tables : "scopes" + users ||--o{ dynamodb_metrics : "scopes" + users ||--o{ nat_gateways : "scopes" + users ||--o{ nat_gateway_metrics : "scopes" + users ||--o{ elb_instances : "scopes" + users ||--o{ elb_metrics : "scopes" + users ||--o{ ai_recommendations : "scopes" + users ||--o{ forecasts : "scopes" + users ||--o{ recommendations : "scopes" + users ||--o{ anomalies : "scopes" + + ec2_instances |o..o{ ec2_metrics : "observed as (logical)" + rds_instances |o..o{ rds_metrics : "observed as (logical)" + elasticache_nodes |o..o{ elasticache_metrics : "observed as (logical)" + ecs_services |o..o{ ecs_metrics : "observed as (logical)" + lambda_functions |o..o{ lambda_metrics : "observed as (logical)" + ebs_volumes |o..o{ ebs_metrics : "observed as (logical)" + s3_buckets |o..o{ s3_metrics : "observed as (logical)" + dynamodb_tables |o..o{ dynamodb_metrics : "observed as (logical)" + nat_gateways |o..o{ nat_gateway_metrics : "observed as (logical)" + elb_instances |o..o{ elb_metrics : "observed as (logical)" + + users { + TEXT user_id PK + TEXT email + TEXT password_hash + TEXT user_type + TEXT created_at + } + aws_connections { + INTEGER id PK + TEXT user_id FK + TEXT connection_name + TEXT aws_account_id + TEXT auth_type + INTEGER access_verified + TEXT sync_status + TEXT last_sync_at + } + daily_costs { + TEXT date PK + TEXT user_id PK + REAL total_cost + } + service_costs { + TEXT date PK + TEXT user_id PK + TEXT service PK + REAL daily_cost + } + service_region_costs { + TEXT date PK + TEXT user_id PK + TEXT service PK + TEXT region PK + REAL daily_cost + } + ec2_instances { + TEXT instance_id PK + TEXT user_id PK + TEXT instance_type + TEXT state + TEXT pricing_model + REAL monthly_cost + } + ec2_metrics { + TEXT timestamp PK + TEXT user_id PK + TEXT instance_id PK + REAL cpu_utilization + } + rds_instances { + TEXT db_instance_id PK + TEXT user_id PK + TEXT db_instance_class + TEXT engine + TEXT pricing_model + REAL monthly_cost + } + rds_metrics { + TEXT timestamp PK + TEXT user_id PK + TEXT db_instance_id PK + REAL cpu_utilization + } + elasticache_nodes { + TEXT cache_cluster_id PK + TEXT user_id PK + TEXT cache_node_type + TEXT engine + REAL monthly_cost + } + elasticache_metrics { + TEXT timestamp PK + TEXT user_id PK + TEXT cache_cluster_id PK + REAL cpu_utilization + } + ecs_services { + TEXT service_name PK + TEXT user_id PK + TEXT cluster_name + TEXT launch_type + REAL monthly_cost + } + ecs_metrics { + TEXT timestamp PK + TEXT user_id PK + TEXT service_name PK + REAL cpu_utilization + } + lambda_functions { + TEXT function_name PK + TEXT user_id PK + TEXT runtime + INTEGER memory_mb + REAL monthly_cost + } + lambda_metrics { + TEXT date PK + TEXT user_id PK + TEXT function_name PK + INTEGER invocations + REAL avg_memory_used_mb + } + ebs_volumes { + TEXT volume_id PK + TEXT user_id PK + TEXT volume_type + INTEGER size_gb + TEXT state + REAL monthly_cost + } + ebs_metrics { + TEXT timestamp PK + TEXT user_id PK + TEXT volume_id PK + REAL read_ops + REAL write_ops + } + s3_buckets { + TEXT bucket_name PK + TEXT user_id PK + TEXT storage_class + REAL monthly_cost + } + s3_metrics { + TEXT timestamp PK + TEXT user_id PK + TEXT bucket_name PK + REAL bucket_size_bytes + } + dynamodb_tables { + TEXT table_name PK + TEXT user_id PK + TEXT capacity_mode + REAL monthly_cost + } + dynamodb_metrics { + TEXT timestamp PK + TEXT user_id PK + TEXT table_name PK + REAL consumed_read_units + REAL consumed_write_units + } + nat_gateways { + TEXT nat_gateway_id PK + TEXT user_id PK + TEXT vpc_id + REAL monthly_cost + } + nat_gateway_metrics { + TEXT timestamp PK + TEXT user_id PK + TEXT nat_gateway_id PK + REAL bytes_in + REAL bytes_out + } + elb_instances { + TEXT elb_arn PK + TEXT user_id PK + TEXT elb_name + TEXT elb_type + INTEGER target_count + REAL monthly_cost + } + elb_metrics { + TEXT timestamp PK + TEXT user_id PK + TEXT elb_arn PK + INTEGER request_count + } + instance_pricing { + INTEGER id PK + TEXT service + TEXT instance_type + REAL on_demand_monthly + } + ai_recommendations { + INTEGER id PK + TEXT user_id FK + TEXT app_type + REAL estimated_cost + TEXT llm_model + } + forecasts { + INTEGER id PK + TEXT user_id FK + TEXT service + TEXT forecast_date + REAL predicted_cost + TEXT model_type + } + recommendations { + INTEGER id PK + TEXT user_id FK + TEXT service + TEXT resource_id + TEXT recommendation_type + REAL monthly_savings + TEXT confidence + } + anomalies { + INTEGER id PK + TEXT user_id FK + TEXT service + TEXT anomaly_date + REAL deviation_percent + TEXT severity + } diff --git a/docs/architecture/sequence.mmd b/docs/architecture/sequence.mmd new file mode 100644 index 0000000..b263488 --- /dev/null +++ b/docs/architecture/sequence.mmd @@ -0,0 +1,39 @@ +%% Sequence — primary flows on main. +%% Sources: backend_api/routers/connections.py (prefix /api/connections), +%% backend_api/routers/forecast.py (prefix /api), aws_collector/runner.py, +%% docs/architecture/README.md, docs/architecture/data-pipeline.md, +%% docs/architecture/engines/forecasting.md +sequenceDiagram + participant B as Browser (Next.js UI :3000) + participant API as FastAPI (:8000, connections router) + participant F as FastAPI (:8000, forecast router) + participant DB as SQLite (storage) + participant W as Sync daemon thread (CollectorRunner) + participant AWS as AWS APIs + participant M as ml_engine + + %% Flow 1: test, save, then sync a connection + B->>API: POST /api/connections/test (keys) + API->>AWS: STS resolve account for keys + AWS-->>API: account id (or error) + API-->>B: account id + validity + B->>API: POST /api/connections (keys + optional account_id) + API->>DB: store connection (workspace aws-account_id) + API-->>B: connection saved (switches active workspace) + B->>API: POST /api/connections/{id}/sync + API->>DB: sync_status = in_progress (409 if already syncing) + API->>W: start daemon thread (returns HTTP 202) + API-->>B: 202 in_progress (polls status separately) + W->>DB: clear previous data for account + W->>AWS: collect costs, inventory, metrics (newest month first, 12 months) + AWS-->>W: observations + pricing + W->>DB: insert via storage.insert_* (incremental commits) + W->>DB: sync_status = success (or failed + error, max 500 chars) + Note over W,DB: failure can leave partial data (not atomic) + + %% Flow 2: forecast request (separate example) + B->>F: GET /api/forecast (user, horizon 1-180, default Prophet/30d) + F->>DB: load latest 365 daily observations + F->>M: anomaly filter (keep original if under 30 left) + forecast + M-->>F: forecast series (never saved to SQLite) + F-->>B: forecast JSON (in-process cache) diff --git a/docs/architecture/state.mmd b/docs/architecture/state.mmd new file mode 100644 index 0000000..842fc46 --- /dev/null +++ b/docs/architecture/state.mmd @@ -0,0 +1,25 @@ +%% State — connection/sync and recommendation lifecycle on main. +%% Sources: backend_api/routers/connections.py, aws_collector/runner.py, +%% optimizer/engine.py, docs/architecture/README.md, +%% docs/architecture/data-pipeline.md, docs/architecture/engines/optimizer.md +stateDiagram-v2 + [*] --> Unsaved: user opens Account Settings + Unsaved --> Tested: POST /connections/test (STS resolves account) + Tested --> Unsaved: keys rejected + Unsaved --> Saved: POST /connections (stored under aws-account_id) + Tested --> Saved: POST /connections (save after test) + Saved --> Syncing: POST /connections/{id}/sync (background thread) + Syncing --> Ready: collect newest-first completes (incremental commits) + Syncing --> Partial: collector or API failure mid-run + Partial --> Syncing: re-run sync (clears account data, retries) + Ready --> Syncing: re-run sync (full replacement per account) + Ready --> Stale: new months accrue / settings change (no auto re-collect) + + state Recommendations { + [*] --> Empty: optimize() deletes user rows first + Empty --> Generated: LP right-sizing + rule checks run + Generated --> Stored: INSERT into recommendations + COMMIT + Stored --> Empty: next optimize() run deletes again + } + + Ready --> Empty: run optimizer for workspace user diff --git a/docs/decisions/README.md b/docs/decisions/README.md new file mode 100644 index 0000000..4dc7c7d --- /dev/null +++ b/docs/decisions/README.md @@ -0,0 +1,24 @@ +# Decisions + +No standalone Architecture Decision Records (ADRs) have been adopted yet — +that is the N/A status being recorded here, not a missing folder. + +Reason: cross-cutting choices on `main` are currently documented inline where +they are implemented and explained: + +- Optimizer approach (LP for compute right-sizing, rules for the rest), + headroom, and clear-then-replace semantics — + [engines/optimizer.md](../architecture/engines/optimizer.md) (`Design rationale`, `Usage`). +- Trust boundary (client-supplied `user_id` is filtering, not authorization) + and connection identity/verification — + [architecture README](../architecture/README.md) (`Trust boundary`, + `Connection identity and verification`). +- SQLite access rules (storage facade preferred, some routes use direct SQL; + transaction ownership) — [Storage API](../api/storage.md). +- Local runtime exposure, persistence, and recovery — + [operations](../operations/local-runtime.md). + +Rule going forward: any new cross-cutting or hard-to-reverse decision must add +a dated note under this folder (`docs/decisions/YYYY-MM-DD-.md`) in the +same change as the code/docs it affects, and link it from the affected +architecture page. diff --git a/ml_engine/__main__.py b/ml_engine/__main__.py index d2a058c..a40f5fa 100644 --- a/ml_engine/__main__.py +++ b/ml_engine/__main__.py @@ -7,28 +7,71 @@ import sys from pathlib import Path -# TODO: Import forecaster when ready -# from storage import db -# from ml_engine import forecaster +from storage import db +from ml_engine.data_prep import load_cost_data +from ml_engine.forecaster import ETSForecaster, NaiveForecaster, SeasonalNaiveForecaster def main(): - """Run ML cost forecasting (not yet implemented).""" + """Run ML cost forecasting and print a short summary.""" parser = argparse.ArgumentParser( - description="Run ML cost forecasting (placeholder)" + description="Run ML cost forecasting from stored daily costs" ) parser.add_argument( "--user-id", required=True, help="User ID to forecast for", ) + parser.add_argument( + "--horizon", + type=int, + default=30, + help="Forecast horizon in days (default: 30)", + ) + parser.add_argument( + "--model", + choices=["naive", "seasonal", "ets"], + default="naive", + help="Forecaster to use (default: naive)", + ) + parser.add_argument( + "--db-path", + type=Path, + default=None, + help="Path to database (default: data/cloud_optimizer.db)", + ) args = parser.parse_args() - print("⚠️ ML Engine CLI not yet implemented") - print(f" User: {args.user_id}") - print(" Use dashboard Forecasts page to run forecasts interactively") - return 1 + conn = db.get_connection(args.db_path) + try: + df = load_cost_data(conn, args.user_id) + if df.empty: + print(f"No daily cost data for user: {args.user_id}", file=sys.stderr) + print("Sync a connection first, then re-run the forecast.") + return 1 + + if args.model == "seasonal": + forecaster = SeasonalNaiveForecaster() + elif args.model == "ets": + forecaster = ETSForecaster() + else: + forecaster = NaiveForecaster() + + forecaster.fit(df) + preds = forecaster.predict(horizon=args.horizon) + + total = float(preds["forecast"].sum()) + print(f"Forecast for user: {args.user_id}") + print(f" Model: {forecaster.name}") + print(f" Horizon: {args.horizon} days") + print(f" Total forecasted cost: ${total:,.2f}") + return 0 + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + finally: + conn.close() if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index 3bf6508..942bc71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,3 +9,58 @@ omit = ["tests/*"] [tool.coverage.report] show_missing = true fail_under = 0 + +# Task verbs (Starter §Task verbs, §CI Same checks locally). +# Short shell steps calling standard tools directly, no glue scripts. +# Absent suites omitted, never stubs; missing tool fails closed (set -eu, no || true). +# Omitted (absent in repo, never stubbed): +# - test:contract: no versioned external contract (no contracts/openapi.yaml; docs/api/http.md states no versioned contract). +# Frontend/backend contract tests in tests/test_frontend_backend_contract.py run as part of `test`. +# - check:sast: no pinned SAST rules/config in repo (no bandit/ruff/mypy/semgrep config). +# - check:secrets: no scan config/baseline in repo (no .gitleaks.toml). +# - check:audit: no audit tool/config in repo (no pip-audit; only `pip check` consistency). + +[tool.poe] +executor = "simple" + +[tool.poe.tasks.dev] +help = "Start the system locally with dependencies" +cmd = "docker compose up --build" + +[tool.poe.tasks.test] +help = "Full suite via the stack's runner" +cmd = "python -m pytest tests/ -v" + +[tool.poe.tasks.migrate] +help = "Migration gate: ensure_schema on disposable DB (real DB migrates on backend startup)" +shell = """ +set -eu +python -c "import sqlite3; from storage import ensure_schema; c=sqlite3.connect(':memory:'); ensure_schema(c); c.close(); print('migrate ok')" +""" + +[tool.poe.tasks.build] +help = "Build the repeatable delivery artifact" +cmd = "docker compose build" + +[tool.poe.tasks."check:hygiene"] +help = "Hygiene check: no placeholders or swallowed failures" +shell = """ +set -eu +git diff --check +python -m compileall -q backend_api cloud_optimizer aws_collector storage data_generation ml_engine optimizer ai_module dashboard +test -d frontend/node_modules || exit 1 +npm --prefix frontend run lint +if grep -Rni --include='*.py' -E 'not yet implemented|TODO: Import forecaster' ml_engine backend_api storage optimizer aws_collector cloud_optimizer data_generation dashboard; then echo "stub markers found" >&2; exit 1; fi +""" + +[tool.poe.tasks.diagrams] +help = "Mermaid-cli validation of .mmd sources; fails on pasted blocks" +shell = """ +set -eu +if grep -Rq '```mermaid' docs/architecture docs/api docs/development docs/operations docs/reference docs/security; then echo "pasted mermaid blocks found (canonical is .mmd)" >&2; exit 1; fi +for f in docs/architecture/*.mmd; do mmdc -i "$f" -o "/tmp/$(basename "$f" .mmd).svg"; done +""" + +[tool.poe.tasks.verify] +help = "Full local gate combining present checks" +sequence = ["test", "migrate", "build", "check:hygiene", "diagrams"] From db3091142219c3e33c7aee439c10629690a7fe4f Mon Sep 17 00:00:00 2001 From: Ibrahim Date: Tue, 20 Oct 2026 06:36:59 +0300 Subject: [PATCH 2/5] fix(ci): invoke poe via binary (python -m poe unavailable on runner) Refs: main audit (PR #8) --- .github/workflows/verify.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2704010..69c2958 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,5 +1,5 @@ # verify — sole CI gate for `main` (Starter §CI Gate job). -# Local parity: `python -m poe verify` runs test → migrate → build → +# Local parity: `poe verify` runs test → migrate → build → # check:hygiene → diagrams (see pyproject.toml [tool.poe.tasks.verify]). # CI mirrors the same task verbs; `gate` is the single aggregating check # required in branch protection. No `|| true`, no `continue-on-error`. @@ -73,7 +73,7 @@ jobs: - name: Run verify (test → migrate → build → check:hygiene → diagrams) run: | set -eu - python -m poe verify + poe verify # Full-history secrets scan (gitleaks). Uses .gitleaks.toml allowlist for # synthetic SYNTHETIC-001 placeholders only; real leaks fail closed. From c3d671029e609cadff95953c738677eff63d9bc5 Mon Sep 17 00:00:00 2001 From: Ibrahim Date: Tue, 20 Oct 2026 06:40:02 +0300 Subject: [PATCH 3/5] fix(deps): pin pulp <4 (4.x removes LpVariable + PULP_CBC_CMD) CI on PR #8 showed 5 optimizer failures with pulp 4.1.0: LpVariable.__init__() unexpected cat. Optimizer uses 2.x/3.x API. Pin to >=2.7,<4 for reproducible verify + docker builds. Refs: #8 --- requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dbbd053..35e06d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,10 @@ statsmodels==0.14.6 pmdarima==2.1.1 # Optimization -pulp>=2.7.0 +# Pinned <4: PuLP 4.x removes LpVariable(...) construction and PULP_CBC_CMD +# (variables must come from prob.add_variable*; CBC via COIN_CMD). The +# optimizer still uses the 2.x/3.x API, so allow 3.x patches but block 4.x. +pulp>=2.7,<4 # AI openai>=1.10.0 From 00561aefd2daa7d4c94d7abc8b52c9708e4be784 Mon Sep 17 00:00:00 2001 From: Ibrahim Date: Tue, 20 Oct 2026 06:43:31 +0300 Subject: [PATCH 4/5] fix(docs): remove pasted mermaid block (canonical is .mmd) diagrams task fails closed on pasted blocks; overview now points to versioned .mmd sources validated by mmdc. Refs: #8 --- docs/architecture/README.md | 44 ++++--------------------------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/docs/architecture/README.md b/docs/architecture/README.md index cdc1827..c6893e1 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -24,46 +24,10 @@ reason; add dated notes there for new hard-to-reverse decisions). ## Overview -```mermaid -flowchart LR - Browser["Browser"] - NextUI["Next.js UI
(port 3000)"] - - subgraph BE["FastAPI backend (port 8000)"] - API["FastAPI routers
(trust-based user_id)"] - Storage["storage facade
(shared data API)"] - Optimizer["optimizer
(PuLP right-sizing + rules)"] - ML["ml_engine
(forecasts / anomalies)"] - AImod["ai_module
(Gemini advisor)"] - Collector["aws_collector
(boto3 pipeline)"] - end - - DB[("SQLite DB
data/cloud_optimizer.db")] - Synthetic["data_generation.synthetic
(demo seed)"] - AWS["AWS
(Cost Explorer / CloudWatch via boto3)"] - Gemini["Google Gemini
(AI recommendations)"] - - Browser --> NextUI - NextUI -->|"HTTP / JSON"| API - API --> Storage - API --> Optimizer - API --> ML - API --> AImod - API --> Collector - Optimizer --> Storage - ML --> Storage - AImod --> Storage - Collector --> Storage - Storage --> DB - API -->|"route-specific SQL"| DB - - Collector -. "real mode" .-> AWS - AImod -. "AI calls" .-> Gemini - Synthetic -. "demo mode" .-> Storage - - classDef ext stroke-dasharray:5 5,stroke:#888,color:#444; - class AWS,Gemini,Synthetic ext; -``` +Render the versioned sources above with `poe diagrams` (CI validates every +`.mmd` with `mmdc`). Pasted `mermaid` blocks are not used — the canonical +diagrams are the `.mmd` files indexed above (C1/C2/C3, sequence, state, +deployment, ER). ## Boundaries From 7689a421250282dbb34762221d106c4c4b893d30 Mon Sep 17 00:00:00 2001 From: Ibrahim Date: Tue, 20 Oct 2026 06:50:32 +0300 Subject: [PATCH 5/5] fix(ci): use puppeteer no-sandbox for mmdc (CI chrome sandbox) Poe diagrams now passes --puppeteerConfigFile ./puppeteer.json so both local and CI render .mmd via mmdc without sandbox failures. Update task comments to reflect .gitleaks.toml secrets scan in CI. Refs: #8 --- puppeteer.json | 1 + pyproject.toml | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 puppeteer.json diff --git a/puppeteer.json b/puppeteer.json new file mode 100644 index 0000000..0034ff8 --- /dev/null +++ b/puppeteer.json @@ -0,0 +1 @@ +{"args":["--no-sandbox","--disable-setuid-sandbox"]} diff --git a/pyproject.toml b/pyproject.toml index 942bc71..c7c70ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,8 +17,9 @@ fail_under = 0 # - test:contract: no versioned external contract (no contracts/openapi.yaml; docs/api/http.md states no versioned contract). # Frontend/backend contract tests in tests/test_frontend_backend_contract.py run as part of `test`. # - check:sast: no pinned SAST rules/config in repo (no bandit/ruff/mypy/semgrep config). -# - check:secrets: no scan config/baseline in repo (no .gitleaks.toml). # - check:audit: no audit tool/config in repo (no pip-audit; only `pip check` consistency). +# Secrets history scan runs in CI (verify.yml → secrets-history via gitleaks + +# .gitleaks.toml allowlisting synthetic placeholders only); no poe task needed. [tool.poe] executor = "simple" @@ -58,7 +59,7 @@ help = "Mermaid-cli validation of .mmd sources; fails on pasted blocks" shell = """ set -eu if grep -Rq '```mermaid' docs/architecture docs/api docs/development docs/operations docs/reference docs/security; then echo "pasted mermaid blocks found (canonical is .mmd)" >&2; exit 1; fi -for f in docs/architecture/*.mmd; do mmdc -i "$f" -o "/tmp/$(basename "$f" .mmd).svg"; done +for f in docs/architecture/*.mmd; do mmdc --puppeteerConfigFile ./puppeteer.json -i "$f" -o "/tmp/$(basename "$f" .mmd).svg"; done """ [tool.poe.tasks.verify]