diff --git a/.gitignore b/.gitignore index cd0dd169..2e1a47df 100644 --- a/.gitignore +++ b/.gitignore @@ -224,3 +224,13 @@ PARITY_TODOs.md backup rebuild run_interpreter.py +run_harbor_single.py +run_neighbor_synthesis.py +run_single_synthesis.py +run_single_synthesis_test.py + +# Local artifacts +.claude/ +.python-version +backups/ +jobs/ diff --git a/CLAUDE.md b/CLAUDE.md index d34cecac..ef6a2e7e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,6 +82,8 @@ Each task lives in `dataset/formulacode_verified///` with a mul After making a feature change, decide whether the change is significant enough to warrant updating the documentation in `docs/`. Changes that affect user-facing behavior, CLI flags, configuration knobs, pipeline stages, agent backends, or architectural decisions should be reflected in the relevant guide or design doc. Internal refactors, bug fixes, and implementation details generally do not need doc updates unless they change observable behavior. +**Diagrams**: use Mermaid (`` ```mermaid `` fenced blocks) for any architecture, flow, or state diagram in `.md` files. Do not use ASCII box-drawing art (`┌ ─ │ └ ──▶`). Mermaid renders natively on GitHub and in the docs site; ASCII does not, and is harder to edit. + ### Tunable constants Any module-level constant that is a knob — timeouts, retries, caps, windows, @@ -130,6 +132,10 @@ The Supabase PostgREST API is also available at `https://db.formulacode.org` via When both `DATASMITH_CF_ACCESS_*` vars are set, `get_client()` and `get_async_client()` in `utils/db.py` automatically inject the required headers. See `docs/guide/remote-access.md` for full setup instructions. +### Public read-only access (RLS) + +Four tables have Row Level Security enabled with `public_read` SELECT-only policies for the `anon` role: `repositories`, `pull_requests`, `candidate_containers`, `harbor_runs`. The service-role key bypasses RLS, so pipeline processes are unaffected. See `supabase/migrations/00012_public_read_rls.sql`. + ### Key tables | Table | Purpose | Populated by | diff --git a/docs/design/components/datasmith.agents.synthesizer.md b/docs/design/components/datasmith.agents.synthesizer.md index 5cf94862..f6ba3a50 100644 --- a/docs/design/components/datasmith.agents.synthesizer.md +++ b/docs/design/components/datasmith.agents.synthesizer.md @@ -56,22 +56,21 @@ ctx = synth.run( The synthesizer progresses through five states in order. Each state either returns a `DockerContext` (success) or falls through to the next state. -``` -CHECK_CACHE ──hit──> return DockerContext - │ - │ miss - v -FIND_SIMILAR ──> TRY_SIMILAR ──any pass──> return DockerContext - │ - │ all fail / none found - v - LLM_GENERATE (sandbox, up to max_attempts) - │ - │ any pass ──> return DockerContext - │ - │ all fail - v - FAIL ──> return None +```mermaid +flowchart TD + Cache["CHECK_CACHE"] + Find["FIND_SIMILAR"] + Try["TRY_SIMILAR"] + Gen["LLM_GENERATE
(sandbox, up to max_attempts)"] + Ok(["return DockerContext"]) + Fail(["FAIL: return None"]) + Cache -- hit --> Ok + Cache -- miss --> Find + Find --> Try + Try -- "any pass" --> Ok + Try -- "all fail / none found" --> Gen + Gen -- "any pass" --> Ok + Gen -- "all fail" --> Fail ``` ### State details diff --git a/docs/design/components/datasmith.resolution.md b/docs/design/components/datasmith.resolution.md index 3ba95dd2..583190ce 100644 --- a/docs/design/components/datasmith.resolution.md +++ b/docs/design/components/datasmith.resolution.md @@ -175,90 +175,72 @@ A persistent JSON blocklist (`{CACHE_DIR}/package_blocklist.json`) tracks packag ### How `prepare_commits_for_building_reports.py` called the resolution module (archive) -``` -prepare_commits_for_building_reports.py - │ - ├─ Load parquet → DataFrame with commit data - ├─ crude_perf_filter(df) → filtered_df - │ - ├─ Build (sha, repo_name) pairs from filtered_df["pr_base"]["sha"] - ├─ _thread_map(safe_analyze_commit, pairs, max_workers=200) - │ │ - │ └─ safe_analyze_commit((sha, repo_name)) - │ │ - │ └─ analyze_commit(sha, repo_name) # from resolution.__init__ - │ │ - │ └─ orchestrator.analyze_commit(sha, repo_name) - │ │ - │ ├─ prepare_repo_checkout(repo_name, sha) [git_utils] - │ ├─ asv_finder(commit) [git_utils] - │ ├─ filter_python_versions_by_commit_date() [python_manager] - │ ├─ discover_candidates(commit) [metadata_parser] - │ ├─ analyze_candidate_meta(candidate) [metadata_parser] - │ ├─ select_primary_candidate(...) [metadata_parser] - │ │ - │ ├─ STRATEGY 1: For each source file × python version: - │ │ ├─ run_uv(["venv", ...]) [python_manager] - │ │ ├─ uv_compile_from_pyproject(...) [dependency_resolver] - │ │ ├─ uv_dry_run_install(...) [dependency_resolver] - │ │ └─ uv_install_real(...) [dependency_resolver] - │ │ - │ ├─ STRATEGY 2: Aggregate requirements - │ │ ├─ extract_requested_extras(...) [package_filters] - │ │ ├─ split_shell_command(cmd) [package_filters] - │ │ ├─ normalize_requirement(tok) [package_filters] - │ │ ├─ resolve_requirements_file(...) [package_filters] - │ │ ├─ uv_build_and_read_metadata(...) [dependency_resolver] - │ │ ├─ infer_runtime_from_imports(...) [import_analyzer] - │ │ ├─ filter_requirements_for_pypi(...) [package_filters] - │ │ ├─ clean_pinned(...) [package_filters] - │ │ ├─ uv_compile(...) [dependency_resolver] - │ │ │ └─ Self-healing retry loop: - │ │ │ ├─ extract_failing_package() [blocklist] - │ │ │ ├─ add_to_blocklist() [blocklist] - │ │ │ └─ remove_package_from_requirements() [blocklist] - │ │ ├─ uv_dry_run_install(...) [dependency_resolver] - │ │ │ └─ Self-healing retry loop (same) - │ │ └─ uv_install_real(...) [dependency_resolver] - │ │ - │ └─ Return dict with resolution results - │ - ├─ pd.DataFrame(analysis_dicts).add_prefix("analysis_") - ├─ Filter: analysis_can_install == True - ├─ Filter: analysis_resolution_strategy not startswith "unresolved" - └─ Save enriched parquet +```mermaid +flowchart TD + Entry["prepare_commits_for_building_reports.py"] + Load["Load parquet → DataFrame"] + Filter1["crude_perf_filter(df)"] + Pairs["Build (sha, repo_name) pairs"] + TMap["_thread_map(safe_analyze_commit, pairs, max_workers=200)"] + Analyze["orchestrator.analyze_commit(sha, repo_name)"] + subgraph Prep["Per-commit preparation"] + direction TB + Checkout["prepare_repo_checkout [git_utils]"] + ASV["asv_finder [git_utils]"] + PyVers["filter_python_versions_by_commit_date [python_manager]"] + Discover["discover_candidates [metadata_parser]"] + AnalyzeMeta["analyze_candidate_meta [metadata_parser]"] + SelectCand["select_primary_candidate [metadata_parser]"] + end + subgraph S1["STRATEGY 1: per source file × python version"] + direction TB + Venv["run_uv(['venv', ...]) [python_manager]"] + Pyproj["uv_compile_from_pyproject [dependency_resolver]"] + DryRun1["uv_dry_run_install [dependency_resolver]"] + Install1["uv_install_real [dependency_resolver]"] + end + subgraph S2["STRATEGY 2: aggregate requirements"] + direction TB + Extras["extract_requested_extras [package_filters]"] + Split["split_shell_command / normalize_requirement [package_filters]"] + Resolve["resolve_requirements_file [package_filters]"] + BuildMeta["uv_build_and_read_metadata [dependency_resolver]"] + Imports["infer_runtime_from_imports [import_analyzer]"] + FilterPyPI["filter_requirements_for_pypi / clean_pinned [package_filters]"] + Compile["uv_compile [dependency_resolver]"] + Heal["Self-healing retry loop:
extract_failing_package → add_to_blocklist →
remove_package_from_requirements [blocklist]"] + DryRun2["uv_dry_run_install (same heal loop)"] + Install2["uv_install_real"] + Compile --> Heal + end + Result["Return dict with resolution results"] + Post["pd.DataFrame(...).add_prefix('analysis_')
filter can_install == True
filter resolution_strategy not startswith 'unresolved'
save enriched parquet"] + + Entry --> Load --> Filter1 --> Pairs --> TMap --> Analyze + Analyze --> Prep --> S1 + Analyze --> S2 + S1 --> Result + S2 --> Result + Result --> Post ``` ### How the new pipeline will call resolution -``` -Pipeline._run_stage("resolve_packages") - │ - ├─ Query pull_requests WHERE is_performance_commit = TRUE - │ AND NOT EXISTS (SELECT 1 FROM packages WHERE packages.sha = pr.merge_commit_sha - │ AND packages.owner = pr.owner AND packages.repo = pr.repo) - │ - ├─ Deduplicate by (owner, repo, merge_commit_sha) - │ (multiple PRs may share the same base commit) - │ - ├─ ResolvePackagesRunner.run(items, n_concurrent=N) - │ │ - │ └─ For each (owner, repo, sha): - │ ├─ analyze_commit(sha, f"{owner}/{repo}") - │ ├─ If result and can_install: - │ │ └─ Upsert into packages table - │ └─ If None or not can_install: - │ └─ Log to runner_failures - │ - └─ Pipeline._synthesize_images() now reads: - SELECT pr.*, pkg.env_payload, pkg.python_version - FROM pull_requests pr - JOIN packages pkg ON pr.owner = pkg.owner - AND pr.repo = pkg.repo - AND pr.merge_commit_sha = pkg.sha - WHERE pr.is_performance_commit = TRUE - AND pr.container_name IS NULL - AND pkg.can_install = TRUE +```mermaid +flowchart TD + Stage["Pipeline._run_stage('resolve_packages')"] + Query["Query pull_requests WHERE is_performance_commit = TRUE
AND NOT EXISTS (packages row for same owner/repo/sha)"] + Dedup["Deduplicate by (owner, repo, merge_commit_sha)"] + Runner["ResolvePackagesRunner.run(items, n_concurrent=N)"] + PerItem["For each (owner, repo, sha):
analyze_commit(sha, owner/repo)"] + Upsert["Upsert into packages table"] + Failure["Log to runner_failures"] + Synth["Pipeline._synthesize_images() joins packages on
(owner, repo, merge_commit_sha) and filters can_install"] + + Stage --> Query --> Dedup --> Runner --> PerItem + PerItem -- "result and can_install" --> Upsert + PerItem -- "None or not can_install" --> Failure + Upsert --> Synth ``` ## Key data models (from archive, ported as-is) diff --git a/docs/guide/monitoring.md b/docs/guide/monitoring.md index 370bda4d..7639b2ed 100644 --- a/docs/guide/monitoring.md +++ b/docs/guide/monitoring.md @@ -39,16 +39,13 @@ cloudflared tunnel run datasmith-grafana Grafana runs as a Docker container that joins the existing Supabase Docker network. It connects to Postgres via a **read-only** `grafana_ro` database role that can only `SELECT` — no writes are possible, even from the Explore SQL editor. -``` -┌──────────────┐ ┌──────────────────────────┐ -│ Browser │────▶│ Grafana (port 3001) │ -└──────────────┘ │ Anonymous Viewer access │ - └───────────┬──────────────┘ - │ SELECT only - ┌───────────▼──────────────┐ - │ Supabase PostgreSQL │ - │ (grafana_ro role) │ - └──────────────────────────┘ +```mermaid +flowchart TD + Browser["Browser"] + Grafana["Grafana (port 3001)
Anonymous Viewer access"] + PG["Supabase PostgreSQL
(grafana_ro role)"] + Browser --> Grafana + Grafana -- "SELECT only" --> PG ``` ## Dashboard Panels diff --git a/docs/guide/remote-access.md b/docs/guide/remote-access.md index 01c507e7..891b3437 100644 --- a/docs/guide/remote-access.md +++ b/docs/guide/remote-access.md @@ -1,72 +1,124 @@ -# Remote Access via Cloudflare Tunnel +# Remote Access -fc-data stores all persistent state in a local Supabase instance. By -default this is only reachable from the host machine (`127.0.0.1:54321`). -A **Cloudflare Tunnel** lets a remote machine run the same fc-data -pipeline against the same database — no VPN, no open ports, no firewall -rules. +The fc-data Supabase instance is not exposed on the public internet. Two paths +reach it from outside the host: + +1. **Full read/write access** over a Cloudflare Tunnel, gated by Cloudflare + Access service tokens. Intended for pipeline operators running `fc-data` + against the shared database. +2. **Public read-only access** via the Supabase anon key. Intended for + client-side websites that display dataset statistics. ## Architecture +```mermaid +flowchart LR + User["Remote fc-data
(service token)"] + Web["Public website
(anon key)"] + CFE["Cloudflare Edge
db.formulacode.org"] + CFA["Cloudflare Access"] + subgraph Host["Host machine"] + CFD["cloudflared"] + SB["Supabase :54321
PostgREST + RLS"] + end + User -->|CF-Access headers + service-role key| CFE + Web -->|anon key| CFE + CFE --> CFA + CFA --> CFD + CFD --> SB +``` + +Cloudflare Access blocks every request at the edge unless it carries a valid +service token. The anon-key path works because the RLS policies in +`supabase/migrations/00012_public_read_rls.sql` allow `SELECT` for the `anon` +role on four tables; writes and all other tables are rejected by RLS. + +--- + +## For users + +### Full read/write access + +You need two things in `tokens.env`: + +```bash +SUPABASE_URL=https://db.formulacode.org +SUPABASE_KEY= + +DATASMITH_CF_ACCESS_CLIENT_ID= +DATASMITH_CF_ACCESS_CLIENT_SECRET= ``` -Remote machine Host machine -┌──────────────┐ ┌──────────────────────┐ -│ fc-data │── HTTPS ──▶ Cloudflare Edge ──▶ cloudflared │──▶ Supabase -│ tokens.env: │ (db.formulacode.org) (tunnel) │ :54321 -│ SUPABASE_URL│ │ -│ CF headers │ Cloudflare Access │ -└──────────────┘ (service-token auth) └──────────────────────┘ + +Both the service-role key and the Cloudflare Access credentials are issued by +a maintainer. To request them, [open an issue][issues] asking for remote +access; include the machine or project you need the credentials for. + +Verify the connection: + +```bash +fc-data --preflight ``` -**Two layers of auth protect the database:** +### Public read-only access + +Use the Supabase anon key (shown as the "Publishable" key in +`supabase status`). No tunnel credentials are required; the host is the same. + +| Table | Exposed | +|-------|---------| +| `repositories` | Repository metadata | +| `pull_requests` | PR metadata, classification, patches | +| `candidate_containers` | Successful build scripts per SHA | +| `harbor_runs` | Benchmark speedup results | + +Example: + +```js +const SUPABASE_URL = "https://db.formulacode.org"; +const ANON_KEY = "sb_publishable_..."; + +const res = await fetch( + `${SUPABASE_URL}/rest/v1/repositories?select=owner,repo,stars&order=stars.desc&limit=20`, + { + headers: { + "apikey": ANON_KEY, + "Authorization": `Bearer ${ANON_KEY}`, + }, + }, +); +``` -1. **Cloudflare Access** — a service token (`CF-Access-Client-Id` / - `CF-Access-Client-Secret` headers) must be present on every request - or Cloudflare rejects it at the edge before it ever reaches the tunnel. -2. **Supabase service-role key** — the standard `apikey` header required - by PostgREST, unchanged from local usage. +Writes return `HTTP 403: new row violates row-level security policy`. -## Prerequisites +--- -- A **Cloudflare account** (free plan is sufficient) -- A **domain managed by Cloudflare** (e.g., `formulacode.org`) -- `cloudflared` CLI installed on the **host machine** (the one running Supabase) +## For developers (host machine setup) -## Host machine setup +This section covers standing up the tunnel and Access policy. Day-to-day +operation is in the [Makefile targets](#makefile-targets) at the bottom. ### 1. Install cloudflared ```bash -# Debian / Ubuntu curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb \ -o cloudflared.deb sudo dpkg -i cloudflared.deb - -# Verify cloudflared --version ``` -### 2. Authenticate +### 2. Authenticate and create the tunnel ```bash cloudflared login -``` - -This opens a browser to authorize `cloudflared` with your Cloudflare -account. Select the domain you want to use (e.g., `formulacode.org`). - -### 3. Create a tunnel - -```bash cloudflared tunnel create datasmith-db ``` -Note the **Tunnel ID** printed (e.g., `a1b2c3d4-...`). A credentials -file is saved to `~/.cloudflared/.json`. +Note the printed Tunnel ID. Credentials are saved to +`~/.cloudflared/.json`. -### 4. Configure the tunnel +### 3. Configure the tunnel -Create `~/.cloudflared/config.yml`: +`~/.cloudflared/config.yml`: ```yaml tunnel: @@ -78,108 +130,68 @@ ingress: - service: http_status:404 ``` -### 5. Create the DNS record +### 4. DNS record ```bash cloudflared tunnel route dns datasmith-db db.formulacode.org ``` -This creates a CNAME record pointing `db.formulacode.org` to the tunnel. - -### 6. Run the tunnel +### 5. Run the tunnel ```bash -# Using the Makefile target (recommended) -make db-tunnel - -# Or directly -cloudflared tunnel run datasmith-db - -# As a systemd service (persistent, survives reboots) -sudo cloudflared service install -sudo systemctl enable cloudflared -sudo systemctl start cloudflared +make db-tunnel # foreground +sudo cloudflared service install # or as a systemd service +sudo systemctl enable --now cloudflared ``` -At this point, `https://db.formulacode.org` proxies to your local -Supabase PostgREST API — but **Cloudflare Access blocks all requests** -until you create an access policy. - -## Cloudflare Access setup +At this point `https://db.formulacode.org` proxies to local Supabase but +Cloudflare Access blocks everything until the policy is in place. -### 1. Create an application +### 6. Cloudflare Access policy -1. Go to [Cloudflare Zero Trust](https://one.dash.cloudflare.com/) → - **Access** → **Applications** -2. Click **Add an application** → **Self-hosted** -3. Set: - - **Application name**: `datasmith-db` - - **Session duration**: `24 hours` - - **Application domain**: `db.formulacode.org` -4. Under **Policies**, create a policy: - - **Policy name**: `Service Token` - - **Action**: Service Auth - - **Include**: Service Token (select the token you'll create next) -5. Save the application +In [Cloudflare Zero Trust](https://one.dash.cloudflare.com/) → Access → +Applications, add a self-hosted app: -### 2. Create a service token +- Application name: `datasmith-db` +- Application domain: `db.formulacode.org` +- Session duration: 24 hours +- Policy: action `Service Auth`, include the service token below. -1. Go to **Access** → **Service Auth** → **Service Tokens** -2. Click **Create Service Token** -3. Name it (e.g., `datasmith-remote`) -4. **Copy both values immediately** — the Client Secret is only shown once: - - `CF-Access-Client-Id` (e.g., `abc123.access`) - - `CF-Access-Client-Secret` (e.g., `long-secret-string`) +Then under Access → Service Auth → Service Tokens, create a token (e.g. +`datasmith-remote`) and copy both the Client ID and Client Secret. The secret +is only shown once. Hand these to the requesting user along with the +service-role key. -## Remote machine setup +### 7. Apply the RLS migration -On the remote machine, edit `tokens.env`: +The anon-key path requires `supabase/migrations/00012_public_read_rls.sql` to +be applied: ```bash -# Point at the tunnel instead of localhost -SUPABASE_URL=https://db.formulacode.org -SUPABASE_KEY=your-service-role-key # Same key as the host machine - -# Cloudflare Access service token -DATASMITH_CF_ACCESS_CLIENT_ID=abc123.access -DATASMITH_CF_ACCESS_CLIENT_SECRET=long-secret-string +docker exec supabase_db_ psql -U postgres -d postgres \ + -c "$(cat supabase/migrations/00012_public_read_rls.sql)" ``` -The `SUPABASE_KEY` is the same service-role key used on the host — it is -not a Cloudflare credential. - -### Verify connectivity +### How the client picks up the headers -```bash -fc-data --preflight -``` - -The Supabase connection check should show `[OK]`. If it fails: - -- **403 Forbidden** — the CF Access headers are missing or the service - token is invalid. Double-check `DATASMITH_CF_ACCESS_CLIENT_ID` and - `DATASMITH_CF_ACCESS_CLIENT_SECRET`. -- **502 Bad Gateway** — `cloudflared` is not running on the host machine - or Supabase is down. SSH into the host and check - `systemctl status cloudflared` and `supabase status`. -- **Connection refused** — DNS is not resolving. Verify - `cloudflared tunnel route dns` was run and the CNAME exists in your - Cloudflare DNS dashboard. - -## How it works in the code - -When `DATASMITH_CF_ACCESS_CLIENT_ID` and `DATASMITH_CF_ACCESS_CLIENT_SECRET` -are both set, `datasmith.utils.db` automatically injects the +When both `DATASMITH_CF_ACCESS_CLIENT_ID` and +`DATASMITH_CF_ACCESS_CLIENT_SECRET` are set, `datasmith.utils.db` injects the `CF-Access-Client-Id` and `CF-Access-Client-Secret` headers into every -Supabase client request via `ClientOptions`. No other code changes are -needed — every call to `get_client()` or `get_async_client()` picks up -the headers transparently. +Supabase client request via `ClientOptions`. When unset, no extra headers are +added and behavior is identical to local development. -When neither variable is set (the default for local development), no -extra headers are added and behavior is identical to before. +### Troubleshooting -## Makefile Targets +| Symptom | Likely cause | +|---------|-------------| +| `403 Forbidden` | Missing or invalid CF Access headers | +| `502 Bad Gateway` | `cloudflared` not running, or Supabase is down | +| `Connection refused` | DNS not resolving; check the CNAME was created | + +## Makefile targets ```bash -make db-tunnel # Expose Supabase PostgREST API via Cloudflare Tunnel +make db-tunnel # Expose Supabase PostgREST via Cloudflare Tunnel ``` + +[issues]: https://github.com/formula-code/datasmith/issues/new diff --git a/grafana/provisioning/dashboards-json/datasmith-overview.json b/grafana/provisioning/dashboards-json/datasmith-overview.json index b69e5eb0..5a4e65fa 100644 --- a/grafana/provisioning/dashboards-json/datasmith-overview.json +++ b/grafana/provisioning/dashboards-json/datasmith-overview.json @@ -14,114 +14,35 @@ "to": "now" }, "templating": { - "list": [ - { - "name": "table_name", - "label": "Table", - "type": "custom", - "query": "repositories,pull_requests,packages,candidate_containers,candidate_prs,error_logs,runner_progress,runner_failures,hook_cache", - "current": { - "text": "pull_requests", - "value": "pull_requests" - }, - "options": [ - { - "text": "repositories", - "value": "repositories", - "selected": false - }, - { - "text": "pull_requests", - "value": "pull_requests", - "selected": true - }, - { - "text": "packages", - "value": "packages", - "selected": false - }, - { - "text": "candidate_containers", - "value": "candidate_containers", - "selected": false - }, - { - "text": "candidate_prs", - "value": "candidate_prs", - "selected": false - }, - { - "text": "error_logs", - "value": "error_logs", - "selected": false - }, - { - "text": "runner_progress", - "value": "runner_progress", - "selected": false - }, - { - "text": "runner_failures", - "value": "runner_failures", - "selected": false - }, - { - "text": "hook_cache", - "value": "hook_cache", - "selected": false - } - ] - }, - { - "name": "row_limit", - "label": "Limit", - "type": "custom", - "query": "50,100,250,500,1000", - "current": { - "text": "100", - "value": "100" - }, - "options": [ - { - "text": "50", - "value": "50", - "selected": false - }, - { - "text": "100", - "value": "100", - "selected": true - }, - { - "text": "250", - "value": "250", - "selected": false - }, - { - "text": "500", - "value": "500", - "selected": false - }, - { - "text": "1000", - "value": "1000", - "selected": false - } - ] - } - ] + "list": [] }, "annotations": { "list": [] }, "panels": [ + { + "id": 100, + "type": "text", + "title": "", + "transparent": true, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 10 + }, + "options": { + "mode": "markdown", + "content": "
\n \"FormulaCode\n
\n\n
\n \"FormulaCode\n \"FormulaCode\n \"FormulaCode\n \"fc-data\n
\n\n[FormulaCode](https://formula-code.github.io/) is a *continually updating* benchmark for evaluating the holistic ability of LLM agents to optimize codebases. FormulaCode consists of two parts: a [pipeline](https://github.com/formula-code/datasmith) to construct performance optimization tasks, and an [execution harness](https://github.com/formula-code/terminal-bench) that connects a language model to our terminal sandbox. _This dashboard shows live metrics for the task generation pipeline._\n\n`fc-data` is a python package for automatically curating and managing FormulaCode tasks. fc-data is designed to run as a monthly CRON job that updates the FormulaCode dataset with new commits and repositories.\n\nThis dashboard provides insights into the growth of the FormulaCode dataset, the distribution of tasks across repositories, and the performance characteristics of the pull requests that form the basis of our optimization problems. For more details on the dataset and how to contribute, please visit our [GitHub repository](https://github.com/formula-code/)." + } + }, { "type": "row", "title": "Dataset Growth", "collapsed": false, "gridPos": { "x": 0, - "y": 0, + "y": 10, "w": 24, "h": 1 } @@ -132,7 +53,7 @@ "type": "stat", "gridPos": { "x": 14, - "y": 1, + "y": 11, "w": 3, "h": 4 }, @@ -164,11 +85,11 @@ }, { "id": 15, - "title": "Total Containers", + "title": "Total Problems", "type": "stat", "gridPos": { "x": 17, - "y": 1, + "y": 11, "w": 3, "h": 4 }, @@ -200,11 +121,11 @@ }, { "id": 16, - "title": "Total Packages", + "title": "Perf. PRs", "type": "stat", "gridPos": { "x": 14, - "y": 5, + "y": 15, "w": 3, "h": 4 }, @@ -214,7 +135,7 @@ }, "targets": [ { - "rawSql": "SELECT COUNT(*) AS total FROM packages;", + "rawSql": "SELECT COUNT(*) AS total FROM packages WHERE can_install;", "format": "table", "refId": "A" } @@ -240,7 +161,7 @@ "type": "stat", "gridPos": { "x": 17, - "y": 5, + "y": 15, "w": 3, "h": 4 }, @@ -272,13 +193,13 @@ }, { "id": 18, - "title": "PR to Container Rate", + "title": "PR to Problem Rate", "type": "stat", "gridPos": { "x": 20, - "y": 1, + "y": 15, "w": 4, - "h": 8 + "h": 4 }, "datasource": { "uid": "supabase-pg", @@ -321,18 +242,18 @@ "collapsed": false, "gridPos": { "x": 0, - "y": 9, + "y": 19, "w": 24, "h": 1 } }, { "id": 52, - "title": "All Repositories by Stars", + "title": "Repository Distribution", "type": "barchart", "gridPos": { "x": 0, - "y": 10, + "y": 20, "w": 24, "h": 10 }, @@ -342,7 +263,7 @@ }, "targets": [ { - "rawSql": "SELECT owner || '/' || repo AS repository, stars FROM repositories WHERE stars IS NOT NULL ORDER BY stars DESC;", + "rawSql": "SELECT owner || '/' || repo AS repository, stars AS \"stargazer count\" FROM repositories WHERE stars IS NOT NULL ORDER BY stars DESC;", "format": "table", "refId": "A" } @@ -367,11 +288,11 @@ }, { "id": 54, - "title": "Containers by Repository", + "title": "Problems by Repository", "type": "marcusolsson-treemap-panel", "gridPos": { "x": 0, - "y": 20, + "y": 30, "w": 24, "h": 10 }, @@ -398,11 +319,11 @@ }, { "id": 50, - "title": "Containers Built by Month", + "title": "Monthly distribution of Problems", "type": "barchart", "gridPos": { "x": 0, - "y": 1, + "y": 11, "w": 14, "h": 8 }, @@ -425,6 +346,15 @@ } }, "overrides": [] + }, + "options": { + "xTickLabelMaxLength": 0, + "xTickLabelSpacing": 0, + "axisCenteredZero": false, + "axisBorderShow": false, + "xAxis": { + "showLabels": false + } } }, { @@ -433,7 +363,7 @@ "collapsed": false, "gridPos": { "x": 0, - "y": 38, + "y": 48, "w": 24, "h": 1 } @@ -444,7 +374,7 @@ "type": "stat", "gridPos": { "x": 0, - "y": 39, + "y": 49, "w": 4, "h": 8 }, @@ -481,7 +411,7 @@ "type": "table", "gridPos": { "x": 4, - "y": 39, + "y": 49, "w": 5, "h": 8 }, @@ -521,7 +451,7 @@ "type": "piechart", "gridPos": { "x": 9, - "y": 39, + "y": 49, "w": 5, "h": 8 }, @@ -550,11 +480,11 @@ }, { "id": 34, - "title": "PR Volume Over Time", - "type": "timeseries", + "title": "Performance PR Volume Over Time", + "type": "barchart", "gridPos": { "x": 14, - "y": 39, + "y": 49, "w": 10, "h": 8 }, @@ -564,14 +494,28 @@ }, "targets": [ { - "rawSql": "SELECT date_trunc('month', created_at) AS time, COUNT(*) FILTER (WHERE is_performance_commit) AS perf_prs FROM pull_requests WHERE created_at IS NOT NULL GROUP BY 1 ORDER BY 1;", - "format": "time_series", + "rawSql": "SELECT to_char(date_trunc('month', created_at), 'YYYY-MM') AS month, COUNT(*) FILTER (WHERE is_performance_commit) AS perf_prs FROM pull_requests WHERE created_at IS NOT NULL GROUP BY 1 ORDER BY 1;", + "format": "table", "refId": "A" } ], "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, "overrides": [] + }, + "options": { + "xTickLabelMaxLength": 0, + "xTickLabelSpacing": 0, + "axisCenteredZero": false, + "axisBorderShow": false, + "xAxis": { + "showLabels": false + } } }, { @@ -580,7 +524,7 @@ "type": "barchart", "gridPos": { "x": 0, - "y": 47, + "y": 57, "w": 24, "h": 8 }, @@ -613,7 +557,7 @@ "collapsed": false, "gridPos": { "x": 0, - "y": 56, + "y": 66, "w": 24, "h": 1 } @@ -624,7 +568,7 @@ "type": "piechart", "gridPos": { "x": 0, - "y": 57, + "y": 67, "w": 6, "h": 8 }, @@ -657,7 +601,7 @@ "type": "barchart", "gridPos": { "x": 6, - "y": 57, + "y": 67, "w": 10, "h": 8 }, @@ -717,7 +661,7 @@ "type": "piechart", "gridPos": { "x": 16, - "y": 57, + "y": 67, "w": 8, "h": 8 }, @@ -750,7 +694,7 @@ "collapsed": false, "gridPos": { "x": 0, - "y": 65, + "y": 75, "w": 24, "h": 1 } @@ -761,7 +705,7 @@ "type": "table", "gridPos": { "x": 0, - "y": 66, + "y": 76, "w": 12, "h": 7 }, @@ -861,7 +805,7 @@ "type": "gauge", "gridPos": { "x": 12, - "y": 66, + "y": 76, "w": 6, "h": 7 }, @@ -908,7 +852,7 @@ "type": "table", "gridPos": { "x": 18, - "y": 66, + "y": 76, "w": 6, "h": 7 }, @@ -934,7 +878,7 @@ "collapsed": false, "gridPos": { "x": 0, - "y": 73, + "y": 83, "w": 24, "h": 1 } @@ -945,7 +889,7 @@ "type": "timeseries", "gridPos": { "x": 0, - "y": 74, + "y": 84, "w": 12, "h": 8 }, @@ -955,7 +899,7 @@ }, "targets": [ { - "rawSql": "SELECT date_trunc('day', created_at) AS time, COUNT(*) FILTER (WHERE success) AS successes, COUNT(*) FILTER (WHERE NOT success) AS failures FROM error_logs WHERE created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY 1;", + "rawSql": "SELECT date_trunc('day', created_at) AS time, COUNT(*) FILTER (WHERE success) AS successes, COUNT(*) FILTER (WHERE NOT success) AS failures FROM error_logs WHERE created_at BETWEEN (SELECT MIN(created_at) FROM error_logs) AND (SELECT MAX(created_at) FROM error_logs) GROUP BY 1 ORDER BY 1;", "format": "time_series", "refId": "A" } @@ -1001,7 +945,9 @@ ] } ] - } + }, + "timeFrom": "30d", + "timeShift": null }, { "id": 5, @@ -1009,7 +955,7 @@ "type": "piechart", "gridPos": { "x": 12, - "y": 74, + "y": 84, "w": 6, "h": 8 }, @@ -1019,7 +965,7 @@ }, "targets": [ { - "rawSql": "SELECT COALESCE(failure_stage, 'unknown') AS stage, COUNT(*) AS count FROM error_logs WHERE NOT success AND created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY 2 DESC;", + "rawSql": "SELECT COALESCE(failure_stage, 'unknown') AS stage, COUNT(*) AS count FROM error_logs WHERE NOT success AND failure_stage IS DISTINCT FROM 'aborted' AND created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY 2 DESC;", "format": "table", "refId": "A" } @@ -1042,7 +988,7 @@ "type": "table", "gridPos": { "x": 18, - "y": 74, + "y": 84, "w": 6, "h": 8 }, @@ -1068,7 +1014,7 @@ "collapsed": false, "gridPos": { "x": 0, - "y": 82, + "y": 92, "w": 24, "h": 1 } @@ -1079,7 +1025,7 @@ "type": "barchart", "gridPos": { "x": 0, - "y": 83, + "y": 93, "w": 8, "h": 8 }, @@ -1105,7 +1051,7 @@ "type": "table", "gridPos": { "x": 8, - "y": 83, + "y": 93, "w": 8, "h": 8 }, @@ -1131,7 +1077,7 @@ "type": "table", "gridPos": { "x": 16, - "y": 83, + "y": 93, "w": 8, "h": 8 }, @@ -1157,7 +1103,7 @@ "collapsed": false, "gridPos": { "x": 0, - "y": 91, + "y": 101, "w": 24, "h": 1 } @@ -1168,7 +1114,7 @@ "type": "barchart", "gridPos": { "x": 0, - "y": 92, + "y": 102, "w": 12, "h": 8 }, @@ -1228,7 +1174,7 @@ "type": "barchart", "gridPos": { "x": 12, - "y": 92, + "y": 102, "w": 6, "h": 8 }, @@ -1256,7 +1202,7 @@ "type": "timeseries", "gridPos": { "x": 18, - "y": 92, + "y": 102, "w": 6, "h": 8 }, @@ -1266,7 +1212,7 @@ }, "targets": [ { - "rawSql": "SELECT date_trunc('day', created_at) AS time, ROUND(AVG(duration_s)::numeric, 1) AS avg_duration_s, ROUND(percentile_cont(0.95) WITHIN GROUP (ORDER BY duration_s)::numeric, 1) AS p95_duration_s FROM error_logs WHERE created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY 1;", + "rawSql": "SELECT date_trunc('day', created_at) AS time, ROUND(AVG(duration_s)::numeric, 1) AS avg_duration_s, ROUND(percentile_cont(0.95) WITHIN GROUP (ORDER BY duration_s)::numeric, 1) AS p95_duration_s FROM error_logs WHERE created_at BETWEEN (SELECT MIN(created_at) FROM error_logs) AND (SELECT MAX(created_at) FROM error_logs) GROUP BY 1 ORDER BY 1;", "format": "time_series", "refId": "A" } @@ -1276,7 +1222,9 @@ "unit": "s" }, "overrides": [] - } + }, + "timeFrom": "30d", + "timeShift": null }, { "type": "row", @@ -1284,7 +1232,7 @@ "collapsed": false, "gridPos": { "x": 0, - "y": 100, + "y": 110, "w": 24, "h": 1 } @@ -1295,7 +1243,7 @@ "type": "table", "gridPos": { "x": 0, - "y": 101, + "y": 111, "w": 12, "h": 8 }, @@ -1321,7 +1269,7 @@ "type": "table", "gridPos": { "x": 12, - "y": 101, + "y": 111, "w": 6, "h": 8 }, @@ -1347,7 +1295,7 @@ "type": "timeseries", "gridPos": { "x": 18, - "y": 101, + "y": 111, "w": 6, "h": 8 }, @@ -1357,7 +1305,7 @@ }, "targets": [ { - "rawSql": "SELECT date_trunc('day', created_at) AS time, ROUND(AVG((resource_metrics->>'build_duration_s')::numeric), 1) AS avg_build_s, ROUND(AVG((resource_metrics->>'test_duration_s')::numeric), 1) AS avg_test_s, ROUND(AVG((resource_metrics->>'peak_memory_bytes')::numeric) / 1048576, 0) AS avg_peak_mem_mb FROM error_logs WHERE resource_metrics IS NOT NULL AND created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY 1;", + "rawSql": "SELECT date_trunc('day', created_at) AS time, ROUND(AVG((resource_metrics->>'build_duration_s')::numeric), 1) AS avg_build_s, ROUND(AVG((resource_metrics->>'test_duration_s')::numeric), 1) AS avg_test_s, ROUND(AVG((resource_metrics->>'peak_memory_bytes')::numeric) / 1048576, 0) AS avg_peak_mem_mb FROM error_logs WHERE resource_metrics IS NOT NULL AND created_at BETWEEN (SELECT MIN(created_at) FROM error_logs WHERE resource_metrics IS NOT NULL) AND (SELECT MAX(created_at) FROM error_logs WHERE resource_metrics IS NOT NULL) GROUP BY 1 ORDER BY 1;", "format": "time_series", "refId": "A" } @@ -1365,7 +1313,9 @@ "fieldConfig": { "defaults": {}, "overrides": [] - } + }, + "timeFrom": "30d", + "timeShift": null }, { "type": "row", @@ -1373,7 +1323,7 @@ "collapsed": false, "gridPos": { "x": 0, - "y": 135, + "y": 145, "w": 24, "h": 1 } @@ -1384,7 +1334,7 @@ "type": "table", "gridPos": { "x": 0, - "y": 140, + "y": 150, "w": 24, "h": 12 }, @@ -1394,7 +1344,7 @@ }, "targets": [ { - "rawSql": "SELECT pr.* FROM pull_requests pr JOIN candidate_prs cp ON pr.owner = cp.owner AND pr.repo = cp.repo AND pr.issue_number = cp.issue_number WHERE pr.is_performance_commit AND cp.issues_json IS NOT NULL AND cp.issues_json != '[]'::jsonb ORDER BY pr.merged_at DESC NULLS LAST LIMIT ${row_limit:raw};", + "rawSql": "SELECT pr.* FROM pull_requests pr JOIN candidate_prs cp ON pr.owner = cp.owner AND pr.repo = cp.repo AND pr.issue_number = cp.issue_number WHERE pr.is_performance_commit AND cp.issues_json IS NOT NULL AND cp.issues_json != '[]'::jsonb ORDER BY pr.merged_at DESC NULLS LAST LIMIT 100;", "format": "table", "refId": "A" } @@ -1410,7 +1360,7 @@ "collapsed": false, "gridPos": { "x": 0, - "y": 152, + "y": 162, "w": 24, "h": 1 } @@ -1421,7 +1371,7 @@ "type": "barchart", "gridPos": { "x": 0, - "y": 153, + "y": 163, "w": 24, "h": 8 }, @@ -1447,7 +1397,7 @@ "collapsed": false, "gridPos": { "x": 0, - "y": 109, + "y": 119, "w": 24, "h": 1 } @@ -1458,7 +1408,7 @@ "type": "barchart", "gridPos": { "x": 0, - "y": 110, + "y": 120, "w": 12, "h": 8 }, @@ -1492,7 +1442,7 @@ "type": "barchart", "gridPos": { "x": 12, - "y": 110, + "y": 120, "w": 12, "h": 8 }, @@ -1518,11 +1468,11 @@ }, { "id": 62, - "title": "Best Speedup per Container (Daytona Success)", + "title": "Best Speedup per Problem", "type": "table", "gridPos": { "x": 0, - "y": 118, + "y": 128, "w": 12, "h": 8 }, @@ -1532,23 +1482,59 @@ }, "targets": [ { - "rawSql": "SELECT h.owner || '/' || h.repo AS repository, h.sha, LEFT(pr.title, 80) AS pr_title, ROUND(MAX(h.max_speedup)::numeric, 3) AS best_max_speedup, ROUND(AVG(h.geomean_speedup)::numeric, 3) AS avg_geomean, MAX(h.n_benchmarks) AS n_benchmarks, COUNT(*) AS trials FROM harbor_runs h LEFT JOIN pull_requests pr ON pr.owner = h.owner AND pr.repo = h.repo AND pr.issue_number = h.issue_number WHERE h.status = 'success' AND h.environment = 'daytona' GROUP BY h.owner, h.repo, h.sha, pr.title ORDER BY best_max_speedup DESC LIMIT 50;", + "rawSql": "SELECT h.owner || '/' || h.repo AS repository, h.issue_number AS pr, ROUND(MAX(h.max_speedup) FILTER (WHERE h.environment = 'daytona')::numeric, 3) AS daytona_best, ROUND(MAX(h.max_speedup) FILTER (WHERE h.environment = 'docker')::numeric, 3) AS docker_best, ROUND(MAX(h.max_speedup)::numeric, 3) AS best_overall, COUNT(*) FILTER (WHERE h.environment = 'daytona') AS daytona_trials, COUNT(*) FILTER (WHERE h.environment = 'docker') AS docker_trials FROM harbor_runs h WHERE h.status = 'success' GROUP BY h.owner, h.repo, h.issue_number ORDER BY best_overall DESC NULLS LAST LIMIT 50;", "format": "table", "refId": "A" } ], "fieldConfig": { "defaults": {}, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "daytona_best" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "gradient" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "yellow", + "value": 1.0 + }, + { + "color": "green", + "value": 1.05 + } + ] + } + } + ] + } + ] } }, { "id": 63, - "title": "Publishable Containers", + "title": "Publishable Problems", "type": "stat", "gridPos": { "x": 12, - "y": 118, + "y": 128, "w": 6, "h": 4 }, @@ -1588,7 +1574,7 @@ "type": "stat", "gridPos": { "x": 18, - "y": 118, + "y": 128, "w": 6, "h": 4 }, @@ -1624,7 +1610,7 @@ "type": "stat", "gridPos": { "x": 12, - "y": 122, + "y": 132, "w": 6, "h": 4 }, @@ -1669,7 +1655,7 @@ "type": "stat", "gridPos": { "x": 18, - "y": 122, + "y": 132, "w": 6, "h": 4 }, @@ -1698,6 +1684,57 @@ }, "overrides": [] } + }, + { + "id": 101, + "type": "stat", + "title": "Live Runners", + "gridPos": { + "x": 20, + "y": 11, + "w": 4, + "h": 4 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT GREATEST((SELECT COUNT(*)::int FROM runner_progress WHERE total - completed - failed > 0 AND updated_at > now() - interval '15 minutes'), CASE WHEN EXISTS (SELECT 1 FROM error_logs WHERE created_at > now() - interval '5 minutes') THEN 1 ELSE 0 END, CASE WHEN EXISTS (SELECT 1 FROM harbor_runs WHERE ran_at > now() - interval '5 minutes') THEN 1 ELSE 0 END) AS live_runners;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Idle", + "index": 0 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + } } ] } diff --git a/supabase/migrations/00012_public_read_rls.sql b/supabase/migrations/00012_public_read_rls.sql new file mode 100644 index 00000000..275bb130 --- /dev/null +++ b/supabase/migrations/00012_public_read_rls.sql @@ -0,0 +1,13 @@ +-- Enable Row Level Security on tables exposed to the public API. +-- The service-role key bypasses RLS, so active pipeline processes are unaffected. + +ALTER TABLE repositories ENABLE ROW LEVEL SECURITY; +ALTER TABLE pull_requests ENABLE ROW LEVEL SECURITY; +ALTER TABLE candidate_containers ENABLE ROW LEVEL SECURITY; +ALTER TABLE harbor_runs ENABLE ROW LEVEL SECURITY; + +-- Allow the anon role to SELECT all rows from these tables. +CREATE POLICY "public_read" ON repositories FOR SELECT TO anon USING (true); +CREATE POLICY "public_read" ON pull_requests FOR SELECT TO anon USING (true); +CREATE POLICY "public_read" ON candidate_containers FOR SELECT TO anon USING (true); +CREATE POLICY "public_read" ON harbor_runs FOR SELECT TO anon USING (true); diff --git a/supabase/migrations/00013_grafana_rls_read.sql b/supabase/migrations/00013_grafana_rls_read.sql new file mode 100644 index 00000000..b8c16385 --- /dev/null +++ b/supabase/migrations/00013_grafana_rls_read.sql @@ -0,0 +1,8 @@ +-- Allow the grafana_ro role to SELECT through RLS on tables that have it enabled. +-- The public_read policies (migration 00012) only cover the anon role; +-- grafana_ro connects via direct Postgres and needs its own policies. + +CREATE POLICY "grafana_read" ON repositories FOR SELECT TO grafana_ro USING (true); +CREATE POLICY "grafana_read" ON pull_requests FOR SELECT TO grafana_ro USING (true); +CREATE POLICY "grafana_read" ON candidate_containers FOR SELECT TO grafana_ro USING (true); +CREATE POLICY "grafana_read" ON harbor_runs FOR SELECT TO grafana_ro USING (true); diff --git a/supabase/migrations/00014_runner_progress_updated_at_trigger.sql b/supabase/migrations/00014_runner_progress_updated_at_trigger.sql new file mode 100644 index 00000000..b2249bf7 --- /dev/null +++ b/supabase/migrations/00014_runner_progress_updated_at_trigger.sql @@ -0,0 +1,20 @@ +-- Ensure runner_progress.updated_at is bumped on every UPDATE. +-- Without this, upserts from BaseRunner only refresh total/completed/failed +-- and updated_at stays pinned at the INSERT default, making it impossible +-- to distinguish live runs from zombies via timestamp. + +CREATE OR REPLACE FUNCTION set_updated_at() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + NEW.updated_at = now(); + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS runner_progress_set_updated_at ON runner_progress; +CREATE TRIGGER runner_progress_set_updated_at + BEFORE UPDATE ON runner_progress + FOR EACH ROW + EXECUTE FUNCTION set_updated_at();