From 3b6d168d5b7fe1245a90cfbd4da521ccc6700e3e Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Mon, 11 May 2026 22:36:30 +0000 Subject: [PATCH] feat(supabase): add findings_* tables for paper-figure summaries (formula-code/fc-eval#19) Adds eight public-read tables backing the seven paper-figure summaries on api.formulacode.org/rest/v1/findings_*: findings_global_leaderboard Table 1 findings_stratified_advantage Figure 3 findings_tag_advantage Table 2 findings_repo_quintiles Table 3 findings_cost_pareto Figure 4 / Table 10 findings_workload_tradeoff Figure 5 findings_temporal_generalization Table 4 / Figure 1 findings_metadata sidecar: _source / _generated_at / axis_metadata / row_count per finding Row column names align with the canonical scaffolds at formula-code/formula-code.github.io:src/data/findings/f{1..7}_*.json (rp_rank, advantage_norm, speedup_geomean, worst_workload_speedup, cost_usd_per_task + advantage_weighted). Per-task rows in findings_workload_tradeoff use (owner, repo, issue_number) to match pull_requests / harbor_runs. RLS + anon grants mirror 00012_public_read_rls.sql + 00015_revoke_anon_select.sql: enable RLS, public_read SELECT policy for anon, then explicit GRANT SELECT (since 00015 revoked the schema-wide anon default). Rows are refreshed by analysis/export_website_findings.py in fc-eval (separate; not yet upstreamed). Pending follow-up after merge: 1. supabase migration up on prod 2. uv run analysis/export_website_findings.py --upsert-supabase Co-Authored-By: Claude Opus 4.7 (1M context) --- supabase/migrations/00021_findings_tables.sql | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 supabase/migrations/00021_findings_tables.sql diff --git a/supabase/migrations/00021_findings_tables.sql b/supabase/migrations/00021_findings_tables.sql new file mode 100644 index 00000000..de8d7bbf --- /dev/null +++ b/supabase/migrations/00021_findings_tables.sql @@ -0,0 +1,157 @@ +-- Per-finding tables backing the paper-figure summaries on +-- api.formulacode.org. See formula-code/fc-eval#19. +-- +-- Rows are refreshed by analysis/export_website_findings.py in the +-- formula-code/fc-eval repo (which reads cached Task objects, runs the +-- existing nb_utils helpers, and upserts here via the service-role key). +-- +-- Each table is anon-readable through PostgREST at: +-- GET /rest/v1/findings_ +-- and pairs with findings_metadata which carries the envelope fields the +-- website expects: {_source, _generated_at, axis_metadata, row_count}. +-- +-- Schema names align with the canonical scaffolds in +-- formula-code/formula-code.github.io:src/data/findings/f{1..7}_*.json +-- (e.g. rp_rank not rp_rank_adv; advantage_norm not normalized_advantage; +-- speedup_geomean not mean_speedup; worst_workload_speedup not +-- worst_workload_spd; advantage_weighted not cost_weighted_adv). Per-task +-- rows use (owner, repo, issue_number) to match pull_requests / harbor_runs. + +-- Envelope sidecar — one row per finding. +CREATE TABLE IF NOT EXISTS findings_metadata ( + finding_name TEXT PRIMARY KEY, -- 'f1_leaderboard' | ... + source TEXT NOT NULL, -- notebook/script reference + paper_artifact TEXT, -- 'Table 1', 'Figure 3', ... + arxiv_url TEXT, + generated_at TIMESTAMPTZ NOT NULL, + axis_metadata JSONB, -- levels / tags / quintiles / bins / cutoffs + row_count INT NOT NULL, + notes TEXT +); + +-- f1: Global Leaderboard (Table 1). +CREATE TABLE IF NOT EXISTS findings_global_leaderboard ( + agent TEXT NOT NULL, -- display label, e.g. 'Terminus 2' + model TEXT NOT NULL, -- display label, e.g. 'Claude 4.0 Sonnet' + rp_rank INT, -- Ranked-Pairs rank; 0 reserved for oracle baseline + advantage DOUBLE PRECISION, + advantage_norm DOUBLE PRECISION, + speedup_geomean DOUBLE PRECISION, + is_baseline BOOLEAN DEFAULT FALSE, -- TRUE for the Human Expert / oracle row + PRIMARY KEY (agent, model) +); + +-- f2: Stratified Advantage (Figure 3). Note: scaffold exposes overall + level2/3/4. +-- We keep level1 too so downstream consumers that want full hierarchy can use it, +-- but the website's slope chart reads from overall/level2/level3/level4 only. +CREATE TABLE IF NOT EXISTS findings_stratified_advantage ( + agent TEXT NOT NULL, + model TEXT NOT NULL, + overall DOUBLE PRECISION, -- agent_advantage (level 4, all benchmarks) + level1 DOUBLE PRECISION, -- agent_advantage_level1 (module) + level2 DOUBLE PRECISION, -- agent_advantage_level2 (class) + level3 DOUBLE PRECISION, -- agent_advantage_level3 (function) + level4 DOUBLE PRECISION, -- agent_advantage_level4 (params; same as overall) + PRIMARY KEY (agent, model) +); + +-- f3: Per-Tag Advantage (Table 2). Tag keys match scaffold taxonomy +-- (parallelization, batching, io, caching, algorithmic, data_structure, +-- reduce_work, approximation, scale, db, higher_level, micro, lower_level, +-- uncategorized). +CREATE TABLE IF NOT EXISTS findings_tag_advantage ( + agent TEXT NOT NULL, + model TEXT NOT NULL, + tag TEXT NOT NULL, + advantage DOUBLE PRECISION, + speedup DOUBLE PRECISION, + n_tasks INT, + PRIMARY KEY (agent, model, tag) +); + +-- f4: Repo Quintiles by stars (Table 3). Quintile bin edges live in +-- findings_metadata.axis_metadata.quintile_ranges so the website doesn't have +-- to inspect rows to render axis labels. +CREATE TABLE IF NOT EXISTS findings_repo_quintiles ( + agent TEXT NOT NULL, + model TEXT NOT NULL, + quintile TEXT NOT NULL, -- 'Q1' | 'Q2' | 'Q3' | 'Q4' | 'Q5' + advantage DOUBLE PRECISION, + speedup DOUBLE PRECISION, + n_tasks INT, + PRIMARY KEY (agent, model, quintile) +); + +-- f5: Cost-Performance Pareto (Figure 4 / Table 10). +CREATE TABLE IF NOT EXISTS findings_cost_pareto ( + agent TEXT NOT NULL, + model TEXT NOT NULL, + cost_usd_per_task DOUBLE PRECISION, + advantage DOUBLE PRECISION, + advantage_weighted DOUBLE PRECISION, -- advantage / cost_usd_per_task + is_pareto BOOLEAN, + PRIMARY KEY (agent, model) +); + +-- f6: Workload Tradeoff (Figure 5). One row per (agent, model, task). +-- The expert (oracle) cluster is flagged via is_expert=TRUE. +CREATE TABLE IF NOT EXISTS findings_workload_tradeoff ( + agent TEXT NOT NULL, + model TEXT NOT NULL, + owner TEXT NOT NULL, + repo TEXT NOT NULL, + issue_number INT NOT NULL, + global_speedup DOUBLE PRECISION, + worst_workload_speedup DOUBLE PRECISION, + is_expert BOOLEAN DEFAULT FALSE, + PRIMARY KEY (agent, model, owner, repo, issue_number) +); + +-- f7: Temporal Generalization (Table 4 / Figure 1). 5 bins around each model's +-- knowledge cutoff: pre6 | pre3 | cutoff | post3 | post6. +CREATE TABLE IF NOT EXISTS findings_temporal_generalization ( + model TEXT NOT NULL, + bin TEXT NOT NULL, + knowledge_cutoff DATE, + speedup DOUBLE PRECISION, + advantage DOUBLE PRECISION, + n_tasks INT, + PRIMARY KEY (model, bin) +); + +-- ---- RLS + grants (mirrors 00012_public_read_rls + 00015_revoke_anon_select) ---- +-- Pattern: enable RLS, add a public_read SELECT policy for the anon role, then +-- explicitly GRANT SELECT since 00015 revoked the schema-wide anon grant. + +ALTER TABLE findings_metadata ENABLE ROW LEVEL SECURITY; +ALTER TABLE findings_global_leaderboard ENABLE ROW LEVEL SECURITY; +ALTER TABLE findings_stratified_advantage ENABLE ROW LEVEL SECURITY; +ALTER TABLE findings_tag_advantage ENABLE ROW LEVEL SECURITY; +ALTER TABLE findings_repo_quintiles ENABLE ROW LEVEL SECURITY; +ALTER TABLE findings_cost_pareto ENABLE ROW LEVEL SECURITY; +ALTER TABLE findings_workload_tradeoff ENABLE ROW LEVEL SECURITY; +ALTER TABLE findings_temporal_generalization ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "public_read" ON findings_metadata FOR SELECT TO anon USING (true); +CREATE POLICY "public_read" ON findings_global_leaderboard FOR SELECT TO anon USING (true); +CREATE POLICY "public_read" ON findings_stratified_advantage FOR SELECT TO anon USING (true); +CREATE POLICY "public_read" ON findings_tag_advantage FOR SELECT TO anon USING (true); +CREATE POLICY "public_read" ON findings_repo_quintiles FOR SELECT TO anon USING (true); +CREATE POLICY "public_read" ON findings_cost_pareto FOR SELECT TO anon USING (true); +CREATE POLICY "public_read" ON findings_workload_tradeoff FOR SELECT TO anon USING (true); +CREATE POLICY "public_read" ON findings_temporal_generalization FOR SELECT TO anon USING (true); + +GRANT SELECT ON + findings_metadata, + findings_global_leaderboard, + findings_stratified_advantage, + findings_tag_advantage, + findings_repo_quintiles, + findings_cost_pareto, + findings_workload_tradeoff, + findings_temporal_generalization +TO anon; + +-- Helpful index for the workload-tradeoff page if it ever filters by repo. +CREATE INDEX IF NOT EXISTS idx_findings_workload_tradeoff_repo + ON findings_workload_tradeoff (owner, repo);