From 08cf4860cd709abf955c290f9df3a21a7594fa40 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 9 Apr 2026 04:48:43 +0000 Subject: [PATCH 1/8] Add Grafana dashboard and read-only Postgres role for Supabase monitoring --- grafana/docker-compose.yml | 37 + .../dashboards-json/datasmith-overview.json | 1445 +++++++++++++++++ .../dashboards/dashboard-provider.yml | 13 + .../datasources/supabase-postgres.yml | 21 + .../migrations/00009_grafana_readonly.sql | 15 + 5 files changed, 1531 insertions(+) create mode 100644 grafana/docker-compose.yml create mode 100644 grafana/provisioning/dashboards-json/datasmith-overview.json create mode 100644 grafana/provisioning/dashboards/dashboard-provider.yml create mode 100644 grafana/provisioning/datasources/supabase-postgres.yml create mode 100644 supabase/migrations/00009_grafana_readonly.sql diff --git a/grafana/docker-compose.yml b/grafana/docker-compose.yml new file mode 100644 index 00000000..67fd98cd --- /dev/null +++ b/grafana/docker-compose.yml @@ -0,0 +1,37 @@ +services: + grafana: + image: grafana/grafana-oss:11.6.0 + container_name: datasmith_grafana + restart: unless-stopped + ports: + - "3001:3000" + env_file: + - ../tokens.env + environment: + # Anonymous read-only access (no login required) + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer + # Admin account for dashboard editing — password from tokens.env + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin} + # Home dashboard + - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_UID=datasmith-overview + # Minimum refresh interval + - GF_DASHBOARDS_MIN_REFRESH_INTERVAL=1m + # Plugins + - GF_INSTALL_PLUGINS=netsage-sankey-panel,marcusolsson-treemap-panel + volumes: + - grafana-data:/var/lib/grafana + - ./provisioning/datasources:/etc/grafana/provisioning/datasources:ro + - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro + - ./provisioning/dashboards-json:/var/lib/grafana/dashboards:ro + networks: + - supabase_net + +volumes: + grafana-data: + +networks: + supabase_net: + external: true + name: supabase_network_datasmith_new diff --git a/grafana/provisioning/dashboards-json/datasmith-overview.json b/grafana/provisioning/dashboards-json/datasmith-overview.json new file mode 100644 index 00000000..d16545e7 --- /dev/null +++ b/grafana/provisioning/dashboards-json/datasmith-overview.json @@ -0,0 +1,1445 @@ +{ + "uid": "datasmith-overview", + "title": "DataSmith Pipeline Overview", + "tags": [ + "datasmith", + "pipeline" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "1m", + "time": { + "from": "2015-01-01T00:00:00.000Z", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "table_name", + "label": "Table", + "type": "custom", + "query": "repositories,pull_requests,packages,candidate_containers,candidate_prs,error_logs,runner_progress,runner_failures,hook_cache", + "current": { + "text": "pull_requests", + "value": "pull_requests" + }, + "options": [ + { + "text": "repositories", + "value": "repositories", + "selected": false + }, + { + "text": "pull_requests", + "value": "pull_requests", + "selected": true + }, + { + "text": "packages", + "value": "packages", + "selected": false + }, + { + "text": "candidate_containers", + "value": "candidate_containers", + "selected": false + }, + { + "text": "candidate_prs", + "value": "candidate_prs", + "selected": false + }, + { + "text": "error_logs", + "value": "error_logs", + "selected": false + }, + { + "text": "runner_progress", + "value": "runner_progress", + "selected": false + }, + { + "text": "runner_failures", + "value": "runner_failures", + "selected": false + }, + { + "text": "hook_cache", + "value": "hook_cache", + "selected": false + } + ] + }, + { + "name": "row_limit", + "label": "Limit", + "type": "custom", + "query": "50,100,250,500,1000", + "current": { + "text": "100", + "value": "100" + }, + "options": [ + { + "text": "50", + "value": "50", + "selected": false + }, + { + "text": "100", + "value": "100", + "selected": true + }, + { + "text": "250", + "value": "250", + "selected": false + }, + { + "text": "500", + "value": "500", + "selected": false + }, + { + "text": "1000", + "value": "1000", + "selected": false + } + ] + } + ] + }, + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "Dataset Growth", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + } + }, + { + "id": 14, + "title": "Total PRs", + "type": "stat", + "gridPos": { + "x": 14, + "y": 1, + "w": 3, + "h": 4 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT COUNT(*) AS total FROM pull_requests;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 15, + "title": "Total Containers", + "type": "stat", + "gridPos": { + "x": 17, + "y": 1, + "w": 3, + "h": 4 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT COUNT(*) AS total FROM candidate_containers;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 16, + "title": "Total Packages", + "type": "stat", + "gridPos": { + "x": 14, + "y": 5, + "w": 3, + "h": 4 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT COUNT(*) AS total FROM packages;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 17, + "title": "Total Repos", + "type": "stat", + "gridPos": { + "x": 17, + "y": 5, + "w": 3, + "h": 4 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT COUNT(*) AS total FROM repositories;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 18, + "title": "PR to Container Rate", + "type": "stat", + "gridPos": { + "x": 20, + "y": 1, + "w": 4, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT ROUND((SELECT COUNT(*)::numeric FROM candidate_containers) / NULLIF((SELECT COUNT(*) FROM pull_requests), 0) * 100, 1) AS container_rate_pct;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 80 + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Repository Overview", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 9, + "w": 24, + "h": 1 + } + }, + { + "id": 52, + "title": "All Repositories by Stars", + "type": "barchart", + "gridPos": { + "x": 0, + "y": 10, + "w": 24, + "h": 10 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT owner || '/' || repo AS repository, stars FROM repositories WHERE stars IS NOT NULL ORDER BY stars DESC;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "xTickLabelMaxLength": 0, + "xTickLabelSpacing": 0, + "axisCenteredZero": false, + "axisBorderShow": false, + "xAxis": { + "showLabels": false + } + } + }, + { + "id": 54, + "title": "Containers by Repository", + "type": "marcusolsson-treemap-panel", + "gridPos": { + "x": 0, + "y": 20, + "w": 24, + "h": 10 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT owner || '/' || repo AS repository, COUNT(*) AS containers FROM candidate_containers GROUP BY 1 ORDER BY 2 DESC;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "sizeByField": "containers", + "textByField": "repository", + "colorByField": "containers" + } + }, + { + "id": 50, + "title": "Containers Built by Month", + "type": "barchart", + "gridPos": { + "x": 0, + "y": 1, + "w": 14, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT to_char(pr.merged_at, 'MM-YYYY') AS month, COUNT(*) AS containers FROM candidate_containers cc JOIN pull_requests pr ON cc.owner = pr.owner AND cc.repo = pr.repo AND cc.issue_number = pr.issue_number WHERE pr.merged_at IS NOT NULL GROUP BY 1, date_trunc('month', pr.merged_at) ORDER BY date_trunc('month', pr.merged_at);", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "green" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Pull Request Insights", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 38, + "w": 24, + "h": 1 + } + }, + { + "id": 31, + "title": "Performance Commit Ratio", + "type": "stat", + "gridPos": { + "x": 0, + "y": 39, + "w": 4, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT ROUND(COUNT(*) FILTER (WHERE is_performance_commit)::numeric / NULLIF(COUNT(*), 0) * 100, 1) AS perf_pct FROM pull_requests;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 53, + "title": "Dataset Time Span", + "type": "table", + "gridPos": { + "x": 4, + "y": 39, + "w": 5, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT 'Earliest' AS metric, to_char(MIN(merged_at), 'YYYY-MM-DD') AS value FROM pull_requests WHERE is_performance_commit AND merged_at IS NOT NULL UNION ALL SELECT 'Latest', to_char(MAX(merged_at), 'YYYY-MM-DD') FROM pull_requests WHERE is_performance_commit AND merged_at IS NOT NULL UNION ALL SELECT 'Span', COUNT(DISTINCT date_trunc('year', merged_at)) || ' years' FROM pull_requests WHERE is_performance_commit AND merged_at IS NOT NULL;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "textMode": "value", + "colorMode": "background" + } + }, + { + "id": 33, + "title": "Difficulty Distribution", + "type": "piechart", + "gridPos": { + "x": 9, + "y": 39, + "w": 5, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT COALESCE(difficulty, 'unset') AS difficulty, COUNT(*) AS count FROM pull_requests WHERE is_performance_commit GROUP BY 1 ORDER BY 2 DESC;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": true, + "calcs": [] + }, + "pieType": "pie" + } + }, + { + "id": 34, + "title": "PR Volume Over Time", + "type": "timeseries", + "gridPos": { + "x": 14, + "y": 39, + "w": 10, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT date_trunc('month', created_at) AS time, COUNT(*) FILTER (WHERE is_performance_commit) AS perf_prs FROM pull_requests WHERE created_at IS NOT NULL GROUP BY 1 ORDER BY 1;", + "format": "time_series", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 51, + "title": "Optimization Type Distribution", + "type": "barchart", + "gridPos": { + "x": 0, + "y": 47, + "w": 24, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT REPLACE(COALESCE(classification, 'unclassified'), '_', ' ') AS optimization_type, COUNT(*) AS count FROM pull_requests WHERE is_performance_commit AND classification != 'legacy-verified' GROUP BY 1 ORDER BY 2 DESC;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal" + } + }, + { + "type": "row", + "title": "Package Resolution", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 56, + "w": 24, + "h": 1 + } + }, + { + "id": 35, + "title": "Python Version Distribution", + "type": "piechart", + "gridPos": { + "x": 0, + "y": 57, + "w": 6, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT COALESCE(python_version, 'unknown') AS python_version, COUNT(*) AS count FROM packages GROUP BY 1 ORDER BY 2 DESC;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": true, + "calcs": [] + }, + "pieType": "pie" + } + }, + { + "id": 36, + "title": "Installability by Repo (Worst)", + "type": "barchart", + "gridPos": { + "x": 6, + "y": 57, + "w": 10, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT owner || '/' || repo AS repository, COUNT(*) FILTER (WHERE can_install) AS installable, COUNT(*) FILTER (WHERE NOT can_install) AS failed FROM packages GROUP BY 1 HAVING COUNT(*) FILTER (WHERE NOT can_install) > 0 ORDER BY failed DESC LIMIT 20;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "installable" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "stacking": "normal" + } + }, + { + "id": 37, + "title": "Resolution Strategy Breakdown", + "type": "piechart", + "gridPos": { + "x": 16, + "y": 57, + "w": 8, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT COALESCE(resolution_strategy, 'unknown') AS strategy, COUNT(*) AS count FROM packages GROUP BY 1 ORDER BY 2 DESC;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": true, + "calcs": [] + }, + "pieType": "pie" + } + }, + { + "type": "row", + "title": "Pipeline Status", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 65, + "w": 24, + "h": 1 + } + }, + { + "id": 1, + "title": "Active Runners", + "type": "table", + "gridPos": { + "x": 0, + "y": 66, + "w": 12, + "h": 7 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT runner_name, runner_id, total, completed, failed, total - completed - failed AS remaining, ROUND(completed::numeric / NULLIF(total, 0) * 100, 1) AS pct_complete, started_at, updated_at FROM runner_progress ORDER BY updated_at DESC LIMIT 20;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "pct_complete" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "gradient" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 100 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 90 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + } + }, + { + "id": 2, + "title": "Latest Runner Completion", + "type": "gauge", + "gridPos": { + "x": 12, + "y": 66, + "w": 6, + "h": 7 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT ROUND(completed::numeric / NULLIF(total, 0) * 100, 1) AS completion_pct FROM runner_progress ORDER BY updated_at DESC LIMIT 1;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "min": 0, + "max": 100, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 90 + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 3, + "title": "Recent Runner Failures", + "type": "table", + "gridPos": { + "x": 18, + "y": 66, + "w": 6, + "h": 7 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT rp.runner_name, rf.item_id, LEFT(rf.error_message, 200) AS error_message, rf.created_at FROM runner_failures rf JOIN runner_progress rp ON rf.runner_id = rp.runner_id ORDER BY rf.created_at DESC LIMIT 20;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "type": "row", + "title": "Synthesis", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 73, + "w": 24, + "h": 1 + } + }, + { + "id": 4, + "title": "Synthesis Success Rate Over Time", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 74, + "w": 12, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT date_trunc('day', created_at) AS time, COUNT(*) FILTER (WHERE success) AS successes, COUNT(*) FILTER (WHERE NOT success) AS failures FROM error_logs WHERE created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY 1;", + "format": "time_series", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "successes" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failures" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + } + }, + { + "id": 5, + "title": "Failure Stage Distribution", + "type": "piechart", + "gridPos": { + "x": 12, + "y": 74, + "w": 6, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT COALESCE(failure_stage, 'unknown') AS stage, COUNT(*) AS count FROM error_logs WHERE NOT success AND created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY 2 DESC;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": true, + "calcs": [] + }, + "pieType": "pie" + } + }, + { + "id": 6, + "title": "Top Error Messages", + "type": "table", + "gridPos": { + "x": 18, + "y": 74, + "w": 6, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT COALESCE(failure_stage, 'unknown') AS stage, LEFT(error_message, 150) AS error_message, COUNT(*) AS occurrences FROM error_logs WHERE NOT success AND created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1, 2 ORDER BY 3 DESC LIMIT 15;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "type": "row", + "title": "Synthesis Deeper Analysis", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 82, + "w": 24, + "h": 1 + } + }, + { + "id": 38, + "title": "First-Attempt vs Multi-Attempt Success", + "type": "barchart", + "gridPos": { + "x": 0, + "y": 83, + "w": 8, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "WITH first_attempts AS ( SELECT owner, repo, sha, success FROM error_logs WHERE attempt_index = 0), retry_success AS ( SELECT DISTINCT owner, repo, sha FROM error_logs WHERE attempt_index > 0 AND success) SELECT 'First attempt' AS category, COUNT(*) FILTER (WHERE success) AS successes, COUNT(*) FILTER (WHERE NOT success) AS failures FROM first_attempts UNION ALL SELECT 'Needed retries', COUNT(*), 0 FROM retry_success;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 39, + "title": "Hardest Repos (Lowest Success Rate)", + "type": "table", + "gridPos": { + "x": 8, + "y": 83, + "w": 8, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT owner || '/' || repo AS repository, COUNT(*) AS attempts, COUNT(*) FILTER (WHERE success) AS successes, ROUND(COUNT(*) FILTER (WHERE success)::numeric / NULLIF(COUNT(*), 0) * 100, 1) AS success_rate_pct FROM error_logs GROUP BY 1 HAVING COUNT(*) >= 3 ORDER BY success_rate_pct ASC, attempts DESC LIMIT 20;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 40, + "title": "Agent Head-to-Head", + "type": "table", + "gridPos": { + "x": 16, + "y": 83, + "w": 8, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT COALESCE(agent_name, 'unknown') AS agent, COUNT(*) AS attempts, COUNT(*) FILTER (WHERE success) AS successes, ROUND(COUNT(*) FILTER (WHERE success)::numeric / NULLIF(COUNT(*), 0) * 100, 1) AS success_rate_pct, ROUND(AVG(duration_s)::numeric, 1) AS avg_duration_s FROM error_logs WHERE created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY success_rate_pct DESC;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "type": "row", + "title": "Attempt Details", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 91, + "w": 24, + "h": 1 + } + }, + { + "id": 7, + "title": "Attempts per Repository", + "type": "barchart", + "gridPos": { + "x": 0, + "y": 92, + "w": 12, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT owner || '/' || repo AS repository, COUNT(*) FILTER (WHERE success) AS successes, COUNT(*) FILTER (WHERE NOT success) AS failures FROM error_logs WHERE created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY COUNT(*) DESC LIMIT 20;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "successes" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failures" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "stacking": "normal" + } + }, + { + "id": 8, + "title": "Avg Duration by Agent", + "type": "barchart", + "gridPos": { + "x": 12, + "y": 92, + "w": 6, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT COALESCE(agent_name, 'unknown') AS agent, ROUND(AVG(duration_s)::numeric, 1) AS avg_duration_s, COUNT(*) AS attempts FROM error_logs WHERE created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY 2 DESC;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 9, + "title": "Synthesis Duration Over Time", + "type": "timeseries", + "gridPos": { + "x": 18, + "y": 92, + "w": 6, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT date_trunc('day', created_at) AS time, ROUND(AVG(duration_s)::numeric, 1) AS avg_duration_s, ROUND(percentile_cont(0.95) WITHIN GROUP (ORDER BY duration_s)::numeric, 1) AS p95_duration_s FROM error_logs WHERE created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY 1;", + "format": "time_series", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Resource Metrics", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 100, + "w": 24, + "h": 1 + } + }, + { + "id": 10, + "title": "Build Metrics (Successful Attempts)", + "type": "table", + "gridPos": { + "x": 0, + "y": 101, + "w": 12, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT owner || '/' || repo AS repository, sha, ROUND((resource_metrics->>'build_duration_s')::numeric, 1) AS build_s, ROUND((resource_metrics->>'test_duration_s')::numeric, 1) AS test_s, ROUND((resource_metrics->>'image_size_bytes')::numeric / 1048576, 1) AS image_mb, ROUND((resource_metrics->>'peak_memory_bytes')::numeric / 1048576, 1) AS peak_mem_mb, created_at FROM error_logs WHERE resource_metrics IS NOT NULL AND success AND created_at >= $__timeFrom() AND created_at <= $__timeTo() ORDER BY created_at DESC LIMIT 50;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 11, + "title": "Image Size vs Build Time", + "type": "table", + "gridPos": { + "x": 12, + "y": 101, + "w": 6, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT ROUND((resource_metrics->>'build_duration_s')::numeric, 1) AS build_s, ROUND((resource_metrics->>'image_size_bytes')::numeric / 1048576, 0) AS image_mb, ROUND((resource_metrics->>'test_duration_s')::numeric, 1) AS test_s, owner || '/' || repo AS repository FROM error_logs WHERE resource_metrics IS NOT NULL AND resource_metrics->>'build_duration_s' IS NOT NULL AND created_at >= $__timeFrom() AND created_at <= $__timeTo() ORDER BY build_s DESC;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 12, + "title": "Avg Resource Metrics Over Time", + "type": "timeseries", + "gridPos": { + "x": 18, + "y": 101, + "w": 6, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT date_trunc('day', created_at) AS time, ROUND(AVG((resource_metrics->>'build_duration_s')::numeric), 1) AS avg_build_s, ROUND(AVG((resource_metrics->>'test_duration_s')::numeric), 1) AS avg_test_s, ROUND(AVG((resource_metrics->>'peak_memory_bytes')::numeric) / 1048576, 0) AS avg_peak_mem_mb FROM error_logs WHERE resource_metrics IS NOT NULL AND created_at >= $__timeFrom() AND created_at <= $__timeTo() GROUP BY 1 ORDER BY 1;", + "format": "time_series", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "type": "row", + "title": "Data Explorer", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 118, + "w": 24, + "h": 1 + } + }, + { + "id": 22, + "title": "Performance PRs with Referenced Issues", + "type": "table", + "gridPos": { + "x": 0, + "y": 123, + "w": 24, + "h": 12 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT pr.* FROM pull_requests pr JOIN candidate_prs cp ON pr.owner = cp.owner AND pr.repo = cp.repo AND pr.issue_number = cp.issue_number WHERE pr.is_performance_commit AND cp.issues_json IS NOT NULL AND cp.issues_json != '[]'::jsonb ORDER BY pr.merged_at DESC NULLS LAST LIMIT ${row_limit:raw};", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "type": "row", + "title": "Pipeline Funnel", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 135, + "w": 24, + "h": 1 + } + }, + { + "id": 30, + "title": "End-to-End Pipeline Funnel", + "type": "barchart", + "gridPos": { + "x": 0, + "y": 136, + "w": 24, + "h": 8 + }, + "datasource": { + "uid": "supabase-pg", + "type": "postgres" + }, + "targets": [ + { + "rawSql": "SELECT stage, count FROM (SELECT 1 AS ord, 'Repositories' AS stage, COUNT(*) AS count FROM repositories UNION ALL SELECT 2, 'Pull Requests', COUNT(*) FROM pull_requests UNION ALL SELECT 3, 'Performance PRs', COUNT(*) FROM pull_requests WHERE is_performance_commit UNION ALL SELECT 4, 'Packages Resolved', COUNT(*) FROM packages WHERE can_install UNION ALL SELECT 5, 'Containers Built', COUNT(*) FROM candidate_containers UNION ALL SELECT 6, 'Published', COUNT(*) FROM pull_requests WHERE published_at IS NOT NULL) sub ORDER BY ord;", + "format": "table", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + ] +} diff --git a/grafana/provisioning/dashboards/dashboard-provider.yml b/grafana/provisioning/dashboards/dashboard-provider.yml new file mode 100644 index 00000000..030b0df6 --- /dev/null +++ b/grafana/provisioning/dashboards/dashboard-provider.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: DataSmith + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/grafana/provisioning/datasources/supabase-postgres.yml b/grafana/provisioning/datasources/supabase-postgres.yml new file mode 100644 index 00000000..d0adc6af --- /dev/null +++ b/grafana/provisioning/datasources/supabase-postgres.yml @@ -0,0 +1,21 @@ +apiVersion: 1 + +datasources: + - name: Supabase PostgreSQL + type: postgres + uid: supabase-pg + access: proxy + url: supabase_db_datasmith_new:5432 + database: postgres + user: grafana_ro + jsonData: + sslmode: disable + maxOpenConns: 5 + maxIdleConns: 2 + connMaxLifetime: 14400 + postgresVersion: 1500 + timescaledb: false + secureJsonData: + password: grafana_readonly + isDefault: true + editable: false diff --git a/supabase/migrations/00009_grafana_readonly.sql b/supabase/migrations/00009_grafana_readonly.sql new file mode 100644 index 00000000..e78113e4 --- /dev/null +++ b/supabase/migrations/00009_grafana_readonly.sql @@ -0,0 +1,15 @@ +-- Read-only role for Grafana datasource. +-- Safe to expose via ngrok — can only SELECT, never mutate. + +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'grafana_ro') THEN + CREATE ROLE grafana_ro WITH LOGIN PASSWORD 'grafana_readonly'; + END IF; +END +$$; + +GRANT CONNECT ON DATABASE postgres TO grafana_ro; +GRANT USAGE ON SCHEMA public TO grafana_ro; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO grafana_ro; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO grafana_ro; From 8fdcf8ac542cd9925728aa951a845f71bbda603d Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 9 Apr 2026 04:48:51 +0000 Subject: [PATCH 2/8] Add grafana/data/ to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 3522c404..cd0dd169 100644 --- a/.gitignore +++ b/.gitignore @@ -200,6 +200,9 @@ scratch/artifacts/ supabase/.temp/ supabase/.branches/ +# Grafana +grafana/data/ + # MkDocs build output site/ From 4e2eff89afe40a562a70f13375631b3bc100ea61 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 9 Apr 2026 04:48:55 +0000 Subject: [PATCH 3/8] Add supabase and grafana Makefile targets --- Makefile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Makefile b/Makefile index f9da319a..1ebdcc86 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,36 @@ docker-clean: ## Clean up dangling Docker images and containers @echo "Cleaning up dangling Docker images and containers" @docker system prune -f +.PHONY: supabase-up +supabase-up: ## Start local Supabase instance + @npx supabase start + +.PHONY: supabase-down +supabase-down: ## Stop local Supabase instance + @npx supabase stop + +.PHONY: supabase-status +supabase-status: ## Show Supabase service status and URLs + @npx supabase status + +.PHONY: grafana-migrate +grafana-migrate: ## Apply the grafana_ro read-only database role + @docker exec supabase_db_datasmith_new psql -U postgres -d postgres -f /dev/stdin < supabase/migrations/00009_grafana_readonly.sql + @echo "grafana_ro role created" + +.PHONY: grafana-up +grafana-up: ## Start Grafana dashboard (http://localhost:3001) + @docker compose -f grafana/docker-compose.yml up -d + @echo "Grafana is running at http://localhost:3001" + +.PHONY: grafana-down +grafana-down: ## Stop Grafana dashboard + @docker compose -f grafana/docker-compose.yml down + +.PHONY: grafana-logs +grafana-logs: ## Tail Grafana container logs + @docker compose -f grafana/docker-compose.yml logs -f + .PHONY: help help: From ac4108926a3aeb8494a19b6746e734c30e5ef4ca Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 9 Apr 2026 04:49:00 +0000 Subject: [PATCH 4/8] Add Grafana monitoring documentation --- docs/guide/monitoring.md | 90 ++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 3 +- 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 docs/guide/monitoring.md diff --git a/docs/guide/monitoring.md b/docs/guide/monitoring.md new file mode 100644 index 00000000..17df5bda --- /dev/null +++ b/docs/guide/monitoring.md @@ -0,0 +1,90 @@ +# Monitoring with Grafana + +fc-data includes a pre-configured Grafana dashboard for monitoring the pipeline, browsing data, and tracking dataset growth. + +## Prerequisites + +- Local Supabase instance running (`make supabase-up`) +- Docker available on the host + +## Quick Start + +```bash +# Apply the read-only database role (first time only) +make grafana-migrate + +# Start Grafana +make grafana-up +``` + +Grafana is now available at **** with no login required (anonymous read-only access). + +To expose it publicly via ngrok: + +```bash +ngrok http 3001 +``` + +## Architecture + +Grafana runs as a Docker container that joins the existing Supabase Docker network. It connects to Postgres via a **read-only** `grafana_ro` database role that can only `SELECT` — no writes are possible, even from the Explore SQL editor. + +``` +┌──────────────┐ ┌──────────────────────────┐ +│ Browser │────▶│ Grafana (port 3001) │ +└──────────────┘ │ Anonymous Viewer access │ + └───────────┬──────────────┘ + │ SELECT only + ┌───────────▼──────────────┐ + │ Supabase PostgreSQL │ + │ (grafana_ro role) │ + └──────────────────────────┘ +``` + +## Dashboard Panels + +The provisioned dashboard ("DataSmith Pipeline Overview") includes: + +| Section | Panels | +|---------|--------| +| **Dataset Growth** | Containers built by month, stat counts (PRs, containers, packages, repos), PR-to-container rate | +| **Repository Overview** | All repos by stars, containers-by-repo treemap | +| **Pull Request Insights** | Performance commit ratio, dataset time span, difficulty distribution, PR volume over time, optimization type distribution | +| **Package Resolution** | Python version distribution, installability by repo, resolution strategy breakdown | +| **Pipeline Status** | Active runners table, completion gauge, recent failures | +| **Synthesis** | Success rate over time, failure stage distribution, top error messages | +| **Synthesis Deeper Analysis** | First-attempt vs retry success, hardest repos, agent head-to-head | +| **Attempt Details** | Attempts per repo, duration by agent, duration over time | +| **Resource Metrics** | Build metrics table, image size vs build time, avg resources over time | +| **Pipeline Funnel** | End-to-end conversion: Repos → PRs → Perf PRs → Resolved → Built → Published | +| **Data Explorer** | Filtered table of performance PRs with referenced issues | + +## Ad-hoc SQL Queries + +Use Grafana's **Explore** mode (compass icon in the sidebar) to run arbitrary read-only SQL against the database. The `grafana_ro` role has `SELECT` access to all tables. + +## Admin Access + +To edit dashboards in the Grafana UI, log in with: + +- **Username:** `admin` +- **Password:** value of `GRAFANA_ADMIN_PASSWORD` in `tokens.env` (defaults to `admin`) + +## Configuration Files + +| File | Purpose | +|------|---------| +| `grafana/docker-compose.yml` | Grafana service definition | +| `grafana/provisioning/datasources/supabase-postgres.yml` | PostgreSQL datasource (read-only) | +| `grafana/provisioning/dashboards/dashboard-provider.yml` | Dashboard file provider config | +| `grafana/provisioning/dashboards-json/datasmith-overview.json` | Dashboard panels and queries | +| `supabase/migrations/00009_grafana_readonly.sql` | Read-only Postgres role | + +## Makefile Targets + +```bash +make grafana-up # Start Grafana +make grafana-down # Stop Grafana +make grafana-logs # Tail container logs +make grafana-migrate # Apply the grafana_ro database role +``` diff --git a/mkdocs.yml b/mkdocs.yml index 12fab276..10a0a6fb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: 🔧 fc-data +site_name: fc-data site_url: https://formula-code.github.io/datasmith/ site_description: Python toolchain for building the FormulaCode benchmark repo_url: https://github.com/formula-code/datasmith @@ -89,3 +89,4 @@ nav: - Publishing: guide/publishing.md - Verification: guide/verification.md - Python API Quickstart: getting-started/quickstart.md + - Monitoring: guide/monitoring.md From 7ab0277de1465a0b34b20a44775205efd261c586 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 9 Apr 2026 04:49:07 +0000 Subject: [PATCH 5/8] Loosen symbolic compliance filter: AND to OR, add new keywords --- src/datasmith/filters.py | 37 ++++++++++++++++++++++++++++++------- tests/test_filters.py | 16 +++++++++------- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/datasmith/filters.py b/src/datasmith/filters.py index 7ce6bf93..32667517 100644 --- a/src/datasmith/filters.py +++ b/src/datasmith/filters.py @@ -34,7 +34,27 @@ r"vectori[sz](?:e[ds]?|ation)|" r"accelerat(?:e[ds]?|ion)|" r"efficien(?:t|cy)|" - r"regress(?:ion|ed)?" + r"regress(?:ion|ed)?|" + r"memory|mem(?:ory)?|" + r"reduc(?:e[ds]?|ing|tion)|" + r"batch(?:ed|ing)?|" + r"inline[ds]?|inlining|" + r"async(?:io)?|" + r"lazy|" + r"gpu|cpu|" + r"chunk(?:ed|ing|s)?|" + r"compil(?:e[ds]?|ation)|jit|" + r"simd|sse|avx|neon|" + r"overhead|" + r"allocat(?:e[ds]?|ion[s]?)|preallocat(?:e[ds]?|ion)|" + r"memoi[sz](?:e[ds]?|ation)|precomput(?:e[ds]?|ation)|" + r"concurren(?:t|cy)|" + r"buffer(?:ed|ing|s)?|" + r"compress(?:ed|ion)?|" + r"unroll(?:ed|ing)?|" + r"dedup(?:licat(?:e[ds]?|ion))?|" + r"pool(?:ing|ed)?|" + r"streaming" r")\b", re.IGNORECASE, ) @@ -60,7 +80,10 @@ r"annotations?|" r"deprecat(?:e[ds]?|ion)|" r"revert(?:ed|ing)?|" - r"backport" + r"backport|" + r"docstring[s]?|" + r"autoupdate|" + r"pre-commit" r")\b", re.IGNORECASE, ) @@ -85,12 +108,12 @@ def message_filter(title: str) -> bool: """Return True if the PR title suggests a performance improvement. - Passes if the title matches at least one positive keyword and - does NOT match any negative keyword. + Passes if the title matches at least one positive keyword OR + does NOT match any negative keyword. This is intentionally loose: + a positive keyword is strong signal, and the absence of negative + keywords indicates ambiguity worth sending to the LLM classifier. """ - if _NEGATIVE_RE.search(title): - return False - return bool(_POSITIVE_RE.search(title)) + return bool(_POSITIVE_RE.search(title)) or not bool(_NEGATIVE_RE.search(title)) def has_core_file(filenames: list[str]) -> bool: diff --git a/tests/test_filters.py b/tests/test_filters.py index bcc507c9..95432a61 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -78,22 +78,24 @@ def test_rejects_non_perf(self, title: str) -> None: "Support Python 3.12", ], ) - def test_rejects_no_keyword(self, title: str) -> None: - assert message_filter(title) is False + def test_passes_ambiguous_no_keyword(self, title: str) -> None: + """Titles with no positive AND no negative keyword pass (ambiguous → let LLM decide).""" + assert message_filter(title) is True class TestMessageFilterEdgeCases: - def test_negative_overrides_positive(self) -> None: - """If title has both positive and negative keywords, reject it.""" - assert message_filter("Revert performance optimization") is False - assert message_filter("Documentation for benchmark suite") is False + def test_positive_overrides_negative(self) -> None: + """If title has both positive and negative keywords, pass (positive is sufficient).""" + assert message_filter("Revert performance optimization") is True + assert message_filter("Documentation for benchmark suite") is True def test_case_insensitive(self) -> None: assert message_filter("OPTIMIZE memory usage") is True assert message_filter("Performance Improvement") is True def test_empty_title(self) -> None: - assert message_filter("") is False + """Empty title has no negative keyword, so it passes (ambiguous).""" + assert message_filter("") is True # ── has_core_file ─────────────────────────────────────────────────── From f7f193bb1951b5b295aff51ff61b283ece6fb9e6 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 9 Apr 2026 04:49:13 +0000 Subject: [PATCH 6/8] Add live dashboard link to README --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index c1b70cf5..9d0cd866 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,9 @@ fc-data Documentation + + Live Dashboard +

[FormulaCode](https://formula-code.github.io/) is a *continually updating* benchmark for evaluating the holistic ability of LLM agents to optimize codebases. FormulaCode consists of two parts: a [pipeline](https://github.com/formula-code/datasmith) to construct performance optimization tasks, and an [execution harness](https://github.com/formula-code/terminal-bench) that connects a language model to our terminal sandbox. _This repository contains the task generation pipeline._ From ab9e807ecb6e9e606bb3b50ea97629fcc67a844e Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 9 Apr 2026 04:49:19 +0000 Subject: [PATCH 7/8] Add dry-run summary logging to pipeline stages --- src/datasmith/update/pipeline.py | 226 ++++++++++++++++++++++++------- tests/update/test_pipeline.py | 20 ++- 2 files changed, 192 insertions(+), 54 deletions(-) diff --git a/src/datasmith/update/pipeline.py b/src/datasmith/update/pipeline.py index 738ebcc7..41bf7d25 100644 --- a/src/datasmith/update/pipeline.py +++ b/src/datasmith/update/pipeline.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections import Counter from typing import Any from datasmith.utils import get_client, get_logger @@ -87,6 +88,56 @@ def __init__( self._min_stars = min_stars self._completed_stages: list[str] = [] + def _log_dry_run_summary( + self, + stage_name: str, + items: list[Any], + extra: dict[str, Any] | None = None, + ) -> None: + """Log an informative dry-run summary for a stage.""" + lines = [f"[DRY RUN] Stage: {stage_name}", f" Items to process: {len(items)}"] + + if not items: + lines.append(" Nothing to do.") + logger.info("\n".join(lines)) + return + + lines.extend(self._repo_breakdown(items)) + lines.extend(self._settings_lines()) + + for k, v in (extra or {}).items(): + lines.append(f" {k}: {v}") + + logger.info("\n".join(lines)) + + def _repo_breakdown(self, items: list[Any]) -> list[str]: + repo_counts: Counter[tuple[str, str]] = Counter() + for it in items: + if isinstance(it, dict): + repo_counts[(it["owner"], it["repo"])] += 1 + elif isinstance(it, tuple) and len(it) >= 2: + repo_counts[(it[0], it[1])] += 1 + + lines = [f" Unique repos: {len(repo_counts)}"] + top = repo_counts.most_common(10) + if top: + lines.append(" Top repos:") + for (owner, repo), count in top: + lines.append(f" {owner}/{repo}: {count}") + if len(repo_counts) > 10: + lines.append(f" ... and {len(repo_counts) - 10} more") + return lines + + def _settings_lines(self) -> list[str]: + settings: list[str] = [] + if self._n_concurrent: + settings.append(f"concurrency={self._n_concurrent}") + if self._force: + settings.append("force=True") + if self._tasks_per_repo: + settings.append(f"tasks_per_repo={self._tasks_per_repo}") + return [f" Settings: {', '.join(settings)}"] if settings else [] + async def run( self, start_date: str, @@ -125,15 +176,15 @@ async def run( for stage_name in stages_to_run: if self._dry_run: - logger.info("[DRY RUN] Would run stage: %s", stage_name) - continue - - logger.info("Starting stage: %s", stage_name) + logger.info("[DRY RUN] Collecting summary for stage: %s", stage_name) + else: + logger.info("Starting stage: %s", stage_name) try: await self._run_stage(stage_name, start_date, end_date) - self._completed_stages.append(stage_name) - self._mark_stage_completed(stage_name) - logger.info("Completed stage: %s", stage_name) + if not self._dry_run: + self._completed_stages.append(stage_name) + self._mark_stage_completed(stage_name) + logger.info("Completed stage: %s", stage_name) except Exception: logger.exception("Stage %s failed", stage_name) raise @@ -162,7 +213,6 @@ async def _scrape_repos(self) -> None: pool = TokenPool() gh = GitHubClient(pool) - runner = ScrapeReposRunner(gh, **({"n_concurrent": self._n_concurrent} if self._n_concurrent else {})) seen: set[tuple[str, str]] = set() items: list[tuple[str, str]] = [] @@ -197,6 +247,17 @@ async def _scrape_repos(self) -> None: items.append(pair) logger.info("Total repos to process: %d", len(items)) + + if self._dry_run: + self._log_dry_run_summary( + "scrape_repos", + items, + extra={"From GitHub search": len(discovered), "From DB (refresh)": len(rows)}, + ) + await gh.close() + return + + runner = ScrapeReposRunner(gh, **({"n_concurrent": self._n_concurrent} if self._n_concurrent else {})) await runner.run(items) await gh.close() @@ -205,6 +266,17 @@ async def _scrape_commits(self, start_date: str, end_date: str) -> None: from datasmith.runners.scrape_commits import ScrapeCommitsRunner from datasmith.utils.tokens import TokenPool + rows = fetch_all("repositories", select="owner, repo") + items = [(r["owner"], r["repo"]) for r in rows] + + if self._dry_run: + self._log_dry_run_summary( + "scrape_commits", + items, + extra={"Date range": f"{start_date} to {end_date}"}, + ) + return + pool = TokenPool() gh = GitHubClient(pool) kwargs: dict[str, Any] = {"since": start_date, "until": end_date} @@ -212,8 +284,6 @@ async def _scrape_commits(self, start_date: str, end_date: str) -> None: kwargs["n_concurrent"] = self._n_concurrent runner = ScrapeCommitsRunner(gh, **kwargs) - rows = fetch_all("repositories", select="owner, repo") - items = [(r["owner"], r["repo"]) for r in rows] await runner.run(items) await gh.close() @@ -227,20 +297,6 @@ async def _scrape_commits(self, start_date: str, end_date: str) -> None: logger.info("Imported %d pull request records from offline source", n) async def _classify_prs(self) -> None: - from datasmith.agents.classifiers import ClassifyJudge, PerfClassifier - from datasmith.agents.config import AgentConfig, configure_dspy - from datasmith.runners.classify_prs import ClassifyPRsRunner - - configure_dspy(AgentConfig.from_env()) - - classifier = PerfClassifier() - judge = ClassifyJudge() - runner = ClassifyPRsRunner( - classifier, - judge, - **({"n_concurrent": self._n_concurrent} if self._n_concurrent else {}), - ) - classify_kwargs: dict[str, Any] = { "select": "owner, repo, issue_number, title, body, patch, file_changes", "filters": {"is_performance_commit_symbolic": True}, @@ -259,15 +315,31 @@ async def _classify_prs(self) -> None: } for r in rows ] - await runner.run(items) - async def _resolve_packages(self, start_date: str, end_date: str) -> None: - from datasmith.runners.resolve_packages import ResolvePackagesRunner + if self._dry_run: + self._log_dry_run_summary( + "classify_prs", + items, + extra={"Filter": "unclassified only" if not self._force else "all (force=True)"}, + ) + return - runner = ResolvePackagesRunner( + from datasmith.agents.classifiers import ClassifyJudge, PerfClassifier + from datasmith.agents.config import AgentConfig, configure_dspy + from datasmith.runners.classify_prs import ClassifyPRsRunner + + configure_dspy(AgentConfig.from_env()) + + classifier = PerfClassifier() + judge = ClassifyJudge() + runner = ClassifyPRsRunner( + classifier, + judge, **({"n_concurrent": self._n_concurrent} if self._n_concurrent else {}), ) + await runner.run(items) + async def _resolve_packages(self, start_date: str, end_date: str) -> None: # Get performance-classified PRs within the date range rows = fetch_all( "pull_requests", @@ -291,29 +363,34 @@ async def _resolve_packages(self, start_date: str, end_date: str) -> None: items.append({"owner": r["owner"], "repo": r["repo"], "sha": sha}) # Skip items already in the packages table (unless --force) + skipped = 0 if items and not self._force: existing_rows = fetch_all("packages", select="owner, repo, sha") existing_keys = {(e["owner"], e["repo"], e["sha"]) for e in existing_rows} + before = len(items) items = [it for it in items if (it["owner"], it["repo"], it["sha"]) not in existing_keys] + skipped = before - len(items) logger.info("Resolving packages for %d commits", len(items)) - await runner.run(items) - async def _render_problems(self) -> None: - from datasmith.agents.config import AgentConfig, configure_dspy - from datasmith.github.client import GitHubClient - from datasmith.runners.render_problems import RenderProblemsRunner - from datasmith.utils.tokens import TokenPool + if self._dry_run: + extra: dict[str, Any] = { + "Date range": f"{start_date} to {end_date}", + "Unique commits from PRs": len(seen), + } + if skipped: + extra["Already resolved (skipped)"] = skipped + self._log_dry_run_summary("resolve_packages", items, extra=extra) + return - configure_dspy(AgentConfig.from_env()) + from datasmith.runners.resolve_packages import ResolvePackagesRunner - pool = TokenPool() - gh = GitHubClient(pool) - runner = RenderProblemsRunner( - gh=gh, + runner = ResolvePackagesRunner( **({"n_concurrent": self._n_concurrent} if self._n_concurrent else {}), ) + await runner.run(items) + async def _render_problems(self) -> None: # Fetch performance-classified PRs rows = fetch_all( "pull_requests", @@ -338,12 +415,15 @@ async def _render_problems(self) -> None: existing_rows = fetch_all("candidate_prs", select="owner, repo, issue_number") existing_keys = {(e["owner"], e["repo"], e["issue_number"]) for e in existing_rows} + skipped_no_pkg = 0 + skipped_existing = 0 items = [] for r in rows: sha = r.get("merge_commit_sha", "") if not sha: continue if (r["owner"], r["repo"], sha) not in installable: + skipped_no_pkg += 1 logger.debug( "Skipping %s/%s#%d: no can_install package for sha %s", r["owner"], @@ -353,6 +433,7 @@ async def _render_problems(self) -> None: ) continue if (r["owner"], r["repo"], r["issue_number"]) in existing_keys: + skipped_existing += 1 continue items.append({ "owner": r["owner"], @@ -369,25 +450,33 @@ async def _render_problems(self) -> None: items = _cap_per_repo(items, self._tasks_per_repo) logger.info("Rendering problem contexts for %d PRs", len(items)) - await runner.run(items) - await gh.close() - async def _synthesize_images(self) -> None: - from datasmith.agents.synthesizer import Synthesizer + if self._dry_run: + extra: dict[str, Any] = {"Performance PRs in DB": len(rows)} + if skipped_no_pkg: + extra["Skipped (no installable package)"] = skipped_no_pkg + if skipped_existing: + extra["Skipped (already rendered)"] = skipped_existing + self._log_dry_run_summary("render_problems", items, extra=extra) + return + + from datasmith.agents.config import AgentConfig, configure_dspy from datasmith.github.client import GitHubClient - from datasmith.runners.synthesize_images import SynthesizeImagesRunner + from datasmith.runners.render_problems import RenderProblemsRunner from datasmith.utils.tokens import TokenPool + configure_dspy(AgentConfig.from_env()) + pool = TokenPool() gh = GitHubClient(pool) - - synth = Synthesizer(agent=self._agent, force=self._force) - runner = SynthesizeImagesRunner( - synth, + runner = RenderProblemsRunner( gh=gh, **({"n_concurrent": self._n_concurrent} if self._n_concurrent else {}), ) + await runner.run(items) + await gh.close() + async def _synthesize_images(self) -> None: query_kwargs: dict[str, Any] = { "select": "owner, repo, issue_number, merge_commit_sha, title, body, created_at, rendered_problem", "filters": {"is_performance_commit": True, "is_performance_commit_symbolic": True}, @@ -421,12 +510,15 @@ async def _synthesize_images(self) -> None: if c.get("issues_json") or c.get("initial_observations") } + skipped_no_pkg = 0 + skipped_no_ctx = 0 items = [] for r in rows: sha = r.get("merge_commit_sha", "") pkg = pkg_lookup.get((r["owner"], r["repo"], sha), {}) # Skip PRs without resolved packages if not pkg: + skipped_no_pkg += 1 logger.debug( "Skipping %s/%s#%d: no resolved packages for sha %s", r["owner"], @@ -437,6 +529,7 @@ async def _synthesize_images(self) -> None: continue # Skip PRs without a rendered context (non-empty issues + observations) if (r["owner"], r["repo"], r["issue_number"]) not in eligible_prs: + skipped_no_ctx += 1 logger.debug( "Skipping %s/%s#%d: no eligible pr_context (empty issues_json or initial_observations)", r["owner"], @@ -461,10 +554,47 @@ async def _synthesize_images(self) -> None: items = _cap_per_repo(items, self._tasks_per_repo) logger.info("Synthesizing images for %d PRs", len(items)) + + if self._dry_run: + extra: dict[str, Any] = {"Candidate PRs in DB": len(rows)} + if skipped_no_pkg: + extra["Skipped (no resolved packages)"] = skipped_no_pkg + if skipped_no_ctx: + extra["Skipped (no rendered context)"] = skipped_no_ctx + self._log_dry_run_summary("synthesize_images", items, extra=extra) + return + + from datasmith.agents.synthesizer import Synthesizer + from datasmith.github.client import GitHubClient + from datasmith.runners.synthesize_images import SynthesizeImagesRunner + from datasmith.utils.tokens import TokenPool + + pool = TokenPool() + gh = GitHubClient(pool) + + synth = Synthesizer(agent=self._agent, force=self._force) + runner = SynthesizeImagesRunner( + synth, + gh=gh, + **({"n_concurrent": self._n_concurrent} if self._n_concurrent else {}), + ) await runner.run(items) await gh.close() async def _publish(self, start_date: str, end_date: str) -> None: + if self._dry_run: + # Query what would be published without running the pipeline + rows = fetch_all( + "candidate_containers", + select="owner, repo, issue_number", + ) + self._log_dry_run_summary( + "publish", + [{"owner": r["owner"], "repo": r["repo"]} for r in rows], + extra={"Date range": f"{start_date} to {end_date}"}, + ) + return + from datasmith.publish.pipeline import publish_pipeline await publish_pipeline(start_date, end_date) diff --git a/tests/update/test_pipeline.py b/tests/update/test_pipeline.py index 6dd51673..dd76e77c 100644 --- a/tests/update/test_pipeline.py +++ b/tests/update/test_pipeline.py @@ -61,19 +61,27 @@ async def mock_run_stage(name, start, end): assert "scrape_commits" in calls assert "classify_prs" not in calls - async def test_dry_run_no_execution(self): + async def test_dry_run_no_stage_completion(self): pipeline = Pipeline(dry_run=True) calls = [] async def mock_run_stage(name, start, end): calls.append(name) - with patch.object(pipeline, "_mark_stage_completed"): - pipeline._run_stage = mock_run_stage # type: ignore[assignment] - await pipeline.run("2024-01-01", "2024-12-31") + mark_calls = [] + original_mark = pipeline._mark_stage_completed - # All stages are skipped in dry_run mode. - assert calls == [] + def tracking_mark(name): + mark_calls.append(name) + + pipeline._run_stage = mock_run_stage # type: ignore[assignment] + pipeline._mark_stage_completed = tracking_mark # type: ignore[assignment] + await pipeline.run("2024-01-01", "2024-12-31") + + # Stages are still dispatched (for summary collection) but never marked completed. + assert calls == STAGES + assert mark_calls == [] + assert pipeline._completed_stages == [] async def test_single_stage_execution(self): pipeline = Pipeline() From c2d5d5a45be004ff639434eaf7ec08273d96de73 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 9 Apr 2026 04:49:24 +0000 Subject: [PATCH 8/8] Fix typo in docs index --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index d16510aa..a4d6d53f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,6 @@ ![banner](https://github.com/formula-code/datasmith/raw/main/static/formula-code-datasmith.png) -

+I

FormulaCode Website