diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..621ccae7 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,24 @@ +name: Publish to PyPI + +on: + release: + types: [published] + +jobs: + publish: + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up the environment + uses: ./.github/actions/setup-python-env + + - name: Build package + run: uv run python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/AGENTS.md b/AGENTS.md index fe9fc7d2..54dcf6a0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # Repository Guidelines -This guide helps contributors work effectively in the datasmith repository. +This guide helps contributors work effectively in the fc-data repository. ## Project Structure & Module Organization - Source: `src/datasmith/` — current modules: `agents/`, `docker/`, `github/`, `publish/`, `resolution/`, `runners/`, `update/`, `utils/`, plus helper modules like `filters.py` and `preflight.py`. @@ -17,7 +17,7 @@ This guide helps contributors work effectively in the datasmith repository. - `uv run python -m pytest --doctest-modules` — run doctests with the standard test suite. - `uvx tox -q` — run the tox matrix (py39–py312) as defined in `tox.ini`. - To run commands using the same environment variables as the user, use `uv run `. -- Pipeline entrypoint: `ds-update --help` and `ds-update --start-date ... --end-date ...`. +- Pipeline entrypoint: `fc-data --help` and `fc-data --start-date ... --end-date ...`. ## Coding Style & Naming Conventions - Python 3.9–3.12. 4‑space indentation, type hints required (mypy strict; see `pyproject.toml`). diff --git a/CLAUDE.md b/CLAUDE.md index 14f5d1a7..368ed18a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What is this project? -DataSmith is the Python toolchain for building the **FormulaCode** dataset — a benchmark of 67+ repositories with 964+ performance-improving commits, designed to evaluate LLMs' ability to optimize real-world codebases. It scores LLMs relative to the human-authored speedup using ASV (Airspeed Velocity) benchmarks rather than binary pass/fail tests. +fc-data is the Python toolchain for building the **FormulaCode** dataset — a benchmark of 67+ repositories with 964+ performance-improving commits, designed to evaluate LLMs' ability to optimize real-world codebases. It scores LLMs relative to the human-authored speedup using ASV (Airspeed Velocity) benchmarks rather than binary pass/fail tests. ## Common commands @@ -24,11 +24,11 @@ uv run pre-commit run -a # Run all pre-commit hooks # Dataset verification (iterative Docker build debugging) python dataset/verify.py --task dataset/formulacode_verified// -# Pipeline update (monthly, the primary entrypoint for DataSmith) -ds-update --start-date YYYY-MM-DD --end-date YYYY-MM-DD # Run all 6 stages -ds-update --start-date 2026-01-01 --end-date 2026-01-31 --stage 4 # Run a single stage -ds-update --start-date 2026-01-01 --end-date 2026-01-31 --resume # Resume from last completed -ds-update --help # See all options +# Pipeline update (monthly, the primary entrypoint for fc-data) +fc-data --start-date YYYY-MM-DD --end-date YYYY-MM-DD # Run all 6 stages +fc-data --start-date 2026-01-01 --end-date 2026-01-31 --stage 4 # Run a single stage +fc-data --start-date 2026-01-01 --end-date 2026-01-31 --resume # Resume from last completed +fc-data --help # See all options ``` ## Architecture @@ -47,7 +47,7 @@ ds-update --help # See all op | `benchmark/` | ASV benchmark collection | | `collation/` | Data aggregation | -### Pipeline stages (`ds-update`) +### Pipeline stages (`fc-data`) 1. **scrape_repos** — Fetch repository metadata from GitHub 2. **scrape_commits** — Scrape merged PR commits and patches @@ -73,7 +73,7 @@ Each task lives in `dataset/formulacode_verified///` with a mul ## Supabase (local) -DataSmith uses a **local Supabase** instance for all persistent state. Connection details live in `tokens.env`: +fc-data uses a **local Supabase** instance for all persistent state. Connection details live in `tokens.env`: - `SUPABASE_URL=http://127.0.0.1:54321` (PostgREST API) - `SUPABASE_KEY=sb_secret_...` (service-role key) diff --git a/README.md b/README.md index 051fcfd1..147d4322 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,23 @@ -# FormulaCode - DataSmith - -[![Build status](https://img.shields.io/github/actions/workflow/status/formula-code/datasmith/main.yml?branch=main)](https://github.com/formula-code/datasmith/actions/workflows/main.yml?query=branch%3Amain) -[![License](https://img.shields.io/github/license/formula-code/datasmith)](https://img.shields.io/github/license/formula-code/datasmith) - -## Abstract - -Datasmith is a package for automatically building and maintaining FormulaCode tasks. The package is engineered to support any repository-level, verification-by-execution based coding benchmark that heavily uses Docker and GitHub. - -FormulaCode is a benchmark of 67+ repositories with 964+ performance-improving commits, designed to evaluate LLMs' ability to optimize real-world codebases. It scores optimizers relative to the human-authored speedup using ASV (Airspeed Velocity) benchmarks — providing a dense performance signal instead of binary pass/fail. +![banner](static/formula-code-datasmith.png) + +

+ + FormulaCode Website + + + FormulaCode Paper + + + FormulaCode Leaderboard + + + FormulaCode Live Task Registry + +

+ +[FormulaCode](https://formula-code.github.io/) is a *continually updating* benchmark for evaluating the holistic ability of LLM agents to optimize codebases. FormulaCode consists of two parts: a [pipeline](https://github.com/formula-code/datasmith) to construct performance optimization tasks, and an [execution harness](https://github.com/formula-code/terminal-bench) that connects a language model to our terminal sandbox. _This repository contains the task generation pipeline._ + +`fc-data` is a python package for automatically curating and managing [FormulaCode](https://formula-code.github.io/) tasks. After installation, fc-data is designed to run as a monthly CRON job that updates the FormulaCode dataset with new commits and repositories. ## High level overview @@ -20,7 +30,7 @@ graph LR A[Github] A2[Supabase] - B["`Datasmith + B["`fc-data (This repository)`"] C[DockerHub] D[HuggingFace] @@ -28,7 +38,17 @@ graph LR ## Use cases -`datasmith` helps you manage your github-centric benchmark. Each benchmark contains a task which revolves around a GitHub Issue (or Pull request; which is just an issue with extra details). We include some helpful properties to start off: +`fc-data` is designed primarily to enable continual dataset updates for FormulaCode. After [installation](#installation), the monthly update is a single command: + +```bash +$ pip install fc-data +$ fc-data --start-date 2026-02-01 --end-date 2026-03-01 +``` + +This runs six stages in order: scrape repos, scrape commits, classify PRs, resolve packages, synthesize Docker images, and publish the docker images to DockerHub and the PRs to HuggingFace The dataset is versioned by month (e.g. `formulacode@2026-03`). In our servers, this command runs as a monthly CRON job. + + +However, this isn't the only use case for `fc-data`. We've designed `fc-data` to helps you manage your custom github-centric benchmark. Each benchmark contains a task which revolves around a GitHub Issue (or Pull request; which is just an issue with extra details). We include some helpful properties to start off: ```python from datasmith.github import PR, GitHubClient @@ -62,15 +82,13 @@ diff = await gh.get_diff("pandas-dev", "pandas", 16222) events = await gh.get_timeline("pandas-dev", "pandas", 16222) ``` -Want to make a problem statement? A pull request can be rendered into one: +Want to extract structured information from the PR? Use our built-in agents or define your own! ```python from datasmith.github import render_problem_statement, scrape_links # Render a problem statement from the PR and its linked issues. statement = render_problem_statement(pr, anonymize=True) -# Don't want to expose the users? anonymize=True maps @alice -> @user_1, strips emails. - # You can also scrape for linked issues via BFS. issues = await scrape_links(pr, gh.get_issue, depth=2, only_issues=True, limit=6) @@ -113,7 +131,7 @@ await runner.run(pr_items) By default, each operation is cached in Supabase so you don't keep hitting expensive hooks. -A pull request is useless if you cannot build a reproducible environment for it. Datasmith supports building docker images for any pull request using a three-tier hierarchy: +A pull request is useless if you cannot build a reproducible environment for it. fc-data supports building docker images for any pull request using a three-tier hierarchy: ```python from datasmith.docker import ImageManager, MultiObjVerifier, SmokeVerifier, ProfileVerifier @@ -140,7 +158,7 @@ result = verifier.verify("formulacode/pandas-dev-pandas:16222") # result.ok, result.rc, result.stdout, result.stderr, result.duration_s ``` -One of the main features of `datasmith` is the ability to automatically synthesize docker containers for a pull request. The synthesizer is a state machine that checks Supabase for cached contexts, tries similar build scripts, then falls back to an installed CLI agent (Claude Code, Codex, or Gemini — auto-detected): +One of the main features of `fc-data` is the ability to automatically synthesize docker containers for a pull request. The synthesizer is a state machine that checks Supabase for cached contexts, tries similar build scripts, then falls back to an installed CLI agent (Claude Code, Codex, or Gemini — auto-detected): ```python from datasmith.agents import Synthesizer @@ -232,79 +250,9 @@ run = Harness( print(run.results[0].is_resolved) # Did the oracle get a speedup > 1.00 over baseline? ``` +## Database schema -## Architecture - -Datasmith contains seven high-level modules. FormulaCode-specific logic lives directly in the appropriate `ds.*` module (no separate examples folder) since FormulaCode is the primary consumer. - -* `ds.utils`: Shared utilities. - * `ds.utils.db`: Supabase client initialization, common query helpers, and the `@supabase_cached` decorator. Cache keys are deterministic (SHA-256 of canonical JSON). - * `ds.utils.tokens`: GitHub token pool with round-robin selection per request. Blocks on rate limits using GitHub's `X-RateLimit-Reset` header. Thread-safe via `threading.Lock`. - * `ds.utils.core`: `Settings` (pydantic-settings from `tokens.env`), structured logging via `get_logger()`, `@with_backoff` retry decorator. -* `ds.github`: Pydantic v2 models for GitHub issues and PRs. All expensive methods use the `@supabase_cached` decorator (from `ds.utils.db`). GitHub API access via `httpx` + `ds.utils.tokens`. - * `ds.github.models`: Frozen `Issue`, `PR` (extends Issue), `FormulaCodeRecord`, `IssueExpanded`, `PRFileChange`, `PRChangeSummary`. - * `ds.github.client`: Async `GitHubClient` wrapping `httpx.AsyncClient` with `TokenPool`. Methods: `get_pr()`, `get_issue()`, `get_timeline()`, `get_diff()`, `get_files()`, `graphql()`. Handles 429/403 with automatic token rotation. - * `ds.github.hooks`: `HookRegistry` — class-level registry for custom hooks. `register()` auto-wraps with `@supabase_cached`. - * `ds.github.links`: `extract_references()` parses `#N`, `owner/repo#N`, and full GitHub URLs. `scrape_links()` does BFS traversal with depth, limit, and `only_issues` controls. - * `ds.github.render`: Jinja2-based `render_problem_statement()` with `Anonymizer` (`@username` -> `@user_N`, email stripping). -* `ds.docker`: Dependencies for constructing and maintaining docker tasks. - * `ds.docker.images`: `ImageManager` wrapping `python-on-whales` (not `docker-py`) — subprocess-based, thread-safe by design, scales to 40-50+ concurrent threads without connection pool issues. Three-tier hierarchy: base -> repo -> PR. - * `ds.docker.context`: `DockerContext` Pydantic model holding Dockerfile + shell scripts. `to_tar_bytes()` produces reproducible tarballs (zero mtimes, deterministic order). `from_directory()` loads from a task directory. - * `ds.docker.verifiers`: Abstract `Verifier` base class. Concrete: `SmokeVerifier` (`import {package}`), `ProfileVerifier` (runs `profile.sh`, timeout exit code 124 treated as success), `PytestVerifier` (runs `run-tests.sh`), `MultiObjVerifier` (chains verifiers, short-circuits on failure). - * `ds.docker.publish`: `DockerHubPublisher` with lazy login, `@with_backoff` retry, version tagging (`@YYYY-MM`), remote tag listing, and delta publish. -* `ds.agents`: Agents for dynamic filtering and automatic build script generation. Simple agents use `dspy`; complex agents use an installed agent (like `codex`). - * `ds.agents.extractors`: `ProblemExtractor` — DSPy module that extracts structured `ProblemExtraction` (initial_observations, triage_attempts, solution_overview, solution_observations) from PR text. - * `ds.agents.classifiers`: `PerfClassifier` — binary YES/NO on whether a PR is performance-improving. `ClassifyJudge` — classifies by `OptimizationType` (14 categories with descriptions) + difficulty (easy/medium/hard) + confidence. Patch truncation via tiktoken. - * `ds.agents.codex`: `codex_exec()` wraps `codex exec --full-auto --json -m {model}` via subprocess. Parses JSON stream for output and files_changed. Handles timeout, missing CLI, and errors. - * `ds.agents.synthesizer`: State machine: `CHECK_CACHE` -> `FIND_SIMILAR` -> `TRY_SIMILAR` -> `LLM_GENERATE` -> `FAIL`. All attempts logged to `build_attempts` table. -* `ds.runners`: Scalable async runners that store inputs/outputs in Supabase. Each runner takes `n_concurrent` and manages its own progress tracking. - * `ds.runners.base`: `BaseRunner` ABC — `asyncio.Semaphore` for concurrency, upserts to `runner_progress` every 10 items or 30s, per-item failures to `runner_failures`. Single item errors never abort the runner. - * `ds.runners.scrape_repos`: Finds compliant GitHub repositories using the REST API. Skips existing repos. - * `ds.runners.scrape_commits`: For a given repository, scrapes merged PRs and stores metadata in `pull_requests`. - * `ds.runners.synthesize_images`: Runs `ds.agents.synthesizer` for each PR. Docker operations via `asyncio.to_thread()`. - * `ds.runners.classify_prs`: Runs `PerfClassifier` + `ClassifyJudge` across PRs concurrently. Updates `pull_requests` table in-place. -* `ds.publish`: Publishes verified tasks to DockerHub and HuggingFace. - * `ds.publish.records`: `records_to_parquet()` / `records_from_parquet()` via pyarrow. `records_from_supabase()` queries unpublished perf PRs. - * `ds.publish.huggingface`: `HuggingFacePublisher` — reads token from file path, uploads Parquet via `huggingface_hub`, generates dataset cards. Append-only: never overwrites prior versions. - * `ds.publish.pipeline`: `publish_pipeline()` orchestrates: query DB -> push DockerHub -> upload HuggingFace -> mark `published_at`. - * Dataset versioning follows `@YYYY-MM` (e.g. `formulacode@2026-03`). The dataset is updated monthly. -* `ds.update`: Pipeline orchestrator. Updates datasmith to find compliant tasks within a given date range. - 1. `ds.runners.scrape_repos` - 2. `ds.runners.scrape_commits` - 3. `ds.runners.classify_prs` - 4. `ds.runners.resolve_packages` - 5. `ds.runners.synthesize_images` - 6. `ds.publish` - - Supports `--resume`, `--stage N`, `--dry-run`, `--n-concurrent`, `--tasks-per-repo`, `--agent`, and `--force`. - -### FormulaCodeRecord - -The bridge between datasmith's PR model and `terminal-bench`'s evaluation harness: - -```python -class FormulaCodeRecord(BaseModel): - owner: str - repo: str - issue_number: int - task_id: str # "{owner}__{repo}-{issue_number}" - gt_hash: str # pr.merge_commit_sha - base_commit: str # pr.base_sha - date: datetime | None # pr.merged_at - instructions: str # rendered problem statement (from ds.github.render) - classification: str # from ds.agents.classifiers (14 categories) - difficulty: str # easy / medium / hard - container_name: str # Docker container reference - patch: str # ground truth diff - image_name: str # full Docker image reference -``` - -`PR.to_record()` constructs this from the PR model. Returns `None` if the merge commit SHA is missing. - - -### Database schema - -Six tables in Supabase (Postgres), defined in `supabase/migrations/00001_initial_schema.sql`: +There are xix tables in Supabase (Postgres): | Table | Primary key | Purpose | |-------|-------------|---------| @@ -372,21 +320,9 @@ $ npx supabase stop # stop all containers Studio is available at the URL printed by `supabase status` (default `http://127.0.0.1:54323`) — use it to browse tables, run SQL, and inspect data. -To seed the `build_attempts` table with proven-good build scripts from the legacy registry: -```bash -$ python scratch/scripts/prefill/transform_registry.py # legacy JSON -> contexts_by_sha.json -$ python scratch/scripts/prefill/prefill_build_attempts.py # insert 73 rows into build_attempts -# Both scripts support --dry-run to preview without writing. -``` - - -## Verification - -We have a simple test to ensure that all the variables we need are properly defined: +Running `preflight` ensures that all the variables are properly defined: ```bash $ python -m datasmith.preflight -DataSmith Preflight Check -======================================== == Environment == [OK] SUPABASE_URL — http://127.0.0.1:54... @@ -410,26 +346,25 @@ All checks passed! After that works, run the tests locally. Each new functionality MUST have a test: ```bash $ make check # ruff lint + mypy type check -$ make test # pytest — 224 tests +$ make test # pytest ``` - ## Updating FormulaCode The monthly update is a single command: ```bash -$ ds-update --start-date 2026-02-01 --end-date 2026-03-01 +$ fc-data --start-date 2026-02-01 --end-date 2026-03-01 ``` This runs six stages in order: scrape repos, scrape commits, classify PRs, resolve packages, synthesize Docker images, and publish to DockerHub + HuggingFace. Options: ```bash -$ ds-update --start-date 2026-02-01 --end-date 2026-03-01 --resume # skip completed stages -$ ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 4 # run only package resolution -$ ds-update --start-date 2026-02-01 --end-date 2026-03-01 --dry-run # log without executing -$ ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 \ +$ fc-data --start-date 2026-02-01 --end-date 2026-03-01 --resume # skip completed stages +$ fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 4 # run only package resolution +$ fc-data --start-date 2026-02-01 --end-date 2026-03-01 --dry-run # log without executing +$ fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 \ --agent codex --n-concurrent 5 --tasks-per-repo 5 # synthesis with codex, capped -$ ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 \ +$ fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 \ --force # re-run synthesis for all tasks ``` @@ -469,4 +404,4 @@ $ python scratch/scripts/prepare_formulacode_dataset.py \ ### Evaluation -Evaluation is done in FormulaCode's fork of the [terminal-bench](https://github.com/formula-code/terminal-bench) evaluation framework. +Evaluation is done in FormulaCode's fork of the [terminal-bench](https://github.com/formula-code/fc-eval) evaluation framework. diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index cf1b3dc5..7b6bb2bb 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -33,7 +33,7 @@ This creates a virtual environment with `uv`, installs all dependencies, and set ## 3. Configure `tokens.env` -DataSmith reads all configuration from a `tokens.env` file in the repo root. The `Settings` class (powered by `pydantic-settings`) loads it automatically — no manual `source` or `export` needed. +fc-data reads all configuration from a `tokens.env` file in the repo root. The `Settings` class (powered by `pydantic-settings`) loads it automatically — no manual `source` or `export` needed. Create the file: @@ -55,7 +55,7 @@ SUPABASE_KEY= # === GitHub (required) === # One or more GitHub personal access tokens, comma-separated. -# DataSmith rotates tokens automatically when one hits the rate limit. +# fc-data rotates tokens automatically when one hits the rate limit. # Create tokens at https://github.com/settings/tokens with `repo` scope. GH_TOKENS=github_pat_xxx ``` @@ -98,7 +98,7 @@ See [Configuration](../guide/configuration.md) for a complete reference of all e ## 4. Set up Supabase -DataSmith uses a local Supabase instance for all persistent state (no cloud account needed). +fc-data uses a local Supabase instance for all persistent state (no cloud account needed). ### Start the instance @@ -129,7 +129,7 @@ This prints connection details. Copy the **service_role key** (not the anon key) ### Apply migrations -DataSmith's schema is defined in numbered SQL migrations: +fc-data's schema is defined in numbered SQL migrations: ```bash npx supabase migration up --local @@ -187,7 +187,7 @@ make test # pytest You're ready to run the pipeline: ```bash -ds-update --start-date 2026-03-01 --end-date 2026-04-01 +fc-data --start-date 2026-03-01 --end-date 2026-04-01 ``` See the **[Pipeline guide](../guide/pipeline.md)** for the full CLI reference and stage descriptions. diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index f1159aeb..c2b55db8 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -1,6 +1,6 @@ # Quickstart -This guide walks through common DataSmith operations with code examples. +This guide walks through common fc-data operations with code examples. ## Working with pull requests diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 42747c97..e5caf928 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1,6 +1,6 @@ # Configuration -DataSmith is configured primarily through a `tokens.env` file in the repository root. The `Settings` class (powered by `pydantic-settings`) loads these automatically. +fc-data is configured primarily through a `tokens.env` file in the repository root. The `Settings` class (powered by `pydantic-settings`) loads these automatically. ## Environment variables diff --git a/docs/guide/docker-images.md b/docs/guide/docker-images.md index a7bfa60b..2e8ee322 100644 --- a/docs/guide/docker-images.md +++ b/docs/guide/docker-images.md @@ -1,6 +1,6 @@ # Docker Images -DataSmith uses a three-tier Docker image hierarchy to build reproducible environments for each pull request. +fc-data uses a three-tier Docker image hierarchy to build reproducible environments for each pull request. ## Image hierarchy diff --git a/docs/guide/pipeline.md b/docs/guide/pipeline.md index c2ff0d3f..38a83624 100644 --- a/docs/guide/pipeline.md +++ b/docs/guide/pipeline.md @@ -1,22 +1,22 @@ -# Pipeline (`ds-update`) +# Pipeline (`fc-data`) -`ds-update` is the primary command for running DataSmith. It discovers performance-improving commits from GitHub, classifies them with LLM agents, resolves dependencies, synthesizes Docker build contexts, and publishes verified images. +`fc-data` is the primary command for running fc-data. It discovers performance-improving commits from GitHub, classifies them with LLM agents, resolves dependencies, synthesizes Docker build contexts, and publishes verified images. ## Quick reference ```bash # Run all 7 stages for a date range -ds-update --start-date 2026-02-01 --end-date 2026-03-01 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 # Resume from where you left off (skips completed stages) -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --resume +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --resume # Run only specific stages -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 3 -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 --stage 6 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 3 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 --stage 6 # Preview what would run without executing -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --dry-run +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --dry-run ``` ## CLI flags @@ -53,10 +53,10 @@ For each repository found, the runner fetches and stores metadata including desc ```bash # Discover repos with at least 1000 stars -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 1 --min-stars 1000 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 1 --min-stars 1000 # Include repos from an offline dataset -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 1 --offline-source data.parquet +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 1 --offline-source data.parquet ``` **Writes to:** `repositories` table @@ -76,7 +76,7 @@ For every repository in the `repositories` table, scrapes all merged pull reques If `--offline-source` is provided, also bulk-imports PR records from the Parquet file (useful for seeding the database with historical data). ```bash -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 2 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 2 ``` **Writes to:** `pull_requests` table (one row per PR) @@ -95,10 +95,10 @@ Only PRs that pass a symbolic pre-filter (`is_performance_commit_symbolic = True ```bash # Classify all unclassified PRs -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 3 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 3 # Re-classify all PRs (including already-classified ones) -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 3 --force +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 3 --force ``` **Updates:** `pull_requests` table (sets `is_performance_commit`, `classification`) @@ -118,7 +118,7 @@ For each performance-classified PR, checks out the repository at the merge commi The resolved dependency set (`env_payload`) and the Python version used are stored in the `packages` table. These are consumed by stage 6 to build `docker_build_env.sh` — the shell script that installs dependencies inside the Docker image. ```bash -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 4 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 4 ``` **Writes to:** `packages` table (`env_payload`, `python_version`, `can_install`) @@ -139,13 +139,13 @@ Builds a rich, deconstructed problem context for each PR. This context is what t ```bash # Render problem contexts -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 # Limit to 3 PRs per repo (useful for testing) -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 --tasks-per-repo 3 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 --tasks-per-repo 3 # Re-render already-processed PRs -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 --force +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 5 --force ``` **Writes to:** `candidate_prs` table @@ -169,19 +169,19 @@ Each attempt (success or failure) is logged to the `error_logs` table with the a ```bash # Use the default auto-detected agent -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 6 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 6 # Force a specific agent -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 6 --agent claude +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 6 --agent claude # Skip LLM generation entirely — only use cached/similar scripts -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 6 --agent none +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 6 --agent none # Limit concurrency and tasks per repo (controls cost) -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 6 --n-concurrent 2 --tasks-per-repo 5 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 6 --n-concurrent 2 --tasks-per-repo 5 # Re-synthesize already-completed PRs -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 6 --force +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 6 --force ``` **Writes to:** `candidate_containers` table (on success), `error_logs` table (every attempt) @@ -207,7 +207,7 @@ The publish pipeline: 6. **Mark published** — Sets `published_at` timestamp on each published PR row ```bash -ds-update --start-date 2026-02-01 --end-date 2026-03-01 --stage 7 +fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 7 ``` **Reads from:** `pull_requests`, `packages`, `candidate_containers` @@ -221,13 +221,13 @@ A monthly update usually looks like this: ```bash # 1. Run the full pipeline -ds-update --start-date 2026-03-01 --end-date 2026-04-01 +fc-data --start-date 2026-03-01 --end-date 2026-04-01 # 2. If it gets interrupted, resume where it left off -ds-update --start-date 2026-03-01 --end-date 2026-04-01 --resume +fc-data --start-date 2026-03-01 --end-date 2026-04-01 --resume # 3. After fixing a synthesis issue, re-run just stages 6-7 -ds-update --start-date 2026-03-01 --end-date 2026-04-01 --stage 6 --stage 7 --force +fc-data --start-date 2026-03-01 --end-date 2026-04-01 --stage 6 --stage 7 --force ``` ## Monitoring progress diff --git a/docs/guide/publishing.md b/docs/guide/publishing.md index 10679c78..2926affa 100644 --- a/docs/guide/publishing.md +++ b/docs/guide/publishing.md @@ -1,6 +1,6 @@ # Publishing -DataSmith publishes verified tasks to DockerHub (Docker images) and HuggingFace (Parquet datasets). +fc-data publishes verified tasks to DockerHub (Docker images) and HuggingFace (Parquet datasets). ## Publishing workflow @@ -49,7 +49,7 @@ rows = sb.table("pull_requests") \ ## Pipeline integration -Publishing is the final stage of the `ds-update` pipeline (stage 7). The `publish_pipeline()` function orchestrates: +Publishing is the final stage of the `fc-data` pipeline (stage 7). The `publish_pipeline()` function orchestrates: 1. Query DB for unpublished, verified PRs 2. Push Docker images to DockerHub diff --git a/docs/index.md b/docs/index.md index 6a8927c1..7b0917b8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,11 +1,11 @@ -# 🔧 DataSmith +# 🔧 fc-data **Python toolchain for building the [FormulaCode](https://github.com/formula-code) benchmark.** [![Build status](https://img.shields.io/github/actions/workflow/status/formula-code/datasmith/main.yml?branch=main)](https://github.com/formula-code/datasmith/actions/workflows/main.yml?query=branch%3Amain) [![License](https://img.shields.io/github/license/formula-code/datasmith)](https://img.shields.io/github/license/formula-code/datasmith) -DataSmith is a package for automatically building and maintaining FormulaCode tasks. It is engineered to support any repository-level, verification-by-execution based coding benchmark that heavily uses Docker and GitHub. +fc-data is a package for automatically building and maintaining FormulaCode tasks. It is engineered to support any repository-level, verification-by-execution based coding benchmark that heavily uses Docker and GitHub. FormulaCode is a benchmark of **67+ repositories** with **964+ performance-improving commits**, designed to evaluate LLMs' ability to optimize real-world codebases. It scores optimizers relative to the human-authored speedup using ASV (Airspeed Velocity) benchmarks — providing a dense performance signal instead of binary pass/fail. @@ -20,24 +20,24 @@ graph LR A[GitHub] A2[Supabase] - B["DataSmith
(This repository)"] + B["fc-data
(This repository)"] C[DockerHub] D[HuggingFace] ``` ## Get started -Most interaction with DataSmith is through a single command: +Most interaction with fc-data is through a single command: ```bash -ds-update --start-date 2026-03-01 --end-date 2026-04-01 +fc-data --start-date 2026-03-01 --end-date 2026-04-01 ``` This runs all 7 pipeline stages: repo discovery, PR scraping, LLM classification, dependency resolution, problem rendering, Docker synthesis, and publishing. See the **[Pipeline guide](guide/pipeline.md)** for the full CLI reference. ## Key features -- **Single-command pipeline** — `ds-update` runs all stages with `--resume`, `--stage`, and `--dry-run` support +- **Single-command pipeline** — `fc-data` runs all stages with `--resume`, `--stage`, and `--dry-run` support - **GitHub scraping** — Async `httpx` client with automatic token rotation across multiple `GH_TOKENS` - **LLM classification** — DSPy-based agents classify PRs by performance category and difficulty - **Docker synthesis** — Automatically generate Docker build contexts using coding agents (Claude, Codex, Gemini) @@ -47,6 +47,6 @@ This runs all 7 pipeline stages: repo discovery, PR scraping, LLM classification ## Quick links - [Installation](getting-started/installation.md) — Set up your development environment -- [Pipeline guide (`ds-update`)](guide/pipeline.md) — **The primary entrypoint** — full CLI reference and stage descriptions +- [Pipeline guide (`fc-data`)](guide/pipeline.md) — **The primary entrypoint** — full CLI reference and stage descriptions - [Quickstart](getting-started/quickstart.md) — Python API examples - [Configuration](guide/configuration.md) — `tokens.env` and environment variables diff --git a/docs/stage6-failure-analysis.md b/docs/stage6-failure-analysis.md index 0e965879..38110936 100644 --- a/docs/stage6-failure-analysis.md +++ b/docs/stage6-failure-analysis.md @@ -1,7 +1,7 @@ # Stage 6 (synthesize_images) Failure Analysis **Date**: 2026-04-04 -**Pipeline invocation**: `ds-update --start-date 2017-01-01 --end-date 2026-03-01 --stage 6 --n-concurrent 4 --tasks-per-repo 1` +**Pipeline invocation**: `fc-data --start-date 2017-01-01 --end-date 2026-03-01 --stage 6 --n-concurrent 4 --tasks-per-repo 1` ## High-level numbers diff --git a/mkdocs.yml b/mkdocs.yml index f40e284c..12fab276 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: 🔧 DataSmith +site_name: 🔧 fc-data site_url: https://formula-code.github.io/datasmith/ site_description: Python toolchain for building the FormulaCode benchmark repo_url: https://github.com/formula-code/datasmith @@ -81,7 +81,7 @@ nav: - Home: index.md - Getting Started: - Installation: getting-started/installation.md - - Pipeline (ds-update): guide/pipeline.md + - Pipeline (fc-data): guide/pipeline.md - Configuration: guide/configuration.md - User Guide: - Docker Images: guide/docker-images.md diff --git a/pyproject.toml b/pyproject.toml index 89fbe0eb..5b4cbc93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] -name = "datasmith" +name = "fc-data" version = "0.1.0" -description = "Python codebase for preparing formulacode's hugginface repositories." +description = "Python toolchain for building and maintaining FormulaCode benchmark tasks." authors = [{ name = "Atharva Sehgal", email = "atharva.sehgal@gmail.com" }] readme = "README.md" keywords = ['python'] @@ -33,7 +33,7 @@ dependencies = [ ] [project.scripts] -ds-update = "datasmith.update.cli:main" +fc-data = "datasmith.update.cli:main" [project.urls] Homepage = "https://formula-code.github.io/datasmith/" @@ -42,6 +42,7 @@ Documentation = "https://formula-code.github.io/datasmith/" [dependency-groups] dev = [ + "build>=1.0", "deptry>=0.23.0", "ipykernel", "ipython", @@ -59,6 +60,12 @@ dev = [ "types-tqdm", ] +docs = [ + "mkdocs>=1.6", + "mkdocs-material>=9.5", + "mkdocstrings[python]>=0.24", +] + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/src/datasmith/update/cli.py b/src/datasmith/update/cli.py index d78c112e..e41fa4d0 100644 --- a/src/datasmith/update/cli.py +++ b/src/datasmith/update/cli.py @@ -34,7 +34,7 @@ def _stages_epilog() -> str: def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( - prog="ds-update", + prog="fc-data", description="Run the FormulaCode update pipeline — discovers performance-improving " "commits from GitHub, classifies them with LLM agents, and builds Docker images " "for benchmark evaluation.", diff --git a/tests/test_website_snippets.py b/tests/test_website_snippets.py index 512c882f..cfe45510 100644 --- a/tests/test_website_snippets.py +++ b/tests/test_website_snippets.py @@ -481,15 +481,15 @@ def test_supabase_client_table_query(self) -> None: # --------------------------------------------------------------------------- -# Guide — "Pipeline" (ds-update CLI) +# Guide — "Pipeline" (fc-data CLI) # --------------------------------------------------------------------------- class TestPipelineSnippets: - """Snippets from the Pipeline guide page: ds-update CLI.""" + """Snippets from the Pipeline guide page: fc-data CLI.""" def test_ds_update_entry_point_exists(self) -> None: - """Website: ``ds-update --start-date 2026-02-01 --end-date 2026-03-01``""" + """Website: ``fc-data --start-date 2026-02-01 --end-date 2026-03-01``""" from datasmith.update.cli import main assert callable(main)