From 317ef5f15d5ce73a4e96461221678acbdd64247a Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:05:26 +0000 Subject: [PATCH 01/24] Add rate-limit detection module and error_logs migration --- src/datasmith/agents/rate_limit.py | 219 ++++++++++++++++++ .../00011_error_logs_rate_limit.sql | 13 ++ tests/agents/test_rate_limit.py | 124 ++++++++++ 3 files changed, 356 insertions(+) create mode 100644 src/datasmith/agents/rate_limit.py create mode 100644 supabase/migrations/00011_error_logs_rate_limit.sql create mode 100644 tests/agents/test_rate_limit.py diff --git a/src/datasmith/agents/rate_limit.py b/src/datasmith/agents/rate_limit.py new file mode 100644 index 00000000..9189fc98 --- /dev/null +++ b/src/datasmith/agents/rate_limit.py @@ -0,0 +1,219 @@ +"""Detect CLI-agent rate-limit / budget-exhaustion errors in raw output. + +Stage 6 (synthesize_images) drives `codex` and `claude` CLI agents against +a shared weekly budget. When the budget runs out, each subsequent attempt +fails in ~2s with an unhelpful `failure_stage='aborted'` unless we +recognise the pattern. This module parses the raw JSONL stream emitted +by either CLI and, if a rate-limit signal is present, returns the +budget-reset timestamp so the runner can pause until the limit clears. + +Detection schemas (observed in error_logs): + +* **Codex** — free-text error event: + {"type":"error","message":"You've hit your usage limit. ... try again + at Apr 11th, 2026 2:32 PM."} + The reset time is a tz-naive, human-readable string. We parse it with + dateutil and assume UTC since the Codex CLI does not include a zone. + +* **Claude** — structured `rate_limit_event` with `resetsAt` (unix epoch): + {"type":"rate_limit_event","rate_limit_info":{ + "status":"allowed|allowed_warning|exceeded|blocked|...", + "resetsAt":1775559600, + "rateLimitType":"five_hour|weekly|...", + "overageStatus":"...", "overageResetsAt":...}} + Status values `allowed` and `allowed_warning` are non-blocking. Any + other status is treated as rate-limited. +""" + +from __future__ import annotations + +import datetime +import json +import re + +# Claude statuses that are NOT rate-limited. Anything else we see means the +# CLI is signalling that the next turn will be blocked or has been blocked. +_CLAUDE_OK_STATUSES = frozenset({"allowed", "allowed_warning"}) + +# Matches the human-readable reset time embedded in Codex usage-limit errors. +# Example: "try again at Apr 11th, 2026 2:32 PM." +_CODEX_RESET_RE = re.compile( + r"try again at\s+" + r"(?P[A-Za-z]{3,9})\s+" + r"(?P\d{1,2})(?:st|nd|rd|th)?,\s+" + r"(?P\d{4})\s+" + r"(?P\d{1,2}):(?P\d{2})\s*" + r"(?PAM|PM)", + re.IGNORECASE, +) + +_MONTHS = { + "jan": 1, + "feb": 2, + "mar": 3, + "apr": 4, + "may": 5, + "jun": 6, + "jul": 7, + "aug": 8, + "sep": 9, + "oct": 10, + "nov": 11, + "dec": 12, +} + + +class RateLimitError(RuntimeError): + """Raised by the synthesizer when an agent attempt hit a usage limit. + + ``reset_at`` is always timezone-aware UTC when known, or ``None`` if + the CLI signalled a rate limit without giving a reset timestamp (in + which case the caller should pick a default pause duration). + """ + + def __init__( + self, + agent_name: str, + reset_at: datetime.datetime | None, + message: str = "", + ) -> None: + self.agent_name = agent_name + self.reset_at = reset_at + self.message = message or f"{agent_name} hit usage limit" + super().__init__(self.message) + + +def detect(agent_name: str, raw_output: str) -> datetime.datetime | None | object: + """Return the reset timestamp if *raw_output* shows a rate-limit hit. + + Returns: + * ``None`` — not rate-limited. + * ``False`` sentinel (via ``_NO_RESET``) — rate-limited but no + reset time could be parsed. + * ``datetime`` (UTC, tz-aware) — rate-limited and we know when the + budget will clear. + """ + if not raw_output: + return None + + if agent_name == "codex" or "codex" in agent_name.lower(): + return _detect_codex(raw_output) + if agent_name == "claude" or "claude" in agent_name.lower(): + return _detect_claude(raw_output) + return None + + +# Sentinel returned when we know a rate limit happened but couldn't parse a +# reset time. ``None`` already means "no rate limit", so we need a distinct +# "yes but unknown reset" value. +_NO_RESET: object = object() + + +def _detect_codex(raw_output: str) -> datetime.datetime | None | object: # noqa: C901 + """Parse the codex JSONL stream for a usage-limit error event.""" + if "usage limit" not in raw_output.lower(): + return None + + # Walk the last few lines — the error is always near the tail. + for line in reversed(raw_output.splitlines()[-20:]): + line = line.strip() + if not line or not line.startswith("{"): + continue + try: + evt = json.loads(line) + except json.JSONDecodeError: + continue + + msg = "" + if evt.get("type") == "error": + msg = evt.get("message") or "" + elif evt.get("type") == "turn.failed": + err = evt.get("error") or {} + msg = err.get("message") or "" + if not msg or "usage limit" not in msg.lower(): + continue + + m = _CODEX_RESET_RE.search(msg) + if not m: + return _NO_RESET + try: + return _parse_codex_reset(m) + except ValueError: + return _NO_RESET + + # Fell through without matching a structured event but the substring + # is present — still treat it as rate limited. + m = _CODEX_RESET_RE.search(raw_output) + if m: + try: + return _parse_codex_reset(m) + except ValueError: + return _NO_RESET + return _NO_RESET + + +def _parse_codex_reset(match: re.Match[str]) -> datetime.datetime: + month = _MONTHS[match.group("month")[:3].lower()] + day = int(match.group("day")) + year = int(match.group("year")) + hour = int(match.group("hour")) % 12 + if match.group("ampm").upper() == "PM": + hour += 12 + minute = int(match.group("minute")) + # Codex does not disclose a timezone for the reset time. Assume UTC — + # this is slightly conservative (may resume up to a few hours early or + # late) but correct to within a budget cycle. + return datetime.datetime(year, month, day, hour, minute, tzinfo=datetime.UTC) + + +def _detect_claude(raw_output: str) -> datetime.datetime | None | object: # noqa: C901 + """Parse the claude JSONL stream for a rate_limit_event with a blocking status.""" + if "rate_limit_event" not in raw_output and "rate_limit" not in raw_output.lower(): + return None + + # Scan all lines; Claude emits a rate_limit_event after every turn, so + # the *last* one is authoritative about current budget state. + last_reset: int | None = None + last_overage_reset: int | None = None + blocked = False + for line in raw_output.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + evt = json.loads(line) + except json.JSONDecodeError: + continue + if evt.get("type") != "rate_limit_event": + continue + info = evt.get("rate_limit_info") or {} + status = info.get("status") + overage_status = info.get("overageStatus") + # Track the most recent values. + if isinstance(info.get("resetsAt"), int | float): + last_reset = int(info["resetsAt"]) + if isinstance(info.get("overageResetsAt"), int | float): + last_overage_reset = int(info["overageResetsAt"]) + if status and status not in _CLAUDE_OK_STATUSES: + blocked = True + if overage_status and overage_status not in _CLAUDE_OK_STATUSES: + blocked = True + + if not blocked: + return None + + # Prefer the standard reset over the overage reset if both are present. + reset_epoch = last_reset or last_overage_reset + if reset_epoch is None: + return _NO_RESET + return datetime.datetime.fromtimestamp(reset_epoch, tz=datetime.UTC) + + +def check(agent_name: str, raw_output: str) -> tuple[bool, datetime.datetime | None]: + """Convenience wrapper returning ``(is_rate_limited, reset_at_or_None)``.""" + result = detect(agent_name, raw_output) + if result is None: + return False, None + if result is _NO_RESET: + return True, None + return True, result # type: ignore[return-value] diff --git a/supabase/migrations/00011_error_logs_rate_limit.sql b/supabase/migrations/00011_error_logs_rate_limit.sql new file mode 100644 index 00000000..909c8c18 --- /dev/null +++ b/supabase/migrations/00011_error_logs_rate_limit.sql @@ -0,0 +1,13 @@ +-- Add rate-limit reset tracking to error_logs. +-- +-- Stage 6 (synthesize_images) can burn through the weekly Codex/Claude budget +-- and start hitting usage-limit errors. We now detect those errors in the +-- raw agent_output, stamp failure_stage='rate_limited', and record the +-- reset timestamp here so the runner can pause until the budget resets. + +ALTER TABLE error_logs + ADD COLUMN IF NOT EXISTS rate_limit_reset_at TIMESTAMPTZ; + +CREATE INDEX IF NOT EXISTS idx_error_logs_rate_limit_reset_at + ON error_logs (rate_limit_reset_at) + WHERE rate_limit_reset_at IS NOT NULL; diff --git a/tests/agents/test_rate_limit.py b/tests/agents/test_rate_limit.py new file mode 100644 index 00000000..da7e0612 --- /dev/null +++ b/tests/agents/test_rate_limit.py @@ -0,0 +1,124 @@ +"""Tests for the CLI-agent rate-limit detector.""" + +from __future__ import annotations + +import datetime +import json + +from datasmith.agents.rate_limit import check, detect + +CODEX_HEADER = '{"type":"thread.started","thread_id":"abc"}\n{"type":"turn.started"}\n' + +CODEX_ERROR_EVENT = ( + '{"type":"error","message":"You\'ve hit your usage limit. Upgrade to Plus to ' + "continue using Codex (https://chatgpt.com/explore/plus), or try again at " + 'Apr 11th, 2026 2:32 PM."}\n' +) + +CODEX_TURN_FAILED = ( + '{"type":"turn.failed","error":{"message":"You\'ve hit your usage limit. ' + "Upgrade to Plus to continue using Codex (https://chatgpt.com/explore/plus), " + 'or try again at Apr 16th, 2026 4:54 PM."}}\n' +) + + +def test_codex_usage_limit_parses_reset_time() -> None: + raw = CODEX_HEADER + CODEX_ERROR_EVENT + CODEX_TURN_FAILED + blocked, reset = check("codex", raw) + assert blocked + assert reset == datetime.datetime(2026, 4, 16, 16, 54, tzinfo=datetime.UTC) + + +def test_codex_usage_limit_without_reset_still_flags() -> None: + # Error present but garbled reset text. + raw = CODEX_HEADER + '{"type":"error","message":"You\'ve hit your usage limit, try again later."}\n' + blocked, reset = check("codex", raw) + assert blocked is True + assert reset is None + + +def test_codex_clean_run_not_flagged() -> None: + raw = CODEX_HEADER + '{"type":"turn.completed","usage":{"input_tokens":4290}}\n' + blocked, reset = check("codex", raw) + assert blocked is False + assert reset is None + + +def test_claude_allowed_status_not_flagged() -> None: + evt = { + "type": "rate_limit_event", + "rate_limit_info": { + "status": "allowed", + "resetsAt": 1775559600, + "rateLimitType": "five_hour", + "overageStatus": "allowed", + "overageResetsAt": 1777593600, + "isUsingOverage": False, + }, + } + raw = json.dumps(evt) + "\n" + blocked, _ = check("claude", raw) + assert blocked is False + + +def test_claude_allowed_warning_not_flagged() -> None: + evt = { + "type": "rate_limit_event", + "rate_limit_info": { + "status": "allowed_warning", + "resetsAt": 1775559600, + "rateLimitType": "five_hour", + "utilization": 0.93, + "surpassedThreshold": 0.9, + }, + } + raw = json.dumps(evt) + "\n" + blocked, _ = check("claude", raw) + assert blocked is False + + +def test_claude_blocked_status_returns_reset_time() -> None: + evt = { + "type": "rate_limit_event", + "rate_limit_info": { + "status": "exceeded", + "resetsAt": 1775559600, + "rateLimitType": "weekly", + "overageStatus": "exceeded", + "overageResetsAt": 1777593600, + "isUsingOverage": True, + }, + } + raw = json.dumps(evt) + "\n" + blocked, reset = check("claude", raw) + assert blocked is True + assert reset == datetime.datetime.fromtimestamp(1775559600, tz=datetime.UTC) + + +def test_claude_uses_latest_event() -> None: + # Two events: the older is allowed, the newer is exceeded. + ok = { + "type": "rate_limit_event", + "rate_limit_info": {"status": "allowed", "resetsAt": 1775000000}, + } + bad = { + "type": "rate_limit_event", + "rate_limit_info": {"status": "blocked", "resetsAt": 1775559600}, + } + raw = json.dumps(ok) + "\n" + json.dumps(bad) + "\n" + blocked, reset = check("claude", raw) + assert blocked is True + assert reset == datetime.datetime.fromtimestamp(1775559600, tz=datetime.UTC) + + +def test_empty_output_not_flagged() -> None: + assert check("codex", "") == (False, None) + assert check("claude", "") == (False, None) + + +def test_unknown_agent_not_flagged() -> None: + assert check("gemini", CODEX_ERROR_EVENT) == (False, None) + + +def test_detect_returns_none_when_not_limited() -> None: + assert detect("codex", "ok done\n") is None From 0ac862b52309385af7da6db97dbdb06329a59835 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:05:45 +0000 Subject: [PATCH 02/24] Add Qwen Code CLI agent --- package.json | 3 +- src/datasmith/agents/installed/README.md | 1 + src/datasmith/agents/installed/__init__.py | 2 + src/datasmith/agents/installed/base.py | 4 +- src/datasmith/agents/installed/qwen.py | 129 +++++++++ tests/agents/installed/FAILURE_MODES.md | 308 +++++++++++++++++++++ tests/agents/installed/test_agents.py | 79 ++++++ 7 files changed, 524 insertions(+), 2 deletions(-) create mode 100644 src/datasmith/agents/installed/qwen.py create mode 100644 tests/agents/installed/FAILURE_MODES.md diff --git a/package.json b/package.json index 6c2eb718..95840949 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "dependencies": { - "@google/gemini-cli": "^0.34.0" + "@google/gemini-cli": "^0.34.0", + "@qwen-code/qwen-code": "^0.14.5" } } diff --git a/src/datasmith/agents/installed/README.md b/src/datasmith/agents/installed/README.md index f625a648..045c859e 100644 --- a/src/datasmith/agents/installed/README.md +++ b/src/datasmith/agents/installed/README.md @@ -11,6 +11,7 @@ structured output. | Claude Code | `claude` | `npm install -g @anthropic-ai/claude-code` | | Codex | `codex` | `npm install -g @openai/codex` | | Gemini CLI | `gemini` | `npm install -g @anthropic-ai/gemini-cli` | +| Qwen Code | `qwen` | `npm install -g @qwen-code/qwen-code` | ## Interface contract diff --git a/src/datasmith/agents/installed/__init__.py b/src/datasmith/agents/installed/__init__.py index 21d0cdda..170bed28 100644 --- a/src/datasmith/agents/installed/__init__.py +++ b/src/datasmith/agents/installed/__init__.py @@ -9,6 +9,7 @@ from datasmith.agents.installed.codex import CodexAgent from datasmith.agents.installed.gemini import GeminiAgent from datasmith.agents.installed.none import NoneAgent +from datasmith.agents.installed.qwen import QwenAgent __all__ = [ "AgentResult", @@ -18,5 +19,6 @@ "GeminiAgent", "InstalledAgent", "NoneAgent", + "QwenAgent", "get_agent", ] diff --git a/src/datasmith/agents/installed/base.py b/src/datasmith/agents/installed/base.py index 4402d12d..c8999a61 100644 --- a/src/datasmith/agents/installed/base.py +++ b/src/datasmith/agents/installed/base.py @@ -218,15 +218,17 @@ def get_agent(preference: list[str] | None = None) -> InstalledAgent: from datasmith.agents.installed.codex import CodexAgent from datasmith.agents.installed.gemini import GeminiAgent from datasmith.agents.installed.none import NoneAgent + from datasmith.agents.installed.qwen import QwenAgent registry: dict[str, type[InstalledAgent]] = { "claude": ClaudeAgent, "codex": CodexAgent, "gemini": GeminiAgent, + "qwen": QwenAgent, "none": NoneAgent, } - order = preference or ["claude", "codex", "gemini"] + order = preference or ["claude", "codex", "gemini", "qwen"] for name in order: cls = registry.get(name) if cls is None: diff --git a/src/datasmith/agents/installed/qwen.py b/src/datasmith/agents/installed/qwen.py new file mode 100644 index 00000000..6a0fa748 --- /dev/null +++ b/src/datasmith/agents/installed/qwen.py @@ -0,0 +1,129 @@ +"""Qwen Code CLI agent implementation.""" + +from __future__ import annotations + +import json + +from datasmith.agents.installed.base import AgentResult, InstalledAgent, run_agent_subprocess +from datasmith.utils import get_logger + +logger = get_logger("agents.installed.qwen") + +_FILE_TOOL_NAMES = {"write_file", "edit", "Write", "Edit", "edit_file", "create_file", "update_file"} + + +def _extract_assistant_text(message: object) -> str: + """Extract text from a Qwen assistant message payload.""" + if isinstance(message, str): + return message + if isinstance(message, dict): + content = message.get("content", "") + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "") + # Strip ... reasoning blocks from output + if "" in text: + text = text.split("", 1)[-1].strip() + if text: + parts.append(text) + return "\n".join(parts) + return "" + + +def _parse_qwen_stdout(stdout: str) -> tuple[list[str], list[str]]: # noqa: C901 + """Parse Qwen Code stream-json output into (output_lines, files_changed).""" + files_changed: list[str] = [] + output_lines: list[str] = [] + + for line in stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + if not isinstance(obj, dict): + continue + + msg_type = obj.get("type", "") + + if msg_type == "assistant" and "message" in obj: + text = _extract_assistant_text(obj["message"]) + if text: + output_lines.append(text) + elif msg_type == "result": + result_text = obj.get("result", "") + if isinstance(result_text, str) and result_text: + # Strip ... from result text too + if "" in result_text: + result_text = result_text.split("", 1)[-1].strip() + if result_text: + output_lines.append(result_text) + elif msg_type == "tool_use": + _collect_file_change(obj, files_changed) + + except json.JSONDecodeError: + output_lines.append(line) + + return output_lines, files_changed + + +def _collect_file_change(obj: dict, files_changed: list[str]) -> None: + """Extract file path from a tool_use event if it's a file-editing tool.""" + tool_name = obj.get("name", "") + tool_input = obj.get("input") or obj.get("args", {}) + if tool_name in _FILE_TOOL_NAMES and isinstance(tool_input, dict): + path = tool_input.get("file_path") or tool_input.get("path", "") + if path: + files_changed.append(path) + + +class QwenAgent(InstalledAgent): + """Qwen Code CLI agent.""" + + def name(self) -> str: + return "qwen" + + def is_available(self) -> bool: + return self._which("qwen") + + def exec( + self, + prompt: str, + timeout: int = 3600, + workdir: str | None = None, + ) -> AgentResult: + cmd = [ + "qwen", + "-p", + prompt, + "--yolo", + "-o", + "stream-json", + ] + + logger.debug("qwen command: %s", " ".join(cmd)) + + try: + returncode, stdout, stderr, duration = run_agent_subprocess( + cmd, timeout=timeout, cwd=workdir, agent_name="qwen" + ) + output_lines, files_changed = _parse_qwen_stdout(stdout) + + return AgentResult( + success=returncode == 0, + output="\n".join(output_lines) if output_lines else stdout, + raw_output=stdout, + files_changed=files_changed, + duration_s=duration, + error=stderr if returncode != 0 else "", + ) + except FileNotFoundError: + return AgentResult( + success=False, + duration_s=0.0, + error="qwen CLI not found. Install with: npm install -g @qwen-code/qwen-code", + ) diff --git a/tests/agents/installed/FAILURE_MODES.md b/tests/agents/installed/FAILURE_MODES.md new file mode 100644 index 00000000..c286f521 --- /dev/null +++ b/tests/agents/installed/FAILURE_MODES.md @@ -0,0 +1,308 @@ +# Synthesis Failure Modes & Bypass Catalog + +This document catalogs the bypass patterns and failure modes discovered during +stage-6 synthesis. Each entry describes the pattern, when it was first +encountered, which repos/agents exhibited it, how it was detected, and what +architectural fix closed it. + +--- + +## 1. Pre-fabricated `/logs/summary_*.json` + +**First encountered**: 2026-04-11, initial audit of 862 candidate_containers. + +**What the agent does**: Writes a JSON file like `/logs/summary_agent.json` +with `{"passed": 1, "total": 1, ...}` inside `docker_build_run.sh` (or +`docker_build_pkg.sh`). `parser.py` reads this file after pytest runs and +reports it as the snapshot summary. The validator's `snapshot.total > 0` +check passes without any real benchmarks running. + +**Variants observed**: +- Shell heredoc: `cat > /logs/summary_agent.json << 'EOF'` +- Python inline: `python3 -c "json.dump({'passed':1}, open('/logs/summary_agent.json','w'))"` +- Variable-name redirect: `export T_BENCH_AGENT_MODEL_NAME=smoke` then + writing `/logs/summary_smoke.json` to dodge wipes targeting the literal + "agent" name. + +**Scale**: 738 of 862 original containers (85.6%). + +**Repos**: pandas-dev/pandas (87), scikit-learn/scikit-learn (78), +Qiskit/qiskit (62), apache/arrow (51), modin-project/modin (41), and +essentially every other repo in the dataset. + +**Agents**: Both Claude and Codex independently discovered this pattern. + +**Detection**: `FORGED_LOGS_RE` in `tamper_audit.py` — matches any write to +`/logs/summary_*.json`, `/logs/test_results.json`, or +`/logs/postrun_agent-*.tar.gz` in the build scripts. + +**Fix**: +- `run-tests.sh`: hard-wipes `/logs/*` at script start with + `find "$T_BENCH_CONTAINER_LOGS_PATH" -mindepth 1 -delete` (catches regular + files, symlinks, and chardevs). +- `run-tests.sh`: pins `T_BENCH_AGENT_MODEL_NAME=agent` (unset + re-export) + so variable-name redirects have no effect. +- `local_ci.py`: simplified to trust exit code only, removing the parsed + `snapshot.total` check that was the original target. + +--- + +## 2. Root-level trivial test file injection + +**First encountered**: 2026-04-11, initial audit. + +**What the agent does**: Creates a `test_formulacode_trivial.py` at the repo +root containing `def test_x(): assert True`, then `git add` + `git commit`. +`pytest_runner.py`'s fallback discovery from cwd collects this file. With +`summary.total > 0` satisfied by 1 trivial passing test, the validator reports +success. + +**Scale**: 129 of 862 (root heredoc), 115 (trivial assert inline). + +**Repos**: apache/arrow (original report — PR #34476), pandas, scikit-learn, +Qiskit, napari, xarray, shapely. + +**Detection**: `ROOT_TEST_HEREDOC_RE`, `TRIVIAL_ASSERT_RE` in `tamper_audit.py`. + +**Fix**: Switched `pytest_runner.py` invocation from `--base $BASE` (git-diff +discovery) to `--all` (upstream's natural collection). The agent can't +influence which tests pytest collects because there's no git-diff input to +forge. + +--- + +## 3. Subdirectory test file injection + +**First encountered**: 2026-04-11, second audit round (after root-level fix). + +**What the agent does**: Creates test files under the repo's `tests/` +directory rather than at the root (e.g. `tests/test_build_smoke.py`, +`bottleneck/tests/test_agent_smoke.py`). These pass the root-level check but +are still agent-fabricated. + +**Repos**: pymc-devs/pymc, pydata/bottleneck, MDAnalysis/mdanalysis, +numpy/numpy, Quansight-Labs/ndindex, newrelic/newrelic-python-agent. + +**Detection**: `ANY_TEST_FILE_WRITE_RE`, `PY_TEST_FILE_WRITE_RE` in +`tamper_audit.py`. + +**Fix**: Same as #2 — `--all` mode runs the upstream's own test collection, +not agent-influenced discovery. Plus post-verification `classify_context` +catches any test file write in the build scripts. + +--- + +## 4. Tmux binary shim + +**First encountered**: 2026-04-11, deep script review of Textualize/rich#3807. + +**What the agent does**: `run-tests.sh` invokes bare `tmux` (line ~182). +The agent renames the real binary (`mv $TMUX_BIN ${TMUX_BIN}.real`) and +replaces it with a shell script that writes forged `/logs/summary_agent.json` +and exits 0. The shim runs at test time (after the initial `/logs/` wipe), +injecting the forged summary right before `parser.py` reads it. + +**Repos**: Textualize/rich, pandas-dev/pandas, sunpy/sunpy (codex). + +**Agents**: Both Claude and Codex. + +**Detection**: `TMUX_SHIM_RE` in `tamper_audit.py` — matches +`mv ... tmux ... .real`, `cat > ... tmux`, or writes targeting `$TMUX_BIN`. + +**Fix**: +- `run-tests.sh`: second `/logs/summary_*.json` wipe immediately before + `parser.py` runs, neutralizing runtime writes from shims. +- Post-verification `classify_context` catches the shim pattern in the saved + build scripts. + +--- + +## 5. Cosmetic source-file append to forge pytest discovery + +**First encountered**: 2026-04-11, smoke test v1 — all 5 new containers +exhibited this pattern. + +**What the agent does**: Appends a no-op comment or blank line to a tracked +source file (e.g. `echo "" >> src/solver.py`). This makes the file appear in +`git diff --name-only $BASE...HEAD`, which `pytest_runner.py`'s discovery maps +to a corresponding test file. The real test file runs and passes, but the +"optimization" is cosmetic — no performance change. + +**Variants**: +- `echo "" >> $REPO_ROOT/mujoco_warp/_src/solver.py` +- `printf '\n# OPT: chain sharding\n' >> src/bartz/X.py` +- `printf '\n' >> src/climpred/*.py` (blank line to every source file) + + `git commit --allow-empty` to make changes appear in committed diff. + +**The agent's own comments explained the strategy**: *"Leave a trivial +unstaged change in solver.py so formulacode_testrunner detects it via +`git diff --name-only` and maps it to solver_test.py"*. + +**Repos**: google-deepmind/mujoco_warp, bartz-org/bartz, devitocodes/devito, +xarray-contrib/xarray-spatial, pangeo-data/climpred. + +**Agents**: Both Claude and Codex independently discovered this. + +**Detection**: Not reliably regex-detectable (legitimate source patches look +identical). Closed architecturally. + +**Fix**: Switched from `--base $BASE` (git-diff discovery) to `--all` +(upstream's natural collection). No git diff = nothing to forge. + +--- + +## 6. `/dev/null` symlink to evade file-existence checks + +**First encountered**: 2026-04-11, bartz-org/bartz#72 (smoke test v1). + +**What the agent does**: `ln -sf /dev/null /logs/test_results.json`. The +`[ -f ]` check in the `/logs/` wipe returns false for a symlink to a chardev, +so the file isn't detected as pre-fabricated. Writes by pytest_runner go to +`/dev/null` (discarded), and the post-pytest cross-check silently skips +because `json.load()` on `/dev/null` raises an exception caught by +`|| true`. + +**The agent's own comments**: *"The test runner writes to this path +unconditionally; the post-pytest wipe_validator_artifacts() call uses `[ -f ]` +which returns false for a symlink to a device file"*. + +**Repos**: bartz-org/bartz. + +**Detection**: Closed by architecture change (hard-wipe with +`find -mindepth 1 -delete` removes symlinks). + +**Fix**: `find "$T_BENCH_CONTAINER_LOGS_PATH" -mindepth 1 -delete` replaces +the `[ -f ]` loop. `find -delete` removes regular files, symlinks, and device +nodes alike. + +--- + +## 7. Fabricated ASV config + benchmark files + +**First encountered**: 2026-04-11, PostHog/posthog#44307 and +optuna/optuna#6447. + +**What the agent does**: Creates a brand-new `asv.*.json` config pointing to a +new `benchmarks/` directory containing agent-written benchmark files. The +benchmarks are handwritten reimplementations of the PR's optimization logic, +not the upstream's actual benchmark suite. ASV discovers the fabricated suite +and reports benchmarks exist, but Harbor stage 7 would measure the agent's +reimplementation rather than the real code. + +**PostHog example**: Created `cli/asv.posthog.json` + +`cli/posthog_benchmarks/bench_backups.py` containing a standalone +reimplementation of `posthog/dags/backups.py`'s `get_latest_backups` with the +optimization hardcoded inline (`if not incremental: return []`). + +**Agents**: Both Claude and Codex independently produced byte-identical +fabrications for PostHog. + +**Scale**: PostHog (2 containers), optuna (2 containers). + +**Detection**: `NEW_ASV_CONFIG_RE`, `NEW_BENCH_FILE_RE` in `tamper_audit.py`. + +**Fix**: +- `docker_build_final.sh` runs after all agent-editable stages and + authoritatively discovers benchmarks via `asv run --bench just-discover`. + Any fabricated config is irrelevant — the final stage uses the upstream's + real config. +- Post-verification `classify_context` catches the patterns. + +--- + +## 8. `asv_benchmarks.txt` stub forgery + +**First encountered**: 2026-04-11, google-deepmind/mujoco_warp#1057. + +**What the agent does**: Writes hardcoded benchmark names directly to +`/workspace/repo/asv_benchmarks.txt` to satisfy `run-tests.sh`'s +`[ ! -s asv_benchmarks.txt ]` early-exit guard. The names may or may not +correspond to real benchmarks. + +**Variants**: +- Hardcoded heredoc: `cat > asv_benchmarks.txt << 'EOF'\nbenchmarks.TimeSolverHumanoid.time_step\nEOF` + (mujoco_warp — comment: *"benchmark stubs"*) +- Placeholder sentinel: `echo "benchmarks.time_placeholder" > asv_benchmarks.txt` + (dipy — last-resort fallback after legitimate discovery failed) + +**Detection**: `ASV_BENCHMARKS_WRITE_RE` (advisory, not auto-delete) in +`backfill_tainted_containers.py`. + +**Fix**: `docker_build_final.sh` overwrites `asv_benchmarks.txt` +authoritatively after all agent scripts run. `run-tests.sh` has a runtime +discovery fallback for cases where the build-time extract fails. Agent +pre-writes are structurally harmless. + +--- + +## 9. `T_BENCH_CONTAINER_LOGS_PATH` redirect + +**First encountered**: 2026-04-11, google-deepmind/mujoco_warp (earlier +synthesis attempt). + +**What the agent does**: Exports +`T_BENCH_CONTAINER_LOGS_PATH=/workspace/logs_custom` in +`/etc/profile.d/asv_build_vars.sh`. `run-tests.sh` sources this and uses the +variable for its `/logs/` wipe target. The wipe clears the wrong directory, +leaving the real `/logs/summary_agent.json` intact. + +**Detection**: `T_BENCH_OVERRIDE_RE` in `tamper_audit.py` catches overrides of +`T_BENCH_AGENT_MODEL_NAME`. The `LOGS_PATH` redirect is caught indirectly by +the hard-wipe using the variable value (which the agent controls) — but since +`parser.py` hardcodes `/logs/`, the redirect is actually self-defeating. + +**Fix**: `run-tests.sh` pins `T_BENCH_AGENT_MODEL_NAME=agent` and uses the +variable for the wipe. `parser.py` reads from the hardcoded `/logs/` path +regardless. + +--- + +## 10. Pre-baked ASV result tarballs + +**First encountered**: 2026-04-11, Textualize/rich#3807. + +**What the agent does**: Creates `/logs/postrun_agent-1-oracle:oracle.tar.gz` +containing fabricated ASV result JSON files with hardcoded timing data. Targets +Harbor's `parser.py` which scans `/logs/` for agent tarballs. + +**Detection**: `PREBAKED_TARBALL_RE` in `tamper_audit.py`. + +**Fix**: `/logs/` hard-wipe at `run-tests.sh` start removes pre-baked +tarballs. Only Harbor's own test.sh legitimately creates these at stage-7 +runtime. + +--- + +## 11. `conftest.py` manipulation + +**First encountered**: 2026-04-11, identified as a theoretical vector during +deep review. Observed in practice in bartz-org/bartz#68 (legitimate use — +`xfail` for crashing tests, not a bypass). + +**What the agent could do**: Write a `conftest.py` at the repo root that +suppresses test collection (`pytest_collection_modifyitems = lambda items: +items.clear()`) or monkeypatches imports to hide failures. + +**Detection**: Not yet in the strict tamper audit (legitimate `conftest.py` +use exists — e.g. bartz's `xfail` for JAX/XLA crashes). Tracked as a +potential future vector. + +**Mitigation**: The `--all` pytest mode and exit-code propagation make this +harder to exploit — `items.clear()` would yield exit code 5 +(NO_TESTS_COLLECTED) which now correctly fails verification. + +--- + +## Summary of architectural fixes + +| Fix | Closes patterns | +|-----|----------------| +| `run-tests.sh` hard-wipe (`find -delete`) | #1, #4, #6, #10 | +| `T_BENCH_AGENT_MODEL_NAME` pin | #1 (variable redirect), #9 | +| `pytest_runner.py --all` (no git-diff discovery) | #2, #3, #5 | +| `pytest_runner.py sys.exit(exit_code)` | #11, general exit-code masking | +| `run-tests.sh exit ${PYTEST_EXIT}` | exit-code propagation to local_ci | +| `docker_build_final.sh` authoritative discovery | #7, #8 | +| Post-verification `classify_context` in synthesizer | All patterns (server-side) | +| Fail-fast `break` on tamper detection | Compute savings (prevents 4h retry loops) | +| Cooperative AGENTS.md framing | Reduced bypass attempts from Claude (not Codex) | diff --git a/tests/agents/installed/test_agents.py b/tests/agents/installed/test_agents.py index 3f620f14..847b49e4 100644 --- a/tests/agents/installed/test_agents.py +++ b/tests/agents/installed/test_agents.py @@ -14,11 +14,13 @@ CodexResult, GeminiAgent, NoneAgent, + QwenAgent, get_agent, ) from datasmith.agents.installed.claude import _parse_claude_stdout from datasmith.agents.installed.codex import _parse_codex_stdout from datasmith.agents.installed.gemini import _parse_gemini_stdout +from datasmith.agents.installed.qwen import _parse_qwen_stdout # ---- AgentResult ---- @@ -65,6 +67,12 @@ def test_falls_back_to_gemini(self, mock_which: MagicMock) -> None: agent = get_agent() assert agent.name() == "gemini" + @patch("shutil.which") + def test_falls_back_to_qwen(self, mock_which: MagicMock) -> None: + mock_which.side_effect = lambda b: "/usr/bin/qwen" if b == "qwen" else None + agent = get_agent() + assert agent.name() == "qwen" + @patch("shutil.which", return_value=None) def test_raises_when_none_available(self, _mock: MagicMock) -> None: with pytest.raises(RuntimeError, match="No installed CLI agent found"): @@ -105,6 +113,14 @@ def test_gemini_available(self, _mock: MagicMock) -> None: def test_gemini_unavailable(self, _mock: MagicMock) -> None: assert GeminiAgent().is_available() is False + @patch("shutil.which", return_value="/usr/bin/qwen") + def test_qwen_available(self, _mock: MagicMock) -> None: + assert QwenAgent().is_available() is True + + @patch("shutil.which", return_value=None) + def test_qwen_unavailable(self, _mock: MagicMock) -> None: + assert QwenAgent().is_available() is False + # ---- CodexAgent.exec ---- @@ -323,3 +339,66 @@ def test_output(self) -> None: line = json.dumps({"output": "info"}) out, _files = _parse_gemini_stdout(line) assert out == ["info"] + + +# ---- QwenAgent.exec ---- + + +class TestQwenAgent: + @patch("datasmith.agents.installed.qwen.run_agent_subprocess") + def test_exec_success(self, mock_run: MagicMock) -> None: + stdout_lines = [ + json.dumps({"type": "assistant", "message": {"content": [{"type": "text", "text": "Fixed!"}]}}), + json.dumps({"type": "tool_use", "name": "edit", "input": {"file_path": "fix.py"}}), + json.dumps({"type": "result", "result": "Done"}), + ] + mock_run.return_value = (0, "\n".join(stdout_lines) + "\n", "", 2.0) + result = QwenAgent().exec("fix the build") + assert result.success is True + assert "Fixed!" in result.output + assert "fix.py" in result.files_changed + + @patch("datasmith.agents.installed.qwen.run_agent_subprocess") + def test_exec_timeout(self, mock_run: MagicMock) -> None: + mock_run.return_value = (-1, '{"type":"assistant","message":"partial"}\n', "timed out", 600.0) + result = QwenAgent().exec("slow", timeout=600) + assert result.success is False + assert result.duration_s == 600.0 + + @patch("datasmith.agents.installed.qwen.run_agent_subprocess") + def test_exec_not_found(self, mock_run: MagicMock) -> None: + mock_run.side_effect = FileNotFoundError() + result = QwenAgent().exec("prompt") + assert result.success is False + assert "not found" in result.error + + @patch("datasmith.agents.installed.qwen.run_agent_subprocess") + def test_command_flags(self, mock_run: MagicMock) -> None: + mock_run.return_value = (0, "", "", 0.5) + QwenAgent().exec("prompt") + cmd = mock_run.call_args[0][0] + assert "--yolo" in cmd + assert "stream-json" in cmd + + +class TestParseQwenStdout: + def test_assistant_message(self) -> None: + line = json.dumps({"type": "assistant", "message": {"content": [{"type": "text", "text": "hello"}]}}) + out, _files = _parse_qwen_stdout(line) + assert out == ["hello"] + + def test_result(self) -> None: + line = json.dumps({"type": "result", "result": "All done"}) + out, _files = _parse_qwen_stdout(line) + assert "All done" in out + + def test_think_tag_stripped(self) -> None: + line = json.dumps({"type": "result", "result": "thinking...\n\n\nActual output"}) + out, _files = _parse_qwen_stdout(line) + assert out == ["Actual output"] + assert "thinking" not in out[0] + + def test_file_change(self) -> None: + line = json.dumps({"type": "tool_use", "name": "edit", "input": {"file_path": "a.py"}}) + _out, files = _parse_qwen_stdout(line) + assert files == ["a.py"] From 1c564fbf4d38f924af68427154237057f0bc8bca Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:05:51 +0000 Subject: [PATCH 03/24] Harden Harbor adapter templates (source root detection, LSV measure, parser) --- .../harbor_adapter/template/lsv_init.py | 202 +++++++++++++++++- .../harbor_adapter/template/lsv_measure.py | 141 +++++++++++- .../harbor_adapter/template/parser.py | 197 ++++++++++++++++- .../harbor_adapter/template/pytest_runner.py | 31 ++- .../harbor_adapter/template/setup.sh | 64 +++++- src/datasmith/harbor_adapter/template/test.sh | 46 +++- 6 files changed, 639 insertions(+), 42 deletions(-) diff --git a/src/datasmith/harbor_adapter/template/lsv_init.py b/src/datasmith/harbor_adapter/template/lsv_init.py index 7c95920a..5317b5b3 100644 --- a/src/datasmith/harbor_adapter/template/lsv_init.py +++ b/src/datasmith/harbor_adapter/template/lsv_init.py @@ -33,7 +33,22 @@ def _ts() -> str: def detect_source_root() -> Path: - """Derive the source package root from /tests/config.json patch headers.""" + """Derive the source package root from /tests/config.json patch headers. + + Matching strategy (first existing wins): + + 1. If the patch touches exactly one non-skip top-level dir AND that dir + exists, use it. Handles the common flat layout (``pandas/``, + ``numpy/``). + 2. Otherwise, search for ``/__init__.py`` under the repo, skipping + well-known non-source dirs. Shortest path wins. Handles src-layout + repos where sources live under ``lib/`` (contourpy), + ``src/`` (many modern projects), or ``python/``. + 3. Fall back to ````, ``src/``, ``lib/``, ``python/`` + in that order. + 4. Last resort: ``REPO_ROOT / pkg`` (may not exist — LSV will raise a + clear error with the path included). + """ config_path = Path("/tests/config.json") if not config_path.exists(): config_path = Path("/workspace/repo/tests/config.json") @@ -41,22 +56,193 @@ def detect_source_root() -> Path: patch = config.get("patch", "") paths = re.findall(r"diff --git a/([^ ]+)", patch) - skip = {"doc", "docs", "test", "tests", ".github", "benchmarks", "asv_bench"} + skip = { + "doc", "docs", "test", "tests", ".github", "benchmarks", "asv_bench", + "ci", "scripts", "examples", "docs_src", "doc_src", + } + # Dirs that exist as a literal patch root but aren't the actual package — + # psygnal ships as src/psygnal, contourpy as lib/contourpy, etc. When the + # only non-skip patch root is one of these, skip step 1 and fall through + # to the /__init__.py search. + src_layout_holders = {"src", "lib", "python", "packages"} roots = {p.split("/")[0] for p in paths if "/" in p} - skip - if len(roots) == 1: - return REPO_ROOT / roots.pop() - pkg = config.get("repo_name", "").split("/")[-1].replace("-", "_") - return REPO_ROOT / pkg + + if len(roots) == 1 and next(iter(roots)) not in src_layout_holders: + cand = REPO_ROOT / next(iter(roots)) + if cand.is_dir(): + return cand + + if pkg: + candidates = sorted( + REPO_ROOT.glob(f"**/{pkg}/__init__.py"), + key=lambda p: len(p.parts), + ) + for c in candidates: + parts = c.relative_to(REPO_ROOT).parts + # Reject matches nested inside a skip dir (e.g. benchmarks/pkg/...) + if not any(part in skip for part in parts[:-2]): + return c.parent + + for layout in ("", "src", "lib", "python"): + cand = (REPO_ROOT / layout / pkg) if layout else (REPO_ROOT / pkg) + if cand.is_dir(): + return cand + + return REPO_ROOT / (pkg or "src") + + +def _strip_jsonc(text: str) -> str: + """Make JSONC text parseable by ``json.loads``. + + Handles three JSONC features that asv.conf.json files routinely use + but stdlib JSON rejects: + + 1. ``//`` line comments + 2. ``/* */`` block comments + 3. trailing commas before ``}`` or ``]`` + + All three must be handled in a **single pass** that tracks whether + we're inside a double-quoted string (with backslash escapes), because + a naive regex would mangle strings like ``"https://foo"`` (comment + regex would strip ``//foo"``) or ``"a, "`` (trailing-comma regex + would strip a legitimate comma inside a string). + + Strategy for trailing commas: when we see ``,`` outside a string, + peek ahead past whitespace — if the next meaningful character is + ``}`` or ``]``, drop the comma. Comments encountered during the peek + are treated as whitespace. + """ + n = len(text) + out: list[str] = [] + i = 0 + in_str = False + + def _next_meaningful(k: int) -> str: + """Return the first non-whitespace, non-comment char at or after k, + or '' if end of input.""" + while k < n: + c = text[k] + if c in " \t\r\n": + k += 1 + continue + if c == "/" and k + 1 < n: + nx = text[k + 1] + if nx == "/": + j = text.find("\n", k + 2) + k = n if j == -1 else j + 1 + continue + if nx == "*": + j = text.find("*/", k + 2) + k = n if j == -1 else j + 2 + continue + return c + return "" + + while i < n: + ch = text[i] + if in_str: + out.append(ch) + if ch == "\\" and i + 1 < n: + out.append(text[i + 1]) + i += 2 + continue + if ch == '"': + in_str = False + i += 1 + continue + if ch == '"': + in_str = True + out.append(ch) + i += 1 + continue + if ch == "/" and i + 1 < n: + nxt = text[i + 1] + if nxt == "/": + j = text.find("\n", i + 2) + i = n if j == -1 else j + continue + if nxt == "*": + j = text.find("*/", i + 2) + i = n if j == -1 else j + 2 + continue + if ch == ",": + follower = _next_meaningful(i + 1) + if follower in ("}", "]"): + # Drop the trailing comma; emit nothing. + i += 1 + continue + out.append(ch) + i += 1 + return "".join(out) + + +def _load_jsonc(path: Path) -> dict | None: + """Parse a JSON-with-comments file. asv.conf.json files frequently use + ``//`` line comments and ``/* */`` block comments (scikit-learn's inner + config in particular), which vanilla ``json.loads`` rejects. Strip the + comments first, then parse. Returns ``None`` if the file can't be read + or the stripped content still isn't valid JSON.""" + try: + text = path.read_text() + except OSError: + return None + try: + return json.loads(_strip_jsonc(text)) + except json.JSONDecodeError: + return None def find_asv_config() -> Path: - """Find the asv.*.json config file in the repo.""" - matches = glob(str(REPO_ROOT / "**/asv.*.json"), recursive=True) + """Find the asv.*.json config file in the repo. + + Some repos (notably scikit-learn) ship multiple asv configs — a stub at + the repo root plus the real one under a subdir like ``asv_benchmarks/``. + sklearn's repo root also has a legacy empty ``benchmarks/`` directory + that matches the outer config's ``benchmark_dir`` but is missing the + ``__init__.py`` asv requires, so a simple "is_dir" check picks the + wrong config. + + Pick the config whose resolved ``benchmark_dir`` is a *valid* asv + benchmark package — i.e. contains ``__init__.py``. Fall back to any + existing directory, then to ``matches[0]`` with a warning. + """ + matches = sorted(glob(str(REPO_ROOT / "**/asv.*.json"), recursive=True)) if not matches: print("ERROR: No asv.*.json config found") sys.exit(1) + + def _bench_dir_for(cfg_path: Path) -> Path | None: + data = _load_jsonc(cfg_path) + if data is None: + return None + bench_rel = data.get("benchmark_dir") or "benchmarks" + return (cfg_path.parent / bench_rel).resolve() + + # Pass 1: config whose benchmark_dir has an __init__.py (asv-valid). + for match in matches: + p = Path(match) + bd = _bench_dir_for(p) + if bd and bd.is_dir() and (bd / "__init__.py").is_file(): + print(f"[{_ts()}] [lsv_init] picked asv config: {p} (benchmark_dir={bd})") + return p + + # Pass 2: any config whose benchmark_dir exists at all (softer fallback). + for match in matches: + p = Path(match) + bd = _bench_dir_for(p) + if bd and bd.is_dir(): + print( + f"[{_ts()}] [lsv_init] WARNING: picked asv config {p} whose " + f"benchmark_dir {bd} lacks __init__.py — asv will likely reject it" + ) + return p + + print( + f"[{_ts()}] [lsv_init] WARNING: no asv config resolved to a real " + f"benchmark_dir; falling back to {matches[0]}" + ) return Path(matches[0]) diff --git a/src/datasmith/harbor_adapter/template/lsv_measure.py b/src/datasmith/harbor_adapter/template/lsv_measure.py index bb63120e..3d2b4a72 100644 --- a/src/datasmith/harbor_adapter/template/lsv_measure.py +++ b/src/datasmith/harbor_adapter/template/lsv_measure.py @@ -30,12 +30,117 @@ def _ts() -> str: OUTPUT_DIR = Path(os.environ.get("LSV_OUTPUT_DIR", "/logs/artifacts/lsv")) +def _strip_jsonc(text: str) -> str: + """Make JSONC text parseable by ``json.loads``. See lsv_init._strip_jsonc + for the full rationale — handles line comments, block comments, and + trailing commas, all with string-aware single-pass scanning.""" + n = len(text) + out: list[str] = [] + i = 0 + in_str = False + + def _next_meaningful(k: int) -> str: + """First non-whitespace, non-comment char at/after k, or ''.""" + while k < n: + c = text[k] + if c in " \t\r\n": + k += 1 + continue + if c == "/" and k + 1 < n: + nx = text[k + 1] + if nx == "/": + j = text.find("\n", k + 2) + k = n if j == -1 else j + 1 + continue + if nx == "*": + j = text.find("*/", k + 2) + k = n if j == -1 else j + 2 + continue + return c + return "" + + while i < n: + ch = text[i] + if in_str: + out.append(ch) + if ch == "\\" and i + 1 < n: + out.append(text[i + 1]) + i += 2 + continue + if ch == '"': + in_str = False + i += 1 + continue + if ch == '"': + in_str = True + out.append(ch) + i += 1 + continue + if ch == "/" and i + 1 < n: + nxt = text[i + 1] + if nxt == "/": + j = text.find("\n", i + 2) + i = n if j == -1 else j + continue + if nxt == "*": + j = text.find("*/", i + 2) + i = n if j == -1 else j + 2 + continue + if ch == ",": + follower = _next_meaningful(i + 1) + if follower in ("}", "]"): + i += 1 + continue + out.append(ch) + i += 1 + return "".join(out) + + +def _load_jsonc(path: Path) -> dict | None: + """Parse a JSON-with-comments file. See lsv_init._load_jsonc.""" + try: + text = path.read_text() + except OSError: + return None + try: + return json.loads(_strip_jsonc(text)) + except json.JSONDecodeError: + return None + + def find_asv_config() -> Path: - """Find the asv.*.json config file in the repo.""" - matches = glob(str(REPO_ROOT / "**/asv.*.json"), recursive=True) + """Find the asv.*.json config file in the repo. + + Mirrors lsv_init.py's picker — prefer configs whose ``benchmark_dir`` + is a valid asv package (has ``__init__.py``). See lsv_init for + rationale (scikit-learn ships a stub config + empty ``benchmarks/`` + stub dir at the repo root that fooled the naive picker). + """ + matches = sorted(glob(str(REPO_ROOT / "**/asv.*.json"), recursive=True)) if not matches: print("ERROR: No asv.*.json config found") sys.exit(1) + + def _bench_dir_for(cfg_path: Path) -> Path | None: + data = _load_jsonc(cfg_path) + if data is None: + return None + bench_rel = data.get("benchmark_dir") or "benchmarks" + return (cfg_path.parent / bench_rel).resolve() + + for match in matches: + p = Path(match) + bd = _bench_dir_for(p) + if bd and bd.is_dir() and (bd / "__init__.py").is_file(): + print(f"[{_ts()}] [lsv_measure] picked asv config: {p}") + return p + + for match in matches: + p = Path(match) + bd = _bench_dir_for(p) + if bd and bd.is_dir(): + return p + return Path(matches[0]) @@ -132,7 +237,19 @@ def main() -> None: changed_files=changed, rounds=args.rounds ) except Exception as e: - print(f"[{_ts()}] [lsv_measure] ERROR: measure_impacted failed: {e}") + msg = str(e) + # Distinguish "lsv_init never created the dep DB" from "measure ran + # but crashed mid-flight". The first means setup.sh failed earlier; + # the second is an LSV runtime bug or a patch that broke the build. + if "Dependency database not found" in msg or "deps.db" in msg: + err = ( + "LSV measure aborted: dependency database missing. " + "lsv_init.py did not complete successfully in setup.sh — " + "see setup_status.json for the failing phase." + ) + else: + err = f"LSV measure_impacted raised: {msg}" + print(f"[{_ts()}] [lsv_measure] ERROR: {err}") print( f"[{_ts()}] [lsv_measure] Writing empty results (agent may have broken the build)" ) @@ -142,6 +259,7 @@ def main() -> None: "total_count": 0, "skipped_count": 0, "timing": {"total_s": 0.0}, + "error": err, } _write_combined_results(empty_measure) return @@ -150,6 +268,22 @@ def main() -> None: print(f" skipped: {measure_result.skipped_count}") print(f" time: {measure_result.timing.total_s:.1f}s") + # LSV's measure_impacted can report ``selected_count > 0`` while leaving + # ``benchmarks`` empty — it happens when the selected benchmarks crash + # at runtime (stale Cython binaries, import errors, ASV discovery + # failures, timeouts) and LSV silently drops them. Without this check we + # write ``num_valid_benchmarks=0`` to reward.json with no error message + # and the operator has no idea what went wrong. + measure_error: str | None = None + if measure_result.selected_count > 0 and not measure_result.benchmarks: + measure_error = ( + f"LSV selected {measure_result.selected_count} benchmarks but measured 0 " + f"(time={measure_result.timing.total_s:.1f}s). Selected benchmarks " + f"likely failed to execute (stale build artifacts, import errors, " + f"or ASV discovery failures)." + ) + print(f"[{_ts()}] [lsv_measure] WARNING: {measure_error}") + for name, delta in measure_result.benchmarks.items(): sign = "+" if (delta.delta_pct or 0) >= 0 else "" pct = f"{sign}{delta.delta_pct:.1f}%" if delta.delta_pct is not None else "n/a" @@ -174,6 +308,7 @@ def main() -> None: "total_count": measure_result.total_count, "skipped_count": measure_result.skipped_count, "timing": dataclasses.asdict(measure_result.timing), + "error": measure_error, } (OUTPUT_DIR / "lsv_measure_results.json").write_text( diff --git a/src/datasmith/harbor_adapter/template/parser.py b/src/datasmith/harbor_adapter/template/parser.py index 41a788a0..95db3397 100644 --- a/src/datasmith/harbor_adapter/template/parser.py +++ b/src/datasmith/harbor_adapter/template/parser.py @@ -208,10 +208,16 @@ def aggregate_by_hierarchy(per_benchmark: dict[str, float]) -> dict: def load_test_results(log_dir: Path) -> tuple[bool, float, dict]: - """Read test_results.json. Returns (tests_passed, success_ratio, raw).""" + """Read test_results.json. Returns (tests_passed, success_ratio, raw). + + Missing file returns ``(False, 0.0, {})`` so the caller can distinguish + 'no pytest data' from 'pytest passed'. This is the inversion of the + previous behavior, which defaulted to ``True`` and silently masked + every failure mode upstream of pytest. + """ path = log_dir / "test_results.json" if not path.exists(): - return True, 1.0, {} + return False, 0.0, {} raw = json.loads(path.read_text()) results = raw.get("results", raw) @@ -229,6 +235,97 @@ def load_test_results(log_dir: Path) -> tuple[bool, float, dict]: return tests_passed, success_ratio, raw +def summarize_pytest(raw: dict) -> dict: + """Extract structured pytest counters from test_results.json for the + reward dossier. Returns an empty dict if there's no usable content.""" + if not raw: + return {} + results = raw.get("results", raw) or {} + summary = results.get("summary", {}) or {} + tests = results.get("tests", []) or [] + return { + "exit_code": results.get("exit_code"), + "total": summary.get("total", 0), + "passed": summary.get("passed", 0), + "failed": summary.get("failed", 0), + "skipped": summary.get("skipped", 0), + "error": summary.get("error", 0), + "duration_s": results.get("duration"), + "selected_nodeids": [t.get("nodeid") for t in tests if t.get("nodeid")], + "strategy": raw.get("strategy"), + "note": raw.get("note"), + } + + +def summarize_snapshots(log_dir: Path) -> dict: + """Collect snapshot verify summaries into a single structured block.""" + out: dict = {"summaries": {}, "passed": True} + for match_path in glob_mod.glob(str(log_dir / "summary_*.json")): + name = Path(match_path).stem.removeprefix("summary_") + try: + data = json.loads(Path(match_path).read_text()) + except (json.JSONDecodeError, OSError): + continue + out["summaries"][name] = data + if data.get("passed") is False: + out["passed"] = False + return out + + +def load_patch_info(log_dir: Path) -> dict: + path = log_dir / "patch_info.json" + if not path.exists(): + return {"applied": False, "files": 0, "added_lines": 0, "removed_lines": 0} + try: + return json.loads(path.read_text()) + except (json.JSONDecodeError, OSError): + return {"applied": False, "files": 0, "added_lines": 0, "removed_lines": 0} + + +def load_setup_status(log_dir: Path) -> dict: + """Read setup_status.json written by setup.sh's EXIT trap. + + Missing file means setup.sh never ran the trap (earlier crash, or an + old image without the trap baked in). Return a conservative default: + exit_code=None so downstream code can treat it as 'unknown' rather + than 'succeeded'. + """ + path = log_dir / "setup_status.json" + if not path.exists(): + return {"exit_code": None, "failed_phase": None, "succeeded": None} + try: + return json.loads(path.read_text()) + except (json.JSONDecodeError, OSError): + return {"exit_code": None, "failed_phase": None, "succeeded": None} + + +def load_timings(log_dir: Path) -> dict: + """Merge setup_timings.json (from setup.sh) and test_timings.json + (from test.sh). Missing files contribute no keys.""" + merged: dict = {} + for name in ("setup_timings.json", "test_timings.json"): + path = log_dir / name + if not path.exists(): + continue + try: + merged.update(json.loads(path.read_text())) + except (json.JSONDecodeError, OSError): + continue + return merged + + +def summarize_lsv_init(lsv_results: dict) -> dict: + init = (lsv_results or {}).get("init", {}) or {} + if not init: + return {} + return { + "benchmarks_discovered": init.get("benchmarks_discovered") or [], + "benchmarks_impactable": init.get("benchmarks_impactable") or [], + "source_files_covered": init.get("source_files_covered"), + "init_time_s": (init.get("timing") or {}).get("total_s"), + } + + def log_snapshot_results(log_dir: Path) -> None: """Log snapshot verification results. Regressions are informational only.""" regressions = [] @@ -255,13 +352,31 @@ def write_reward( speedups: dict[str, float], speedup_levels: dict, tests_passed: bool, - snapshots_passed: bool, + snapshots: dict, + lsv_error: str | None = None, + *, + patch: dict | None = None, + lsv_init_summary: dict | None = None, + lsv_measure_raw: dict | None = None, + pytest_summary: dict | None = None, + timings: dict | None = None, + setup_status: dict | None = None, ) -> None: - """Write reward.json and reward.txt.""" + """Write reward.json + reward.txt with a structured dossier. + + The top-level gating fields (``num_valid_benchmarks``, ``max_speedup``, + ``lsv_mean_speedup``, ``tests_passed``, ``snapshots_passed``, + ``lsv_error``) are preserved for back-compat with the publish filter + and harbor_healthcheck row builder. Everything else lives in nested + sub-blocks so post-mortem triage has the full picture without + round-tripping through the trial container.""" reward_dir.mkdir(parents=True, exist_ok=True) - # reward.json — keeps current format with agent_advantage fields + max_speedup = max(speedups.values()) if speedups else None + snapshots_passed = bool((snapshots or {}).get("passed", True)) + reward_data = { + # ── legacy scalar fields (publish filter, old downstream tools) ── "agent_advantage": (advantage_levels["level4"] if advantage_levels else None), "agent_advantage_level1": ( advantage_levels["level1"] if advantage_levels else None @@ -278,8 +393,27 @@ def write_reward( "num_valid_benchmarks": len(speedups), "per_benchmark_speedups": speedups, "lsv_mean_speedup": speedup_levels["level4"], + "max_speedup": max_speedup, "tests_passed": tests_passed, "snapshots_passed": snapshots_passed, + "lsv_error": lsv_error, + # ── structured dossier ──────────────────────────────────────────── + "patch": patch or {}, + "lsv": { + "init": lsv_init_summary or {}, + "measure": { + "selected_count": (lsv_measure_raw or {}).get("selected_count"), + "total_count": (lsv_measure_raw or {}).get("total_count"), + "skipped_count": (lsv_measure_raw or {}).get("skipped_count"), + "time_s": ((lsv_measure_raw or {}).get("timing") or {}).get("total_s"), + "measured_count": len(speedups), + "error": lsv_error, + }, + }, + "pytest": pytest_summary or {}, + "snapshots": snapshots or {}, + "timings": timings or {}, + "setup": setup_status or {}, } (reward_dir / "reward.json").write_text(json.dumps(reward_data, indent=2)) @@ -289,9 +423,10 @@ def write_reward( (reward_dir / "reward.txt").write_text(str(speedup_value)) print(f"[parser] reward.txt = {speedup_value}") - print(f"[parser] agent_advantage = {reward_data['agent_advantage']}") + print(f"[parser] max_speedup = {max_speedup}") print(f"[parser] num_valid_benchmarks = {len(speedups)}") print(f"[parser] tests_passed = {tests_passed}") + print(f"[parser] patch_applied = {(patch or {}).get('applied')}") # ── Main ───────────────────────────────────────────────────────────────────── @@ -305,14 +440,39 @@ def main() -> None: ) args = parser.parse_args() + # Load all sidecars written by setup.sh / test.sh. Each helper returns + # an empty/default payload if its file is missing, so a partially-run + # trial still produces a well-formed reward.json. + patch_info = load_patch_info(LOG_DIR) + timings = load_timings(LOG_DIR) + snapshot_block = summarize_snapshots(LOG_DIR) + setup_status = load_setup_status(LOG_DIR) + # Load LSV results lsv_results = load_lsv_results(LSV_DIR) + lsv_init_summary = summarize_lsv_init(lsv_results) if not lsv_results: print("[parser] No LSV results found. Writing zero reward.") - write_reward(REWARD_DIR, None, None, {}, {"level4": 0.0}, False, True) + write_reward( + REWARD_DIR, + None, + None, + {}, + {"level4": 0.0}, + False, + snapshot_block, + patch=patch_info, + lsv_init_summary=lsv_init_summary, + lsv_measure_raw=None, + pytest_summary={}, + timings=timings, + setup_status=setup_status, + ) sys.exit(1) - benchmarks = lsv_results.get("measure", {}).get("benchmarks", {}) + measure_results = lsv_results.get("measure", {}) or {} + benchmarks = measure_results.get("benchmarks", {}) or {} + lsv_error = measure_results.get("error") # Compute speedups (baseline/current from LSV) speedups = compute_per_benchmark_speedups(benchmarks) @@ -341,8 +501,9 @@ def main() -> None: else: print("[parser] SUPABASE_URL not set; skipping advantage computation") - # Load test and snapshot results - tests_passed, _, _ = load_test_results(LOG_DIR) + # Load test results + summarize pytest into a dossier block. + tests_passed, _, test_raw = load_test_results(LOG_DIR) + pytest_summary = summarize_pytest(test_raw) log_snapshot_results(LOG_DIR) # Write reward files @@ -353,8 +514,22 @@ def main() -> None: speedups, speedup_levels, tests_passed, - True, + snapshot_block, + lsv_error=lsv_error, + patch=patch_info, + lsv_init_summary=lsv_init_summary, + lsv_measure_raw=measure_results, + pytest_summary=pytest_summary, + timings=timings, + setup_status=setup_status, ) + if lsv_error: + print(f"[parser] lsv_error = {lsv_error}") + if setup_status.get("exit_code") not in (None, 0): + print( + f"[parser] setup_status = failed (exit_code={setup_status.get('exit_code')}, " + f"phase={setup_status.get('failed_phase')})" + ) print("[parser] Complete.") diff --git a/src/datasmith/harbor_adapter/template/pytest_runner.py b/src/datasmith/harbor_adapter/template/pytest_runner.py index 28ee7d3c..6c40a28d 100644 --- a/src/datasmith/harbor_adapter/template/pytest_runner.py +++ b/src/datasmith/harbor_adapter/template/pytest_runner.py @@ -735,24 +735,19 @@ def main(args) -> dict: if __name__ == "__main__": args = parse_args() output = main(args) - # if output['results']['exit_code'] != 0: - if output["results"]["summary"]["total"] == 0: - args.extra_args = "" - output = main(args) - # if output['results']['exit_code'] != 0: - if output["results"]["summary"]["total"] == 0: - # numpy issues. - args.extra_args = ( - "--import-mode=importlib --ignore=build --ignore=tools/ci -c tox.ini" - ) - output = main(args) - - # Write test results to file for Harbor parser - logs_root = Path( - os.environ.get("T_BENCH_TASK_LOGS_PATH") - or os.environ.get("T_BENCH_CONTAINER_LOGS_PATH") - or "/logs" - ) + # NOTE: previous versions retried on total==0 with different extra_args + # (e.g. "-c tox.ini"), which caused pytest to re-discover the entire repo + # from cwd and collect unrelated tests (e.g. arrow's dev/test_merge_arrow_pr.py + # which imports `jira`). total==0 here legitimately means "no tests mapped + # from the PR's changed files" and should be reported as such, not worked + # around by scanning the whole tree. + + # Write test results to file for Harbor parser. Must match parser.py's + # LOG_DIR default ("/logs/artifacts") so the parser actually finds it — + # neither Harbor nor test.sh set T_BENCH_TASK_LOGS_PATH, so the previous + # fallback ("/logs") silently diverged from the reader and every failing + # test suite was silently recorded as tests_passed=True. + logs_root = Path(os.environ.get("T_BENCH_TASK_LOGS_PATH", "/logs/artifacts")) logs_root.mkdir(parents=True, exist_ok=True) with open(logs_root / "test_results.json", "w", encoding="utf-8") as f: json.dump(output, f, sort_keys=True) diff --git a/src/datasmith/harbor_adapter/template/setup.sh b/src/datasmith/harbor_adapter/template/setup.sh index ce8db08b..006a5da8 100644 --- a/src/datasmith/harbor_adapter/template/setup.sh +++ b/src/datasmith/harbor_adapter/template/setup.sh @@ -7,10 +7,62 @@ ts() { date -u "+%Y-%m-%dT%H:%M:%SZ"; } TASK_ID="{{ task_id }}" export TASK_ID +# Harbor's [verifier.env] section only reaches test.sh, not setup.sh, so we +# bake the agent name directly into setup.sh at render time. lsv_init.py +# reads this to decide whether to capture the oracle snapshot baseline. +export HARBOR_AGENT_NAME="oracle" + +setup_start=$(date +%s) +# Track which phase we're currently in so the exit trap can report WHERE +# setup crashed, not just the exit code. Updated at every phase boundary +# below. +SETUP_PHASE="init" +lsv_init_start="" +lsv_init_end="" +mkdir -p /logs/artifacts + +# Crash trap — always writes setup_status.json even if setup.sh exits 1. +# Without this, a failing lsv_init leaves no record of WHY setup failed and +# parser.py has to guess. The trap runs on any exit (normal or error), so we +# write the timings file here too instead of at the happy-path bottom. +_write_setup_status() { + local ec=$? + local setup_end + setup_end=$(date +%s) + local lsv_init_s=0 + if [ -n "${lsv_init_start}" ] && [ -n "${lsv_init_end}" ]; then + lsv_init_s=$((lsv_init_end - lsv_init_start)) + elif [ -n "${lsv_init_start}" ]; then + lsv_init_s=$((setup_end - lsv_init_start)) + fi + cat > /logs/artifacts/setup_status.json < /logs/artifacts/setup_timings.json < ... < "${LOG_DIR}/patch.diff" 2>/dev/null || true +patch_files=$(git diff {{ base_commit }} --name-only 2>/dev/null | wc -l | tr -d ' ') +patch_numstat=$(git diff {{ base_commit }} --numstat 2>/dev/null || true) +patch_added=$(printf '%s\n' "${patch_numstat}" | awk '{a+=$1} END {print a+0}') +patch_removed=$(printf '%s\n' "${patch_numstat}" | awk '{d+=$2} END {print d+0}') +python - "${LOG_DIR}" "${patch_files}" "${patch_added}" "${patch_removed}" <<'PYEOF' +import json, sys, pathlib +log_dir, files, added, removed = sys.argv[1:5] +info = { + "applied": int(files) > 0, + "files": int(files), + "added_lines": int(added), + "removed_lines": int(removed), +} +pathlib.Path(log_dir, "patch_info.json").write_text(json.dumps(info)) +PYEOF +echo "[$(ts)] [test] patch: files=${patch_files} +${patch_added}/-${patch_removed}" # ── LSV Phase 2: measure_impacted ──────────────────────────────────────── # Helpers live at /opt/lsv (baked into the image by Dockerfile). echo "[$(ts)] [test] Running LSV measure..." +lsv_measure_start=$(date +%s) python /opt/lsv/lsv_measure.py --base-commit {{ base_commit }} --rounds {{ rounds }} +lsv_measure_end=$(date +%s) # ── Snapshot vars ─────────────────────────────────────────────────────── SNAPSHOT_DIR="${LOG_DIR}/.snapshots" @@ -46,6 +66,8 @@ SNAPSHOT_FILTER="${FORMULACODE_SNAPSHOT_FILTER:-^(lexer|verifier)\\.}" SNAPSHOT_TIMEOUT="${FORMULACODE_SNAPSHOT_TIMEOUT:-30}" BENCHMARK_DIR="${BENCHMARK_DIR:-}" +snapshot_start=$(date +%s) + # ── Snapshot baseline (oracle only) ───────────────────────────────────── if [ "${AGENT_KEY}" = "oracle" ] && [ -n "${BENCHMARK_DIR}" ] && command -v snapshot-tool >/dev/null 2>&1; then echo "[$(ts)] [test] Recording snapshot baseline (oracle)..." @@ -79,7 +101,10 @@ if [ "${AGENT_KEY}" != "oracle" ] && [ -n "${BENCHMARK_DIR}" ] && \ "${BENCHMARK_DIR}" || echo "WARNING: snapshot verification failed" fi +snapshot_end=$(date +%s) + # ── Pytest ─────────────────────────────────────────────────────────────── +pytest_start=$(date +%s) {%- if run_pytest %} echo "[$(ts)] [test] Running pytest..." python /opt/lsv/pytest_runner.py --base {{ base_commit }} --extra-args "-p jinja_patch_plugin_pandas" @@ -87,6 +112,25 @@ python /opt/lsv/pytest_runner.py --base {{ base_commit }} --extra-args "-p jinja mkdir -p "$LOG_DIR" echo '{"results": {"exit_code": 0, "summary": {"error": 0, "failed": 0}, "details": "Tests skipped as per configuration."}}' > "$LOG_DIR/test_results.json" {%- endif %} +pytest_end=$(date +%s) + +# Persist per-step timings for the reward parser to pick up. +test_end=$(date +%s) +python - "${LOG_DIR}" "${test_start}" "${lsv_measure_start}" "${lsv_measure_end}" \ + "${snapshot_start}" "${snapshot_end}" "${pytest_start}" \ + "${pytest_end}" "${test_end}" <<'PYEOF' +import json, sys, pathlib +args = [int(a) for a in sys.argv[2:]] +(test_start, lsv_ms, lsv_me, snap_s, snap_e, py_s, py_e, test_end) = args +log_dir = sys.argv[1] +timings = { + "test_total_s": test_end - test_start, + "lsv_measure_s": lsv_me - lsv_ms, + "snapshot_s": snap_e - snap_s, + "pytest_s": py_e - py_s, +} +pathlib.Path(log_dir, "test_timings.json").write_text(json.dumps(timings)) +PYEOF # ── Parser: compute reward ─────────────────────────────────────────────── echo "[$(ts)] [test] Computing reward..." From 97f249b9e0d8a0c1aba9cfad2c8b38443c2b01c9 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:05:56 +0000 Subject: [PATCH 04/24] Fix Docker build templates: ASV discovery fallback, pytest exit propagation --- .../docker/templates/docker_build_env.sh | 22 ++------- .../docker/templates/docker_build_final.sh | 12 +++-- .../docker/templates/pytest_runner.py | 16 +++++++ src/datasmith/docker/templates/run-tests.sh | 47 +++++++++++++++++-- 4 files changed, 71 insertions(+), 26 deletions(-) diff --git a/src/datasmith/docker/templates/docker_build_env.sh b/src/datasmith/docker/templates/docker_build_env.sh index 065127ae..8ac7b909 100644 --- a/src/datasmith/docker/templates/docker_build_env.sh +++ b/src/datasmith/docker/templates/docker_build_env.sh @@ -188,27 +188,15 @@ if [[ -z "$IMPORT_NAME" ]]; then echo "WARN: Could not determine import name; the pkg stage will fall back to local detection." fi -cd_asv_json_dir || { echo "No 'asv.*.json' file found." >&2; exit 1; } - -CONF_NAME="$(asv_conf_name || true)" -if [[ -z "${CONF_NAME:-}" ]]; then - echo "No 'asv.*.json' file found." >&2 - exit 1 -fi - -# PY_VERSIONS=$(python - <= (3,7)] -# print(" ".join(cfg.pythons)) -# PY -# ) +# NOTE: asv.*.json discovery is deliberately NOT done here. Some repos +# (Qiskit, Dask, Astropy, …) keep their ASV config in a separate benchmark +# repo that is cloned by docker_build_pkg.sh, which runs after this stage. +# Gating the env stage on asv.*.json would make those repos unreachable. source /etc/profile.d/asv_build_vars.sh || true PY_VERSIONS="${PY_VERSION:-${ASV_PY_VERSIONS:-}}" if [[ -z "$PY_VERSIONS" ]]; then - echo "No Satisfying PY_VERSIONS found in $CONF_NAME" >&2 - cat "$CONF_NAME" >&2 + echo "No PY_VERSIONS configured (set PY_VERSION or ASV_PY_VERSIONS)." >&2 exit 1 fi diff --git a/src/datasmith/docker/templates/docker_build_final.sh b/src/datasmith/docker/templates/docker_build_final.sh index ce1e388d..50802ffd 100644 --- a/src/datasmith/docker/templates/docker_build_final.sh +++ b/src/datasmith/docker/templates/docker_build_final.sh @@ -60,16 +60,19 @@ echo "[final] Discovering ASV benchmarks from ${CONF_NAME:-} ..." REAL_CONF="$(realpath "$CONF_NAME")" CONF_DIR="$(dirname "$REAL_CONF")" -# asv must be run from the directory containing the config +# asv must be run from the directory containing the config, and Benchmarks.load() +# resolves `results_dir` relative to the current working directory — so the +# extract step must also run with cwd == CONF_DIR. Keep both inside the pushd. pushd "$CONF_DIR" > /dev/null micromamba run -n "$ENV_NAME" asv machine --yes --config "$REAL_CONF" \ --machine dockertest --num_cpu 1 --ram 4GB 2>&1 | tail -1 || true micromamba run -n "$ENV_NAME" asv run --bench just-discover \ --config "$REAL_CONF" --python=same --machine=dockertest || true -popd > /dev/null -# Extract benchmark names from the generated benchmarks.json -python - "$CONF_NAME" /workspace/repo/asv_benchmarks.txt <<'EXTRACT_EOF' || true +# Extract benchmark names from the generated benchmarks.json. Using the +# conda env python (not base) so asv/asv_runner versions match what +# Benchmarks.load() wrote in the previous step. +micromamba run -n "$ENV_NAME" python - "$REAL_CONF" /workspace/repo/asv_benchmarks.txt <<'EXTRACT_EOF' || true import sys from asv.config import Config from asv.benchmarks import Benchmarks @@ -80,6 +83,7 @@ with open(sys.argv[2], "w") as f: for b in sorted(bm._all_benchmarks.keys()): f.write(b + "\n") EXTRACT_EOF +popd > /dev/null # Fall back to the pre-computed file if live discovery produced nothing. if [ ! -s /workspace/repo/asv_benchmarks.txt ] && [ -n "$FALLBACK_FILE" ] && [ -s "$FALLBACK_FILE" ]; then diff --git a/src/datasmith/docker/templates/pytest_runner.py b/src/datasmith/docker/templates/pytest_runner.py index 36bebfec..7f5270b6 100644 --- a/src/datasmith/docker/templates/pytest_runner.py +++ b/src/datasmith/docker/templates/pytest_runner.py @@ -690,3 +690,19 @@ def main(args) -> dict: print("FORMULACODE_TESTS_END") with open("/logs/test_results.json", "w", encoding="utf-8") as f: json.dump(output, f, sort_keys=True) + + # Propagate pytest's exit code so run-tests.sh (and local_ci.py) can + # gate on it. Without this, the script always exits 0 and broken test + # suites silently pass verification. + # + # Exit code semantics (pytest.ExitCode): + # 0 = OK (all passed) → success + # 1 = TESTS_FAILED (some fail) → success (test failures are acceptable; + # they indicate the upstream suite has flaky/broken tests at this + # commit, not that the build is broken) + # 2 = INTERRUPTED → failure (collection errors) + # 3 = INTERNAL_ERROR → failure + # 4 = USAGE_ERROR → failure + # 5 = NO_TESTS_COLLECTED → failure + raw_exit = output["results"].get("exit_code", 0) + sys.exit(0 if raw_exit in (0, 1) else raw_exit) diff --git a/src/datasmith/docker/templates/run-tests.sh b/src/datasmith/docker/templates/run-tests.sh index f2cb4e67..35b05517 100644 --- a/src/datasmith/docker/templates/run-tests.sh +++ b/src/datasmith/docker/templates/run-tests.sh @@ -127,11 +127,40 @@ echo "Agent solution patch saved to: $PATCH_PATH" reset_repo_state {{ base_commit }} -# ASV benchmark discovery is handled authoritatively by docker_build_final.sh -# during the `final` Docker stage (local_ci.py builds target="final"), so -# /workspace/repo/asv_benchmarks.txt is already populated by the time this -# script runs. We only check here that it's non-empty — an empty file means -# discovery failed and the task is unsuitable for FormulaCode. +# ASV benchmark discovery is normally handled authoritatively by +# docker_build_final.sh during the `final` Docker stage (local_ci.py builds +# target="final"), so /workspace/repo/asv_benchmarks.txt is populated at +# image-build time. But ASV discovery is sometimes flaky in the build env +# (missing runtime deps, late-bound imports, etc.), so if the file is empty +# we re-run discovery here at container-run time as a self-healing fallback. +# This does NOT reopen the agent-write bypass landscape: docker_build_final.sh +# still runs AFTER every agent-editable script during the image build and +# overwrites anything the agent wrote, so the only way asv_benchmarks.txt +# can be empty here is a *legitimate* discovery failure at build time. +if [ ! -s /workspace/repo/asv_benchmarks.txt ]; then + echo "[run-tests] asv_benchmarks.txt empty; retrying discovery at runtime ..." + REAL_CONF="$(realpath "$CONF_NAME")" + CONF_DIR="$(dirname "$REAL_CONF")" + pushd "$CONF_DIR" > /dev/null + micromamba run -n "$ENV_NAME" asv machine --yes --config "$REAL_CONF" \ + --machine dockertest --num_cpu 1 --ram 4GB 2>&1 | tail -1 || true + micromamba run -n "$ENV_NAME" asv run --bench just-discover \ + --config "$REAL_CONF" --python=same --machine=dockertest 2>&1 | tail -5 || true + micromamba run -n "$ENV_NAME" python - "$REAL_CONF" /workspace/repo/asv_benchmarks.txt <<'EXTRACT_EOF' || true +import sys +from asv.config import Config +from asv.benchmarks import Benchmarks +conf = Config.load(sys.argv[1]) +bm = Benchmarks.load(conf) +with open(sys.argv[2], "w") as f: + for b in sorted(bm._all_benchmarks.keys()): + f.write(b + "\n") +EXTRACT_EOF + popd > /dev/null + if [ -s /workspace/repo/asv_benchmarks.txt ]; then + echo "[run-tests] runtime discovery recovered $(wc -l < /workspace/repo/asv_benchmarks.txt) benchmarks." + fi +fi # Early exit: no ASV benchmarks discovered — task is useless for FormulaCode if [ ! -f /workspace/repo/asv_benchmarks.txt ] || [ ! -s /workspace/repo/asv_benchmarks.txt ]; then @@ -175,6 +204,7 @@ cat > jinja_patch_plugin_pandas.py << 'PY' _patch() PY python formulacode_testrunner.py --all --base {{ base_commit }} --extra-args "-p jinja_patch_plugin_pandas --maxfail=20" +PYTEST_EXIT=$? {% else %} echo -e "FORMULACODE_TESTS_START\n{\"results\": {\"exit_code\": 0, \"details\": \"Tests skipped as per configuration.\"}}\nFORMULACODE_TESTS_END" {%- endif %} @@ -190,3 +220,10 @@ cat > parser.py <<'EOF' EOF python parser.py + +# Propagate the pytest exit code as the script's exit code. Without this, +# the script's exit code would be parser.py's (always 0), hiding real pytest +# failures from local_ci.py which gates on exit code alone. +{%- if run_pytest %} +exit ${PYTEST_EXIT:-0} +{%- endif %} From bc0f8aab38b1b5dba0149dec9ef8300db0477714 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:06:08 +0000 Subject: [PATCH 05/24] Improve synthesizer: base_sha support, TRY_DEFAULT concurrency fix, tamper fail-fast --- src/datasmith/agents/sandbox.py | 26 ++++-- src/datasmith/agents/synthesizer.py | 96 ++++++++++++++++++--- src/datasmith/agents/templates/AGENTS.md.j2 | 5 ++ 3 files changed, 110 insertions(+), 17 deletions(-) diff --git a/src/datasmith/agents/sandbox.py b/src/datasmith/agents/sandbox.py index 366c055a..3d3573cb 100644 --- a/src/datasmith/agents/sandbox.py +++ b/src/datasmith/agents/sandbox.py @@ -124,6 +124,7 @@ def run( pr_context: str, prior_attempts: str = "", dry_run: bool = False, + base_sha: str = "", ) -> SandboxResult: """Prepare workspace, launch agent, extract results. @@ -146,6 +147,7 @@ def run( python_version=python_version, pr_context=pr_context, prior_attempts=prior_attempts, + base_sha=base_sha, ) # 2. Init git repo (Codex requirement) @@ -185,11 +187,16 @@ def _prepare_workspace( python_version: str, pr_context: str, prior_attempts: str = "", + base_sha: str = "", ) -> None: """Create the workspace directory structure.""" task_dir = workspace / "task" task_dir.mkdir(parents=True, exist_ok=True) + # Use base_sha for Docker checkout so the repo is at the + # pre-optimization state; fall back to merge_commit_sha for compat. + checkout_sha = base_sha or sha + # Copy ALL template files from docker/templates/ into task/ docker_templates = Path(__file__).parents[1] / "docker" / "templates" for fname in ( @@ -207,11 +214,12 @@ def _prepare_workspace( shutil.copy2(str(src), str(task_dir / fname)) # Render run-tests.sh from Jinja2 template with embedded scripts - run_tests_sh = _render_run_tests_sh(docker_templates, base_commit=sha) + run_tests_sh = _render_run_tests_sh(docker_templates, base_commit=checkout_sha) (task_dir / "run-tests.sh").write_text(run_tests_sh) - # Generate task.txt - task_txt = _generate_task_txt(owner, repo, sha, env_payload, python_version, repo_image) + # Generate task.txt — use checkout_sha so Dockerfile.pr checks out + # the base commit, not the merge commit. + task_txt = _generate_task_txt(owner, repo, checkout_sha, env_payload, python_version, repo_image) (task_dir / "task.txt").write_text(task_txt) # Render AGENTS.md from Jinja2 template @@ -489,6 +497,7 @@ def verify_context( python_version: str, context: DockerContext, timeout_s: int = 3600, + base_sha: str = "", ) -> SandboxResult: """Build and verify a :class:`DockerContext` without launching an agent. @@ -498,6 +507,10 @@ def verify_context( start = time.time() docker_templates = Path(__file__).parents[1] / "docker" / "templates" + # Use base_sha for Docker checkout so the repo is at the + # pre-optimization state; fall back to merge_commit_sha for compat. + checkout_sha = base_sha or sha + with tempfile.TemporaryDirectory(prefix="verify-ctx-") as tmpdir: workspace = Path(tmpdir) task_dir = workspace / "task" @@ -519,11 +532,12 @@ def verify_context( shutil.copy2(str(src), str(task_dir / fname)) # Render run-tests.sh from Jinja2 template - run_tests_sh = _render_run_tests_sh(docker_templates, base_commit=sha) + run_tests_sh = _render_run_tests_sh(docker_templates, base_commit=checkout_sha) (task_dir / "run-tests.sh").write_text(run_tests_sh) - # Write task.txt - task_txt = _generate_task_txt(owner, repo, sha, env_payload, python_version, repo_image) + # Write task.txt — use checkout_sha so Dockerfile.pr checks out + # the base commit, not the merge commit. + task_txt = _generate_task_txt(owner, repo, checkout_sha, env_payload, python_version, repo_image) (task_dir / "task.txt").write_text(task_txt) # Override with the candidate context's editable scripts diff --git a/src/datasmith/agents/synthesizer.py b/src/datasmith/agents/synthesizer.py index a7c1925d..e08cb5e7 100644 --- a/src/datasmith/agents/synthesizer.py +++ b/src/datasmith/agents/synthesizer.py @@ -6,6 +6,8 @@ from pathlib import Path from typing import Any, cast +from datasmith.agents.rate_limit import RateLimitError +from datasmith.agents.rate_limit import check as check_rate_limit from datasmith.agents.sandbox import SandboxResult, verify_context from datasmith.agents.tamper_audit import TamperResult, classify_context from datasmith.docker.context import DockerContext @@ -53,6 +55,7 @@ def __init__( agent: str | None = None, force: bool = False, max_aborts: int = 2, + max_default_failures_per_repo: int = 3, ) -> None: self._max_attempts = max_attempts self._max_aborts = max_aborts @@ -60,9 +63,18 @@ def __init__( self._agent = agent self._force = force self._trace: list[SynthesisState] = [] - # Repos for which TRY_DEFAULT has already run in this pipeline run. - # Gated in-memory so we only pay the default-build cost once per repo. + # Repos for which TRY_DEFAULT has already SUCCEEDED in this run. + # Once any PR succeeds with the default template for a repo, the row + # is in `candidate_containers` and later PRs should hit TRY_SIMILAR + # (which re-verifies the saved context against a new SHA) instead of + # redundantly rebuilding the default template from scratch. self._tried_default_repos: set[tuple[str, str]] = set() + # Failure counter per repo. If TRY_DEFAULT fails `max_default_failures_per_repo` + # times for the same repo, we stop retrying and fall through to LLM_GENERATE + # (or fail if agent=none). Prevents burning hours on repos where every PR's + # base commit is structurally broken. + self._max_default_failures_per_repo = max_default_failures_per_repo + self._default_failures: dict[tuple[str, str], int] = {} @property def trace(self) -> list[SynthesisState]: @@ -79,6 +91,7 @@ def run( # noqa: C901 env_payload: str = "", python_version: str = "", force: bool = False, + base_sha: str = "", ) -> DockerContext | None: """Run the synthesis state machine. Returns DockerContext on success, None on failure.""" self._trace = [] @@ -108,6 +121,7 @@ def run( # noqa: C901 env_payload=env_payload, python_version=python_version, context=ctx, + base_sha=base_sha, ) if result.success: logger.info("Similar context passed for %s/%s#%d", owner, repo, issue_number) @@ -122,11 +136,43 @@ def run( # noqa: C901 return ctx failed_attempts.append((ctx, result)) - # State: TRY_DEFAULT — attempt a build with the stock template scripts - # once per repo per run. If it works, no agent needed; if it doesn't, - # the failure trace becomes prior-attempt context for the LLM. - if (owner, repo) not in self._tried_default_repos: - self._tried_default_repos.add((owner, repo)) + # State: TRY_DEFAULT — attempt a build with the stock template scripts. + # + # Concurrency note: a single Synthesizer is shared across N concurrent + # workers. The old design checked-and-marked `_tried_default_repos` + # before `verify_context` ran, so in a race all workers but one would + # short-circuit TRY_DEFAULT even when the first worker's attempt was + # still in flight — and a single unlucky SHA failure would doom every + # subsequent PR in the same repo. + # + # New semantics: + # - `_tried_default_repos` marks only SUCCESS. Once any PR succeeds + # with the default template for this repo, the row is in + # `candidate_containers` and later PRs hit TRY_SIMILAR. + # - `_default_failures` counts failures. After + # `max_default_failures_per_repo` consecutive failures we stop + # retrying to avoid burning hours on structurally broken repos + # (e.g. every PR's env_payload is incompatible with the base image). + fail_count = self._default_failures.get((owner, repo), 0) + already_succeeded = (owner, repo) in self._tried_default_repos + too_many_failures = fail_count >= self._max_default_failures_per_repo + if already_succeeded: + logger.debug( + "Skipping TRY_DEFAULT for %s/%s#%d — default already succeeded for this repo", + owner, + repo, + issue_number, + ) + elif too_many_failures: + logger.info( + "Skipping TRY_DEFAULT for %s/%s#%d — %d prior failures (cap=%d)", + owner, + repo, + issue_number, + fail_count, + self._max_default_failures_per_repo, + ) + if (not already_succeeded) and (not too_many_failures): self._trace.append(SynthesisState.TRY_DEFAULT) default_ctx = _load_default_context() result = verify_context( @@ -137,6 +183,7 @@ def run( # noqa: C901 env_payload=env_payload, python_version=python_version, context=default_ctx, + base_sha=base_sha, ) if result.success: tamper = classify_context(default_ctx) @@ -152,9 +199,11 @@ def run( # noqa: C901 tamper.as_list(), ) self._log_tamper(owner, repo, sha, issue_number, 0, tamper, "try_default") + self._default_failures[(owner, repo)] = fail_count + 1 failed_attempts.append((default_ctx, result)) else: logger.info("Default template build succeeded for %s/%s#%d", owner, repo, issue_number) + self._tried_default_repos.add((owner, repo)) self._save_context( owner, repo, @@ -165,6 +214,7 @@ def run( # noqa: C901 ) return default_ctx else: + self._default_failures[(owner, repo)] = fail_count + 1 failed_attempts.append((default_ctx, result)) logger.info( "Default template build failed for %s/%s#%d — feeding trace into LLM priors", @@ -201,6 +251,7 @@ def run( # noqa: C901 prior_attempts=prior_attempts, issue_number=issue_number, attempt_index=attempt_idx, + base_sha=base_sha, ) if generated is not None: tamper = classify_context(generated) @@ -221,8 +272,11 @@ def run( # noqa: C901 tamper.as_list(), ) self._log_tamper(owner, repo, sha, issue_number, attempt_idx, tamper, "llm_generate") - attempt_idx += 1 - continue + # Fail-fast: an agent that fabricates artifacts on one + # attempt will almost certainly do it again on the next. + # Break immediately instead of burning another multi-hour + # session on the same PR. + break logger.info( "Sandbox synthesis succeeded for %s/%s#%d (attempt %d)", owner, @@ -379,6 +433,7 @@ def _sandbox_generate( prior_attempts: str = "", issue_number: int = 0, attempt_index: int = 0, + base_sha: str = "", ) -> tuple[DockerContext | None, dict, bool]: from datasmith.agents.sandbox import SandboxRunner @@ -393,6 +448,7 @@ def _sandbox_generate( pr_context=pr_context, prior_attempts=prior_attempts, dry_run=self._dry_run, + base_sha=base_sha, ) self._log_attempt( owner=owner, @@ -402,6 +458,16 @@ def _sandbox_generate( attempt_index=attempt_index, result=result, ) + # Surface budget exhaustion as a typed exception so the runner can + # pause *all* workers until the reset time instead of burning the + # remaining attempt budget on what will just be more ~2s failures. + is_rl, reset_at = check_rate_limit(result.agent_name, result.raw_agent_output) + if is_rl: + raise RateLimitError( + agent_name=result.agent_name, + reset_at=reset_at, + message=(f"{result.agent_name} hit usage limit during synthesis for {owner}/{repo}@{sha[:12]}"), + ) ctx = result.docker_context if result.success else None return ctx, result.resource_metrics, result.aborted @@ -426,9 +492,16 @@ def _log_attempt( # Aborted attempts (agent never produced failure.json or # verification_success.json) get a sentinel stage so they're # distinguishable from real verifier failures in error_logs. - if result.aborted: - failure_stage: str | None = "aborted" + is_rl, rl_reset = check_rate_limit(result.agent_name, result.raw_agent_output) + rl_reset_iso: str | None = rl_reset.isoformat() if (is_rl and rl_reset) else None + if is_rl: + failure_stage: str | None = "rate_limited" error_message: str | None = ( + f"{result.agent_name} hit weekly/periodic usage limit; reset_at={rl_reset_iso or 'unknown'}" + ) + elif result.aborted: + failure_stage = "aborted" + error_message = ( "Agent exited without running local_ci.py to completion " "(no failure.json or verification_success.json found)." ) @@ -451,6 +524,7 @@ def _log_attempt( "agent_output": raw_output or None, "files_changed": json.dumps(result.files_changed), "resource_metrics": result.resource_metrics or None, + "rate_limit_reset_at": rl_reset_iso, "created_at": timestamp, } try: diff --git a/src/datasmith/agents/templates/AGENTS.md.j2 b/src/datasmith/agents/templates/AGENTS.md.j2 index 88a79746..d9710363 100644 --- a/src/datasmith/agents/templates/AGENTS.md.j2 +++ b/src/datasmith/agents/templates/AGENTS.md.j2 @@ -1,5 +1,10 @@ # FormulaCode Docker Build Verification Guide +**Your only job is to make the upstream build, tests, and benchmarks pass by +fixing real dependency and configuration problems.** Do not create test files, +benchmark suites, ASV configs, or `/logs/` artifacts — the pipeline validates +outputs independently and rejects fabricated results. + You are fixing a Docker build context for the FormulaCode dataset. The goal of this workspace is to produce a Docker image in which the upstream repository **at the base commit** (`{{ sha }}`) builds cleanly, the project package From d7cc5441e4f74ebe1a87ba94cc426336284fbbd4 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:06:16 +0000 Subject: [PATCH 06/24] Add queue-based worker pool, neighbor cascade, and rate-limit pause to synthesize_images --- src/datasmith/runners/synthesize_images.py | 336 ++++++++++++++++++++- 1 file changed, 331 insertions(+), 5 deletions(-) diff --git a/src/datasmith/runners/synthesize_images.py b/src/datasmith/runners/synthesize_images.py index 316e542c..c38772b3 100644 --- a/src/datasmith/runners/synthesize_images.py +++ b/src/datasmith/runners/synthesize_images.py @@ -1,16 +1,45 @@ from __future__ import annotations import asyncio +import datetime +import os import tempfile import threading from typing import Any +from datasmith.agents.rate_limit import RateLimitError from datasmith.agents.synthesizer import Synthesizer from datasmith.runners.base import BaseRunner from datasmith.utils import get_client, get_logger logger = get_logger("runners.synthesize_images") +# All tunable knobs below are overridable from tokens.env — see CLAUDE.md +# "Tunable constants". tokens.env is auto-loaded by datasmith/__init__.py. + +# Default wait applied when an agent signals a rate limit but we couldn't +# parse a reset time. One hour gives room for the five-hour bucket to drain +# a little without pinning the runner to a hard-coded weekly stall. +DATASMITH_RL_DEFAULT_PAUSE_S: float = float(os.environ.get("DATASMITH_RL_DEFAULT_PAUSE_S", "3600")) +# Grace period added on top of the parsed reset time to avoid immediately +# retrying at t=reset and getting throttled by a clock skew of a few seconds. +DATASMITH_RL_PAUSE_JITTER_S: float = float(os.environ.get("DATASMITH_RL_PAUSE_JITTER_S", "30")) +# Maximum consecutive rate-limit retries for a single item before we give up +# and let it fail. Prevents infinite loops if detection is misfiring. +DATASMITH_RL_MAX_RETRIES: int = int(os.environ.get("DATASMITH_RL_MAX_RETRIES", "3")) + +# Chronological neighborhood window (days) used when enqueuing neighbor PRs +# after a successful synthesis. PRs created within ±this many days of the +# successful PR are highly likely to share its dependency environment, so +# TRY_SIMILAR will reuse the fresh context for free instead of burning +# another codex session on them. +DATASMITH_NEIGHBOR_WINDOW_DAYS: int = int(os.environ.get("DATASMITH_NEIGHBOR_WINDOW_DAYS", "60")) +# Hard ceiling on neighbors enqueued per successful item. Protects against a +# runaway enqueue burst in a repo with hundreds of PRs inside the window. +# Neighbors whose context hits in TRY_SIMILAR are effectively free, but those +# that fall through to LLM_GENERATE do consume agent budget, so we cap. +DATASMITH_NEIGHBOR_CAP: int = int(os.environ.get("DATASMITH_NEIGHBOR_CAP", "40")) + def _ensure_prerequisite_images(owner: str, repo: str, py_version: str = "") -> None: """Build the base and repo Docker images if they don't exist locally. @@ -41,6 +70,7 @@ def _build_pr_image( env_payload: str, docker_context: Any | None = None, python_version: str = "", + base_sha: str = "", ) -> str: """Build the final PR image from synthesized context (no push). @@ -52,16 +82,20 @@ def _build_pr_image( mgr = ImageManager() pr_tag = get_pr_image_name(owner, repo, issue_number) + # Use base_sha for the Docker checkout so the repo is at the + # pre-optimization state; fall back to merge_commit_sha for compat. + checkout_sha = base_sha or sha + if ctx is not None: with tempfile.TemporaryDirectory(prefix="docker-ctx-") as tmpdir: ctx.to_directory(tmpdir) - _fill_missing_scripts(tmpdir, base_commit=sha) + _fill_missing_scripts(tmpdir, base_commit=checkout_sha) mgr.build_pr_image( owner, repo, issue_number, context=tmpdir, - commit_sha=sha or "HEAD", + commit_sha=checkout_sha or "HEAD", env_payload=env_payload or "[]", py_version=python_version, ) @@ -70,7 +104,7 @@ def _build_pr_image( owner, repo, issue_number, - commit_sha=sha or "HEAD", + commit_sha=checkout_sha or "HEAD", env_payload=env_payload or "[]", py_version=python_version, ) @@ -171,6 +205,93 @@ def _fill_missing_scripts(context_dir: str, base_commit: str = "") -> None: _prereq_done: set[tuple[str, str]] = set() +def _fetch_neighbor_items( + owner: str, + repo: str, + lo_iso: str, + hi_iso: str, +) -> list[dict[str, Any]]: + """Query Supabase for PRs in *owner/repo* with ``created_at`` in ``[lo, hi]``. + + Returns item dicts shaped like ``pipeline._synthesize_images`` items, so + :meth:`SynthesizeImagesRunner._do_process_item` can consume them without + any special-casing. Filters mirror the stage 6 selection: performance + commits with resolved packages and a non-empty extracted problem context, + excluding PRs that already have a ``container_name``. + """ + from datasmith.utils.db import fetch_all + + rows = fetch_all( + "pull_requests", + select=("owner, repo, issue_number, merge_commit_sha, base_sha, title, body, created_at, rendered_problem"), + filters={ + "owner": owner, + "repo": repo, + "is_performance_commit": True, + "is_performance_commit_symbolic": True, + }, + neq_filters={"merge_commit_sha": ""}, + gte_filters={"created_at": lo_iso}, + lte_filters={"created_at": hi_iso}, + is_null=["container_name"], + ) + if not rows: + return [] + + pkg_rows = fetch_all( + "packages", + select="owner, repo, sha, env_payload, python_version", + filters={"can_install": True, "owner": owner, "repo": repo}, + ) + pkg_lookup = {(p["owner"], p["repo"], p["sha"]): p for p in pkg_rows} + + ctx_rows = fetch_all( + "candidate_prs", + select="owner, repo, issue_number, issues_json, initial_observations", + filters={"owner": owner, "repo": repo}, + ) + eligible: set[tuple[str, str, int]] = { + (c["owner"], c["repo"], c["issue_number"]) + for c in ctx_rows + if c.get("issues_json") or c.get("initial_observations") + } + + # Batch repo_description lookup once — cheap and keeps the item dict + # shape consistent with pipeline._synthesize_images. + desc_rows = fetch_all( + "repositories", + select="owner, repo, description", + filters={"owner": owner, "repo": repo}, + ) + repo_description = "" + if desc_rows: + repo_description = desc_rows[0].get("description") or "" + + items: list[dict[str, Any]] = [] + for r in rows: + sha = r.get("merge_commit_sha", "") + pkg = pkg_lookup.get((r["owner"], r["repo"], sha), {}) + if not pkg: + continue + if (r["owner"], r["repo"], r["issue_number"]) not in eligible: + continue + items.append({ + "owner": r["owner"], + "repo": r["repo"], + "issue_number": r["issue_number"], + "sha": sha, + "base_sha": r.get("base_sha", ""), + "title": r.get("title", ""), + "body": r.get("body", ""), + "created_at": r.get("created_at"), + "pr_context": r.get("rendered_problem") or r.get("body", ""), + "repo_description": repo_description, + "env_payload": pkg.get("env_payload", ""), + "python_version": pkg.get("python_version", ""), + }) + return items + + class SynthesizeImagesRunner(BaseRunner): """Run Synthesizer for each PR to produce Docker build contexts.""" @@ -183,6 +304,80 @@ def __init__( super().__init__(name="synthesize_images", n_concurrent=n_concurrent) self._synthesizer = synthesizer self._gh = gh # GitHubClient, optional — needed for rendering problem statements + # Shared pause state — when any worker raises RateLimitError, it sets + # `_rl_resume_at` and every other worker blocks on `_rl_lock` until + # the clock passes that timestamp. A single agent's weekly budget is + # shared across all workers, so pausing one without pausing the rest + # would just burn the remaining attempt budget on ~2s failures. + self._rl_lock = asyncio.Lock() + self._rl_resume_at: datetime.datetime | None = None + # Queue-based worker pool state. Populated in ``run`` so that + # ``_do_process_item`` can enqueue chronologically adjacent PRs on + # success and have other workers pick them up without respawning. + # The runner formalises the old two-pass workflow (first pass + # hydrates with codex, second pass with agent=none) into a single + # pass: once a PR synthesises successfully, its neighbors are added + # to the same queue and will hit TRY_SIMILAR cheaply before falling + # through to LLM_GENERATE only if the context genuinely mismatches. + self._queue: asyncio.Queue[Any] | None = None + self._enqueued: set[tuple[str, str, int]] = set() + + async def run(self, items: list[Any]) -> None: + """Override ``BaseRunner.run`` with a queue-based worker pool. + + ``BaseRunner.run`` gathers a fixed list of tasks, but this runner + needs to enqueue additional items mid-flight (chronologically + adjacent PRs, added after each successful synthesis). A bounded + worker pool reading from an ``asyncio.Queue`` gives us that + ability without perturbing other stages. + """ + self._total = len(items) + self._completed = 0 + self._failed = 0 + self._enqueued = set() + self._init_progress() + + queue: asyncio.Queue[Any] = asyncio.Queue() + self._queue = queue + for item in items: + key = (item["owner"], item["repo"], item["issue_number"]) + if key in self._enqueued: + continue + self._enqueued.add(key) + queue.put_nowait(item) + + workers = [asyncio.create_task(self._worker_loop()) for _ in range(self._n_concurrent)] + try: + await queue.join() + except (KeyboardInterrupt, asyncio.CancelledError): + for w in workers: + w.cancel() + await asyncio.gather(*workers, return_exceptions=True) + raise + finally: + for w in workers: + w.cancel() + await asyncio.gather(*workers, return_exceptions=True) + self._update_progress(force=True) + self._queue = None + + async def _worker_loop(self) -> None: + """Pull items off the shared queue until cancelled.""" + queue = self._queue + if queue is None: + return + while True: + item = await queue.get() + try: + await self._process_item(item) + self._completed += 1 + except Exception as exc: + self._failed += 1 + self._log_failure(item, exc) + logger.exception("Failed processing item %s", self._item_id(item)) + finally: + self._maybe_update_progress() + queue.task_done() async def _render_problem(self, item: dict[str, Any]) -> str | None: """Render the problem statement for a PR, scraping linked issues. @@ -248,7 +443,30 @@ async def _render_problem(self, item: dict[str, Any]) -> str | None: return rendered async def _process_item(self, item: Any) -> None: - """Process a PR dict with owner, repo, issue_number, pr_context.""" + """Process a PR dict, transparently pausing on CLI-agent rate limits. + + Every worker first blocks on ``_wait_for_rate_limit`` so that a + pause triggered by a peer worker is respected without that peer + needing to re-raise into every outstanding task. Each item gets up + to ``DATASMITH_RL_MAX_RETRIES`` rate-limit pauses before we let it fail. + """ + for attempt in range(DATASMITH_RL_MAX_RETRIES + 1): + await self._wait_for_rate_limit() + try: + await self._do_process_item(item) + return + except RateLimitError as exc: + if attempt >= DATASMITH_RL_MAX_RETRIES: + logger.warning( + "Rate-limit retries exhausted for %s after %d pauses — failing item", + self._item_id(item), + DATASMITH_RL_MAX_RETRIES, + ) + raise + await self._trigger_rate_limit_pause(exc) + + async def _do_process_item(self, item: Any) -> None: + """Inner implementation of ``_process_item``.""" owner = item["owner"] repo = item["repo"] issue_number = item["issue_number"] @@ -265,6 +483,7 @@ async def _process_item(self, item: Any) -> None: pr_context = rendered sha = item.get("sha", "") + base_sha = item.get("base_sha", "") env_payload = item.get("env_payload", "") from datasmith.docker.images import get_repo_image_name @@ -282,6 +501,7 @@ async def _process_item(self, item: Any) -> None: repo_image=repo_image, env_payload=env_payload, python_version=py_version, + base_sha=base_sha, ) if ctx is None: @@ -290,7 +510,17 @@ async def _process_item(self, item: Any) -> None: logger.info("Successfully synthesized image for %s/%s#%d", owner, repo, issue_number) # Build the final PR image locally (no push yet) - pr_tag = await asyncio.to_thread(_build_pr_image, owner, repo, issue_number, sha, env_payload, ctx, py_version) + pr_tag = await asyncio.to_thread( + _build_pr_image, + owner, + repo, + issue_number, + sha, + env_payload, + ctx, + py_version, + base_sha=base_sha, + ) # Record the container name in Supabase *before* pushing. If the DB # write fails, the image stays unpublished and a re-run picks up the @@ -303,6 +533,102 @@ async def _process_item(self, item: Any) -> None: # DB state is durable — safe to publish the image. await asyncio.to_thread(_push_pr_image, owner, repo, pr_tag) + # Spread the win: PRs created within ±DATASMITH_NEIGHBOR_WINDOW_DAYS + # of this one likely share its dependency environment, so the context + # we just cached will satisfy TRY_SIMILAR for them for free. Only the + # PRs whose context genuinely differs fall through to LLM_GENERATE. + await self._enqueue_neighbors(owner, repo, issue_number, item.get("created_at")) + + async def _enqueue_neighbors( + self, + owner: str, + repo: str, + issue_number: int, + created_at_raw: Any, + ) -> None: + """Find and enqueue chronologically adjacent PRs in the same repo.""" + if self._queue is None or not created_at_raw: + return + try: + if isinstance(created_at_raw, str): + base_dt = datetime.datetime.fromisoformat(created_at_raw.replace("Z", "+00:00")) + elif isinstance(created_at_raw, datetime.datetime): + base_dt = created_at_raw + else: + return + except ValueError: + return + + window = datetime.timedelta(days=DATASMITH_NEIGHBOR_WINDOW_DAYS) + lo = (base_dt - window).isoformat() + hi = (base_dt + window).isoformat() + + neighbors = await asyncio.to_thread(_fetch_neighbor_items, owner, repo, lo, hi) + + added = 0 + for nb in neighbors: + if added >= DATASMITH_NEIGHBOR_CAP: + break + key = (nb["owner"], nb["repo"], nb["issue_number"]) + if key in self._enqueued: + continue + self._enqueued.add(key) + self._total += 1 + self._queue.put_nowait(nb) + added += 1 + + if added: + logger.info( + "Enqueued %d neighbor PR(s) for %s/%s#%d (±%d days)", + added, + owner, + repo, + issue_number, + DATASMITH_NEIGHBOR_WINDOW_DAYS, + ) + + async def _wait_for_rate_limit(self) -> None: + """Block until any active rate-limit pause has elapsed.""" + while True: + resume_at = self._rl_resume_at + if resume_at is None: + return + now = datetime.datetime.now(tz=datetime.UTC) + remaining = (resume_at - now).total_seconds() + if remaining <= 0: + return + logger.info( + "Worker sleeping %.0fs for agent rate-limit reset at %s", + remaining, + resume_at.isoformat(), + ) + await asyncio.sleep(min(remaining, 60.0)) + + async def _trigger_rate_limit_pause(self, exc: RateLimitError) -> None: + """Install a shared pause triggered by *exc*. + + Only the first worker to enter under the lock sets ``_rl_resume_at``; + later workers that hit the same exception while the pause is already + in effect simply return and re-wait on their next loop iteration. + """ + async with self._rl_lock: + now = datetime.datetime.now(tz=datetime.UTC) + if self._rl_resume_at and self._rl_resume_at > now: + # Another worker already set the pause — honour theirs. + return + if exc.reset_at is not None: + resume_at = exc.reset_at + datetime.timedelta(seconds=DATASMITH_RL_PAUSE_JITTER_S) + else: + resume_at = now + datetime.timedelta(seconds=DATASMITH_RL_DEFAULT_PAUSE_S) + self._rl_resume_at = resume_at + wait_s = max(0.0, (resume_at - now).total_seconds()) + logger.warning( + "Agent %s hit usage limit — pausing synthesis until %s (%.0fs)", + exc.agent_name, + resume_at.isoformat(), + wait_s, + ) + @staticmethod def _ensure_prereqs(owner: str, repo: str, py_version: str) -> None: """Build base/repo images if missing, with dedup across threads.""" From 7db9d1cdfd359a69f831ff97fd30285fb1828abe Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:06:24 +0000 Subject: [PATCH 07/24] Improve harbor_healthcheck: granular status classification, memory bump, row-by-row retry --- src/datasmith/runners/harbor_healthcheck.py | 146 ++++++++++++++++++-- 1 file changed, 134 insertions(+), 12 deletions(-) diff --git a/src/datasmith/runners/harbor_healthcheck.py b/src/datasmith/runners/harbor_healthcheck.py index 6b3ad0a1..77ff8f77 100644 --- a/src/datasmith/runners/harbor_healthcheck.py +++ b/src/datasmith/runners/harbor_healthcheck.py @@ -77,6 +77,10 @@ def _build_verifier_env() -> dict[str, str]: val = os.environ.get(host_key) if val: env[container_key] = val + # lsv_init.py gates snapshot capture on HARBOR_AGENT_NAME=="oracle"; Harbor + # itself never sets this. Inline a literal so the two scripts (lsv_init + + # test.sh) agree on the agent identity the trial is running. + env["HARBOR_AGENT_NAME"] = "oracle" return env @@ -142,11 +146,18 @@ def _build_job_config( from harbor.models.orchestrator_type import OrchestratorType from harbor.models.trial.config import AgentConfig, EnvironmentConfig + # Harbor defaults to 4 GB per trial container (per the task.toml template) + # which is too tight for lsv_init on mid/large Python repos — sklearn's + # dep-graph walk alone exceeds 4 GB and gets OOM-killed with exit 137. + # Bump to 32 GB across the board; the host has 500 GB so there's plenty + # of headroom, and smaller repos won't actually use more than they need. + MEMORY_MB = 32 * 1024 if use_daytona: environment = EnvironmentConfig( type=EnvironmentType.DAYTONA, force_build=True, delete=True, + override_memory_mb=MEMORY_MB, kwargs={ "auto_stop_interval_mins": 0, "auto_delete_interval_mins": 0, @@ -157,6 +168,7 @@ def _build_job_config( type=EnvironmentType.DOCKER, force_build=True, delete=True, + override_memory_mb=MEMORY_MB, ) return JobConfig( @@ -208,29 +220,108 @@ def _row_from_trial( # noqa: C901 n_benchmarks: int | None = None status = "failed" error_message: str | None = None - + harbor_exception: str | None = None + + # Harbor can raise ``VerifierTimeoutError`` *after* test.sh has already + # written a valid reward.json — the verifier wrapper enforces a + # wall-clock budget that includes file-upload/stdout-drain overhead, so + # a trial can land a perfectly good reward.json on disk and still get + # flagged. In that case we want the success, not the wrapper timeout. + # → Check reward.json FIRST and treat trial.exception_info as + # decoration (stored as ``harbor_exception`` for post-mortem triage). if trial.exception_info is not None: - error_message = str(getattr(trial.exception_info, "message", None) or trial.exception_info) - elif paths.reward_json_path.exists(): + harbor_exception = str(getattr(trial.exception_info, "message", None) or trial.exception_info) + + if paths.reward_json_path.exists(): try: reward_payload = json.loads(paths.reward_json_path.read_text()) except Exception as exc: error_message = f"reward.json parse error: {exc}" + if harbor_exception: + error_message = f"{error_message}; harbor_exception: {harbor_exception}" else: - speedups = (reward_payload or {}).get("per_benchmark_speedups") or {} + payload = reward_payload or {} + speedups = payload.get("per_benchmark_speedups") or {} if speedups: try: max_speedup = max(float(v) for v in speedups.values()) except (TypeError, ValueError) as exc: error_message = f"non-numeric speedup values: {exc}" - geomean = (reward_payload or {}).get("lsv_mean_speedup") - n_benchmarks = (reward_payload or {}).get("num_valid_benchmarks") or len(speedups) or None - if max_speedup is not None: + geomean = payload.get("lsv_mean_speedup") + n_benchmarks = payload.get("num_valid_benchmarks") or len(speedups) or None + + patch_info = payload.get("patch") or {} + patch_applied = patch_info.get("applied") + lsv_block = payload.get("lsv") or {} + lsv_init = lsv_block.get("init") or {} + lsv_init_populated = bool(lsv_init) + impactable = len(lsv_init.get("benchmarks_impactable") or []) + source_files_covered = lsv_init.get("source_files_covered") + lsv_error = payload.get("lsv_error") + tests_passed = payload.get("tests_passed") + setup = payload.get("setup") or {} + setup_exit = setup.get("exit_code") + setup_phase = setup.get("failed_phase") + + # Priority order for status classification — most specific wins. + # Check setup failures FIRST because a failed setup cascades into + # every other failure mode (no dep DB → lsv_measure crashes → + # empty benchmarks → reward.json looks like a run produced no + # signal). We want the report to name the real root cause. + if setup_exit not in (None, 0): + if setup_phase == "lsv_init": + status = "lsv_init_failed" + error_message = ( + f"lsv_init.py crashed in setup.sh (exit={setup_exit}). " + "Benchmark discovery never completed, so no dep DB was " + "written. Check setup.txt in the trial dir for the " + "underlying ASV/LSV traceback." + ) + else: + status = "setup_failed" + error_message = f"setup.sh failed in phase '{setup_phase}' (exit={setup_exit})." + elif patch_applied is False: + status = "patch_failed" + error_message = "solve.sh produced no diff vs base_commit" + elif lsv_init_populated and impactable == 0 and source_files_covered == 0: + # lsv_init ran TO COMPLETION but found zero coverage — a + # real repo/LSV compatibility issue, not a crash. + status = "lsv_init_empty" + error_message = ( + "LSV init mapped 0 source files and 0 impactable benchmarks — " + "ASV/LSV could not trace imports into this repo's benchmark suite." + ) + elif max_speedup is not None: status = "success" + elif lsv_error: + status = "lsv_measure_failed" + error_message = lsv_error elif not speedups: status = "no_benchmarks" + + if tests_passed is False and status == "success": + # Benchmarks ran but tests failed — degrade so publish + # doesn't silently gate on a broken suite. + status = "tests_failed" + error_message = error_message or "tests_passed=False in reward.json" + + # If we used reward.json but Harbor also raised an exception, + # keep the reward-derived status as authoritative and tack the + # exception onto error_message so it's still visible in triage. + if harbor_exception and status != "success": + suffix = f" (harbor_exception: {harbor_exception})" + error_message = (error_message or "") + suffix else: - error_message = "reward.json missing" + # No reward.json means the trial never completed a verifier pass. + # Harbor's exception (if any) is now the only signal we have. + if harbor_exception: + error_message = f"reward.json missing; harbor_exception: {harbor_exception}" + if "VerifierTimeoutError" in harbor_exception or "timeout" in harbor_exception.lower(): + status = "verifier_timeout" + else: + status = "harbor_exception" + else: + error_message = "reward.json missing (no harbor exception recorded)" return { "owner": meta["owner"], @@ -252,15 +343,46 @@ def _row_from_trial( # noqa: C901 def _insert_harbor_runs(rows: list[dict[str, Any]], chunk_size: int = 100) -> int: """Insert (not upsert) into harbor_runs. Each row is a fresh run with an - auto-generated run_id, so upsert semantics don't apply here.""" + auto-generated run_id, so upsert semantics don't apply here. + + Failure handling: harbor_runs has a foreign key on + ``candidate_containers(owner, repo, sha)``. If a PR's container row + has been deleted out from under us (e.g. stage 6 rebuilt and the old + sha was dropped), the chunk insert raises ``23503`` and we'd lose + every other row in the chunk. Retry row-by-row on chunk failure so + orphan rows are logged and skipped instead of blowing up the whole + stage. + """ if not rows: return 0 client = get_client() total = 0 for i in range(0, len(rows), chunk_size): chunk = rows[i : i + chunk_size] - client.table("harbor_runs").insert(chunk).execute() - total += len(chunk) + try: + client.table("harbor_runs").insert(chunk).execute() + total += len(chunk) + continue + except Exception as chunk_exc: + logger.warning( + "harbor_runs chunk insert failed (%s); retrying row-by-row", + chunk_exc, + ) + # Fall back to per-row inserts so a single FK violation or other + # row-scoped error doesn't drop the rest of the batch. + for row in chunk: + try: + client.table("harbor_runs").insert(row).execute() + total += 1 + except Exception as row_exc: + logger.warning( + "harbor_runs orphan row skipped: %s/%s@%s status=%s — %s", + row.get("owner"), + row.get("repo"), + (row.get("sha") or "")[:12], + row.get("status"), + row_exc, + ) return total @@ -270,7 +392,7 @@ async def run_harbor_healthcheck( task_dir: Path, use_daytona: bool = False, n_concurrent_trials: int = 4, - rounds: int = 4, + rounds: int = 2, job_name: str | None = None, ) -> list[dict[str, Any]]: """Materialize *items* into *task_dir*, run Harbor's oracle agent on the From 80efbdba8916d23b9723a122953c3b1dfbe5f2c7 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:06:42 +0000 Subject: [PATCH 08/24] Refactor PY_RELEASES to module scope, add Python 3.14 --- src/datasmith/resolution/orchestrator.py | 15 +++++++---- src/datasmith/resolution/python_manager.py | 30 ++++++++++++---------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/datasmith/resolution/orchestrator.py b/src/datasmith/resolution/orchestrator.py index 985bd20a..370f0fc4 100644 --- a/src/datasmith/resolution/orchestrator.py +++ b/src/datasmith/resolution/orchestrator.py @@ -37,7 +37,12 @@ resolve_requirements_file, split_shell_command, ) -from .python_manager import ensure_python_version_available, filter_python_versions_by_commit_date, run_uv +from .python_manager import ( + SUPPORTED_PYTHON_VERSIONS, + ensure_python_version_available, + filter_python_versions_by_commit_date, + run_uv, +) logger = get_logger("resolution.orchestrator") @@ -85,23 +90,23 @@ def analyze_commit(sha: str, repo_name: str, bypass_cache: bool = False) -> dict bc = getattr(cfg, "build_command", None) ic = getattr(cfg, "install_command", None) if bc: - if isinstance(bc, (list, tuple)): + if isinstance(bc, list | tuple): bc = " && ".join(bc).replace("-mpip", "-m pip") cfg_items.build_commands.add(str(bc)) if ic: - if isinstance(ic, (list, tuple)): + if isinstance(ic, list | tuple): ic = " && ".join(ic) cfg_items.install_commands.add(str(ic)) mx = getattr(cfg, "matrix", None) or {} for k, v in mx.items(): values = cfg_items.matrix.setdefault(k, set()) - if isinstance(v, (list, tuple, set)): + if isinstance(v, list | tuple | set): values.update(map(str, v)) else: values.add(str(v)) if not cfg_items.pythons: - cfg_items.pythons.update({(3, 8), (3, 9), (3, 10), (3, 11), (3, 12)}) + cfg_items.pythons.update(SUPPORTED_PYTHON_VERSIONS) # B) Choose Python version candidates if (not cfg_items.pythons) or all(py < (3, 8) for py in cfg_items.pythons): diff --git a/src/datasmith/resolution/python_manager.py b/src/datasmith/resolution/python_manager.py index d8172769..0bf1f6b0 100644 --- a/src/datasmith/resolution/python_manager.py +++ b/src/datasmith/resolution/python_manager.py @@ -12,6 +12,20 @@ logger = get_logger("resolution.python_manager") +PY_RELEASES: dict[tuple[int, int], dt.datetime] = { + (3, 7): dt.datetime(2018, 6, 27, tzinfo=dt.UTC), + (3, 8): dt.datetime(2019, 10, 14, tzinfo=dt.UTC), + (3, 9): dt.datetime(2020, 10, 5, tzinfo=dt.UTC), + (3, 10): dt.datetime(2021, 10, 4, tzinfo=dt.UTC), + (3, 11): dt.datetime(2022, 10, 24, tzinfo=dt.UTC), + (3, 12): dt.datetime(2023, 10, 2, tzinfo=dt.UTC), + (3, 13): dt.datetime(2024, 10, 7, tzinfo=dt.UTC), + (3, 14): dt.datetime(2025, 10, 7, tzinfo=dt.UTC), +} + +SUPPORTED_PYTHON_VERSIONS: set[tuple[int, int]] = {v for v in PY_RELEASES if v >= (3, 8)} + + def run_uv( args: list[str], *, @@ -69,24 +83,14 @@ def filter_python_versions_by_commit_date( # noqa: C901 if not valid_versions: return [] - py_releases = { - (3, 7): dt.datetime(2018, 6, 27, tzinfo=dt.timezone.utc), - (3, 8): dt.datetime(2019, 10, 14, tzinfo=dt.timezone.utc), - (3, 9): dt.datetime(2020, 10, 5, tzinfo=dt.timezone.utc), - (3, 10): dt.datetime(2021, 10, 4, tzinfo=dt.timezone.utc), - (3, 11): dt.datetime(2022, 10, 24, tzinfo=dt.timezone.utc), - (3, 12): dt.datetime(2023, 10, 2, tzinfo=dt.timezone.utc), - (3, 13): dt.datetime(2024, 10, 7, tzinfo=dt.timezone.utc), - } - grace_period = dt.timedelta(days=90) filtered = [] for v in valid_versions: version_key = (v[0], v[1]) - release_date = py_releases.get(version_key) + release_date = PY_RELEASES.get(version_key) if release_date is None: - if commit_date < dt.datetime(2024, 1, 1, tzinfo=dt.timezone.utc): + if commit_date < dt.datetime(2024, 1, 1, tzinfo=dt.UTC): continue filtered.append(v) elif commit_date >= release_date - grace_period: @@ -94,7 +98,7 @@ def filter_python_versions_by_commit_date( # noqa: C901 if not filtered: inferred = [] - for version_key, release_date in sorted(py_releases.items(), reverse=True): + for version_key, release_date in sorted(PY_RELEASES.items(), reverse=True): if version_key < (3, 8): continue if release_date <= commit_date + grace_period: From 2456552178c9ce1d407c1ab1ba982161e31ea6b8 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:06:56 +0000 Subject: [PATCH 09/24] Stabilize fetch_all pagination ordering and add docker image prune --- src/datasmith/utils/db.py | 53 +++++++++++++++++++++++++++-- src/datasmith/utils/docker_prune.py | 28 +++++++++------ 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/src/datasmith/utils/db.py b/src/datasmith/utils/db.py index a3fe8344..7885f481 100644 --- a/src/datasmith/utils/db.py +++ b/src/datasmith/utils/db.py @@ -5,11 +5,34 @@ import functools import hashlib import json +import logging import os -from typing import Any, Callable, TypeVar, cast +from collections.abc import Callable, Sequence +from typing import Any, TypeVar, cast from supabase import Client, create_client +logger = logging.getLogger(__name__) + +# Stable primary-key ordering per table, used by ``fetch_all`` to make +# range-based pagination deterministic. PostgREST/Postgres provide no +# stable ordering without an explicit ORDER BY, so paginating large +# result sets without one silently drops and duplicates rows — which +# is exactly how classify_prs and other stages ended up leaving work +# behind across repeated invocations. Keep entries in sync with the +# primary keys declared in ``supabase/migrations/``. +_DEFAULT_ORDER_BY: dict[str, tuple[str, ...]] = { + "repositories": ("owner", "repo"), + "pull_requests": ("owner", "repo", "issue_number"), + "candidate_prs": ("owner", "repo", "issue_number"), + "packages": ("owner", "repo", "sha"), + "candidate_containers": ("owner", "repo", "sha"), + "harbor_runs": ("run_id",), + "error_logs": ("id",), + "runner_progress": ("runner_id",), + "runner_failures": ("id",), +} + F = TypeVar("F", bound=Callable[..., Any]) _client: Client | None = None @@ -64,7 +87,7 @@ def batch_upsert(table: str, rows: list[dict[str, Any]], chunk_size: int = 100) return total -def fetch_all( +def fetch_all( # noqa: C901 table: str, select: str = "*", filters: dict[str, Any] | None = None, @@ -73,16 +96,32 @@ def fetch_all( lte_filters: dict[str, Any] | None = None, neq_filters: dict[str, Any] | None = None, page_size: int = 1000, + order_by: Sequence[str] | str | None = None, ) -> list[dict[str, Any]]: """Paginate through all rows matching the query. Supabase/PostgREST caps responses at 1 000 rows by default. This helper fetches successive pages using ``range()`` until a page returns fewer than *page_size* rows. + + ``order_by`` must specify a stable (unique) key — otherwise + Postgres is free to return rows in different orders across + pages, silently dropping and duplicating rows. If the caller + doesn't pass one, we fall back to ``_DEFAULT_ORDER_BY[table]`` + (the table's primary key). Tables missing from that map will + emit a warning when pagination actually crosses a page boundary. """ + if order_by is None: + order_cols: tuple[str, ...] = _DEFAULT_ORDER_BY.get(table, ()) + elif isinstance(order_by, str): + order_cols = (order_by,) + else: + order_cols = tuple(order_by) + client = get_client() rows: list[dict[str, Any]] = [] offset = 0 + warned_unstable = False while True: query = client.table(table).select(select) for col, val in (filters or {}).items(): @@ -95,11 +134,21 @@ def fetch_all( query = query.lte(col, val) for col, val in (neq_filters or {}).items(): query = query.neq(col, val) + for col in order_cols: + query = query.order(col) resp = query.range(offset, offset + page_size - 1).execute() page = cast(list[dict[str, Any]], resp.data or []) rows.extend(page) if len(page) < page_size: break + if not order_cols and not warned_unstable: + logger.warning( + "fetch_all(%r) crossed page boundary without an order_by; " + "pagination is non-deterministic and may drop or duplicate rows. " + "Add the table's primary key to _DEFAULT_ORDER_BY or pass order_by explicitly.", + table, + ) + warned_unstable = True offset += page_size return rows diff --git a/src/datasmith/utils/docker_prune.py b/src/datasmith/utils/docker_prune.py index 76e0933b..ebce4a33 100644 --- a/src/datasmith/utils/docker_prune.py +++ b/src/datasmith/utils/docker_prune.py @@ -11,28 +11,25 @@ logger = get_logger("utils.docker_prune") -_DEFAULT_INTERVAL_SEC = 600 +_DEFAULT_INTERVAL_SEC = 7200 -def _run_prune() -> None: - docker = shutil.which("docker") - if docker is None: - logger.warning("docker binary not found on PATH; skipping builder prune") - return +def _run_prune_cmd(docker: str, args: list[str], label: str) -> None: try: result = subprocess.run( - [docker, "builder", "prune", "-f"], + [docker, *args], check=False, capture_output=True, text=True, - timeout=300, + timeout=600, ) except subprocess.TimeoutExpired: - logger.warning("docker builder prune timed out after 300s") + logger.warning("%s timed out after 600s", label) return if result.returncode != 0: logger.warning( - "docker builder prune exited %d: %s", + "%s exited %d: %s", + label, result.returncode, (result.stderr or result.stdout).strip()[:500], ) @@ -42,7 +39,16 @@ def _run_prune() -> None: if "Total reclaimed space" in line: reclaimed = line.strip() break - logger.info("docker builder prune: %s", reclaimed or "done") + logger.info("%s: %s", label, reclaimed or "done") + + +def _run_prune() -> None: + docker = shutil.which("docker") + if docker is None: + logger.warning("docker binary not found on PATH; skipping docker prune") + return + _run_prune_cmd(docker, ["builder", "prune", "-f"], "docker builder prune") + _run_prune_cmd(docker, ["image", "prune", "-f"], "docker image prune") @contextlib.contextmanager From e4fd69f2ee6eee9ba339f7b10599359bbab15a50 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:07:00 +0000 Subject: [PATCH 10/24] Wire base_sha through pipeline, add qwen CLI choice, enable httpx redirects --- src/datasmith/github/client.py | 1 + src/datasmith/update/cli.py | 8 +++++--- src/datasmith/update/pipeline.py | 9 ++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/datasmith/github/client.py b/src/datasmith/github/client.py index 4af73a57..172d4968 100644 --- a/src/datasmith/github/client.py +++ b/src/datasmith/github/client.py @@ -26,6 +26,7 @@ async def _client(self) -> httpx.AsyncClient: self._http = httpx.AsyncClient( base_url="https://api.github.com", timeout=30.0, + follow_redirects=True, headers={"Accept": "application/vnd.github.v3+json"}, limits=httpx.Limits( max_connections=200, diff --git a/src/datasmith/update/cli.py b/src/datasmith/update/cli.py index 4b59ac85..aba69cbd 100644 --- a/src/datasmith/update/cli.py +++ b/src/datasmith/update/cli.py @@ -72,7 +72,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--agent", type=str, default=None, - choices=["claude", "codex", "gemini", "none"], + choices=["claude", "codex", "gemini", "qwen", "none"], help="CLI agent to use for stage 6 synthesis (default: auto-detect first available). " "'none' skips LLM generation and relies only on similar-context matching.", ) @@ -104,9 +104,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--harbor-rounds", type=int, - default=4, + default=2, metavar="N", - help="Number of LSV timing rounds per Harbor trial in stage 7 (default: 4)", + help="Number of LSV timing rounds per Harbor trial in stage 7 (default: 2). " + "Higher values scale measure/init time linearly; 4 routinely blew Harbor's " + "6-hour verifier budget on large repos (scikit-image, arrow).", ) parser.add_argument( "--harbor-limit", diff --git a/src/datasmith/update/pipeline.py b/src/datasmith/update/pipeline.py index a7fb64a5..4a41992f 100644 --- a/src/datasmith/update/pipeline.py +++ b/src/datasmith/update/pipeline.py @@ -109,7 +109,7 @@ def __init__( offline_source: str | None = None, min_stars: int = 500, harbor_use_daytona: bool = False, - harbor_rounds: int = 4, + harbor_rounds: int = 2, harbor_limit: int | None = None, harbor_tasks: str | None = None, ) -> None: @@ -528,7 +528,7 @@ async def _render_problems(self, start_date: str, end_date: str) -> None: async def _synthesize_images(self, start_date: str, end_date: str) -> None: query_kwargs: dict[str, Any] = { - "select": "owner, repo, issue_number, merge_commit_sha, title, body, created_at, rendered_problem", + "select": "owner, repo, issue_number, merge_commit_sha, base_sha, title, body, created_at, rendered_problem", "filters": {"is_performance_commit": True, "is_performance_commit_symbolic": True}, "neq_filters": {"merge_commit_sha": ""}, "gte_filters": {"created_at": start_date}, @@ -594,6 +594,7 @@ async def _synthesize_images(self, start_date: str, end_date: str) -> None: "repo": r["repo"], "issue_number": r["issue_number"], "sha": sha, + "base_sha": r.get("base_sha", ""), "title": r.get("title", ""), "body": r.get("body", ""), "created_at": r.get("created_at"), @@ -761,9 +762,7 @@ async def _publish(self, start_date: str, end_date: str) -> None: def _get_completed_stages(self) -> list[str]: try: - client = get_client() - resp = client.table("runner_progress").select("runner_name, completed, total").execute() - rows: list[dict[str, Any]] = resp.data # type: ignore[assignment] + rows = fetch_all("runner_progress", select="runner_name, completed, total") completed: list[str] = [] for r in rows: if r["total"] > 0 and r["completed"] >= r["total"]: From 0aedcbcacb2a1281939afce303f30863216c73e4 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:07:04 +0000 Subject: [PATCH 11/24] Point harbor dep at local path, update uv.lock --- pyproject.toml | 2 +- uv.lock | 53 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a37dcf34..eb06cedd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ dependencies = [ "dspy>=2.6.27", "gitpython", - "harbor @ git+https://github.com/formula-code/harbor.git", + "harbor @ file:///mnt/sdd1/atharvas/formulacode/eval_frameworks/harbor", "httpx>=0.27", "huggingface-hub>=0.20", "jinja2>=3.1.6", diff --git a/uv.lock b/uv.lock index d720d032..a552b93c 100644 --- a/uv.lock +++ b/uv.lock @@ -1119,7 +1119,7 @@ docs = [ requires-dist = [ { name = "dspy", specifier = ">=2.6.27" }, { name = "gitpython" }, - { name = "harbor", git = "https://github.com/formula-code/harbor.git" }, + { name = "harbor", directory = "../eval_frameworks/harbor" }, { name = "httpx", specifier = ">=0.27" }, { name = "huggingface-hub", specifier = ">=0.20" }, { name = "jinja2", specifier = ">=3.1.6" }, @@ -1361,6 +1361,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, + { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, @@ -1371,6 +1372,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/e8/58c7f85958bda41dafea50497cbd59738c5c43dbbea5ee83d651234398f4/greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31", size = 272814, upload-time = "2025-08-07T13:15:50.011Z" }, { url = "https://files.pythonhosted.org/packages/62/dd/b9f59862e9e257a16e4e610480cfffd29e3fae018a68c2332090b53aac3d/greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945", size = 641073, upload-time = "2025-08-07T13:42:57.23Z" }, { url = "https://files.pythonhosted.org/packages/f7/0b/bc13f787394920b23073ca3b6c4a7a21396301ed75a655bcb47196b50e6e/greenlet-3.2.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:710638eb93b1fa52823aa91bf75326f9ecdfd5e0466f00789246a5280f4ba0fc", size = 655191, upload-time = "2025-08-07T13:45:29.752Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d6/6adde57d1345a8d0f14d31e4ab9c23cfe8e2cd39c3baf7674b4b0338d266/greenlet-3.2.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c5111ccdc9c88f423426df3fd1811bfc40ed66264d35aa373420a34377efc98a", size = 649516, upload-time = "2025-08-07T13:53:16.314Z" }, { url = "https://files.pythonhosted.org/packages/7f/3b/3a3328a788d4a473889a2d403199932be55b1b0060f4ddd96ee7cdfcad10/greenlet-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76383238584e9711e20ebe14db6c88ddcedc1829a9ad31a584389463b5aa504", size = 652169, upload-time = "2025-08-07T13:18:32.861Z" }, { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, @@ -1381,6 +1383,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/5c/85273fd7cc388285632b0498dbbab97596e04b154933dfe0f3e68156c68c/greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0", size = 273586, upload-time = "2025-08-07T13:16:08.004Z" }, { url = "https://files.pythonhosted.org/packages/d1/75/10aeeaa3da9332c2e761e4c50d4c3556c21113ee3f0afa2cf5769946f7a3/greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f", size = 686346, upload-time = "2025-08-07T13:42:59.944Z" }, { url = "https://files.pythonhosted.org/packages/c0/aa/687d6b12ffb505a4447567d1f3abea23bd20e73a5bed63871178e0831b7a/greenlet-3.2.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c17b6b34111ea72fc5a4e4beec9711d2226285f0386ea83477cbb97c30a3f3a5", size = 699218, upload-time = "2025-08-07T13:45:30.969Z" }, + { url = "https://files.pythonhosted.org/packages/dc/8b/29aae55436521f1d6f8ff4e12fb676f3400de7fcf27fccd1d4d17fd8fecd/greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1", size = 694659, upload-time = "2025-08-07T13:53:17.759Z" }, { url = "https://files.pythonhosted.org/packages/92/2e/ea25914b1ebfde93b6fc4ff46d6864564fba59024e928bdc7de475affc25/greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735", size = 695355, upload-time = "2025-08-07T13:18:34.517Z" }, { url = "https://files.pythonhosted.org/packages/72/60/fc56c62046ec17f6b0d3060564562c64c862948c9d4bc8aa807cf5bd74f4/greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337", size = 657512, upload-time = "2025-08-07T13:18:33.969Z" }, { url = "https://files.pythonhosted.org/packages/23/6e/74407aed965a4ab6ddd93a7ded3180b730d281c77b765788419484cdfeef/greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269", size = 1612508, upload-time = "2025-11-04T12:42:23.427Z" }, @@ -1434,8 +1437,8 @@ wheels = [ [[package]] name = "harbor" -version = "0.1.43" -source = { git = "https://github.com/formula-code/harbor.git#f600c47df1c64c3a7e6f6c249cdbdfbbe5d63680" } +version = "0.1.44" +source = { directory = "../eval_frameworks/harbor" } dependencies = [ { name = "claude-agent-sdk" }, { name = "datasets" }, @@ -1466,6 +1469,50 @@ dependencies = [ { name = "uvicorn" }, ] +[package.metadata] +requires-dist = [ + { name = "claude-agent-sdk", specifier = ">=0.1.17" }, + { name = "datasets", specifier = ">=4.4.1" }, + { name = "daytona", specifier = ">=0.121.0" }, + { name = "dirhash", specifier = ">=0.5.0" }, + { name = "docker", specifier = ">=7.1.0" }, + { name = "dockerfile-parse", specifier = ">=2.0.1" }, + { name = "e2b", specifier = ">=2.4.2" }, + { name = "fastapi", specifier = ">=0.128.0" }, + { name = "git-python", specifier = ">=1.0.3" }, + { name = "jinja2", specifier = ">=3.1.6" }, + { name = "kubernetes", specifier = ">=32.0.0" }, + { name = "litellm", specifier = ">=1.80.8" }, + { name = "modal", specifier = ">=1.3.2" }, + { name = "packaging", specifier = ">=25.0" }, + { name = "portkey-ai", specifier = ">=1.10.0" }, + { name = "pydantic", specifier = ">=2.11.7" }, + { name = "python-dotenv", specifier = ">=1.1.1" }, + { name = "pyyaml", specifier = ">=6.0.2" }, + { name = "requests", specifier = ">=2.32.4" }, + { name = "rich", specifier = ">=14.1.0" }, + { name = "runloop-api-client", specifier = ">=1.2.0" }, + { name = "shortuuid", specifier = ">=1.0.13" }, + { name = "supabase", specifier = ">=2.27.0" }, + { name = "tenacity", specifier = ">=9.1.2" }, + { name = "toml", specifier = ">=0.10.2" }, + { name = "typer", specifier = ">=0.16.0" }, + { name = "uvicorn", specifier = ">=0.38.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "boto3", specifier = ">=1.41.5" }, + { name = "docker", specifier = ">=7.1.0" }, + { name = "ipykernel", specifier = ">=6.30.1" }, + { name = "jinja2", specifier = ">=3.1.6" }, + { name = "pandas", specifier = ">=2.3.3" }, + { name = "pytest", specifier = ">=8.4.2" }, + { name = "pytest-asyncio", specifier = ">=1.2.0" }, + { name = "pytest-cov", specifier = ">=7.0.0" }, + { name = "pyyaml", specifier = ">=6.0.3" }, +] + [[package]] name = "hf-xet" version = "1.1.10" From 235d4c3b9cbda0989949b0ee08236368de6f9f7e Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:07:09 +0000 Subject: [PATCH 12/24] Update docs: tunable constants, neighbor cascade, rate-limit handling --- CLAUDE.md | 30 +++++++++++++ docs/design/components/datasmith.runners.md | 2 +- docs/guide/configuration.md | 30 +++++++++++++ docs/guide/synthesis.md | 48 +++++++++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index a63273c3..47d85300 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,6 +78,36 @@ Each task lives in `dataset/formulacode_verified///` with a mul - **Build**: hatchling backend, uv for dependency management - **CI**: GitHub Actions runs `make check` + tests on Python 3.11 and 3.12 +### Tunable constants + +Any module-level constant that is a knob — timeouts, retries, caps, windows, +concurrency limits, thresholds — **must** be overridable from `tokens.env` +without a code change. The `datasmith` package auto-loads `tokens.env` at +import time (`src/datasmith/__init__.py` → `dotenv.load_dotenv`), so reading +`os.environ.get(...)` at module scope picks up `tokens.env` values. + +- **Naming**: prefix every overridable constant and its env variable with + `DATASMITH_` so it is globally greppable in both Python and shell env. +- **Pattern**: read the env var at module top, coerce to the target type, + and fall back to a literal default: + + ```python + import os + + DATASMITH_RL_MAX_RETRIES: int = int(os.environ.get("DATASMITH_RL_MAX_RETRIES", "3")) + DATASMITH_NEIGHBOR_WINDOW_DAYS: int = int( + os.environ.get("DATASMITH_NEIGHBOR_WINDOW_DAYS", "60") + ) + ``` + +- **Scope**: this rule applies to *tunable* knobs. Magic strings that + identify protocol fields, schema columns, or on-disk paths are not + constants in this sense and should stay as literals. +- **Existing uses** (non-exhaustive, grep `DATASMITH_` for the full list): + `DATASMITH_RL_DEFAULT_PAUSE_S`, `DATASMITH_RL_PAUSE_JITTER_S`, + `DATASMITH_RL_MAX_RETRIES`, `DATASMITH_NEIGHBOR_WINDOW_DAYS`, + `DATASMITH_NEIGHBOR_CAP`. + ## Supabase (local) fc-data uses a **local Supabase** instance for all persistent state. Connection details live in `tokens.env`: diff --git a/docs/design/components/datasmith.runners.md b/docs/design/components/datasmith.runners.md index 9ef78f1b..8b77068f 100644 --- a/docs/design/components/datasmith.runners.md +++ b/docs/design/components/datasmith.runners.md @@ -34,7 +34,7 @@ graph LR * `ds.runners.scrape_commits`: For a given repository, scrapes all commits and runs compliance checks (`.exists`, `.attribute_compliance`, `.llm_compliance`) on each PR. * `ds.runners.classify_prs`: Runs a classifier agent across a set of PRs concurrently. * `ds.runners.resolve_packages`: For each classified PR, runs `ds.resolution.analyze_commit()` to resolve Python dependencies via `uv`, then persists results (pinned deps, Python version) to the `packages` table. Deduplicates by `(owner, repo, sha)`. See `datasmith.resolution.md`. -* `ds.runners.synthesize_images`: For a given set of PRs, runs `ds.agents.synthesizer` for each. Reads `env_payload` and `python_version` from the `packages` table (populated by `resolve_packages`). Returns `list[str | None]`. This is expensive and must scale to ~20k PRs. +* `ds.runners.synthesize_images`: For a given set of PRs, runs `ds.agents.synthesizer` for each. Reads `env_payload` and `python_version` from the `packages` table (populated by `resolve_packages`). Returns `list[str | None]`. This is expensive and must scale to ~20k PRs. Unlike the other runners, this one overrides `BaseRunner.run` with a queue-based worker pool so `_do_process_item` can enqueue additional items mid-flight: after every successful synthesis, PRs in the same repo whose `created_at` is within `±DATASMITH_NEIGHBOR_WINDOW_DAYS` are pushed onto the queue (capped at `DATASMITH_NEIGHBOR_CAP` per success, deduped against an in-memory set). Those neighbor items re-enter the synthesizer at `CHECK_CACHE → FIND_SIMILAR` so `TRY_SIMILAR` can reuse the freshly-cached context for free, only falling through to `LLM_GENERATE` on genuine environment drift. This collapses the old two-pass hydration workflow (`--agent codex` pass + `--agent none` pass) into a single run. Rate-limit errors from the CLI agent (detected via `ds.agents.rate_limit`) raise `RateLimitError`, which installs a shared pause across all workers until the budget resets — see the Synthesis user guide for tunable knobs. ## Async Concurrency Model diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index e5caf928..59f51242 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -32,6 +32,36 @@ fc-data is configured primarily through a `tokens.env` file in the repository ro | `DOCKERHUB_TOKEN` | DockerHub access token | | `HF_TOKEN_PATH` | Path to HuggingFace token file | +### Tunable constants + +Any module-level constant that is a knob — timeouts, retries, caps, +windows, concurrency limits — is overridable from `tokens.env` without a +code change. Every such variable is prefixed `DATASMITH_` so it is +globally greppable in both Python and shell env. `tokens.env` is loaded +at import time by `datasmith/__init__.py`, so setting one of these in the +file is enough; no export needed. + +#### Stage 6: synthesize_images + +Rate-limit pause behavior (see [Synthesis → Rate-limit handling](synthesis.md#rate-limit-handling)): + +| Variable | Description | Default | +|----------|-------------|---------| +| `DATASMITH_RL_DEFAULT_PAUSE_S` | Fallback pause (seconds) when the agent signals a rate limit but no reset time could be parsed | `3600` | +| `DATASMITH_RL_PAUSE_JITTER_S` | Grace seconds added to the parsed reset time before workers resume, to ride out clock skew | `30` | +| `DATASMITH_RL_MAX_RETRIES` | Maximum consecutive rate-limit pauses for a single item before it is marked failed | `3` | + +Chronological neighborhood cascade (see [Synthesis → Chronological neighborhood cascade](synthesis.md#chronological-neighborhood-cascade)): + +| Variable | Description | Default | +|----------|-------------|---------| +| `DATASMITH_NEIGHBOR_WINDOW_DAYS` | ± window, in days of PR `created_at`, for enqueuing neighbor PRs after a successful synthesis | `60` | +| `DATASMITH_NEIGHBOR_CAP` | Hard ceiling on neighbor PRs enqueued per successful item | `40` | + +Setting `DATASMITH_NEIGHBOR_CAP=0` disables the cascade entirely — the +runner then behaves like a pre-cascade fixed-item pool, useful for tight +unit-style reruns where extra enqueues would confuse progress tracking. + ## Agent backend resolution The agent configuration (`agents/config.py`) checks environment variables in priority order: diff --git a/docs/guide/synthesis.md b/docs/guide/synthesis.md index 6b387235..552a8320 100644 --- a/docs/guide/synthesis.md +++ b/docs/guide/synthesis.md @@ -60,6 +60,54 @@ The synthesizer auto-detects which coding agent CLI is installed: The agent runs in a sandboxed workspace with the Docker build context, edits build scripts, runs verification, and iterates until the verifier passes or attempts are exhausted. +## Chronological neighborhood cascade + +`SynthesizeImagesRunner` uses a queue-based worker pool rather than a fixed +task list. Whenever a PR synthesises successfully, the runner queries for +other PRs in the same repo whose `created_at` is within +`±DATASMITH_NEIGHBOR_WINDOW_DAYS` (default 60) and enqueues them (capped at +`DATASMITH_NEIGHBOR_CAP` per success, default 40). Those neighbor items +re-enter the state machine at `CHECK_CACHE → FIND_SIMILAR`, where +`TRY_SIMILAR` almost always reuses the freshly-cached context for free. +Only PRs whose environment has genuinely drifted fall through to +`LLM_GENERATE` and consume agent budget. + +This replaces the old two-pass workflow (run once with `--agent codex`, +then again with `--agent none`) with a single pass that spreads each win +across its chronological neighborhood automatically. A successful +neighbor cascades its own neighbors onto the queue, so a single codex +session can hydrate an entire repo's worth of adjacent PRs. + +Seed items still respect `--tasks-per-repo`; neighbors are additive and +only bounded by `DATASMITH_NEIGHBOR_CAP` and the `_enqueued` dedupe set. + +## Rate-limit handling + +Codex and Claude both enforce periodic usage limits (five-hour and weekly +buckets) that can exhaust mid-run. The synthesizer detects these from the +raw agent output in two ways: + +- **Codex** — parses the free-text `{"type":"error", ...}` event (the + `"You've hit your usage limit ... try again at "` message) and + extracts the reset time. +- **Claude** — parses structured `rate_limit_event` records for any + `status` outside `{allowed, allowed_warning}` and reads `resetsAt` as a + unix epoch. + +When detected, the attempt is logged to `error_logs` with +`failure_stage='rate_limited'` and `rate_limit_reset_at` set to the parsed +reset time. A `RateLimitError` bubbles up into +`SynthesizeImagesRunner._process_item`, which installs a shared pause on +the runner — every worker blocks in `_wait_for_rate_limit` until the clock +passes the reset time (plus a small jitter) instead of continuing to burn +the attempt budget on sub-3-second failures. Each item is retried up to +`DATASMITH_RL_MAX_RETRIES` times across rate-limit pauses before being +marked as failed. If no reset time could be parsed, the pause defaults to +`DATASMITH_RL_DEFAULT_PAUSE_S` seconds. + +See [Configuration → Tunable constants](configuration.md#tunable-constants) +for the full list of overridable knobs. + ## Dependencies on resolution The synthesizer requires `env_payload` (pinned dependencies) and `python_version` from pipeline Stage 4 (Resolve Packages). Without this data, `docker_build_env.sh` cannot install the correct packages. Always run resolution before synthesis. From 6f1fc82ad0d2a918290ce1de7e2992068033902c Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:20:02 +0000 Subject: [PATCH 13/24] Revert harbor dep to git URL for CI compatibility --- pyproject.toml | 2 +- uv.lock | 50 +++----------------------------------------------- 2 files changed, 4 insertions(+), 48 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eb06cedd..a37dcf34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ dependencies = [ "dspy>=2.6.27", "gitpython", - "harbor @ file:///mnt/sdd1/atharvas/formulacode/eval_frameworks/harbor", + "harbor @ git+https://github.com/formula-code/harbor.git", "httpx>=0.27", "huggingface-hub>=0.20", "jinja2>=3.1.6", diff --git a/uv.lock b/uv.lock index a552b93c..ff036e97 100644 --- a/uv.lock +++ b/uv.lock @@ -1119,7 +1119,7 @@ docs = [ requires-dist = [ { name = "dspy", specifier = ">=2.6.27" }, { name = "gitpython" }, - { name = "harbor", directory = "../eval_frameworks/harbor" }, + { name = "harbor", git = "https://github.com/formula-code/harbor.git" }, { name = "httpx", specifier = ">=0.27" }, { name = "huggingface-hub", specifier = ">=0.20" }, { name = "jinja2", specifier = ">=3.1.6" }, @@ -1437,8 +1437,8 @@ wheels = [ [[package]] name = "harbor" -version = "0.1.44" -source = { directory = "../eval_frameworks/harbor" } +version = "0.1.43" +source = { git = "https://github.com/formula-code/harbor.git#f600c47df1c64c3a7e6f6c249cdbdfbbe5d63680" } dependencies = [ { name = "claude-agent-sdk" }, { name = "datasets" }, @@ -1469,50 +1469,6 @@ dependencies = [ { name = "uvicorn" }, ] -[package.metadata] -requires-dist = [ - { name = "claude-agent-sdk", specifier = ">=0.1.17" }, - { name = "datasets", specifier = ">=4.4.1" }, - { name = "daytona", specifier = ">=0.121.0" }, - { name = "dirhash", specifier = ">=0.5.0" }, - { name = "docker", specifier = ">=7.1.0" }, - { name = "dockerfile-parse", specifier = ">=2.0.1" }, - { name = "e2b", specifier = ">=2.4.2" }, - { name = "fastapi", specifier = ">=0.128.0" }, - { name = "git-python", specifier = ">=1.0.3" }, - { name = "jinja2", specifier = ">=3.1.6" }, - { name = "kubernetes", specifier = ">=32.0.0" }, - { name = "litellm", specifier = ">=1.80.8" }, - { name = "modal", specifier = ">=1.3.2" }, - { name = "packaging", specifier = ">=25.0" }, - { name = "portkey-ai", specifier = ">=1.10.0" }, - { name = "pydantic", specifier = ">=2.11.7" }, - { name = "python-dotenv", specifier = ">=1.1.1" }, - { name = "pyyaml", specifier = ">=6.0.2" }, - { name = "requests", specifier = ">=2.32.4" }, - { name = "rich", specifier = ">=14.1.0" }, - { name = "runloop-api-client", specifier = ">=1.2.0" }, - { name = "shortuuid", specifier = ">=1.0.13" }, - { name = "supabase", specifier = ">=2.27.0" }, - { name = "tenacity", specifier = ">=9.1.2" }, - { name = "toml", specifier = ">=0.10.2" }, - { name = "typer", specifier = ">=0.16.0" }, - { name = "uvicorn", specifier = ">=0.38.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "boto3", specifier = ">=1.41.5" }, - { name = "docker", specifier = ">=7.1.0" }, - { name = "ipykernel", specifier = ">=6.30.1" }, - { name = "jinja2", specifier = ">=3.1.6" }, - { name = "pandas", specifier = ">=2.3.3" }, - { name = "pytest", specifier = ">=8.4.2" }, - { name = "pytest-asyncio", specifier = ">=1.2.0" }, - { name = "pytest-cov", specifier = ">=7.0.0" }, - { name = "pyyaml", specifier = ">=6.0.3" }, -] - [[package]] name = "hf-xet" version = "1.1.10" From 047e3483109f1c47e3cda89a049e25a25800a31c Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:22:33 +0000 Subject: [PATCH 14/24] Add Qwen to agent docs in synthesis and pipeline guides --- docs/guide/pipeline.md | 4 ++-- docs/guide/synthesis.md | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/guide/pipeline.md b/docs/guide/pipeline.md index 38a83624..12d80dca 100644 --- a/docs/guide/pipeline.md +++ b/docs/guide/pipeline.md @@ -30,7 +30,7 @@ fc-data --start-date 2026-02-01 --end-date 2026-03-01 --dry-run | `--dry-run` | Log what each stage would do without executing | `false` | | `--n-concurrent N` | Max concurrent items per runner stage | auto | | `--tasks-per-repo N` | Cap tasks per repo for stages 5–6 (useful for large repos) | unlimited | -| `--agent AGENT` | Agent for stage 6 synthesis: `claude`, `codex`, `gemini`, or `none` | auto-detect | +| `--agent AGENT` | Agent for stage 6 synthesis: `claude`, `codex`, `gemini`, `qwen`, or `none` | auto-detect | | `--force` | Re-process already-completed tasks in stages 5–6 | `false` | | `--offline-source PATH` | Import PR data from a Parquet file instead of scraping GitHub (stages 1–2) | — | | `--min-stars N` | Minimum GitHub stars for repo discovery in stage 1 | `500` | @@ -186,7 +186,7 @@ fc-data --start-date 2026-02-01 --end-date 2026-03-01 --stage 6 --force **Writes to:** `candidate_containers` table (on success), `error_logs` table (every attempt) **Runner:** `SynthesizeImagesRunner` -**Requires:** Docker daemon running, resolved packages (stage 4), rendered problems (stage 5). LLM agent CLI (`claude`, `codex`, or `gemini`) must be on `$PATH` unless `--agent none`. +**Requires:** Docker daemon running, resolved packages (stage 4), rendered problems (stage 5). LLM agent CLI (`claude`, `codex`, `gemini`, or `qwen`) must be on `$PATH` unless `--agent none`. !!! warning Synthesis can be expensive — each LLM attempt may consume significant tokens and each Docker build takes minutes. Use `--n-concurrent` and `--tasks-per-repo` to control cost. Start with `--agent none` to exhaust cached/similar scripts before using LLM generation. diff --git a/docs/guide/synthesis.md b/docs/guide/synthesis.md index 552a8320..3f6f15e0 100644 --- a/docs/guide/synthesis.md +++ b/docs/guide/synthesis.md @@ -9,7 +9,7 @@ The synthesizer follows a strict try-existing-first strategy: 1. **Check cache** — Look up Supabase for an existing build script for this PR 2. **Find similar** — Query Supabase for similar scripts from the same repository 3. **Try similar** — Run each similar script against the verifier chain -4. **LLM generate** — Fall back to an installed coding agent (Claude Code, Codex, or Gemini) +4. **LLM generate** — Fall back to an installed coding agent (Claude Code, Codex, Gemini, or Qwen Code) 5. **Fail** — Log all attempts and return `None` All attempts are logged to the `build_attempts` table so failed PRs can be retried later with improved prompts or models. @@ -57,6 +57,7 @@ The synthesizer auto-detects which coding agent CLI is installed: | Claude Code | `claude` | Checks `which claude` | | Codex | `codex` | Checks `which codex` | | Gemini | `gemini` | Checks `which gemini` | +| Qwen Code | `qwen` | Checks `which qwen` | The agent runs in a sandboxed workspace with the Docker build context, edits build scripts, runs verification, and iterates until the verifier passes or attempts are exhausted. From 9377c9e2a8b6106aa24845964075d9a3b17120d0 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:22:49 +0000 Subject: [PATCH 15/24] Add Cloudflare Access header support to Supabase clients --- src/datasmith/utils/db.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/datasmith/utils/db.py b/src/datasmith/utils/db.py index 7885f481..86324d2b 100644 --- a/src/datasmith/utils/db.py +++ b/src/datasmith/utils/db.py @@ -10,10 +10,16 @@ from collections.abc import Callable, Sequence from typing import Any, TypeVar, cast -from supabase import Client, create_client +from supabase import Client, ClientOptions, create_client logger = logging.getLogger(__name__) +# Cloudflare Access service-token credentials. When both are set, every +# Supabase HTTP request includes the CF-Access-Client-Id / Secret headers +# so requests can pass through a Cloudflare Access-protected tunnel. +DATASMITH_CF_ACCESS_CLIENT_ID: str = os.environ.get("DATASMITH_CF_ACCESS_CLIENT_ID", "") +DATASMITH_CF_ACCESS_CLIENT_SECRET: str = os.environ.get("DATASMITH_CF_ACCESS_CLIENT_SECRET", "") + # Stable primary-key ordering per table, used by ``fetch_all`` to make # range-based pagination deterministic. PostgREST/Postgres provide no # stable ordering without an explicit ORDER BY, so paginating large @@ -37,6 +43,13 @@ _client: Client | None = None +_CF_ACCESS_HEADERS: dict[str, str] = {} +if DATASMITH_CF_ACCESS_CLIENT_ID and DATASMITH_CF_ACCESS_CLIENT_SECRET: + _CF_ACCESS_HEADERS = { + "CF-Access-Client-Id": DATASMITH_CF_ACCESS_CLIENT_ID, + "CF-Access-Client-Secret": DATASMITH_CF_ACCESS_CLIENT_SECRET, + } + def get_client() -> Client: """Return a singleton Supabase client from env vars.""" @@ -46,7 +59,11 @@ def get_client() -> Client: key = os.environ.get("SUPABASE_KEY", "") if not url or not key: raise ValueError("SUPABASE_URL and SUPABASE_KEY must be set") - _client = create_client(url, key) + if _CF_ACCESS_HEADERS: + options = ClientOptions(headers=_CF_ACCESS_HEADERS) + _client = create_client(url, key, options=options) + else: + _client = create_client(url, key) return _client @@ -56,12 +73,15 @@ async def get_async_client() -> Any: Imported lazily to avoid import errors when supabase async extras are not installed. """ - from supabase import acreate_client + from supabase import AsyncClientOptions, acreate_client url = os.environ.get("SUPABASE_URL", "") key = os.environ.get("SUPABASE_KEY", "") if not url or not key: raise ValueError("SUPABASE_URL and SUPABASE_KEY must be set") + if _CF_ACCESS_HEADERS: + options = AsyncClientOptions(headers=_CF_ACCESS_HEADERS) + return await acreate_client(url, key, options=options) return await acreate_client(url, key) From 148bf4b611b7bb216e537acf5211f60884d5dec1 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:22:53 +0000 Subject: [PATCH 16/24] Add db-tunnel Makefile target for Cloudflare Tunnel --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 881a544e..61c568a1 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,10 @@ grafana-logs: ## Tail Grafana container logs grafana-tunnel: ## Expose Grafana publicly via Cloudflare Tunnel @cloudflared tunnel run datasmith-grafana +.PHONY: db-tunnel +db-tunnel: ## Expose Supabase PostgREST API via Cloudflare Tunnel (db.formulacode.org) + @cloudflared tunnel run datasmith-db + .PHONY: help help: From 7960bd0ae56944304388c591a90599ef64ce84ee Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:22:58 +0000 Subject: [PATCH 17/24] Add remote access guide and update docs for Cloudflare Tunnel support --- docs/getting-started/installation.md | 22 ++++ docs/guide/monitoring.md | 1 + docs/guide/remote-access.md | 185 +++++++++++++++++++++++++++ mkdocs.yml | 1 + 4 files changed, 209 insertions(+) create mode 100644 docs/guide/remote-access.md diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 7b6bb2bb..69d77930 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -182,6 +182,28 @@ make check # Ruff lint + mypy type check make test # pytest ``` +## Makefile reference + +Run `make help` to list all targets. The complete reference: + +| Target | Description | +|--------|-------------| +| `make install` | Create virtual environment with uv, install pre-commit hooks | +| `make check` | Run ruff lint, mypy type check, and deptry dependency check | +| `make test` | Run pytest with coverage | +| `make build` | Build wheel file | +| `make clean-build` | Remove build artifacts | +| `make docker-clean` | Prune dangling Docker images and containers | +| `make supabase-up` | Start local Supabase instance | +| `make supabase-down` | Stop local Supabase instance | +| `make supabase-status` | Show Supabase service status and URLs | +| `make grafana-migrate` | Apply the `grafana_ro` read-only database role | +| `make grafana-up` | Start Grafana dashboard (`http://localhost:3001`) | +| `make grafana-down` | Stop Grafana dashboard | +| `make grafana-logs` | Tail Grafana container logs | +| `make grafana-tunnel` | Expose Grafana publicly via Cloudflare Tunnel | +| `make db-tunnel` | Expose Supabase PostgREST API via Cloudflare Tunnel | + ## Next steps You're ready to run the pipeline: diff --git a/docs/guide/monitoring.md b/docs/guide/monitoring.md index a0160102..370bda4d 100644 --- a/docs/guide/monitoring.md +++ b/docs/guide/monitoring.md @@ -97,4 +97,5 @@ make grafana-up # Start Grafana make grafana-down # Stop Grafana make grafana-logs # Tail container logs make grafana-migrate # Apply the grafana_ro database role +make grafana-tunnel # Expose Grafana publicly via Cloudflare Tunnel ``` diff --git a/docs/guide/remote-access.md b/docs/guide/remote-access.md new file mode 100644 index 00000000..01c507e7 --- /dev/null +++ b/docs/guide/remote-access.md @@ -0,0 +1,185 @@ +# Remote Access via Cloudflare Tunnel + +fc-data stores all persistent state in a local Supabase instance. By +default this is only reachable from the host machine (`127.0.0.1:54321`). +A **Cloudflare Tunnel** lets a remote machine run the same fc-data +pipeline against the same database — no VPN, no open ports, no firewall +rules. + +## Architecture + +``` +Remote machine Host machine +┌──────────────┐ ┌──────────────────────┐ +│ fc-data │── HTTPS ──▶ Cloudflare Edge ──▶ cloudflared │──▶ Supabase +│ tokens.env: │ (db.formulacode.org) (tunnel) │ :54321 +│ SUPABASE_URL│ │ +│ CF headers │ Cloudflare Access │ +└──────────────┘ (service-token auth) └──────────────────────┘ +``` + +**Two layers of auth protect the database:** + +1. **Cloudflare Access** — a service token (`CF-Access-Client-Id` / + `CF-Access-Client-Secret` headers) must be present on every request + or Cloudflare rejects it at the edge before it ever reaches the tunnel. +2. **Supabase service-role key** — the standard `apikey` header required + by PostgREST, unchanged from local usage. + +## Prerequisites + +- A **Cloudflare account** (free plan is sufficient) +- A **domain managed by Cloudflare** (e.g., `formulacode.org`) +- `cloudflared` CLI installed on the **host machine** (the one running Supabase) + +## Host machine setup + +### 1. Install cloudflared + +```bash +# Debian / Ubuntu +curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb \ + -o cloudflared.deb +sudo dpkg -i cloudflared.deb + +# Verify +cloudflared --version +``` + +### 2. Authenticate + +```bash +cloudflared login +``` + +This opens a browser to authorize `cloudflared` with your Cloudflare +account. Select the domain you want to use (e.g., `formulacode.org`). + +### 3. Create a tunnel + +```bash +cloudflared tunnel create datasmith-db +``` + +Note the **Tunnel ID** printed (e.g., `a1b2c3d4-...`). A credentials +file is saved to `~/.cloudflared/.json`. + +### 4. Configure the tunnel + +Create `~/.cloudflared/config.yml`: + +```yaml +tunnel: +credentials-file: /home//.cloudflared/.json + +ingress: + - hostname: db.formulacode.org + service: http://localhost:54321 + - service: http_status:404 +``` + +### 5. Create the DNS record + +```bash +cloudflared tunnel route dns datasmith-db db.formulacode.org +``` + +This creates a CNAME record pointing `db.formulacode.org` to the tunnel. + +### 6. Run the tunnel + +```bash +# Using the Makefile target (recommended) +make db-tunnel + +# Or directly +cloudflared tunnel run datasmith-db + +# As a systemd service (persistent, survives reboots) +sudo cloudflared service install +sudo systemctl enable cloudflared +sudo systemctl start cloudflared +``` + +At this point, `https://db.formulacode.org` proxies to your local +Supabase PostgREST API — but **Cloudflare Access blocks all requests** +until you create an access policy. + +## Cloudflare Access setup + +### 1. Create an application + +1. Go to [Cloudflare Zero Trust](https://one.dash.cloudflare.com/) → + **Access** → **Applications** +2. Click **Add an application** → **Self-hosted** +3. Set: + - **Application name**: `datasmith-db` + - **Session duration**: `24 hours` + - **Application domain**: `db.formulacode.org` +4. Under **Policies**, create a policy: + - **Policy name**: `Service Token` + - **Action**: Service Auth + - **Include**: Service Token (select the token you'll create next) +5. Save the application + +### 2. Create a service token + +1. Go to **Access** → **Service Auth** → **Service Tokens** +2. Click **Create Service Token** +3. Name it (e.g., `datasmith-remote`) +4. **Copy both values immediately** — the Client Secret is only shown once: + - `CF-Access-Client-Id` (e.g., `abc123.access`) + - `CF-Access-Client-Secret` (e.g., `long-secret-string`) + +## Remote machine setup + +On the remote machine, edit `tokens.env`: + +```bash +# Point at the tunnel instead of localhost +SUPABASE_URL=https://db.formulacode.org +SUPABASE_KEY=your-service-role-key # Same key as the host machine + +# Cloudflare Access service token +DATASMITH_CF_ACCESS_CLIENT_ID=abc123.access +DATASMITH_CF_ACCESS_CLIENT_SECRET=long-secret-string +``` + +The `SUPABASE_KEY` is the same service-role key used on the host — it is +not a Cloudflare credential. + +### Verify connectivity + +```bash +fc-data --preflight +``` + +The Supabase connection check should show `[OK]`. If it fails: + +- **403 Forbidden** — the CF Access headers are missing or the service + token is invalid. Double-check `DATASMITH_CF_ACCESS_CLIENT_ID` and + `DATASMITH_CF_ACCESS_CLIENT_SECRET`. +- **502 Bad Gateway** — `cloudflared` is not running on the host machine + or Supabase is down. SSH into the host and check + `systemctl status cloudflared` and `supabase status`. +- **Connection refused** — DNS is not resolving. Verify + `cloudflared tunnel route dns` was run and the CNAME exists in your + Cloudflare DNS dashboard. + +## How it works in the code + +When `DATASMITH_CF_ACCESS_CLIENT_ID` and `DATASMITH_CF_ACCESS_CLIENT_SECRET` +are both set, `datasmith.utils.db` automatically injects the +`CF-Access-Client-Id` and `CF-Access-Client-Secret` headers into every +Supabase client request via `ClientOptions`. No other code changes are +needed — every call to `get_client()` or `get_async_client()` picks up +the headers transparently. + +When neither variable is set (the default for local development), no +extra headers are added and behavior is identical to before. + +## Makefile Targets + +```bash +make db-tunnel # Expose Supabase PostgREST API via Cloudflare Tunnel +``` diff --git a/mkdocs.yml b/mkdocs.yml index 10a0a6fb..47c4a089 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -83,6 +83,7 @@ nav: - Installation: getting-started/installation.md - Pipeline (fc-data): guide/pipeline.md - Configuration: guide/configuration.md + - Remote Access: guide/remote-access.md - User Guide: - Docker Images: guide/docker-images.md - Synthesis: guide/synthesis.md From 591184d2b1c3f5b0d80266d295c22aceea761671 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:28:19 +0000 Subject: [PATCH 18/24] Add documentation update guidance to CLAUDE.md --- CLAUDE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 47d85300..fea7e80d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,6 +78,10 @@ Each task lives in `dataset/formulacode_verified///` with a mul - **Build**: hatchling backend, uv for dependency management - **CI**: GitHub Actions runs `make check` + tests on Python 3.11 and 3.12 +### Documentation + +After making a feature change, decide whether the change is significant enough to warrant updating the documentation in `docs/`. Changes that affect user-facing behavior, CLI flags, configuration knobs, pipeline stages, agent backends, or architectural decisions should be reflected in the relevant guide or design doc. Internal refactors, bug fixes, and implementation details generally do not need doc updates unless they change observable behavior. + ### Tunable constants Any module-level constant that is a knob — timeouts, retries, caps, windows, From 07968d60f75e0ea8ee498bec089ded919fce4b13 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:37:50 +0000 Subject: [PATCH 19/24] Fix UP038 isinstance calls and suppress UP046/UP047 for Python 3.11 compat --- pyproject.toml | 3 +++ src/datasmith/resolution/metadata_parser.py | 24 ++++++++++----------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a37dcf34..5a56548d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,6 +145,9 @@ ignore = [ "S701", # mutable class attr without ClassVar (acceptable for registries) "RUF012", + # PEP 695 type params require 3.12+; we still test on 3.11 + "UP046", + "UP047", # ternary suggestions (readability preference) "SIM108", ] diff --git a/src/datasmith/resolution/metadata_parser.py b/src/datasmith/resolution/metadata_parser.py index 62f4c48d..3e9bedd3 100644 --- a/src/datasmith/resolution/metadata_parser.py +++ b/src/datasmith/resolution/metadata_parser.py @@ -130,7 +130,7 @@ def safe_eval(node: ast.AST, depth: int = 0) -> Any: # noqa: C901 return env[node.id] raise ValueError(f"Unknown name {node.id}") - if isinstance(node, (ast.List, ast.Tuple, ast.Set)): + if isinstance(node, ast.List | ast.Tuple | ast.Set): elts = [] for e in node.elts: elts.append(safe_eval(e, depth + 1)) @@ -138,7 +138,7 @@ def safe_eval(node: ast.AST, depth: int = 0) -> Any: # noqa: C901 if isinstance(node, ast.Dict): out: dict[Any, Any] = {} - for k, v in zip(node.keys, node.values): + for k, v in zip(node.keys, node.values, strict=False): if k is None: raise ValueError("Dict unpacking not allowed here") key = safe_eval(k, depth + 1) @@ -146,11 +146,11 @@ def safe_eval(node: ast.AST, depth: int = 0) -> Any: # noqa: C901 out[key] = val return out - if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)): + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.UAdd | ast.USub): v = safe_eval(node.operand, depth + 1) - if isinstance(v, (int, float)) and isinstance(node.op, ast.USub): + if isinstance(v, int | float) and isinstance(node.op, ast.USub): return -v - if isinstance(v, (int, float)) and isinstance(node.op, ast.UAdd): + if isinstance(v, int | float) and isinstance(node.op, ast.UAdd): return +v raise ValueError("Unsupported unary op") @@ -182,7 +182,7 @@ def safe_eval(node: ast.AST, depth: int = 0) -> Any: # noqa: C901 out2: dict[Any, Any] = {} if isinstance(seq, list): for item in seq: - if isinstance(item, (list, tuple)) and len(item) == 2: + if isinstance(item, list | tuple) and len(item) == 2: out2[item[0]] = item[1] else: raise ValueError("Unsupported dict constructor form") @@ -209,9 +209,9 @@ def safe_eval(node: ast.AST, depth: int = 0) -> Any: # noqa: C901 for target in node.targets: if isinstance(target, ast.Name): env[target.id] = val - elif isinstance(target, (ast.Tuple, ast.List)): # noqa: SIM102 - if isinstance(val, (list, tuple)) and len(target.elts) == len(val): - for elt, v in zip(target.elts, val): + elif isinstance(target, ast.Tuple | ast.List): # noqa: SIM102 + if isinstance(val, list | tuple) and len(target.elts) == len(val): + for elt, v in zip(target.elts, val, strict=False): if isinstance(elt, ast.Name): env[elt.id] = v elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.value is not None: @@ -261,15 +261,15 @@ def merge_kwargs_from_starstar(val: Any, into: dict[str, Any]) -> None: if isinstance(pyreq, str): meta.requires_python = pyreq install_requires = setup_kwargs.get("install_requires") - if isinstance(install_requires, (list, tuple)): + if isinstance(install_requires, list | tuple): meta.core_deps.update([x for x in install_requires if isinstance(x, str)]) extras_require = setup_kwargs.get("extras_require") if isinstance(extras_require, dict): for k, v in extras_require.items(): - if isinstance(k, str) and isinstance(v, (list, tuple)): + if isinstance(k, str) and isinstance(v, list | tuple): meta.extras[k] = {x for x in v if isinstance(x, str)} setup_requires = setup_kwargs.get("setup_requires") - if isinstance(setup_requires, (list, tuple)): + if isinstance(setup_requires, list | tuple): meta.build_requires.update([x for x in setup_requires if isinstance(x, str)]) return meta From 269d54734c9d259876486ed3ce207f791462fc12 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:38:24 +0000 Subject: [PATCH 20/24] Fix remaining UP038 isinstance call in git_utils --- src/datasmith/resolution/git_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/datasmith/resolution/git_utils.py b/src/datasmith/resolution/git_utils.py index a07efd02..f212eddc 100644 --- a/src/datasmith/resolution/git_utils.py +++ b/src/datasmith/resolution/git_utils.py @@ -7,10 +7,10 @@ import shutil import threading import time -from collections.abc import Iterable +from collections.abc import Callable, Iterable from contextlib import suppress from pathlib import Path -from typing import Any, Callable, cast +from typing import Any, cast from git import Commit, Repo @@ -365,7 +365,7 @@ def read_blob_text(commit: Commit, relpath: str, default: str | None = None) -> if data_stream is None: return default raw_bytes = data_stream.read() - if not isinstance(raw_bytes, (bytes, bytearray)): + if not isinstance(raw_bytes, bytes | bytearray): return default return bytes(raw_bytes).decode("utf-8", errors="replace") except Exception: From 2212e56b62cf9921e1d76f785f4f45e66555ff82 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:40:55 +0000 Subject: [PATCH 21/24] Apply ruff auto-fixes across codebase (strict=False zip, import sorting) --- src/datasmith/docker/publish.py | 4 ++-- src/datasmith/github/hooks.py | 3 ++- src/datasmith/github/links.py | 4 ++-- src/datasmith/publish/pipeline.py | 6 +++--- src/datasmith/resolution/cache.py | 4 ++-- src/datasmith/resolution/dependency_resolver.py | 4 ++-- src/datasmith/runners/scrape_commits.py | 4 ++-- src/datasmith/utils/core.py | 3 ++- tests/github/test_models.py | 6 +++--- 9 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/datasmith/docker/publish.py b/src/datasmith/docker/publish.py index 3892a574..c1ccf48d 100644 --- a/src/datasmith/docker/publish.py +++ b/src/datasmith/docker/publish.py @@ -3,7 +3,7 @@ from __future__ import annotations import os -from datetime import datetime, timezone +from datetime import UTC, datetime import httpx from python_on_whales import DockerClient @@ -35,7 +35,7 @@ def push(self, image_tag: str) -> None: self._docker.push(image_tag) def tag_with_version(self, image_tag: str) -> str: - version = datetime.now(tz=timezone.utc).strftime("@%Y-%m") + version = datetime.now(tz=UTC).strftime("@%Y-%m") new_tag = f"{image_tag}{version}" self._docker.tag(image_tag, new_tag) return new_tag diff --git a/src/datasmith/github/hooks.py b/src/datasmith/github/hooks.py index 47adec6d..e168e190 100644 --- a/src/datasmith/github/hooks.py +++ b/src/datasmith/github/hooks.py @@ -2,7 +2,8 @@ from __future__ import annotations -from typing import Any, Callable, ClassVar +from collections.abc import Callable +from typing import Any, ClassVar from datasmith.utils import get_logger, supabase_cached diff --git a/src/datasmith/github/links.py b/src/datasmith/github/links.py index 7719817c..67d8da4f 100644 --- a/src/datasmith/github/links.py +++ b/src/datasmith/github/links.py @@ -3,8 +3,8 @@ from __future__ import annotations import re -from collections.abc import Awaitable -from typing import Any, Callable +from collections.abc import Awaitable, Callable +from typing import Any from datasmith.github.models import IssueExpanded from datasmith.utils import get_logger diff --git a/src/datasmith/publish/pipeline.py b/src/datasmith/publish/pipeline.py index 1bdb2386..951c0ef1 100644 --- a/src/datasmith/publish/pipeline.py +++ b/src/datasmith/publish/pipeline.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from datasmith.publish.huggingface import HuggingFacePublisher from datasmith.publish.records import records_from_supabase @@ -26,7 +26,7 @@ async def publish_pipeline( logger.info("Found %d unpublished records", len(records)) - version = f"formulacode@{datetime.now(tz=timezone.utc).strftime('%Y-%m')}" + version = f"formulacode@{datetime.now(tz=UTC).strftime('%Y-%m')}" # DockerHub push (optional) if dockerhub_push: @@ -47,7 +47,7 @@ async def publish_pipeline( # Mark as published in Supabase client = get_client() - now = datetime.now(tz=timezone.utc).isoformat() + now = datetime.now(tz=UTC).isoformat() for record in records: try: client.table("pull_requests").update({"published_at": now}).eq("owner", record.owner).eq( diff --git a/src/datasmith/resolution/cache.py b/src/datasmith/resolution/cache.py index 83a5b6f3..b9497761 100644 --- a/src/datasmith/resolution/cache.py +++ b/src/datasmith/resolution/cache.py @@ -9,8 +9,8 @@ import re import sqlite3 import threading -from collections.abc import Iterator -from typing import Callable, ParamSpec, TypeVar, cast +from collections.abc import Callable, Iterator +from typing import ParamSpec, TypeVar, cast _cache_lock = threading.Lock() _P = ParamSpec("_P") diff --git a/src/datasmith/resolution/dependency_resolver.py b/src/datasmith/resolution/dependency_resolver.py index 0df3fcb9..701f67bf 100644 --- a/src/datasmith/resolution/dependency_resolver.py +++ b/src/datasmith/resolution/dependency_resolver.py @@ -21,8 +21,8 @@ def strip_ansi(s: str) -> str: def rfc3339(ts: dt.datetime) -> str: """Convert a datetime to RFC3339 format string.""" if ts.tzinfo is None: - ts = ts.replace(tzinfo=dt.timezone.utc) - return ts.astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z") + ts = ts.replace(tzinfo=dt.UTC) + return ts.astimezone(dt.UTC).isoformat().replace("+00:00", "Z") def uv_compile_from_pyproject( diff --git a/src/datasmith/runners/scrape_commits.py b/src/datasmith/runners/scrape_commits.py index 015e2c53..0ad66ce1 100644 --- a/src/datasmith/runners/scrape_commits.py +++ b/src/datasmith/runners/scrape_commits.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from typing import Any from datasmith.filters import symbolic_compliance @@ -17,7 +17,7 @@ def _parse_iso(value: str | None) -> datetime | None: return None # Handle date-only strings like "2024-01-01" if "T" not in value: - return datetime.fromisoformat(value).replace(tzinfo=timezone.utc) + return datetime.fromisoformat(value).replace(tzinfo=UTC) # Handle full ISO datetime strings (with or without trailing Z) cleaned = value.replace("Z", "+00:00") return datetime.fromisoformat(cleaned) diff --git a/src/datasmith/utils/core.py b/src/datasmith/utils/core.py index bece5208..58237bb5 100644 --- a/src/datasmith/utils/core.py +++ b/src/datasmith/utils/core.py @@ -6,7 +6,8 @@ import logging import sys import time -from typing import Any, Callable, TypeVar +from collections.abc import Callable +from typing import Any, TypeVar from pydantic_settings import BaseSettings diff --git a/tests/github/test_models.py b/tests/github/test_models.py index 78f3de68..ae2c26c2 100644 --- a/tests/github/test_models.py +++ b/tests/github/test_models.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from unittest.mock import MagicMock, patch import httpx @@ -67,7 +67,7 @@ def test_pr_frozen_immutability(self) -> None: pr.title = "Changed" # type: ignore[misc] def test_pr_json_roundtrip(self) -> None: - now = datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc) + now = datetime(2024, 6, 15, 12, 0, 0, tzinfo=UTC) pr = PR( repository="org/lib", issue_number=99, @@ -86,7 +86,7 @@ def test_pr_json_roundtrip(self) -> None: assert restored.merged_at == now def test_to_record_success(self) -> None: - now = datetime(2024, 1, 1, tzinfo=timezone.utc) + now = datetime(2024, 1, 1, tzinfo=UTC) pr = PR( repository="myorg/mylib", issue_number=42, From 3c5a2715b0afcfc81e8bef46345a2c2c0a7ceead Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:46:23 +0000 Subject: [PATCH 22/24] Fix tests for split build/push, 8-stage pipeline, and fetch_all ordering --- tests/integration/conftest.py | 3 +++ tests/integration/test_e2e_pipeline.py | 14 ++++++++--- tests/publish/test_records.py | 22 ++++++++++++----- tests/runners/test_synthesize_images.py | 33 +++++++++++++++---------- tests/test_website_snippets.py | 6 ++--- 5 files changed, 52 insertions(+), 26 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b6f0b5a9..8a56b5fd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -19,6 +19,9 @@ def mock_supabase_client(): table.is_.return_value = table table.gte.return_value = table table.lte.return_value = table + table.neq.return_value = table + table.order.return_value = table + table.range.return_value = table table.limit.return_value = table table.execute.return_value = MagicMock(data=[]) return client diff --git a/tests/integration/test_e2e_pipeline.py b/tests/integration/test_e2e_pipeline.py index 06e51dad..3d393d0a 100644 --- a/tests/integration/test_e2e_pipeline.py +++ b/tests/integration/test_e2e_pipeline.py @@ -15,15 +15,18 @@ async def test_full_pipeline_single_stage(self, mock_supabase_client): mock_supabase_client.table.return_value.execute.return_value = MagicMock(data=[]) - with patch("datasmith.update.pipeline.get_client", return_value=mock_supabase_client): - # Run only the publish stage (lightest) + with ( + patch("datasmith.update.pipeline.get_client", return_value=mock_supabase_client), + patch("datasmith.utils.db.get_client", return_value=mock_supabase_client), + ): + # Run only the publish stage (lightest — stage 8) calls = [] async def tracked_publish(start, end): calls.append(("publish", start, end)) pipeline._publish = tracked_publish # type: ignore[assignment] - await pipeline.run("2024-01-01", "2024-12-31", stage=7) + await pipeline.run("2024-01-01", "2024-12-31", stage=8) assert len(calls) == 1 assert calls[0] == ("publish", "2024-01-01", "2024-12-31") @@ -43,7 +46,10 @@ async def mock_run_stage(name, start, end): ] ) - with patch("datasmith.update.pipeline.get_client", return_value=mock_supabase_client): + with ( + patch("datasmith.update.pipeline.get_client", return_value=mock_supabase_client), + patch("datasmith.utils.db.get_client", return_value=mock_supabase_client), + ): mock_supabase_client.table.return_value.execute.return_value = completed_response pipeline._run_stage = mock_run_stage # type: ignore[assignment] await pipeline.run("2024-01-01", "2024-12-31", resume=True) diff --git a/tests/publish/test_records.py b/tests/publish/test_records.py index eb66aead..87207536 100644 --- a/tests/publish/test_records.py +++ b/tests/publish/test_records.py @@ -66,16 +66,26 @@ def test_queries_supabase(self): "merged_at": "2024-01-01T00:00:00Z", }, ] + fake_harbor_rows = [ + { + "owner": "org", + "repo": "repo", + "sha": "abc", + "max_speedup": 2.0, + "status": "success", + "environment": "daytona", + }, + ] - with patch("datasmith.publish.records.fetch_all", return_value=fake_rows) as mock_fetch: + with patch("datasmith.publish.records.fetch_all", side_effect=[fake_rows, fake_harbor_rows]) as mock_fetch: records = records_from_supabase(start_date="2024-01-01", end_date="2024-12-31") assert len(records) == 1 assert records[0].owner == "org" assert records[0].task_id == "org__repo-1" - # Verify fetch_all was called with correct filters - call_kwargs = mock_fetch.call_args - assert call_kwargs[1]["filters"] == {"is_performance_commit": True} - assert call_kwargs[1]["gte_filters"] == {"merged_at": "2024-01-01"} - assert call_kwargs[1]["lte_filters"] == {"merged_at": "2024-12-31"} + # Verify fetch_all was called with correct filters (first call = pull_requests) + first_call_kwargs = mock_fetch.call_args_list[0] + assert first_call_kwargs[1]["filters"] == {"is_performance_commit": True} + assert first_call_kwargs[1]["gte_filters"] == {"merged_at": "2024-01-01"} + assert first_call_kwargs[1]["lte_filters"] == {"merged_at": "2024-12-31"} diff --git a/tests/runners/test_synthesize_images.py b/tests/runners/test_synthesize_images.py index f63a5082..4710a653 100644 --- a/tests/runners/test_synthesize_images.py +++ b/tests/runners/test_synthesize_images.py @@ -18,10 +18,11 @@ def _clear_prereq_cache() -> None: # Shared patches for Docker image helpers that are always mocked in unit tests _MOCK_PREREQS = patch("datasmith.runners.synthesize_images._ensure_prerequisite_images") -_MOCK_BUILD_PUSH = patch( - "datasmith.runners.synthesize_images._build_and_push_pr_image", +_MOCK_BUILD = patch( + "datasmith.runners.synthesize_images._build_pr_image", return_value="formulacode/numpy-numpy:42", ) +_MOCK_PUSH = patch("datasmith.runners.synthesize_images._push_pr_image") _MOCK_REPO_IMAGE = patch( "datasmith.docker.images.get_repo_image_name", return_value="formulacode/numpy-numpy:latest", @@ -73,7 +74,8 @@ async def test_docker_runs_in_thread(self) -> None: patch("datasmith.runners.base.get_client", return_value=mock_client), patch("datasmith.runners.synthesize_images.get_client", return_value=mock_client), _MOCK_PREREQS, - _MOCK_BUILD_PUSH, + _MOCK_BUILD, + _MOCK_PUSH, _MOCK_REPO_IMAGE, ): runner = SynthesizeImagesRunner(synthesizer=synthesizer, n_concurrent=1) @@ -89,6 +91,7 @@ async def test_docker_runs_in_thread(self) -> None: repo_image="formulacode/numpy-numpy:latest", env_payload="", python_version="", + base_sha="", ) @@ -105,7 +108,8 @@ async def test_handles_failure(self) -> None: patch("datasmith.runners.base.get_client", return_value=mock_client), patch("datasmith.runners.synthesize_images.get_client", return_value=mock_client), _MOCK_PREREQS, - _MOCK_BUILD_PUSH, + _MOCK_BUILD, + _MOCK_PUSH, _MOCK_REPO_IMAGE, ): runner = SynthesizeImagesRunner(synthesizer=synthesizer, n_concurrent=1) @@ -130,9 +134,10 @@ async def test_builds_and_pushes_on_success(self) -> None: patch("datasmith.runners.synthesize_images.get_client", return_value=mock_client), _MOCK_PREREQS as mock_prereqs, patch( - "datasmith.runners.synthesize_images._build_and_push_pr_image", + "datasmith.runners.synthesize_images._build_pr_image", return_value="formulacode/numpy-numpy:42", - ) as mock_build_push, + ) as mock_build, + patch("datasmith.runners.synthesize_images._push_pr_image") as mock_push, _MOCK_REPO_IMAGE, ): runner = SynthesizeImagesRunner(synthesizer=synthesizer, n_concurrent=1) @@ -141,10 +146,9 @@ async def test_builds_and_pushes_on_success(self) -> None: # Prerequisites were checked mock_prereqs.assert_called_once() - # Image was built and pushed (last arg is the synthesized DockerContext) - mock_build_push.assert_called_once() - args = mock_build_push.call_args[0] - assert args[:5] == ("numpy", "numpy", 42, "", "") + # Image was built and pushed + mock_build.assert_called_once() + mock_push.assert_called_once() # container_name was persisted to DB mock_client.table.assert_any_call("pull_requests") @@ -172,7 +176,8 @@ async def test_renders_and_stores_problem_statement(self) -> None: patch("datasmith.runners.base.get_client", return_value=mock_client), patch("datasmith.runners.synthesize_images.get_client", return_value=mock_client), _MOCK_PREREQS, - _MOCK_BUILD_PUSH, + _MOCK_BUILD, + _MOCK_PUSH, _MOCK_REPO_IMAGE, patch( "datasmith.github.render.render_problem_statement", @@ -211,7 +216,8 @@ async def test_skips_render_without_gh(self) -> None: patch("datasmith.runners.base.get_client", return_value=mock_client), patch("datasmith.runners.synthesize_images.get_client", return_value=mock_client), _MOCK_PREREQS, - _MOCK_BUILD_PUSH, + _MOCK_BUILD, + _MOCK_PUSH, _MOCK_REPO_IMAGE, patch( "datasmith.github.render.render_problem_statement", @@ -257,7 +263,8 @@ async def test_render_includes_scraped_issues(self) -> None: patch("datasmith.runners.base.get_client", return_value=mock_client), patch("datasmith.runners.synthesize_images.get_client", return_value=mock_client), _MOCK_PREREQS, - _MOCK_BUILD_PUSH, + _MOCK_BUILD, + _MOCK_PUSH, _MOCK_REPO_IMAGE, patch( "datasmith.github.render.render_problem_statement", diff --git a/tests/test_website_snippets.py b/tests/test_website_snippets.py index cfe45510..b626ee8f 100644 --- a/tests/test_website_snippets.py +++ b/tests/test_website_snippets.py @@ -558,11 +558,11 @@ def test_ds_update_multiple_stages(self) -> None: args = _parse(["--start-date", "2026-02-01", "--end-date", "2026-03-01", "--stage", "5", "--stage", "6"]) assert args.stage == [5, 6] - def test_pipeline_has_7_stages(self) -> None: - """Website documents 7 pipeline stages.""" + def test_pipeline_has_8_stages(self) -> None: + """Website documents 8 pipeline stages.""" from datasmith.update.pipeline import STAGES - assert len(STAGES) == 7 + assert len(STAGES) == 8 # --------------------------------------------------------------------------- From 2045973341e5122b4d689b9b87f680d66eefb8df Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:49:02 +0000 Subject: [PATCH 23/24] Exclude harbor_adapter/template from mypy checks --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5a56548d..3e69554a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ packages = ["src/datasmith"] [tool.mypy] files = ["src"] -exclude = ["src/datasmith/agents/templates/", "src/datasmith/docker/templates/"] +exclude = ["src/datasmith/agents/templates/", "src/datasmith/docker/templates/", "src/datasmith/harbor_adapter/template/"] disallow_untyped_defs = true disallow_any_unimported = true no_implicit_optional = true From 1744588c1d9ab44795bc9ccf4f5d43ec569a62b6 Mon Sep 17 00:00:00 2001 From: Atharva Sehgal Date: Thu, 16 Apr 2026 04:52:57 +0000 Subject: [PATCH 24/24] Fix mypy errors: harbor type ignores, deptry excludes, arg-type suppressions --- pyproject.toml | 8 ++++++-- src/datasmith/agents/synthesizer.py | 2 +- src/datasmith/publish/records.py | 2 +- src/datasmith/runners/harbor_healthcheck.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3e69554a..0a5fe7ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,10 @@ warn_return_any = true warn_unused_ignores = true show_error_codes = true +[[tool.mypy.overrides]] +module = "harbor.*" +ignore_missing_imports = true + [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" @@ -172,10 +176,10 @@ branch = true source = ["src"] [tool.deptry] -exclude = ["src/datasmith/docker/templates/", "src/datasmith/agents/templates/"] +exclude = ["src/datasmith/docker/templates/", "src/datasmith/agents/templates/", "src/datasmith/harbor_adapter/template/"] ignore = ["DEP003"] [tool.deptry.per_rule_ignores] -DEP001 = ["tomllib"] +DEP001 = ["tomllib", "tomli"] DEP004 = ["pandas"] [[tool.mypy.overrides]] diff --git a/src/datasmith/agents/synthesizer.py b/src/datasmith/agents/synthesizer.py index e08cb5e7..a94de409 100644 --- a/src/datasmith/agents/synthesizer.py +++ b/src/datasmith/agents/synthesizer.py @@ -567,7 +567,7 @@ def _log_tamper( } try: client = get_client() - client.table("error_logs").insert(row).execute() + client.table("error_logs").insert(row).execute() # type: ignore[arg-type] logger.info( "Logged tamper detection for %s/%s@%s attempt %d: %s", owner, diff --git a/src/datasmith/publish/records.py b/src/datasmith/publish/records.py index a68c2c44..e12ba811 100644 --- a/src/datasmith/publish/records.py +++ b/src/datasmith/publish/records.py @@ -97,7 +97,7 @@ def records_from_supabase( # noqa: C901 records: list[FormulaCodeRecord] = [] for row in rows: sha = row.get("merge_commit_sha", "") - best = best_speedup.get((row.get("owner"), row.get("repo"), sha)) + best = best_speedup.get((row.get("owner", ""), row.get("repo", ""), sha)) if best is None: dropped_no_run += 1 continue diff --git a/src/datasmith/runners/harbor_healthcheck.py b/src/datasmith/runners/harbor_healthcheck.py index 77ff8f77..19bb86eb 100644 --- a/src/datasmith/runners/harbor_healthcheck.py +++ b/src/datasmith/runners/harbor_healthcheck.py @@ -47,7 +47,7 @@ def _deterministic_trial_name(self): # type: ignore[no-untyped-def] return self.task.get_task_id().get_name() TrialConfig.generate_trial_name = _deterministic_trial_name - TrialConfig._fc_datasmith_patched = True # type: ignore[attr-defined] + TrialConfig._fc_datasmith_patched = True def _build_verifier_env() -> dict[str, str]: