diff --git a/benchmarks/dataset-agent/README.md b/benchmarks/dataset-agent/README.md index 8385edc..5737d79 100644 --- a/benchmarks/dataset-agent/README.md +++ b/benchmarks/dataset-agent/README.md @@ -146,6 +146,24 @@ without committing raw run folders: Agent canary actually emitted browser actions before starting a Playwright compiler. +For browser-action canaries, add `--require-playwright-ready` to make the +benchmark fail with `failureCategory: "capability_gate"` unless the +`playwright-candidate-readiness` artifact is `ready`. This gate uses the +readiness artifact, not raw browser step counts, so it still requires +actionable browser steps, source anchors, and no Agent-disabled diagnostic. + +```bash +COLLECTION_AGENT_ENABLE_AGENT=true \ +COLLECTION_AGENT_POLL_TIMEOUT_MS=480000 \ +COLLECTION_AGENT_PIPELINE_MODULE=./backend/BigSet_Data_Collection_Agent/src/orchestrator/pipeline.ts \ +BIGSET_COLLECTION_BENCHMARK_RUNNER_MODULE=./backend/src/pipeline/collection-agent-runner.ts \ +node benchmarks/dataset-agent/run-benchmark.mjs \ + --require-playwright-ready \ + --prompt-ids mcp-docs-pages \ + --timeout-ms 900000 \ + --system collection-self-heal='node --import ./backend/node_modules/tsx/dist/esm/index.mjs benchmarks/dataset-agent/adapters/collection-self-healing-adapter.mjs' +``` + ## Verify Self-Healing Stack Use this before asking someone else to migrate a new collection agent into the diff --git a/benchmarks/dataset-agent/run-benchmark.mjs b/benchmarks/dataset-agent/run-benchmark.mjs index a52fa96..1fe09c2 100755 --- a/benchmarks/dataset-agent/run-benchmark.mjs +++ b/benchmarks/dataset-agent/run-benchmark.mjs @@ -567,11 +567,19 @@ async function runSystemPrompt(input) { parsedPayload, normalized, }); - const status = infraBlockerReason - ? "blocked" - : execution.exitCode === 0 && parsedPayload && answerKeyScore.passed - ? "ok" - : "failed"; + const capabilityGateReason = infraBlockerReason + ? null + : playwrightReadinessGateReason({ + diagnostics: normalized.diagnostics, + requirePlaywrightReady: input.config.requirePlaywrightReady, + }); + const status = benchmarkStatusForOutcome({ + execution, + parsedPayload, + answerKeyScore, + infraBlockerReason, + capabilityGateReason, + }); const promptRunDirectory = join( input.runDirectory, @@ -597,9 +605,12 @@ async function runSystemPrompt(input) { expectedStress: input.promptDefinition.expectedStress, answerKey: answerKeyForPrompt(input.promptDefinition), status, - failureCategory: status === "ok" ? undefined : ( - infraBlockerReason ? "infra" : answerKeyScore.failureCategory - ), + failureCategory: failureCategoryForOutcome({ + status, + infraBlockerReason, + capabilityGateReason, + answerKeyScore, + }), factualAccuracyScore: answerKeyScore.factualAccuracyScore, entityCoverageRatio: answerKeyScore.entityCoverageRatio, domainAccuracyRatio: answerKeyScore.domainAccuracyRatio, @@ -657,6 +668,7 @@ async function runSystemPrompt(input) { validation, answerKeyScore, infraBlockerReason, + capabilityGateReason, minRequiredCompleteness: input.config.minRequiredCompleteness, validationIssues: normalized.validationIssues, }), @@ -758,6 +770,7 @@ function parseArgs(args) { tinyFishAgentStepUsd: 0.015, minRequiredCompleteness: 0.75, minFactualAccuracy: defaultMinimumFactualAccuracy, + requirePlaywrightReady: false, }; for (let index = 0; index < args.length; index += 1) { @@ -797,6 +810,8 @@ function parseArgs(args) { } else if (arg === "--min-factual-accuracy") { config.minFactualAccuracy = nonNegativeNumber(value, config.minFactualAccuracy); index += 1; + } else if (arg === "--require-playwright-ready") { + config.requirePlaywrightReady = true; } else if (arg === "--help" || arg === "-h") { printHelpAndExit(); } else { @@ -972,6 +987,71 @@ export function normalizePayload(payload) { }; } +export function playwrightReadinessGateReason({ + diagnostics, + requirePlaywrightReady, +}) { + if (!requirePlaywrightReady) { + return null; + } + const readiness = diagnostics?.playwrightCandidateReadiness; + if (!readiness || typeof readiness !== "object") { + return "Playwright readiness gate failed: missing playwrightCandidateReadiness diagnostics."; + } + const reasons = stringArrayValue(readiness.reasons); + if (readiness.status !== "ready") { + return [ + "Playwright readiness gate failed:", + reasons.length > 0 + ? reasons.join("; ") + : `status is ${String(readiness.status ?? "missing")}.`, + ].join(" "); + } + if (numberValue(readiness.browserStepCount) <= 0) { + return "Playwright readiness gate failed: no actionable browser steps."; + } + if (numberValue(readiness.sourceUrlCount) <= 0) { + return "Playwright readiness gate failed: no source URLs to anchor replay."; + } + return null; +} + +export function benchmarkStatusForOutcome({ + execution, + parsedPayload, + answerKeyScore, + infraBlockerReason, + capabilityGateReason, +}) { + if (infraBlockerReason) { + return "blocked"; + } + if (capabilityGateReason) { + return "failed"; + } + return execution.exitCode === 0 && parsedPayload && answerKeyScore.passed + ? "ok" + : "failed"; +} + +export function failureCategoryForOutcome({ + status, + infraBlockerReason, + capabilityGateReason, + answerKeyScore, +}) { + if (status === "ok") { + return undefined; + } + if (infraBlockerReason) { + return "infra"; + } + if (capabilityGateReason) { + return "capability_gate"; + } + return answerKeyScore.failureCategory; +} + function normalizeUsage(value) { return { promptTokens: numberValue(value?.promptTokens ?? value?.inputTokens ?? value?.prompt_tokens), @@ -1041,7 +1121,7 @@ function evaluateRows({ rows, promptDefinition }) { }; } -async function rescoreBenchmarkRun({ runDirectory, prompts, config }) { +export async function rescoreBenchmarkRun({ runDirectory, prompts, config }) { const previousSummary = JSON.parse(await readFile(join(runDirectory, "summary.json"), "utf8")); const promptsById = new Map(prompts.map((promptDefinition) => [ promptDefinition.id, @@ -1089,11 +1169,19 @@ async function rescoreBenchmarkRun({ runDirectory, prompts, config }) { parsedPayload: usablePayload, normalized, }); - const status = infraBlockerReason - ? "blocked" - : execution.exitCode === 0 && usablePayload && answerKeyScore.passed - ? "ok" - : "failed"; + const capabilityGateReason = infraBlockerReason + ? null + : playwrightReadinessGateReason({ + diagnostics: normalized.diagnostics, + requirePlaywrightReady: config.requirePlaywrightReady, + }); + const status = benchmarkStatusForOutcome({ + execution, + parsedPayload: usablePayload, + answerKeyScore, + infraBlockerReason, + capabilityGateReason, + }); rescoredLaneResults.push({ ...laneResult, @@ -1103,9 +1191,12 @@ async function rescoreBenchmarkRun({ runDirectory, prompts, config }) { expectedStress: promptDefinition.expectedStress, answerKey: answerKeyForPrompt(promptDefinition), status, - failureCategory: status === "ok" ? undefined : ( - infraBlockerReason ? "infra" : answerKeyScore.failureCategory - ), + failureCategory: failureCategoryForOutcome({ + status, + infraBlockerReason, + capabilityGateReason, + answerKeyScore, + }), factualAccuracyScore: answerKeyScore.factualAccuracyScore, entityCoverageRatio: answerKeyScore.entityCoverageRatio, domainAccuracyRatio: answerKeyScore.domainAccuracyRatio, @@ -1130,6 +1221,32 @@ async function rescoreBenchmarkRun({ runDirectory, prompts, config }) { needsReviewCount: validation.needsReviewCount, validationIssueCount: normalized.validationIssues.length, validationIssues: normalized.validationIssues, + selfHealingAction: normalized.diagnostics.selfHealingAction, + selfHealingArtifactKinds: normalized.diagnostics.artifactKinds, + processTraceStepCount: normalized.diagnostics.processTrace?.stepCount, + processTraceBrowserStepCount: + normalized.diagnostics.processTrace?.browserStepCount, + playwrightCandidateStatus: + normalized.diagnostics.playwrightCandidateReadiness?.status, + playwrightCandidateBrowserStepCount: + normalized.diagnostics.playwrightCandidateReadiness?.browserStepCount, + playwrightCandidateSourceUrlCount: + normalized.diagnostics.playwrightCandidateReadiness?.sourceUrlCount, + diagnostics: normalized.diagnostics, + usage: normalized.usage, + searchCallCount: normalized.metrics.searchCallCount, + fetchCallCount: normalized.metrics.fetchCallCount, + browserCallCount: normalized.metrics.browserCallCount, + agentRunCount: normalized.metrics.agentRunCount, + agentStepCount: normalized.metrics.agentStepCount, + estimatedModelCostUsd: estimateModelCostUsd(normalized.usage, config), + estimatedTinyFishAgentCostUsd: roundUsd( + normalized.metrics.agentStepCount * config.tinyFishAgentStepUsd + ), + estimatedTotalCostUsd: roundUsd( + estimateModelCostUsd(normalized.usage, config) + + normalized.metrics.agentStepCount * config.tinyFishAgentStepUsd + ), errorMessage: status === "ok" ? undefined : failureReason({ @@ -1138,6 +1255,7 @@ async function rescoreBenchmarkRun({ runDirectory, prompts, config }) { validation, answerKeyScore, infraBlockerReason, + capabilityGateReason, minRequiredCompleteness: config.minRequiredCompleteness, validationIssues: normalized.validationIssues, }), @@ -1652,6 +1770,7 @@ export function failureReason({ validation, answerKeyScore, infraBlockerReason, + capabilityGateReason, minRequiredCompleteness, validationIssues = [], }) { @@ -1659,6 +1778,7 @@ export function failureReason({ if (execution.timedOut) return "Command timed out."; if (execution.exitCode !== 0) return `Command exited ${execution.exitCode}.`; if (!parsedPayload) return "No parseable JSON object found in stdout."; + if (capabilityGateReason) return capabilityGateReason; const capabilityDiagnostic = capabilityDiagnosticReason(validationIssues); if (capabilityDiagnostic) return capabilityDiagnostic; if (answerKeyScore?.failureCategory === "clarification") { @@ -1807,6 +1927,12 @@ node benchmarks/dataset-agent/run-benchmark.mjs \\ Rescore existing artifacts without spending credits: node benchmarks/dataset-agent/run-benchmark.mjs --rescore-dir benchmark-results/ +Require self-healing Playwright readiness for browser-action canaries: +node benchmarks/dataset-agent/run-benchmark.mjs \\ + --require-playwright-ready \\ + --prompt-ids mcp-docs-pages \\ + --system collection-self-heal='node --import ./backend/node_modules/tsx/dist/esm/index.mjs benchmarks/dataset-agent/adapters/collection-self-healing-adapter.mjs' + Agent command contract: - stdout should contain a JSON object. - Preferred shape: { "rows": [], "validationIssues": [], "usage": {}, "metrics": {} } diff --git a/benchmarks/dataset-agent/run-benchmark.test.mjs b/benchmarks/dataset-agent/run-benchmark.test.mjs index 377cdff..e22c910 100644 --- a/benchmarks/dataset-agent/run-benchmark.test.mjs +++ b/benchmarks/dataset-agent/run-benchmark.test.mjs @@ -1,10 +1,17 @@ import assert from "node:assert/strict"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { test } from "node:test"; import { + benchmarkStatusForOutcome, + failureCategoryForOutcome, failureReason, findInfrastructureBlockerReason, normalizePayload, + playwrightReadinessGateReason, + rescoreBenchmarkRun, scoreBenchmarkRows, } from "./run-benchmark.mjs"; import { selfHealingDiagnosticsFromTick } from "./adapters/self-healing-output.mjs"; @@ -233,3 +240,146 @@ test("self-healing diagnostics summarize trace and readiness artifacts", () => { "ready" ); }); + +test("Playwright readiness gate fails otherwise passing benchmark output", () => { + const capabilityGateReason = playwrightReadinessGateReason({ + requirePlaywrightReady: true, + diagnostics: notReadyDiagnostics(), + }); + const answerKeyScore = { passed: true, failureCategory: undefined }; + const status = benchmarkStatusForOutcome({ + execution: { exitCode: 0 }, + parsedPayload: { rows: passingRows() }, + answerKeyScore, + infraBlockerReason: null, + capabilityGateReason, + }); + + assert.equal(status, "failed"); + assert.match(capabilityGateReason, /no actionable browser steps/i); + assert.equal(failureCategoryForOutcome({ + status, + infraBlockerReason: null, + capabilityGateReason, + answerKeyScore, + }), "capability_gate"); + assert.equal(failureReason({ + execution: { exitCode: 0, timedOut: false }, + parsedPayload: { rows: passingRows() }, + validation: passingValidation, + answerKeyScore, + infraBlockerReason: null, + capabilityGateReason, + minRequiredCompleteness: 0.75, + }), capabilityGateReason); +}); + +test("Playwright readiness gate does not override infrastructure blockers", () => { + const infraBlockerReason = "Infrastructure/auth/credits blocker."; + const capabilityGateReason = null; + const answerKeyScore = { passed: true, failureCategory: undefined }; + const status = benchmarkStatusForOutcome({ + execution: { exitCode: 0 }, + parsedPayload: null, + answerKeyScore, + infraBlockerReason, + capabilityGateReason, + }); + + assert.equal(status, "blocked"); + assert.equal(failureCategoryForOutcome({ + status, + infraBlockerReason, + capabilityGateReason, + answerKeyScore, + }), "infra"); +}); + +test("rescore applies Playwright readiness gate semantics", async () => { + const runDirectory = await mkdtemp(join(tmpdir(), "bigset-benchmark-rescore-")); + const artifactDirectory = join(runDirectory, "collection-self-heal", "01-gate-prompt"); + await mkdir(artifactDirectory, { recursive: true }); + + const parsedPayload = { + rows: passingRows(), + validationIssues: [], + diagnostics: notReadyDiagnostics(), + }; + await writeFile( + join(runDirectory, "summary.json"), + JSON.stringify({ + laneResults: [{ + system: "collection-self-heal", + promptId: "gate-prompt", + promptQuality: "good", + artifactDirectory, + exitCode: 0, + timedOut: false, + }], + }) + ); + await writeFile( + join(artifactDirectory, "parsed-output.json"), + JSON.stringify(parsedPayload) + ); + await writeFile(join(artifactDirectory, "stdout.txt"), JSON.stringify(parsedPayload)); + await writeFile(join(artifactDirectory, "stderr.txt"), ""); + + const rescored = await rescoreBenchmarkRun({ + runDirectory, + prompts: [{ + id: "gate-prompt", + quality: "good", + persona: "developer", + prompt: "Find official docs.", + expectedStress: "Browser action gate.", + requiredColumns: ["entity_name", "source_url"], + }], + config: { + promptIds: null, + minRequiredCompleteness: 0.75, + minFactualAccuracy: 0.75, + requirePlaywrightReady: true, + inputUsdPer1M: 0.05, + outputUsdPer1M: 0.5, + tinyFishAgentStepUsd: 0.015, + }, + }); + + assert.equal(rescored.laneResults[0].status, "failed"); + assert.equal(rescored.laneResults[0].failureCategory, "capability_gate"); + assert.match(rescored.laneResults[0].errorMessage, /no actionable browser steps/i); + assert.equal(rescored.laneResults[0].playwrightCandidateStatus, "not_ready"); +}); + +function passingRows() { + return [{ + cells: { + entity_name: "Example", + source_url: "https://example.com/docs", + }, + sourceUrls: ["https://example.com/docs"], + evidence: [{ + columnName: "entity_name", + sourceUrl: "https://example.com/docs", + quote: "Example docs", + }], + }]; +} + +function notReadyDiagnostics() { + return { + playwrightCandidateReadiness: { + status: "not_ready", + reasons: ["Trace has no actionable browser steps with URL/selector/target data."], + browserStepCount: 0, + sourceUrlCount: 1, + }, + processTrace: { + runtime: "collection", + stepCount: 3, + browserStepCount: 0, + sourceUrlCount: 1, + }, + }; +} diff --git a/docs/data-collection-agent-migration-plan.md b/docs/data-collection-agent-migration-plan.md index 359945b..5d0331b 100644 --- a/docs/data-collection-agent-migration-plan.md +++ b/docs/data-collection-agent-migration-plan.md @@ -195,6 +195,8 @@ The current layer does not yet: `processTraceBrowserStepCount`, and `playwrightCandidateBrowserStepCount` so the canary proves browser-action provenance, not only row/evidence quality + - run browser-action canaries with `--require-playwright-ready` so row + quality cannot hide missing replayable browser-action provenance - full benchmark only after the 2-prompt run is not obviously broken - live `--dataset-id` dry-run only after Convex/env prerequisites are ready - `--commit` only on a throwaway dataset first