diff --git a/examples/personal-workspace-browser/research-measurements.mjs b/examples/personal-workspace-browser/research-measurements.mjs index dda2f0ed26..99dcd84475 100644 --- a/examples/personal-workspace-browser/research-measurements.mjs +++ b/examples/personal-workspace-browser/research-measurements.mjs @@ -44,6 +44,11 @@ events.append(build_explore_node_event(goal_id=goal,node_id='single',title='单 events.append(build_explore_finding_event(goal_id=goal,node_id='single',finding_id='single-finding',title='单项测量保持完整来源', summary='A small record uses the same reading hierarchy.',status='tentative',recorded_at='2026-01-01T03:00:00Z', measurements=metrics([8,12,0,None])[:1],evidence_refs=['result:synthetic-0'])) +events.append(build_explore_node_event(goal_id=goal,node_id='paired',title='同一条件下的配对候选',recorded_at='2026-01-01T04:00:00Z')) +events.append(build_explore_finding_event(goal_id=goal,node_id='paired',finding_id='paired-finding',title='配对观测保留两项来源', + summary='Both candidates retain their own receipt under unchanged comparison conditions.',status='refuted', + recorded_at='2026-01-01T04:00:00Z',measurements=[metrics([8,12,0,None],0)[0],metrics([7,10,0,None],1)[0]], + evidence_refs=['result:synthetic-0','result:synthetic-1'])) Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'合成研究 · 量化结果阅读','events':events},ensure_ascii=False)) `, fixture], {cwd: repoRoot}); owner = await startHistoryOwner(fixture); @@ -59,7 +64,7 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'合成 const url = `${owner.base}/chat/`; const open = async node => { await page.goto(new URL(`?goalId=${goal}&view=research&research_goal=${goal}&research_node=${node}&statusUrl=%2Fstatus.json`, url).href, {waitUntil: "domcontentloaded"}); - await page.locator(".research-history-coverage").filter({hasText: "5 / 5"}).waitFor(); + await page.locator(".research-history-coverage").filter({hasText: "7 / 7"}).waitFor(); return page.locator(".research-history-detail"); }; let reader = await open("comparison"); @@ -117,6 +122,23 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'合成 await page.reload({waitUntil: "domcontentloaded"}); await reader.getByRole("heading", {name: "Measured results", exact: true}).waitFor(); assert.equal(await reader.getByRole("button", {name: "About these measurements", exact: true}).count(), 1); + reader = await open("paired"); + const paired = reader.locator(".research-recorded-finding .explore-measurements"); + assert.deepEqual(await paired.locator(".explore-measurement-reading strong").allTextContents(), ["8", "7"]); + const pairSources = paired.locator(":scope > details"); + await pairSources.locator(":scope > summary").click(); + for (const width of [1440, 390]) { + await page.setViewportSize({width, height: 900}); + const groups = pairSources.locator(":scope > section"); + assert.equal(await groups.count(), 2, "Paired candidates retain separate source groups"); + for (const [index, candidate] of ["c", "d"].entries()) { + const text = await groups.nth(index).innerText(); + assert(text.includes("candidate:" + candidate.repeat(64))); + assert(text.includes(`result:synthetic-${index}`), "Each value keeps its own candidate/result association"); + } + assert(await paired.evaluate(node => node.scrollWidth <= node.clientWidth + 1)); + await page.screenshot({path: resolve(outputDir, `research-measurements-paired-${width}.png`), animations: "disabled"}); + } const readback = await (await page.request.get(`${owner.base}/fixture/readback`)).json(); assert(readback.source_unchanged && readback.mutation_attempts === 0, "Reading and help do not rewrite source or execute work"); assert.deepEqual(errors, []); diff --git a/loopx/capabilities/explore/README.md b/loopx/capabilities/explore/README.md index 8af07a525e..3f8102d8a2 100644 --- a/loopx/capabilities/explore/README.md +++ b/loopx/capabilities/explore/README.md @@ -28,8 +28,8 @@ Manual capture uses `explore finding --node --evidence-ref result:a Each result reference must belong to the finding's `evidence_refs`; ordinary attachments also require the same `input_revision`. Use a new finding id per -evaluated candidate, and reuse an id only to correct that finding. Missing values -are explicit `null`; zero and negative values remain valid measurements. +evaluation or paired comparison, and reuse an id only to correct that finding. +Missing values are explicit `null`; zero and negative values remain valid measurements. When an `execution_observation` declares a `result_ref`, each measurement must cite that same result. This applies to manual findings and writeback attachments, @@ -66,7 +66,7 @@ separate. This convenience grants no evaluation, correction or execution authori Keep comparison conditions separate from the candidate and its result: -| Field | Meaning across successive findings | +| Field | Meaning across observations | | --- | --- | | `input_revision` | Evaluated task/data revision; retain it when the evaluated inputs are verified unchanged. | | `evaluation_ref` | Evaluator/scoring protocol version; retain it when the evaluation protocol is verified unchanged. | @@ -86,6 +86,15 @@ Changed or unknown evaluation conditions must stay separate. Do not invent a shared revision merely to obtain a comparison. A growing local corpus does not change the input revision for an unchanged evaluated subset; changed samples do. +A paired comparison may instead record both candidates in **one finding** under +the same verified comparison conditions. Keep both `candidate_ref` and +`evidence_ref` values, and list both results in the finding's `evidence_refs`. +Distinct candidates do not require invented metrics or scopes. A repeated +candidate under the same metric, unit, direction and conditions is still a +duplicate, even with another result reference; record a repeat evaluation in a +new finding. Every measurement still requires a candidate reference. A paired +finding cannot claim two separate receipts as one execution observation's result. + Comparisons require the same node, metric, unit, direction, comparison scope, input revision and evaluator. Producers must change the scope when task, model/resource budget or evaluation conditions change, and separate native @@ -93,8 +102,11 @@ evaluation from local proxies. Explore validates the declared comparison keys, not their truth. It computes the latest recorded measurement, historical best candidate/source and direction-adjusted improvement against the prior best, before applying the context's display limit. Latest follows source-log append -order, including tied or backfilled timestamps; its ordinal is derived, not -another persisted fact. Corrected findings replace earlier +order, including tied or backfilled timestamps. Within a finding, observations +follow their declared array order: latest is the last matching observation and +prior best includes earlier observations in that batch. This order does not +assert execution chronology or identify the retained candidate. The source ordinal +is derived, not another persisted fact. Corrected findings replace earlier versions in comparisons; original versions remain in history. Counts are records, not independent trials. Feedback pages reuse `--result-limit` (1..20, default 3), `--result-offset`, `--result-revision` and the optional `--result-node` filter. diff --git a/loopx/control_plane/capabilities/explore_measurements.ts b/loopx/control_plane/capabilities/explore_measurements.ts index dc73bbba05..8c4e470fa7 100644 --- a/loopx/control_plane/capabilities/explore_measurements.ts +++ b/loopx/control_plane/capabilities/explore_measurements.ts @@ -72,8 +72,10 @@ export function normalizeExploreMeasurements(value: unknown, context?: unknown, value: row.value as number | null, higher_is_better: row.higher_is_better, comparison_scope: ref("comparison_scope"), input_revision: text("input_revision", 160), evaluation_ref: ref("evaluation_ref"), candidate_ref: ref("candidate_ref"), evidence_ref: ref("evidence_ref")}; - const key = comparisonKey("", result); - if (seen.has(key)) throw new EffectRuntimeRequestError("measurement metric/scope is duplicated"); + // A comparison batch can contain distinct candidates under identical + // conditions. Candidate identity distinguishes observations, not groups. + const key = JSON.stringify([comparisonKey("", result), result.candidate_ref]); + if (seen.has(key)) throw new EffectRuntimeRequestError("measurement candidate/metric/scope is duplicated"); seen.add(key); return result; }); @@ -97,7 +99,9 @@ function comparisonKey(node: string, metric: ExploreMeasurement): string { metric.comparison_scope, metric.input_revision, metric.evaluation_ref]); } -/** Latest means latest appended finding, never the current or retained candidate. +/** Latest means the last declared observation in the latest appended finding, + * not execution chronology or the current/retained candidate. Within a batch, + * previous best includes earlier declared observations; order is not inferred. * Keep all matching evidence when computing best; presentation limits apply later. */ export function projectExploreMeasurementFeedback(findings: readonly JsonObject[]): ExploreMeasurementFeedback[] { diff --git a/tests/control_plane/test_explore_measurements.py b/tests/control_plane/test_explore_measurements.py index d2a7eac762..dd4f11984a 100644 --- a/tests/control_plane/test_explore_measurements.py +++ b/tests/control_plane/test_explore_measurements.py @@ -31,6 +31,88 @@ def context(args): ("registry_path", "runtime_root", "goal_id", "agent_id")}) +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_paired_cli_refresh_rejects_duplicates_then_replays_and_reads_both_candidates(tmp_path, monkeypatch, provider): + from canonical_authority_fixture import isolate_sqlite_runtime + from tests.control_plane.test_quota_settlement_cli import ( + _write_fixture, _run_cli, GOAL_ID, AGENT_ID, TODO_ID, TURN_ID, + ) + from loopx.chat_server import ChatHTTPServer, ChatRequestHandler + import threading + from urllib.request import urlopen + + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_fixture(tmp_path, provider=provider) + config = json.loads(registry.read_text()) + config["goals"][0]["explore_graph"] = {"enabled": True} + registry.write_text(json.dumps(config)) + binding = ("--goal-id", GOAL_ID, "--agent-id", AGENT_ID, "--todo-id", TODO_ID, + "--turn-instance-id", TURN_ID) + rc, guard = _run_cli(registry, runtime, "quota", "should-run", "--codex-app", *binding, cwd=project) + assert rc == 0 and guard["should_run"], guard + pair = [{**measurement("baseline", 9), "evidence_ref": "result:baseline"}, + {**measurement("alternative", 6), "evidence_ref": "result:alternative"}] + attachment = {**payload()["explore_result"], "measurements": pair, + "evidence_refs": [row["evidence_ref"] for row in pair]} + source = tmp_path / "paired-result.json" + args = ("refresh-state", *binding, "--classification", "validated_change", + "--delivery-batch-scale", "implementation", "--delivery-outcome", "outcome_progress", + "--no-global-sync", "--suppress-external-sinks", "--explore-result-json", str(source)) + log = explore_result_log_path(runtime, GOAL_ID) + index = runtime / "goals" / GOAL_ID / "runs" / "index.jsonl" + previous = index.read_bytes() if index.exists() else b"" + # Different receipts cannot disguise duplicate candidate observations. + source.write_text(json.dumps({**attachment, "measurements": [pair[0], + {**pair[1], "candidate_ref": pair[0]["candidate_ref"]}]})) + rc, rejected = _run_cli(registry, runtime, *args, cwd=project) + assert rc == 1 and not rejected["appended"], rejected + assert "duplicated" in rejected["error"] + assert (index.read_bytes() if index.exists() else b"") == previous + assert not log.exists() + + source.write_text(json.dumps(attachment)) + rc, captured = _run_cli(registry, runtime, *args, cwd=project) + assert rc == 0 and captured["appended"] and captured["explore_result_delivery"]["ok"], captured + committed = (index.read_bytes(), log.read_bytes()) + rc, replay = _run_cli(registry, runtime, *args, cwd=project) + assert rc == 0 and replay["idempotent_replay"], replay + assert (index.read_bytes(), log.read_bytes()) == committed + rc, received = _run_cli(registry, runtime, "explore", "turn-context", "--goal-id", GOAL_ID, + "--agent-id", AGENT_ID, cwd=project) + assert rc == 0, received + finding = received["graph"]["writeback_results"][0] + assert finding["measurements"] == pair + feedback = received["graph"]["measurement_feedback"][0] + assert feedback["best"]["candidate_ref"] == "candidate:baseline" + assert feedback["latest"]["measurement"]["candidate_ref"] == "candidate:alternative" + assert feedback["improvement"] == -3 and feedback["measured_count"] == 2 + assert len(load_explore_result_events_strict(log, goal_id=GOAL_ID)) == 2 + + # The existing product reader consumes the same persisted finding, not a + # second measurement owner or a synthetic HTTP response. + server = ChatHTTPServer(("127.0.0.1", 0), ChatRequestHandler) + server.registry_path, server.runtime_root = registry, runtime + server.runtime_root_override, server.verbose = None, False + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + base = f"http://127.0.0.1:{server.server_port}/api/chat/explore-results?goal_id={GOAL_ID}" + for suffix in ("", "&view=history"): + with urlopen(base + suffix) as response: + rows = json.load(response)["items"] + measured = next(row for row in rows if row.get("measurements")) + assert measured["measurements"] == pair + finally: + server.shutdown() + worker.join(timeout=5) + server.server_close() + assert (index.read_bytes(), log.read_bytes()) == committed + source.write_text(json.dumps({**attachment, "measurements": [pair[0], {**pair[1], "value": 10}]})) + rc, conflict = _run_cli(registry, runtime, *args, cwd=project) + assert rc == 1, conflict + assert (index.read_bytes(), log.read_bytes()) == committed + + @pytest.mark.parametrize("measured", [False, True]) def test_tied_finding_times_bound_latest_appends_and_corrections(measured): # The production writer records whole seconds. Equal source times must not diff --git a/tests/control_plane_ts/explore_measurements.test.ts b/tests/control_plane_ts/explore_measurements.test.ts index 8259220944..8c794b48e9 100644 --- a/tests/control_plane_ts/explore_measurements.test.ts +++ b/tests/control_plane_ts/explore_measurements.test.ts @@ -106,6 +106,60 @@ test("quantitative evidence keeps zero, negatives and unknown distinct and rejec assert.throws(() => normalizeExploreMeasurements([])); }); +test("one finding retains paired candidates without changing their comparison conditions", () => { + const baseline = {...metric, value: 9}; + const candidate = {...metric, value: 6, candidate_ref: "candidate:b", evidence_ref: "result:b"}; + const pair = [baseline, candidate]; + const attachment = {schema_version: "explore_result_attachment_v0", question: "Which candidate improves quality?", + applicability: "Matched inputs, evaluator and resource budget", input_revision: metric.input_revision, + observation: "The matched candidate regressed.", interpretation: "Retain both observations for the next decision.", + status: "refuted", evidence_refs: pair.map(item => item.evidence_ref), measurements: pair}; + assert.deepEqual(normalizeExploreResultAttachment({attachment}).measurements, pair); + const {comparison_scope, input_revision, evaluation_ref} = metric; + assert.deepEqual(normalizeExploreMeasurements(pair.map(({comparison_scope: _scope, + input_revision: _input, evaluation_ref: _evaluation, ...row}) => row), + {comparison_scope, input_revision, evaluation_ref}), pair); + for (const direction of [true, false]) { + const measured = pair.map(item => ({...item, higher_is_better: direction})); + const [feedback] = projectExploreMeasurementFeedback([{finding_id: "paired", node_id: "question-a", measurements: measured}]); + assert.equal(feedback.measured_count, 2); + assert.equal(feedback.latest.measurement.candidate_ref, "candidate:b"); + assert.equal(feedback.previous_best?.candidate_ref, "candidate:a"); + assert.equal(feedback.best?.candidate_ref, direction ? "candidate:a" : "candidate:b"); + assert.equal(feedback.improvement, direction ? -3 : 3); + const [reordered] = projectExploreMeasurementFeedback([{finding_id: "paired", node_id: "question-a", + measurements: [...measured].reverse()}]); + assert.deepEqual(reordered.best, feedback.best, "declared batch order cannot change the best score"); + assert.equal(reordered.measured_count, 2); + assert.equal(reordered.latest.measurement.candidate_ref, "candidate:a", "latest is the last declared observation, not execution chronology"); + } + const [unknown] = projectExploreMeasurementFeedback([{finding_id: "paired", node_id: "question-a", + measurements: [baseline, {...candidate, value: null}]}]); + assert.equal(unknown.best?.value, 9); + assert.equal(unknown.improvement, null); + assert.equal(unknown.measured_count, 1); + assert.equal(unknown.unknown_count, 1); +}); + +test("paired observations do not weaken duplicate, scope, source or execution checks", () => { + for (const mutation of [{value: 10}, {evidence_ref: "result:another"}, {candidate_ref: " candidate:a "}]) + assert.throws(() => normalizeExploreMeasurements([metric, {...metric, ...mutation}]), /duplicated/); + const {candidate_ref: _candidate, ...missingCandidate} = metric; + assert.throws(() => normalizeExploreMeasurements([missingCandidate]), /fields are missing/); + const changedScope = {...metric, comparison_scope: "validation:other-budget"}; + assert.equal(normalizeExploreMeasurements([metric, changedScope]).length, 2); + assert.equal(projectExploreMeasurementFeedback([{finding_id: "changed", node_id: "question-a", + measurements: [metric, changedScope]}]).length, 2); + const other = {...metric, candidate_ref: "candidate:b", evidence_ref: "result:b"}; + const execution = {schema_version: "explore_execution_observation_v0", + candidate_sha256: "a".repeat(64), result_candidate_sha256: "a".repeat(64), + input_revision: metric.input_revision, planned_method_id: "probe", executed_method_id: "probe", + change_reason: null, elapsed_seconds: 1, limit_seconds: 10, search_iterations: null, + stop_reason: "process_exit:0", evidence_refs: [metric.evidence_ref], result_ref: metric.evidence_ref}; + assert.throws(() => normalizeExploreMeasurementsRequest({measurements: [metric, other], + execution_observation: execution}), /executed candidate's result/); +}); + test("regression returns the earlier best candidate and useful signed feedback, including lower-is-better", () => { const input = [finding("new", 6), finding("best", 9), finding("first", 8)]; const before = JSON.stringify(input);