Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion examples/personal-workspace-browser/research-measurements.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ events.append(build_explore_node_event(goal_id=goal,node_id='single',title='单
events.append(build_explore_finding_event(goal_id=goal,node_id='single',finding_id='single-finding',title='单项测量保持完整来源',
summary='A small record uses the same reading hierarchy.',status='tentative',recorded_at='2026-01-01T03:00:00Z',
measurements=metrics([8,12,0,None])[:1],evidence_refs=['result:synthetic-0']))
events.append(build_explore_node_event(goal_id=goal,node_id='paired',title='同一条件下的配对候选',recorded_at='2026-01-01T04:00:00Z'))
events.append(build_explore_finding_event(goal_id=goal,node_id='paired',finding_id='paired-finding',title='配对观测保留两项来源',
summary='Both candidates retain their own receipt under unchanged comparison conditions.',status='refuted',
recorded_at='2026-01-01T04:00:00Z',measurements=[metrics([8,12,0,None],0)[0],metrics([7,10,0,None],1)[0]],
evidence_refs=['result:synthetic-0','result:synthetic-1']))
Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'合成研究 · 量化结果阅读','events':events},ensure_ascii=False))
`, fixture], {cwd: repoRoot});
owner = await startHistoryOwner(fixture);
Expand All @@ -59,7 +64,7 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'合成
const url = `${owner.base}/chat/`;
const open = async node => {
await page.goto(new URL(`?goalId=${goal}&view=research&research_goal=${goal}&research_node=${node}&statusUrl=%2Fstatus.json`, url).href, {waitUntil: "domcontentloaded"});
await page.locator(".research-history-coverage").filter({hasText: "5 / 5"}).waitFor();
await page.locator(".research-history-coverage").filter({hasText: "7 / 7"}).waitFor();
return page.locator(".research-history-detail");
};
let reader = await open("comparison");
Expand Down Expand Up @@ -117,6 +122,23 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'合成
await page.reload({waitUntil: "domcontentloaded"});
await reader.getByRole("heading", {name: "Measured results", exact: true}).waitFor();
assert.equal(await reader.getByRole("button", {name: "About these measurements", exact: true}).count(), 1);
reader = await open("paired");
const paired = reader.locator(".research-recorded-finding .explore-measurements");
assert.deepEqual(await paired.locator(".explore-measurement-reading strong").allTextContents(), ["8", "7"]);
const pairSources = paired.locator(":scope > details");
await pairSources.locator(":scope > summary").click();
for (const width of [1440, 390]) {
await page.setViewportSize({width, height: 900});
const groups = pairSources.locator(":scope > section");
assert.equal(await groups.count(), 2, "Paired candidates retain separate source groups");
for (const [index, candidate] of ["c", "d"].entries()) {
const text = await groups.nth(index).innerText();
assert(text.includes("candidate:" + candidate.repeat(64)));
assert(text.includes(`result:synthetic-${index}`), "Each value keeps its own candidate/result association");
}
assert(await paired.evaluate(node => node.scrollWidth <= node.clientWidth + 1));
await page.screenshot({path: resolve(outputDir, `research-measurements-paired-${width}.png`), animations: "disabled"});
}
const readback = await (await page.request.get(`${owner.base}/fixture/readback`)).json();
assert(readback.source_unchanged && readback.mutation_attempts === 0, "Reading and help do not rewrite source or execute work");
assert.deepEqual(errors, []);
Expand Down
22 changes: 17 additions & 5 deletions loopx/capabilities/explore/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ Manual capture uses `explore finding --node <id> --evidence-ref result:a

Each result reference must belong to the finding's `evidence_refs`; ordinary
attachments also require the same `input_revision`. Use a new finding id per
evaluated candidate, and reuse an id only to correct that finding. Missing values
are explicit `null`; zero and negative values remain valid measurements.
evaluation or paired comparison, and reuse an id only to correct that finding.
Missing values are explicit `null`; zero and negative values remain valid measurements.

When an `execution_observation` declares a `result_ref`, each measurement must
cite that same result. This applies to manual findings and writeback attachments,
Expand Down Expand Up @@ -66,7 +66,7 @@ separate. This convenience grants no evaluation, correction or execution authori

Keep comparison conditions separate from the candidate and its result:

| Field | Meaning across successive findings |
| Field | Meaning across observations |
| --- | --- |
| `input_revision` | Evaluated task/data revision; retain it when the evaluated inputs are verified unchanged. |
| `evaluation_ref` | Evaluator/scoring protocol version; retain it when the evaluation protocol is verified unchanged. |
Expand All @@ -86,15 +86,27 @@ Changed or unknown evaluation conditions must stay separate. Do not invent a
shared revision merely to obtain a comparison. A growing local corpus does not
change the input revision for an unchanged evaluated subset; changed samples do.

A paired comparison may instead record both candidates in **one finding** under
the same verified comparison conditions. Keep both `candidate_ref` and
`evidence_ref` values, and list both results in the finding's `evidence_refs`.
Distinct candidates do not require invented metrics or scopes. A repeated
candidate under the same metric, unit, direction and conditions is still a
duplicate, even with another result reference; record a repeat evaluation in a
new finding. Every measurement still requires a candidate reference. A paired
finding cannot claim two separate receipts as one execution observation's result.

Comparisons require the same node, metric, unit, direction, comparison scope,
input revision and evaluator. Producers must change the scope when task,
model/resource budget or evaluation conditions change, and separate native
evaluation from local proxies. Explore validates the declared comparison keys,
not their truth. It computes the latest recorded measurement, historical best
candidate/source and direction-adjusted improvement against the prior best,
before applying the context's display limit. Latest follows source-log append
order, including tied or backfilled timestamps; its ordinal is derived, not
another persisted fact. Corrected findings replace earlier
order, including tied or backfilled timestamps. Within a finding, observations
follow their declared array order: latest is the last matching observation and
prior best includes earlier observations in that batch. This order does not
assert execution chronology or identify the retained candidate. The source ordinal
is derived, not another persisted fact. Corrected findings replace earlier
versions in comparisons; original versions remain in history. Counts are records,
not independent trials. Feedback pages reuse `--result-limit` (1..20, default 3),
`--result-offset`, `--result-revision` and the optional `--result-node` filter.
Expand Down
10 changes: 7 additions & 3 deletions loopx/control_plane/capabilities/explore_measurements.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,10 @@ export function normalizeExploreMeasurements(value: unknown, context?: unknown,
value: row.value as number | null, higher_is_better: row.higher_is_better,
comparison_scope: ref("comparison_scope"), input_revision: text("input_revision", 160),
evaluation_ref: ref("evaluation_ref"), candidate_ref: ref("candidate_ref"), evidence_ref: ref("evidence_ref")};
const key = comparisonKey("", result);
if (seen.has(key)) throw new EffectRuntimeRequestError("measurement metric/scope is duplicated");
// A comparison batch can contain distinct candidates under identical
// conditions. Candidate identity distinguishes observations, not groups.
const key = JSON.stringify([comparisonKey("", result), result.candidate_ref]);
if (seen.has(key)) throw new EffectRuntimeRequestError("measurement candidate/metric/scope is duplicated");
seen.add(key);
return result;
});
Expand All @@ -97,7 +99,9 @@ function comparisonKey(node: string, metric: ExploreMeasurement): string {
metric.comparison_scope, metric.input_revision, metric.evaluation_ref]);
}

/** Latest means latest appended finding, never the current or retained candidate.
/** Latest means the last declared observation in the latest appended finding,
* not execution chronology or the current/retained candidate. Within a batch,
* previous best includes earlier declared observations; order is not inferred.
* Keep all matching evidence when computing best; presentation limits apply later.
*/
export function projectExploreMeasurementFeedback(findings: readonly JsonObject[]): ExploreMeasurementFeedback[] {
Expand Down
82 changes: 82 additions & 0 deletions tests/control_plane/test_explore_measurements.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,88 @@ def context(args):
("registry_path", "runtime_root", "goal_id", "agent_id")})


@pytest.mark.parametrize("provider", ["file", "sqlite"])
def test_paired_cli_refresh_rejects_duplicates_then_replays_and_reads_both_candidates(tmp_path, monkeypatch, provider):
from canonical_authority_fixture import isolate_sqlite_runtime
from tests.control_plane.test_quota_settlement_cli import (
_write_fixture, _run_cli, GOAL_ID, AGENT_ID, TODO_ID, TURN_ID,
)
from loopx.chat_server import ChatHTTPServer, ChatRequestHandler
import threading
from urllib.request import urlopen

isolate_sqlite_runtime(tmp_path, monkeypatch)
project, runtime, registry = _write_fixture(tmp_path, provider=provider)
config = json.loads(registry.read_text())
config["goals"][0]["explore_graph"] = {"enabled": True}
registry.write_text(json.dumps(config))
binding = ("--goal-id", GOAL_ID, "--agent-id", AGENT_ID, "--todo-id", TODO_ID,
"--turn-instance-id", TURN_ID)
rc, guard = _run_cli(registry, runtime, "quota", "should-run", "--codex-app", *binding, cwd=project)
assert rc == 0 and guard["should_run"], guard
pair = [{**measurement("baseline", 9), "evidence_ref": "result:baseline"},
{**measurement("alternative", 6), "evidence_ref": "result:alternative"}]
attachment = {**payload()["explore_result"], "measurements": pair,
"evidence_refs": [row["evidence_ref"] for row in pair]}
source = tmp_path / "paired-result.json"
args = ("refresh-state", *binding, "--classification", "validated_change",
"--delivery-batch-scale", "implementation", "--delivery-outcome", "outcome_progress",
"--no-global-sync", "--suppress-external-sinks", "--explore-result-json", str(source))
log = explore_result_log_path(runtime, GOAL_ID)
index = runtime / "goals" / GOAL_ID / "runs" / "index.jsonl"
previous = index.read_bytes() if index.exists() else b""
# Different receipts cannot disguise duplicate candidate observations.
source.write_text(json.dumps({**attachment, "measurements": [pair[0],
{**pair[1], "candidate_ref": pair[0]["candidate_ref"]}]}))
rc, rejected = _run_cli(registry, runtime, *args, cwd=project)
assert rc == 1 and not rejected["appended"], rejected
assert "duplicated" in rejected["error"]
assert (index.read_bytes() if index.exists() else b"") == previous
assert not log.exists()

source.write_text(json.dumps(attachment))
rc, captured = _run_cli(registry, runtime, *args, cwd=project)
assert rc == 0 and captured["appended"] and captured["explore_result_delivery"]["ok"], captured
committed = (index.read_bytes(), log.read_bytes())
rc, replay = _run_cli(registry, runtime, *args, cwd=project)
assert rc == 0 and replay["idempotent_replay"], replay
assert (index.read_bytes(), log.read_bytes()) == committed
rc, received = _run_cli(registry, runtime, "explore", "turn-context", "--goal-id", GOAL_ID,
"--agent-id", AGENT_ID, cwd=project)
assert rc == 0, received
finding = received["graph"]["writeback_results"][0]
assert finding["measurements"] == pair
feedback = received["graph"]["measurement_feedback"][0]
assert feedback["best"]["candidate_ref"] == "candidate:baseline"
assert feedback["latest"]["measurement"]["candidate_ref"] == "candidate:alternative"
assert feedback["improvement"] == -3 and feedback["measured_count"] == 2
assert len(load_explore_result_events_strict(log, goal_id=GOAL_ID)) == 2

# The existing product reader consumes the same persisted finding, not a
# second measurement owner or a synthetic HTTP response.
server = ChatHTTPServer(("127.0.0.1", 0), ChatRequestHandler)
server.registry_path, server.runtime_root = registry, runtime
server.runtime_root_override, server.verbose = None, False
worker = threading.Thread(target=server.serve_forever, daemon=True)
worker.start()
try:
base = f"http://127.0.0.1:{server.server_port}/api/chat/explore-results?goal_id={GOAL_ID}"
for suffix in ("", "&view=history"):
with urlopen(base + suffix) as response:
rows = json.load(response)["items"]
measured = next(row for row in rows if row.get("measurements"))
assert measured["measurements"] == pair
finally:
server.shutdown()
worker.join(timeout=5)
server.server_close()
assert (index.read_bytes(), log.read_bytes()) == committed
source.write_text(json.dumps({**attachment, "measurements": [pair[0], {**pair[1], "value": 10}]}))
rc, conflict = _run_cli(registry, runtime, *args, cwd=project)
assert rc == 1, conflict
assert (index.read_bytes(), log.read_bytes()) == committed


@pytest.mark.parametrize("measured", [False, True])
def test_tied_finding_times_bound_latest_appends_and_corrections(measured):
# The production writer records whole seconds. Equal source times must not
Expand Down
54 changes: 54 additions & 0 deletions tests/control_plane_ts/explore_measurements.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,60 @@ test("quantitative evidence keeps zero, negatives and unknown distinct and rejec
assert.throws(() => normalizeExploreMeasurements([]));
});

test("one finding retains paired candidates without changing their comparison conditions", () => {
const baseline = {...metric, value: 9};
const candidate = {...metric, value: 6, candidate_ref: "candidate:b", evidence_ref: "result:b"};
const pair = [baseline, candidate];
const attachment = {schema_version: "explore_result_attachment_v0", question: "Which candidate improves quality?",
applicability: "Matched inputs, evaluator and resource budget", input_revision: metric.input_revision,
observation: "The matched candidate regressed.", interpretation: "Retain both observations for the next decision.",
status: "refuted", evidence_refs: pair.map(item => item.evidence_ref), measurements: pair};
assert.deepEqual(normalizeExploreResultAttachment({attachment}).measurements, pair);
const {comparison_scope, input_revision, evaluation_ref} = metric;
assert.deepEqual(normalizeExploreMeasurements(pair.map(({comparison_scope: _scope,
input_revision: _input, evaluation_ref: _evaluation, ...row}) => row),
{comparison_scope, input_revision, evaluation_ref}), pair);
for (const direction of [true, false]) {
const measured = pair.map(item => ({...item, higher_is_better: direction}));
const [feedback] = projectExploreMeasurementFeedback([{finding_id: "paired", node_id: "question-a", measurements: measured}]);
assert.equal(feedback.measured_count, 2);
assert.equal(feedback.latest.measurement.candidate_ref, "candidate:b");
assert.equal(feedback.previous_best?.candidate_ref, "candidate:a");
assert.equal(feedback.best?.candidate_ref, direction ? "candidate:a" : "candidate:b");
assert.equal(feedback.improvement, direction ? -3 : 3);
const [reordered] = projectExploreMeasurementFeedback([{finding_id: "paired", node_id: "question-a",
measurements: [...measured].reverse()}]);
assert.deepEqual(reordered.best, feedback.best, "declared batch order cannot change the best score");
assert.equal(reordered.measured_count, 2);
assert.equal(reordered.latest.measurement.candidate_ref, "candidate:a", "latest is the last declared observation, not execution chronology");
}
const [unknown] = projectExploreMeasurementFeedback([{finding_id: "paired", node_id: "question-a",
measurements: [baseline, {...candidate, value: null}]}]);
assert.equal(unknown.best?.value, 9);
assert.equal(unknown.improvement, null);
assert.equal(unknown.measured_count, 1);
assert.equal(unknown.unknown_count, 1);
});

test("paired observations do not weaken duplicate, scope, source or execution checks", () => {
for (const mutation of [{value: 10}, {evidence_ref: "result:another"}, {candidate_ref: " candidate:a "}])
assert.throws(() => normalizeExploreMeasurements([metric, {...metric, ...mutation}]), /duplicated/);
const {candidate_ref: _candidate, ...missingCandidate} = metric;
assert.throws(() => normalizeExploreMeasurements([missingCandidate]), /fields are missing/);
const changedScope = {...metric, comparison_scope: "validation:other-budget"};
assert.equal(normalizeExploreMeasurements([metric, changedScope]).length, 2);
assert.equal(projectExploreMeasurementFeedback([{finding_id: "changed", node_id: "question-a",
measurements: [metric, changedScope]}]).length, 2);
const other = {...metric, candidate_ref: "candidate:b", evidence_ref: "result:b"};
const execution = {schema_version: "explore_execution_observation_v0",
candidate_sha256: "a".repeat(64), result_candidate_sha256: "a".repeat(64),
input_revision: metric.input_revision, planned_method_id: "probe", executed_method_id: "probe",
change_reason: null, elapsed_seconds: 1, limit_seconds: 10, search_iterations: null,
stop_reason: "process_exit:0", evidence_refs: [metric.evidence_ref], result_ref: metric.evidence_ref};
assert.throws(() => normalizeExploreMeasurementsRequest({measurements: [metric, other],
execution_observation: execution}), /executed candidate's result/);
});

test("regression returns the earlier best candidate and useful signed feedback, including lower-is-better", () => {
const input = [finding("new", 6), finding("best", 9), finding("first", 8)];
const before = JSON.stringify(input);
Expand Down
Loading