From 26023a098976ff4e9b3bac27b342111d46247b28 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:47:10 +0800 Subject: [PATCH 1/2] feat(explore): ground work choices in paired goal outcomes Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/capabilities/explore/README.md | 14 ++ .../capabilities/explore_turn_context.ts | 14 +- .../work_items/yardstick_decision.ts | 73 +++++++++-- .../control_plane/test_work_yardstick_cli.py | 120 +++++++++++++++++- .../yardstick_decision.test.ts | 49 +++++++ 5 files changed, 248 insertions(+), 22 deletions(-) diff --git a/loopx/capabilities/explore/README.md b/loopx/capabilities/explore/README.md index 3f8102d8a2..7f6b891dee 100644 --- a/loopx/capabilities/explore/README.md +++ b/loopx/capabilities/explore/README.md @@ -119,6 +119,20 @@ identify the current retained candidate, establish causality, satisfy acceptance or authorize execution. Actual adoption and outcome benefit require subsequent work and evidence; automatic branch rankings remain unchanged. +When an enabled work Yardstick has two different measured candidates on its +declared goal profile, from findings on the same node and Goal/Agent/Todo +binding, Explore context names the observed better-scoring candidate as an +available comparison base and links the scoped evidence. A +tied goal outcome calls for checking headroom. The detail read shows a proxy +ordering only when its exact profile compares the same two candidates, with +separate result references and metric directions for each profile. Weigh +continuation, a distinct mechanism or a compatible combination by expected +end-to-end gain and full implementation and validation cost. Use a bounded +check that can change the choice while honoring the accepted constraints and +budget. The compact cue keeps the recorded next-action reason and adds a +separate comparison cue. The detail read retains the complete work decision. +The comparison is advisory and carries no retention or execution obligation. + Manually captured measured findings also appear in the existing result pages and, when linked to eligible work, its exact-source comparison cards. They need no `writeback-result` origin tag. Read the page before optionally binding a diff --git a/loopx/control_plane/capabilities/explore_turn_context.ts b/loopx/control_plane/capabilities/explore_turn_context.ts index b6a4fa911e..61acb0739f 100644 --- a/loopx/control_plane/capabilities/explore_turn_context.ts +++ b/loopx/control_plane/capabilities/explore_turn_context.ts @@ -358,11 +358,15 @@ export function projectExploreTurnContext(params: JsonObject): JsonObject { target: {summary: "", scope: "", acceptance_ref: ""}, authoritative: {metric: "", unit: "", higher_is_better: true, comparison_scope: "", input_revision: "", evaluation_ref: ""}, proxy: null, counterevidence_refs: [], next_action: {method_id: "", reason: "", cost_bound: ""}, prior_finding_id: null}}, - ...(workCues.length ? {work_yardsticks: workCues.slice(0, 3).map(cue => resultNode !== null ? cue : { - source_finding_id: cue.source_finding_id, node_id: cue.node_id, - todo_id: (cue.binding as JsonObject).todo_id, target: (cue.target as JsonObject).summary, - cue: (cue.next_action as JsonObject).reason, counterevidence_refs: cue.counterevidence_refs, - details_command: resultRead("--result-node", String(cue.node_id)), + ...(workCues.length ? {work_yardsticks: workCues.slice(0, 3).map(cue => { + if (resultNode !== null) return cue; + const comparison = cue.search_comparison as JsonObject | undefined; + return {source_finding_id: cue.source_finding_id, node_id: cue.node_id, + todo_id: (cue.binding as JsonObject).todo_id, target: (cue.target as JsonObject).summary, + cue: (cue.next_action as JsonObject).reason, counterevidence_refs: cue.counterevidence_refs, + ...(comparison ? {comparison_cue: comparison.guidance, + comparison_candidate_ref: (comparison.goal as JsonObject).preferred_candidate_ref} : {}), + details_command: resultRead("--result-node", String(cue.node_id))}; }), omitted_work_yardsticks: Math.max(0, workCues.length - 3), work_yardstick_command: command("turn-context", "--agent-id", agent, "--todo-id", "")} : {}), } : {}), diff --git a/loopx/control_plane/work_items/yardstick_decision.ts b/loopx/control_plane/work_items/yardstick_decision.ts index f236ab6281..8d3c70392c 100644 --- a/loopx/control_plane/work_items/yardstick_decision.ts +++ b/loopx/control_plane/work_items/yardstick_decision.ts @@ -5,6 +5,7 @@ import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; import {requireJsonObject, type JsonObject} from "../runtime_decode.ts"; import {isExploreEvidenceRef} from "../capabilities/explore_evidence_ref.ts"; +import type {ExploreMeasurementFeedback, ExploreMeasurementPoint} from "../capabilities/explore_measurements.ts"; export const WORK_YARDSTICK_DECISION_SCHEMA = "work_yardstick_decision_v0"; export type WorkYardstickProfile = {metric: string; unit: string; higher_is_better: boolean; @@ -77,12 +78,15 @@ export function normalizeWorkYardstickCapture(params: JsonObject): WorkYardstick } return {decision: normalizeWorkYardstickDecision(capture.decision, params.evidence_refs), binding: normalized as WorkYardstickCapture["binding"]}; } +function belongsToWork(row: JsonObject, binding: JsonObject): boolean { + const capture = row.work_yardstick as WorkYardstickCapture | undefined; + const source = capture?.binding; + return !!source && source.goal_id === binding.goal_id && source.agent_id === binding.agent_id + && source.todo_id === binding.todo_id; +} + function currentFinding(findings: JsonObject[], binding: JsonObject): JsonObject | undefined { - return findings.filter(row => { - const capture = row.work_yardstick as JsonObject | undefined; - const source = capture?.binding as JsonObject | undefined; - return source && ["goal_id", "agent_id", "todo_id"].every(key => source[key] === binding[key]); - }).sort((a, b) => { + return findings.filter(row => belongsToWork(row, binding)).sort((a, b) => { // Late delivery reuses the original committed time, not append order. const left = parseTodoTimestampMicros(String(a.last_updated_at ?? "")); const right = parseTodoTimestampMicros(String(b.last_updated_at ?? "")); @@ -107,15 +111,56 @@ export function validateWorkYardstickContinuation(params: JsonObject): JsonObjec } return {decision}; } + +function measuredPair(row: ExploreMeasurementFeedback | undefined) { + const previous = row?.previous_best, latest = row?.latest; + if (!previous || !latest || !Number.isFinite(previous.value)) return null; + const {value, candidate_ref, evidence_ref} = latest.measurement; + if (value === null || !Number.isFinite(value) || candidate_ref === previous.candidate_ref) return null; + const recent: ExploreMeasurementPoint = {finding_id: latest.finding_id, candidate_ref, evidence_ref, value}; + const preferred = value === previous.value ? null : (value > previous.value) === row.higher_is_better + ? candidate_ref : previous.candidate_ref; + return {previous_best: previous, latest: recent, preferred_candidate_ref: preferred}; +} + +function matchesProfile(row: ExploreMeasurementFeedback, profile: WorkYardstickProfile): boolean { + return Object.entries(profile).every(([key, value]) => row[key as keyof WorkYardstickProfile] === value); +} + +function measuredSearchComparison(feedback: ExploreMeasurementFeedback[], decision: WorkYardstickDecision, + scopedFindings: Set) { + const pairFor = (profile: WorkYardstickProfile) => { + const pair = measuredPair(feedback.find(row => matchesProfile(row, profile))); + return pair && scopedFindings.has(pair.previous_best.finding_id) && scopedFindings.has(pair.latest.finding_id) ? pair : null; + }; + const goal = pairFor(decision.authoritative); + if (!goal) return null; + const screen = decision.proxy && pairFor(decision.proxy); + // Both comparisons have two distinct candidates. Order is append/declaration + // order, so join the unordered pair instead of assuming execution chronology. + const goalCandidates = [goal.previous_best.candidate_ref, goal.latest.candidate_ref]; + const proxy = screen && goalCandidates.includes(screen.previous_best.candidate_ref) + && goalCandidates.includes(screen.latest.candidate_ref) ? screen : null; + const reversed = !!(goal.preferred_candidate_ref && proxy?.preferred_candidate_ref + && goal.preferred_candidate_ref !== proxy.preferred_candidate_ref); + const opening = goal.preferred_candidate_ref === null + ? "Observed goal results tie under this scope. Check remaining headroom before another evaluation. " + : (reversed ? "The proxy reverses the observed goal order. " : "") + + "Use the better-scoring candidate under this scope as a comparison base when available. "; + return {goal, ...(proxy ? {proxy, proxy_order_reversed: reversed} : {}), + guidance: opening + "Weigh continuing, a different mechanism or a compatible combination by expected end-to-end gain and full implementation and validation cost. Prefer a bounded check that can change the choice."}; +} + export function projectWorkYardstickCue(params: JsonObject): JsonObject | null { if (params.enabled !== true || !params.todo_id) return null; - const source = currentFinding((params.findings ?? []) as JsonObject[], params); + const findings = (params.findings ?? []) as JsonObject[]; + const source = currentFinding(findings, params); if (!source) return null; const capture = normalizeWorkYardstickCapture({capture: source.work_yardstick, evidence_refs: source.evidence_refs, goal_id: params.goal_id, agent_id: params.agent_id}); - const decision = capture.decision as JsonObject; - const counter = decision.counterevidence_refs as string[]; - const previous = ((params.findings ?? []) as JsonObject[]).find(row => row.finding_id === decision.prior_finding_id); + const decision = capture.decision; + const counter = decision.counterevidence_refs; + const previous = findings.find(row => row.finding_id === decision.prior_finding_id); const priorCapture = previous?.work_yardstick as JsonObject | undefined; const priorBinding = priorCapture?.binding as JsonObject | undefined; const priorDecision = priorCapture?.decision as JsonObject | undefined; @@ -127,11 +172,15 @@ export function projectWorkYardstickCue(params: JsonObject): JsonObject | null { candidate_sha256: execution.candidate_sha256, evidence_refs: execution.evidence_refs, recorded_planned_method_matches: execution.planned_method_id === (priorDecision!.next_action as JsonObject).method_id} : null; - const profiles = [decision.authoritative, decision.proxy].filter(Boolean) as JsonObject[]; - const feedback = ((params.measurement_feedback ?? []) as JsonObject[]).filter(row => row.node_id === source.node_id && profiles.some(p => - Object.entries(p).every(([key, value]) => row[key] === value))); + const profiles = decision.proxy ? [decision.authoritative, decision.proxy] : [decision.authoritative]; + const feedback = ((params.measurement_feedback ?? []) as ExploreMeasurementFeedback[]).filter(row => + row.node_id === source.node_id && profiles.some(p => matchesProfile(row, p))); + const scopedFindings = new Set(findings.filter(row => row.node_id === source.node_id && belongsToWork(row, params)) + .map(row => row.finding_id)); + const searchComparison = measuredSearchComparison(feedback, decision, scopedFindings); return {source_finding_id: source.finding_id, node_id: source.node_id, binding: capture.binding, ...decision, measurement_feedback: feedback, + ...(searchComparison ? {search_comparison: searchComparison} : {}), ...(followthrough ? {previous_decision_followthrough: followthrough} : {}), guidance: counter.length ? "Counterevidence is recorded for this scoped proxy. Reassess its use at the next concrete choice; a local screening heuristic cannot veto an informative authoritative check by itself. Preserve accepted correctness, authorization and budget constraints." diff --git a/tests/control_plane/test_work_yardstick_cli.py b/tests/control_plane/test_work_yardstick_cli.py index 8868d472e9..378c84fe71 100644 --- a/tests/control_plane/test_work_yardstick_cli.py +++ b/tests/control_plane/test_work_yardstick_cli.py @@ -4,11 +4,12 @@ or a causal benchmark gain. No evaluator, credentials or active Goal are used. """ import hashlib +from itertools import groupby import json from tests.control_plane.test_quota_settlement_cli import ( - _write_fixture, _run_cli, GOAL_ID, AGENT_ID, TODO_ID, + _initialize_fixture_authority, _projected_cli_args, _write_fixture, _run_cli, GOAL_ID, AGENT_ID, TODO_ID, ) from loopx.capabilities.explore.result_log import explore_result_log_path, load_explore_result_events from loopx.capabilities.explore.turn_context import explore_turn_context @@ -35,13 +36,18 @@ def attachment(): evidence_refs=["result:reversal"], work_yardstick_decision=decision()) -def setup(tmp_path, enabled=True): - project, runtime, path = _write_fixture(tmp_path, provider="sqlite") +def setup(tmp_path, enabled=True, *, active=False): + project, runtime, path = _write_fixture(tmp_path) config = json.loads(path.read_text()) + if active: + config["goals"][0]["status"] = "active" + state = project / config["goals"][0]["state_file"] + state.write_text(state.read_text().replace("status: active-read-only", "status: active", 1)) config["goals"][0]["explore_graph"] = {"enabled": True} config["goals"][0]["quota"]["allowed_slots"] = 4 config["goals"][0].setdefault("execution_profile", {})["yardstick_enabled"] = enabled path.write_text(json.dumps(config)) + _initialize_fixture_authority(project, runtime, provider="sqlite") return project, runtime, path @@ -55,11 +61,11 @@ def binding(turn): return ["--goal-id", GOAL_ID, "--agent-id", AGENT_ID, "--todo-id", TODO_ID, "--turn-instance-id", turn] -def refresh(project, runtime, path, turn, source): +def refresh(project, runtime, path, turn, source, *options): run(project, runtime, path, "quota", "should-run", "--codex-app", *binding(turn)) return run(project, runtime, path, "refresh-state", *binding(turn), "--classification", "validated_change", "--delivery-batch-scale", "implementation", "--delivery-outcome", "outcome_progress", - "--explore-result-json", str(source), "--no-global-sync", "--suppress-external-sinks") + "--explore-result-json", str(source), "--no-global-sync", "--suppress-external-sinks", *options) def context(runtime, path, *, detail=True): @@ -71,6 +77,110 @@ def context(runtime, path, *, detail=True): return compact +def test_measured_yardstick_changes_next_work_from_local_screening_to_a_whole_goal_comparison(tmp_path): + project, runtime, path = setup(tmp_path, active=True) + acceptance = tmp_path / "acceptance.json" + acceptance.write_text(json.dumps(dict(outcome_id="encoded-artifact", owner_agent_id=AGENT_ID, + objective="Reduce the encoded artifact size", todo_ids=[TODO_ID], + requirements=[{"id": "round-trip", "description": "Decoding must preserve every original entry in order"}], + source={"ref": "message:encoding-request", "summary": "Preserve the complete input"}))) + before = run(project, runtime, path, "goal-acceptance", "inspect", "--goal-id", GOAL_ID) + run(project, runtime, path, "goal-acceptance", "refine", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--document", str(acceptance), "--operation-id", "declare-encoding", "--expected-provider-revision", + before["provider_revision"], "--execute") + run(project, runtime, path, "configure-goal", "--goal-id", GOAL_ID, "--execution-replan-after-turns", "2", + "--execution-stride-strategy", "evidence_conditioned_v1", "--execute") + payload = ["repeated-entry"] * 24 + ["other-entry"] * 24 + + def encode(value): + return json.dumps(value, separators=(",", ":")).encode() + + def dictionary_encode(items): + keys = list(dict.fromkeys(items)) + return {"keys": keys, "indices": [keys.index(item) for item in items]} + + base = dictionary_encode(payload) + trial = attachment() + choice = trial["work_yardstick_decision"] + choice["authoritative"].update(metric="size", unit="bytes", higher_is_better=False) + choice["proxy"].update(metric="size", unit="bytes", higher_is_better=False, comparison_scope="validation:single-entry") + choice["counterevidence_refs"] = [] + choice["next_action"] = dict(method_id="method:plain", reason="Try another plain encoding using the tiny screen", + cost_bound="One artifact and one bounded end-to-end check") + items = [(choice["authoritative"], payload, "goal"), (choice["proxy"], payload[:1], "screen")] + trial["measurements"] = [ + {**profile, "candidate_ref": f"candidate:{name}", "evidence_ref": f"result:{source}-{name}", + "value": len(encode(candidate))} + for profile, data, source in items + for name, candidate in [("dictionary", dictionary_encode(data)), ("plain", data)] + ] + assert trial["measurements"][0]["value"] < trial["measurements"][1]["value"] + assert trial["measurements"][2]["value"] > trial["measurements"][3]["value"] + trial["evidence_refs"] = [row["evidence_ref"] for row in trial["measurements"]] + source = tmp_path / "measured-pair.json" + source.write_text(json.dumps(trial)) + first = refresh(project, runtime, path, "turn-pair", source, "--delivery-boundary", "in_flight_continuation") + assert first["explore_result_delivery"]["ok"] + run(project, runtime, path, *_projected_cli_args(first["settlement_owed"]["command"], turn_instance_id="turn-pair")) + + guard = run(project, runtime, path, "quota", "should-run", "--codex-app", *binding("turn-compare")) + compact = run(project, runtime, path, "explore", "turn-context", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--todo-id", TODO_ID)["graph"]["work_yardsticks"][0] + assert compact["comparison_candidate_ref"] == "candidate:dictionary" + assert compact["cue"] == choice["next_action"]["reason"] + assert "different mechanism or a compatible combination" in compact["comparison_cue"] + assert "full implementation and validation cost" in compact["comparison_cue"] + assert compact["comparison_cue"] in json.dumps(guard) + detailed = run(project, runtime, path, "explore", "turn-context", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--todo-id", TODO_ID, "--result-node", compact["node_id"])["graph"]["work_yardsticks"][0] + assert detailed["next_action"] == choice["next_action"] + comparison = detailed["search_comparison"] + assert comparison["goal"]["preferred_candidate_ref"] == "candidate:dictionary" + assert comparison["proxy"]["preferred_candidate_ref"] == "candidate:plain" + assert comparison["proxy_order_reversed"] is True + + # Deterministic consumer. Combine the evidenced baseline with an independent + # representation change and verify the original end-to-end outcome. + selected_base = {"candidate:dictionary": base}[compact["comparison_candidate_ref"]] + runs = [[index, len(list(group))] for index, group in groupby(selected_base["indices"])] + artifact = project / "combined-encoding.json" + artifact.write_bytes(encode({"keys": selected_base["keys"], "runs": runs})) + result = json.loads(artifact.read_bytes()) + assert [result["keys"][i] for i, count in result["runs"] for _ in range(count)] == payload + assert artifact.stat().st_size < len(encode(base)) + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + follow = attachment() + follow["evidence_refs"] = ["result:combined"] + follow["work_yardstick_decision"] = {**choice, "prior_finding_id": detailed["source_finding_id"]} + follow["execution_observation"] = dict(schema_version="explore_execution_observation_v0", candidate_sha256=digest, + input_revision="inputs-v1", planned_method_id="method:plain", executed_method_id="method:dictionary-runs", + change_reason="The matched outcome favored the dictionary; test a different representation on that base", + elapsed_seconds=None, limit_seconds=None, search_iterations=1, stop_reason="complete", + evidence_refs=["result:combined"], result_ref="result:combined", result_candidate_sha256=digest, exit_code=0) + follow["measurements"] = [{**choice["authoritative"], "value": artifact.stat().st_size, + "candidate_ref": f"sha256:{digest}", "evidence_ref": "result:combined"}] + source.write_text(json.dumps(follow)) + saved = run(project, runtime, path, "refresh-state", *binding("turn-compare"), "--classification", "validated_change", + "--delivery-batch-scale", "implementation", "--delivery-outcome", "outcome_progress", + "--delivery-boundary", "in_flight_continuation", "--explore-result-json", str(source), "--no-global-sync", "--suppress-external-sinks") + assert saved["explore_result_delivery"]["ok"] + cue = context(runtime, path)["graph"]["work_yardsticks"][0] + assert cue["previous_decision_followthrough"]["executed_method_id"] == "method:dictionary-runs" + assert cue["search_comparison"]["goal"]["preferred_candidate_ref"] == f"sha256:{digest}" + assert "proxy" not in cue["search_comparison"] # The old proxy did not measure this new pair. + run(project, runtime, path, *_projected_cli_args(saved["settlement_owed"]["command"], turn_instance_id="turn-compare")) + review = run(project, runtime, path, "quota", "should-run", "--codex-app", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, "--turn-instance-id", "turn-review") + assert "periodic_review_due" in {trigger["kind"] for trigger in review["autonomous_replan_obligation"]["triggers"]} + assert cue["search_comparison"]["guidance"] in json.dumps(review) + assert "comparison_candidate_ref" in json.dumps(review) + before = context(runtime, path, detail=False)["graph"]["measurement_feedback"] + run(project, runtime, path, "configure-goal", "--goal-id", GOAL_ID, "--no-execution-yardstick-enabled", "--execute") + disabled = run(project, runtime, path, "explore", "turn-context", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--todo-id", TODO_ID)["graph"] + assert "work_yardsticks" not in disabled and disabled["measurement_feedback"] == before + + def test_sqlite_current_work_to_ordinary_turn_to_concrete_action_and_readback(tmp_path): project, runtime, path = setup(tmp_path) source = tmp_path / "capture.json" diff --git a/tests/control_plane_ts/yardstick_decision.test.ts b/tests/control_plane_ts/yardstick_decision.test.ts index f63a665c61..ce2d25c7af 100644 --- a/tests/control_plane_ts/yardstick_decision.test.ts +++ b/tests/control_plane_ts/yardstick_decision.test.ts @@ -4,6 +4,7 @@ import {normalizeWorkYardstickDecision, normalizeWorkYardstickCapture, validateWorkYardstickContinuation, projectWorkYardstickCue} from "../../loopx/control_plane/work_items/yardstick_decision.ts"; import {normalizeExploreResultAttachment} from "../../loopx/control_plane/capabilities/explore_result_writeback.ts"; import {projectExploreTurnContext} from "../../loopx/control_plane/capabilities/explore_turn_context.ts"; +import {projectExploreMeasurementFeedback} from "../../loopx/control_plane/capabilities/explore_measurements.ts"; const authoritative = {metric: "quality", unit: "points", higher_is_better: true, comparison_scope: "validation:fixed", input_revision: "inputs-v1", evaluation_ref: "evaluator:outcome-v1"}; @@ -53,6 +54,54 @@ test("a late replay cannot replace a newer committed choice; exact profiles sele assert.equal(Object.hasOwn(cue, "fidelity_validated"), false); }); +test("goal and proxy join the same two candidates regardless of declared order and metric direction", () => { + const timeProxy = {...proxy, higher_is_better: false, unit: "seconds"}; + const measured = {...finding, work_yardstick: {binding, decision: {...decision, proxy: timeProxy}}, measurements: [ + {...authoritative, value: 0, candidate_ref: "candidate:a", evidence_ref: "result:goal-a"}, + {...authoritative, value: 3, candidate_ref: "candidate:b", evidence_ref: "result:goal-b"}, + {...timeProxy, value: 4, candidate_ref: "candidate:b", evidence_ref: "result:screen-b"}, + {...timeProxy, value: 2, candidate_ref: "candidate:a", evidence_ref: "result:screen-a"}, + ]}; + const project = (row = measured) => projectWorkYardstickCue({...binding, enabled: true, findings: [row], + measurement_feedback: projectExploreMeasurementFeedback([row])})!.search_comparison; + const result = project(); + assert.equal(result.goal.preferred_candidate_ref, "candidate:b"); + assert.equal(result.goal.previous_best.value, 0); + assert.equal(result.proxy.preferred_candidate_ref, "candidate:a"); + assert.equal(result.proxy.latest.evidence_ref, "result:screen-a"); + assert.equal(result.proxy_order_reversed, true); + assert.match(result.guidance, /comparison base when available/); + assert.match(result.guidance, /continuing, a different mechanism or a compatible combination/); + + const observations = (changeIndex: number, changed: object) => ({...measured, + measurements: measured.measurements.map((row, index) => index === changeIndex ? {...row, ...changed} : row)}); + assert.equal(project(observations(1, {value: null})), undefined); + assert.equal(project(observations(1, {evaluation_ref: "evaluator:other"})), undefined); + assert.equal(project(observations(2, {candidate_ref: "candidate:unrelated"})).proxy, undefined); + assert.equal(project(observations(2, {comparison_scope: "validation:other"})).proxy, undefined); + assert.equal(project(observations(2, {input_revision: "other-inputs"})).proxy, undefined); + + const cap = project({...measured, measurements: measured.measurements.map((row, index) => + index < 2 ? {...row, value: 10} : row)}); + assert.equal(cap.goal.preferred_candidate_ref, null); + assert.equal(cap.proxy_order_reversed, false); + assert.match(cap.guidance, /goal results tie under this scope/); + const repeat = {...measured, finding_id: "later-retest", recorded_order: 2, + measurements: [{...measured.measurements[1]!, value: 3}]}; + const sameCandidate = projectWorkYardstickCue({...binding, enabled: true, findings: [repeat, measured], + measurement_feedback: projectExploreMeasurementFeedback([repeat, measured])})!; + assert.equal(sameCandidate.search_comparison, undefined); + + for (const other of [{...binding, goal_id: "other"}, {...binding, agent_id: "peer"}, {...binding, todo_id: "todo_other"}]) { + const foreign = {...repeat, work_yardstick: {...repeat.work_yardstick, binding: other}, + measurements: [{...repeat.measurements[0]!, candidate_ref: "candidate:foreign", value: 99}]}; + const shared = projectWorkYardstickCue({...binding, enabled: true, findings: [foreign, measured], + measurement_feedback: projectExploreMeasurementFeedback([foreign, measured])})!; + assert.equal(shared.source_finding_id, measured.finding_id); + assert.equal(shared.search_comparison, undefined); + } +}); + test("only strictly older committed replay preserves a stale predecessor as history", () => { const params = {enabled: true, decision, binding, findings: [finding]}; assert.deepEqual(validateWorkYardstickContinuation({...params, From f7679ebd13123fb69130701f294c3d62288773ae Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:37:18 +0800 Subject: [PATCH 2/2] fix(explore): rank Yardstick evidence within the current acceptance basis Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/capabilities/explore/README.md | 3 ++ .../work_items/yardstick_decision.ts | 23 +++++++----- .../control_plane/test_work_yardstick_cli.py | 37 +++++++++++++++++++ .../yardstick_decision.test.ts | 2 +- 4 files changed, 54 insertions(+), 11 deletions(-) diff --git a/loopx/capabilities/explore/README.md b/loopx/capabilities/explore/README.md index 7f6b891dee..735cb9b76c 100644 --- a/loopx/capabilities/explore/README.md +++ b/loopx/capabilities/explore/README.md @@ -131,6 +131,9 @@ end-to-end gain and full implementation and validation cost. Use a bounded check that can change the choice while honoring the accepted constraints and budget. The compact cue keeps the recorded next-action reason and adds a separate comparison cue. The detail read retains the complete work decision. +The comparison uses findings from the current work, target, and acceptance +reference. A different target or acceptance reference needs its own eligible +pair, even when the measured metric, evaluator and inputs stay the same. The comparison is advisory and carries no retention or execution obligation. Manually captured measured findings also appear in the existing result pages diff --git a/loopx/control_plane/work_items/yardstick_decision.ts b/loopx/control_plane/work_items/yardstick_decision.ts index 8d3c70392c..b27d24953a 100644 --- a/loopx/control_plane/work_items/yardstick_decision.ts +++ b/loopx/control_plane/work_items/yardstick_decision.ts @@ -5,7 +5,8 @@ import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; import {requireJsonObject, type JsonObject} from "../runtime_decode.ts"; import {isExploreEvidenceRef} from "../capabilities/explore_evidence_ref.ts"; -import type {ExploreMeasurementFeedback, ExploreMeasurementPoint} from "../capabilities/explore_measurements.ts"; +import {projectExploreMeasurementFeedback, type ExploreMeasurementFeedback, + type ExploreMeasurementPoint} from "../capabilities/explore_measurements.ts"; export const WORK_YARDSTICK_DECISION_SCHEMA = "work_yardstick_decision_v0"; export type WorkYardstickProfile = {metric: string; unit: string; higher_is_better: boolean; @@ -127,12 +128,8 @@ function matchesProfile(row: ExploreMeasurementFeedback, profile: WorkYardstickP return Object.entries(profile).every(([key, value]) => row[key as keyof WorkYardstickProfile] === value); } -function measuredSearchComparison(feedback: ExploreMeasurementFeedback[], decision: WorkYardstickDecision, - scopedFindings: Set) { - const pairFor = (profile: WorkYardstickProfile) => { - const pair = measuredPair(feedback.find(row => matchesProfile(row, profile))); - return pair && scopedFindings.has(pair.previous_best.finding_id) && scopedFindings.has(pair.latest.finding_id) ? pair : null; - }; +function measuredSearchComparison(feedback: ExploreMeasurementFeedback[], decision: WorkYardstickDecision) { + const pairFor = (profile: WorkYardstickProfile) => measuredPair(feedback.find(row => matchesProfile(row, profile))); const goal = pairFor(decision.authoritative); if (!goal) return null; const screen = decision.proxy && pairFor(decision.proxy); @@ -175,9 +172,15 @@ export function projectWorkYardstickCue(params: JsonObject): JsonObject | null { const profiles = decision.proxy ? [decision.authoritative, decision.proxy] : [decision.authoritative]; const feedback = ((params.measurement_feedback ?? []) as ExploreMeasurementFeedback[]).filter(row => row.node_id === source.node_id && profiles.some(p => matchesProfile(row, p))); - const scopedFindings = new Set(findings.filter(row => row.node_id === source.node_id && belongsToWork(row, params)) - .map(row => row.finding_id)); - const searchComparison = measuredSearchComparison(feedback, decision, scopedFindings); + // An earlier target can share the same evaluator profile and work identity. + // Filter source findings before ranking so its best cannot displace a legal pair. + const scopedFindings = findings.filter(row => { + const target = (row.work_yardstick as WorkYardstickCapture | undefined)?.decision?.target; + return row.node_id === source.node_id && belongsToWork(row, params) && target !== undefined + && target.summary === decision.target.summary && target.scope === decision.target.scope + && target.acceptance_ref === decision.target.acceptance_ref; + }); + const searchComparison = measuredSearchComparison(projectExploreMeasurementFeedback(scopedFindings), decision); return {source_finding_id: source.finding_id, node_id: source.node_id, binding: capture.binding, ...decision, measurement_feedback: feedback, ...(searchComparison ? {search_comparison: searchComparison} : {}), diff --git a/tests/control_plane/test_work_yardstick_cli.py b/tests/control_plane/test_work_yardstick_cli.py index 378c84fe71..b25cc101d5 100644 --- a/tests/control_plane/test_work_yardstick_cli.py +++ b/tests/control_plane/test_work_yardstick_cli.py @@ -77,6 +77,43 @@ def context(runtime, path, *, detail=True): return compact +def test_measured_yardstick_restarts_comparison_when_the_acceptance_basis_changes(tmp_path): + project, runtime, path = setup(tmp_path, active=True) + source = tmp_path / "finding.json" + + def record(turn, acceptance, candidates, previous=None): + finding = attachment() + choice = finding["work_yardstick_decision"] + choice["target"]["acceptance_ref"] = acceptance + choice["counterevidence_refs"] = [] + choice["prior_finding_id"] = previous + finding["measurements"] = [ + {**choice["authoritative"], "candidate_ref": f"candidate:{name}", + "evidence_ref": f"result:{name}", "value": value} + for name, value in candidates + ] + finding["evidence_refs"] = [measurement["evidence_ref"] for measurement in finding["measurements"]] + source.write_text(json.dumps(finding)) + receipt = refresh(project, runtime, path, turn, source, "--delivery-boundary", "in_flight_continuation") + assert receipt["explore_result_delivery"]["ok"], receipt + run(project, runtime, path, *_projected_cli_args(receipt["settlement_owed"]["command"], turn_instance_id=turn)) + return receipt["explore_result_delivery"]["finding_id"] + + old = record("turn-old-basis", "acceptance:task-v1", [("old-a", 10), ("old-b", 20)]) + assert context(runtime, path)["graph"]["work_yardsticks"][0]["search_comparison"]["goal"][ + "preferred_candidate_ref"] == "candidate:old-b" + first = record("turn-first-new-basis", "acceptance:task-v2", [("new-c", 6)], old) + compact = context(runtime, path, detail=False)["graph"]["work_yardsticks"][0] + assert compact["source_finding_id"] == first and "comparison_candidate_ref" not in compact + assert "search_comparison" not in context(runtime, path)["graph"]["work_yardsticks"][0] + latest = record("turn-second-new-basis", "acceptance:task-v2", [("new-d", 9)], first) + cue = context(runtime, path)["graph"]["work_yardsticks"][0] + comparison = cue["search_comparison"]["goal"] + assert cue["target"]["acceptance_ref"] == "acceptance:task-v2" + assert comparison["previous_best"]["finding_id"] == first and comparison["latest"]["finding_id"] == latest + assert comparison["preferred_candidate_ref"] == "candidate:new-d" + + def test_measured_yardstick_changes_next_work_from_local_screening_to_a_whole_goal_comparison(tmp_path): project, runtime, path = setup(tmp_path, active=True) acceptance = tmp_path / "acceptance.json" diff --git a/tests/control_plane_ts/yardstick_decision.test.ts b/tests/control_plane_ts/yardstick_decision.test.ts index ce2d25c7af..2b406e6e08 100644 --- a/tests/control_plane_ts/yardstick_decision.test.ts +++ b/tests/control_plane_ts/yardstick_decision.test.ts @@ -98,7 +98,7 @@ test("goal and proxy join the same two candidates regardless of declared order a const shared = projectWorkYardstickCue({...binding, enabled: true, findings: [foreign, measured], measurement_feedback: projectExploreMeasurementFeedback([foreign, measured])})!; assert.equal(shared.source_finding_id, measured.finding_id); - assert.equal(shared.search_comparison, undefined); + assert.deepEqual(shared.search_comparison, result); } });