From f3ec9eb99c0884f0e849ee0fed3729405554978a Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:59:17 +0800 Subject: [PATCH] fix(workspace): distinguish real native benchmark results Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../fixtures/edgebench-ad-native-results.json | 1217 +++++++++++++++++ .../edgebench-graph-saved-analysis.json | 52 +- .../edgebench-portfolio-native-results.json | 1217 +++++++++++++++++ .../goal-research-progress.tsx | 15 +- .../research-history.md | 59 +- .../research-real-benchmark.mjs | 107 +- .../test_research_history_fixture.py | 56 +- 7 files changed, 2698 insertions(+), 25 deletions(-) create mode 100644 apps/presentation/dashboard/src/data/fixtures/edgebench-ad-native-results.json create mode 100644 apps/presentation/dashboard/src/data/fixtures/edgebench-portfolio-native-results.json diff --git a/apps/presentation/dashboard/src/data/fixtures/edgebench-ad-native-results.json b/apps/presentation/dashboard/src/data/fixtures/edgebench-ad-native-results.json new file mode 100644 index 0000000000..d0e8d523de --- /dev/null +++ b/apps/presentation/dashboard/src/data/fixtures/edgebench-ad-native-results.json @@ -0,0 +1,1217 @@ +{ + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "display_name": "Ad placement · captured native results", + "events": [], + "provenance": { + "kind": "captured_read_only_analysis_api", + "captured_at": "2026-11-04T08:46:22.286770+00:00", + "source_run_id": "ad-official-native3600-integration-20261011-04", + "limitations": [ + "Historical 11–12 October native run from an existing formally registered read-only analysis Goal; not the recent history fixture or a new solver run.", + "The original, Chinese and English analysis API responses and native citation responses are unchanged.", + "Six registered episodes have no saved analysis job or result. No model judgment or analysis usage is synthesized.", + "Initial, best and final native results are measured; retained result and adopted improvement remain unknown.", + "The selected analysis Goal has no Explore records. No solver history or relationship is reconstructed.", + "No authority store, execution state, raw session transcript, model process or solver is restored by this display fixture." + ] + }, + "analysis_readbacks": { + "original": { + "request_path": "/api/chat/research-analysis?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06", + "response_sha256": "56670c1183da44d227be9d15088698baca8bfe800a9bc87ac0e715f072f58c69", + "response": { + "ok": true, + "catalog": { + "ok": true, + "availability": "available", + "reason_code": null, + "default_pack_id": "c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06", + "packs": [ + { + "pack_id": "c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06", + "title": "Ad Official · 18h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "6eaeaa2eed3cae8d30785b60129d72eae3a7dfaa510e38393a8313be514980a5", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/ad/native-outcomes.json", + "artifact_sha256": "7fe357894002389316d493d98d9bc4c642709da42299160840e64232d866ca26", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:50.507Z", + "end": "2026-10-12T06:28:29.005Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 16, + "valid_count": 16, + "initial": { + "submission_index": 1, + "result_id": "0f01f973558f", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:53:51.989Z", + "observed_at": "2026-10-11T12:56:06.845Z", + "value": 49550815819, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 13, + "result_id": "80bee97892b5", + "candidate_ref": "agent-13", + "submitted_at": "2026-10-12T01:34:30.830Z", + "observed_at": "2026-10-12T01:36:57.526Z", + "value": 49653326363, + "citation": { + "source_id": "released-native-inventory", + "first_line": 13, + "last_line": 13 + } + }, + "retained": null, + "final_valid": { + "submission_index": 16, + "result_id": "9be1f24ecc8b", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T05:36:53.759Z", + "observed_at": "2026-10-12T05:39:34.830Z", + "value": 49650446168, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + } + ] + }, + "pack": { + "ok": true, + "pack_id": "c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06", + "title": "Ad Official · 18h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "6eaeaa2eed3cae8d30785b60129d72eae3a7dfaa510e38393a8313be514980a5", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/ad/native-outcomes.json", + "artifact_sha256": "7fe357894002389316d493d98d9bc4c642709da42299160840e64232d866ca26", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:50.507Z", + "end": "2026-10-12T06:28:29.005Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 16, + "valid_count": 16, + "initial": { + "submission_index": 1, + "result_id": "0f01f973558f", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:53:51.989Z", + "observed_at": "2026-10-11T12:56:06.845Z", + "value": 49550815819, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 13, + "result_id": "80bee97892b5", + "candidate_ref": "agent-13", + "submitted_at": "2026-10-12T01:34:30.830Z", + "observed_at": "2026-10-12T01:36:57.526Z", + "value": 49653326363, + "citation": { + "source_id": "released-native-inventory", + "first_line": 13, + "last_line": 13 + } + }, + "retained": null, + "final_valid": { + "submission_index": 16, + "result_id": "9be1f24ecc8b", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T05:36:53.759Z", + "observed_at": "2026-10-12T05:39:34.830Z", + "value": 49650446168, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + }, + "objective": "分析原生 Official 研究如何推进原任务，区分事实、模型解释和独立验证。原始任务见 original-objective。此 Goal 是对真实原生轨迹的新分析生命周期；历史 Official 没有 LoopX 执行 Goal。保留历史 score_countable=false，只作诊断参考，不能和新恢复组作因果效果比较。原生结果及时间由原文件提供；未验证最终保留候选、贡献归因或货币成本，不可用 best 代替 retained。数值过程指标仅对声明样本适用。", + "episodes": [ + { + "id": "native-feedback-1", + "question": "原生提交 agent-1 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-1", + "original-feedback-1", + "original-message-344", + "original-message-378", + "original-message-395", + "original-message-441" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-4", + "question": "原生提交 agent-4 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-4", + "original-feedback-4", + "original-message-2387", + "original-message-2432", + "original-message-2483", + "original-message-2505" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-7", + "question": "原生提交 agent-7 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-7", + "original-feedback-7", + "original-message-4393", + "original-message-4403", + "original-message-4437", + "original-message-4460" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-10", + "question": "原生提交 agent-10 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-10", + "original-feedback-10", + "original-message-7387", + "original-message-7405", + "original-message-7430", + "original-message-7447" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-13", + "question": "原生提交 agent-13 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-13", + "original-feedback-13", + "original-message-10489", + "original-message-10513", + "original-message-10523", + "original-message-10546" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-16", + "question": "原生提交 agent-16 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-16", + "original-feedback-16", + "original-message-14485", + "original-message-14504", + "original-message-14524", + "original-message-14540" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + } + ], + "job": null, + "result": null, + "applicability": "current" + }, + "execution_enabled": false + } + }, + "zh": { + "request_path": "/api/chat/research-analysis?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06&output_language=zh", + "response_sha256": "08d1f35b52f9b58a867e104591a55f885e9c1076a510b9ca92d3e022785e9b6f", + "response": { + "ok": true, + "catalog": { + "ok": true, + "availability": "available", + "reason_code": null, + "default_pack_id": "c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06", + "packs": [ + { + "pack_id": "c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06", + "title": "Ad Official · 18h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "bb5b6a4169f8f541f51b391fd07a2209790741702ce5f5c8ea09178480e9a161", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "zh" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/ad/native-outcomes.json", + "artifact_sha256": "7fe357894002389316d493d98d9bc4c642709da42299160840e64232d866ca26", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:50.507Z", + "end": "2026-10-12T06:28:29.005Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 16, + "valid_count": 16, + "initial": { + "submission_index": 1, + "result_id": "0f01f973558f", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:53:51.989Z", + "observed_at": "2026-10-11T12:56:06.845Z", + "value": 49550815819, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 13, + "result_id": "80bee97892b5", + "candidate_ref": "agent-13", + "submitted_at": "2026-10-12T01:34:30.830Z", + "observed_at": "2026-10-12T01:36:57.526Z", + "value": 49653326363, + "citation": { + "source_id": "released-native-inventory", + "first_line": 13, + "last_line": 13 + } + }, + "retained": null, + "final_valid": { + "submission_index": 16, + "result_id": "9be1f24ecc8b", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T05:36:53.759Z", + "observed_at": "2026-10-12T05:39:34.830Z", + "value": 49650446168, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + } + ] + }, + "pack": { + "ok": true, + "pack_id": "c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06", + "title": "Ad Official · 18h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "bb5b6a4169f8f541f51b391fd07a2209790741702ce5f5c8ea09178480e9a161", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "zh" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/ad/native-outcomes.json", + "artifact_sha256": "7fe357894002389316d493d98d9bc4c642709da42299160840e64232d866ca26", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:50.507Z", + "end": "2026-10-12T06:28:29.005Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 16, + "valid_count": 16, + "initial": { + "submission_index": 1, + "result_id": "0f01f973558f", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:53:51.989Z", + "observed_at": "2026-10-11T12:56:06.845Z", + "value": 49550815819, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 13, + "result_id": "80bee97892b5", + "candidate_ref": "agent-13", + "submitted_at": "2026-10-12T01:34:30.830Z", + "observed_at": "2026-10-12T01:36:57.526Z", + "value": 49653326363, + "citation": { + "source_id": "released-native-inventory", + "first_line": 13, + "last_line": 13 + } + }, + "retained": null, + "final_valid": { + "submission_index": 16, + "result_id": "9be1f24ecc8b", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T05:36:53.759Z", + "observed_at": "2026-10-12T05:39:34.830Z", + "value": 49650446168, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + }, + "objective": "分析原生 Official 研究如何推进原任务，区分事实、模型解释和独立验证。原始任务见 original-objective。此 Goal 是对真实原生轨迹的新分析生命周期；历史 Official 没有 LoopX 执行 Goal。保留历史 score_countable=false，只作诊断参考，不能和新恢复组作因果效果比较。原生结果及时间由原文件提供；未验证最终保留候选、贡献归因或货币成本，不可用 best 代替 retained。数值过程指标仅对声明样本适用。", + "episodes": [ + { + "id": "native-feedback-1", + "question": "原生提交 agent-1 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-1", + "original-feedback-1", + "original-message-344", + "original-message-378", + "original-message-395", + "original-message-441" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-4", + "question": "原生提交 agent-4 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-4", + "original-feedback-4", + "original-message-2387", + "original-message-2432", + "original-message-2483", + "original-message-2505" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-7", + "question": "原生提交 agent-7 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-7", + "original-feedback-7", + "original-message-4393", + "original-message-4403", + "original-message-4437", + "original-message-4460" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-10", + "question": "原生提交 agent-10 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-10", + "original-feedback-10", + "original-message-7387", + "original-message-7405", + "original-message-7430", + "original-message-7447" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-13", + "question": "原生提交 agent-13 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-13", + "original-feedback-13", + "original-message-10489", + "original-message-10513", + "original-message-10523", + "original-message-10546" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-16", + "question": "原生提交 agent-16 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-16", + "original-feedback-16", + "original-message-14485", + "original-message-14504", + "original-message-14524", + "original-message-14540" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + } + ], + "job": null, + "result": null, + "applicability": "current" + }, + "execution_enabled": false + } + }, + "en": { + "request_path": "/api/chat/research-analysis?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06&output_language=en", + "response_sha256": "d145136ee17d870ffbbcf89e45516b74302f71883fd82be92462f62faad263e7", + "response": { + "ok": true, + "catalog": { + "ok": true, + "availability": "available", + "reason_code": null, + "default_pack_id": "c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06", + "packs": [ + { + "pack_id": "c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06", + "title": "Ad Official · 18h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "b47bda9123569d535da8fe09002f817da8c8844bf6367e65fb2eecd51f4f9d1f", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "en" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/ad/native-outcomes.json", + "artifact_sha256": "7fe357894002389316d493d98d9bc4c642709da42299160840e64232d866ca26", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:50.507Z", + "end": "2026-10-12T06:28:29.005Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 16, + "valid_count": 16, + "initial": { + "submission_index": 1, + "result_id": "0f01f973558f", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:53:51.989Z", + "observed_at": "2026-10-11T12:56:06.845Z", + "value": 49550815819, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 13, + "result_id": "80bee97892b5", + "candidate_ref": "agent-13", + "submitted_at": "2026-10-12T01:34:30.830Z", + "observed_at": "2026-10-12T01:36:57.526Z", + "value": 49653326363, + "citation": { + "source_id": "released-native-inventory", + "first_line": 13, + "last_line": 13 + } + }, + "retained": null, + "final_valid": { + "submission_index": 16, + "result_id": "9be1f24ecc8b", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T05:36:53.759Z", + "observed_at": "2026-10-12T05:39:34.830Z", + "value": 49650446168, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + } + ] + }, + "pack": { + "ok": true, + "pack_id": "c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06", + "title": "Ad Official · 18h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "b47bda9123569d535da8fe09002f817da8c8844bf6367e65fb2eecd51f4f9d1f", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "en" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/ad/native-outcomes.json", + "artifact_sha256": "7fe357894002389316d493d98d9bc4c642709da42299160840e64232d866ca26", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-ad-analysis-20261011-04", + "case_id": "ad_placement_optimization", + "run_id": "ad-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4", + "window": "Original captured native session 2026-10-11T12:28:50.507Z to 2026-10-12T06:28:29.005Z; declared solver budget 18.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:50.507Z", + "end": "2026-10-12T06:28:29.005Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 16, + "valid_count": 16, + "initial": { + "submission_index": 1, + "result_id": "0f01f973558f", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:53:51.989Z", + "observed_at": "2026-10-11T12:56:06.845Z", + "value": 49550815819, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 13, + "result_id": "80bee97892b5", + "candidate_ref": "agent-13", + "submitted_at": "2026-10-12T01:34:30.830Z", + "observed_at": "2026-10-12T01:36:57.526Z", + "value": 49653326363, + "citation": { + "source_id": "released-native-inventory", + "first_line": 13, + "last_line": 13 + } + }, + "retained": null, + "final_valid": { + "submission_index": 16, + "result_id": "9be1f24ecc8b", + "candidate_ref": "agent-16", + "submitted_at": "2026-10-12T05:36:53.759Z", + "observed_at": "2026-10-12T05:39:34.830Z", + "value": 49650446168, + "citation": { + "source_id": "released-native-inventory", + "first_line": 16, + "last_line": 16 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + }, + "objective": "分析原生 Official 研究如何推进原任务，区分事实、模型解释和独立验证。原始任务见 original-objective。此 Goal 是对真实原生轨迹的新分析生命周期；历史 Official 没有 LoopX 执行 Goal。保留历史 score_countable=false，只作诊断参考，不能和新恢复组作因果效果比较。原生结果及时间由原文件提供；未验证最终保留候选、贡献归因或货币成本，不可用 best 代替 retained。数值过程指标仅对声明样本适用。", + "episodes": [ + { + "id": "native-feedback-1", + "question": "原生提交 agent-1 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-1", + "original-feedback-1", + "original-message-344", + "original-message-378", + "original-message-395", + "original-message-441" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-4", + "question": "原生提交 agent-4 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-4", + "original-feedback-4", + "original-message-2387", + "original-message-2432", + "original-message-2483", + "original-message-2505" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-7", + "question": "原生提交 agent-7 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-7", + "original-feedback-7", + "original-message-4393", + "original-message-4403", + "original-message-4437", + "original-message-4460" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-10", + "question": "原生提交 agent-10 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-10", + "original-feedback-10", + "original-message-7387", + "original-message-7405", + "original-message-7430", + "original-message-7447" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-13", + "question": "原生提交 agent-13 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-13", + "original-feedback-13", + "original-message-10489", + "original-message-10513", + "original-message-10523", + "original-message-10546" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-16", + "question": "原生提交 agent-16 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-16", + "original-feedback-16", + "original-message-14485", + "original-message-14504", + "original-message-14524", + "original-message-14540" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + } + ], + "job": null, + "result": null, + "applicability": "current" + }, + "execution_enabled": false + } + } + }, + "analysis_citations": [ + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06&analysis_key=6eaeaa2eed3cae8d30785b60129d72eae3a7dfaa510e38393a8313be514980a5&source_id=released-native-inventory&first_line=1&last_line=1", + "response_sha256": "5d36f55807538f00c360d21fe8dbb5c4d7121efbcd5fcf45568e501b873833a0", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/ad/native-observations.jsonl", + "first_line": 1, + "last_line": 1, + "text": "{\"submission_index\":1,\"result_id\":\"0f01f973558f\",\"candidate_ref\":\"agent-1\",\"valid\":true,\"value\":49550815819.0,\"submitted_at\":\"2026-10-11T12:53:51.989Z\",\"observed_at\":\"2026-10-11T12:56:06.845Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/submissions/agent-1/report.json\",\"report_sha256\":\"262bf849bc739f1aa0465ab33becd52256904745e87a3b3f88f62145619e9714\",\"observation_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-45-01a12aef-c280-7042-a2fc-add2d305a19e.jsonl\",\"observation_artifact_sha256\":\"3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4\",\"observation_line\":379}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06&analysis_key=6eaeaa2eed3cae8d30785b60129d72eae3a7dfaa510e38393a8313be514980a5&source_id=released-native-inventory&first_line=13&last_line=13", + "response_sha256": "bc8496f68021fe8373b80a686febc72ea0e3bf1b894bd0e7afbf51b1ea1454a6", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/ad/native-observations.jsonl", + "first_line": 13, + "last_line": 13, + "text": "{\"submission_index\":13,\"result_id\":\"80bee97892b5\",\"candidate_ref\":\"agent-13\",\"valid\":true,\"value\":49653326363.0,\"submitted_at\":\"2026-10-12T01:34:30.830Z\",\"observed_at\":\"2026-10-12T01:36:57.526Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/submissions/agent-13/report.json\",\"report_sha256\":\"1ff915107bfd3a25bdbb111ffa7eb44d0b78fd20a594dc34b773db99b25db3bb\",\"observation_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-45-01a12aef-c280-7042-a2fc-add2d305a19e.jsonl\",\"observation_artifact_sha256\":\"3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4\",\"observation_line\":10516}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06&analysis_key=6eaeaa2eed3cae8d30785b60129d72eae3a7dfaa510e38393a8313be514980a5&source_id=released-native-inventory&first_line=16&last_line=16", + "response_sha256": "9cf78700c7d7691f64b4e69f9b389fec0dd0f622e165f386b3eaecf39eaf097c", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/ad/native-observations.jsonl", + "first_line": 16, + "last_line": 16, + "text": "{\"submission_index\":16,\"result_id\":\"9be1f24ecc8b\",\"candidate_ref\":\"agent-16\",\"valid\":true,\"value\":49650446168.0,\"submitted_at\":\"2026-10-12T05:36:53.759Z\",\"observed_at\":\"2026-10-12T05:39:34.830Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/submissions/agent-16/report.json\",\"report_sha256\":\"f8ad028fc2fe4d0441bb46fb0df6644d5d5eeacbf0b41d02c32c48ceae399a70\",\"observation_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-45-01a12aef-c280-7042-a2fc-add2d305a19e.jsonl\",\"observation_artifact_sha256\":\"3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4\",\"observation_line\":14520}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06&analysis_key=bb5b6a4169f8f541f51b391fd07a2209790741702ce5f5c8ea09178480e9a161&source_id=released-native-inventory&first_line=1&last_line=1&output_language=zh", + "response_sha256": "5d36f55807538f00c360d21fe8dbb5c4d7121efbcd5fcf45568e501b873833a0", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/ad/native-observations.jsonl", + "first_line": 1, + "last_line": 1, + "text": "{\"submission_index\":1,\"result_id\":\"0f01f973558f\",\"candidate_ref\":\"agent-1\",\"valid\":true,\"value\":49550815819.0,\"submitted_at\":\"2026-10-11T12:53:51.989Z\",\"observed_at\":\"2026-10-11T12:56:06.845Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/submissions/agent-1/report.json\",\"report_sha256\":\"262bf849bc739f1aa0465ab33becd52256904745e87a3b3f88f62145619e9714\",\"observation_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-45-01a12aef-c280-7042-a2fc-add2d305a19e.jsonl\",\"observation_artifact_sha256\":\"3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4\",\"observation_line\":379}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06&analysis_key=bb5b6a4169f8f541f51b391fd07a2209790741702ce5f5c8ea09178480e9a161&source_id=released-native-inventory&first_line=13&last_line=13&output_language=zh", + "response_sha256": "bc8496f68021fe8373b80a686febc72ea0e3bf1b894bd0e7afbf51b1ea1454a6", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/ad/native-observations.jsonl", + "first_line": 13, + "last_line": 13, + "text": "{\"submission_index\":13,\"result_id\":\"80bee97892b5\",\"candidate_ref\":\"agent-13\",\"valid\":true,\"value\":49653326363.0,\"submitted_at\":\"2026-10-12T01:34:30.830Z\",\"observed_at\":\"2026-10-12T01:36:57.526Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/submissions/agent-13/report.json\",\"report_sha256\":\"1ff915107bfd3a25bdbb111ffa7eb44d0b78fd20a594dc34b773db99b25db3bb\",\"observation_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-45-01a12aef-c280-7042-a2fc-add2d305a19e.jsonl\",\"observation_artifact_sha256\":\"3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4\",\"observation_line\":10516}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06&analysis_key=bb5b6a4169f8f541f51b391fd07a2209790741702ce5f5c8ea09178480e9a161&source_id=released-native-inventory&first_line=16&last_line=16&output_language=zh", + "response_sha256": "9cf78700c7d7691f64b4e69f9b389fec0dd0f622e165f386b3eaecf39eaf097c", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/ad/native-observations.jsonl", + "first_line": 16, + "last_line": 16, + "text": "{\"submission_index\":16,\"result_id\":\"9be1f24ecc8b\",\"candidate_ref\":\"agent-16\",\"valid\":true,\"value\":49650446168.0,\"submitted_at\":\"2026-10-12T05:36:53.759Z\",\"observed_at\":\"2026-10-12T05:39:34.830Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/submissions/agent-16/report.json\",\"report_sha256\":\"f8ad028fc2fe4d0441bb46fb0df6644d5d5eeacbf0b41d02c32c48ceae399a70\",\"observation_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-45-01a12aef-c280-7042-a2fc-add2d305a19e.jsonl\",\"observation_artifact_sha256\":\"3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4\",\"observation_line\":14520}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06&analysis_key=b47bda9123569d535da8fe09002f817da8c8844bf6367e65fb2eecd51f4f9d1f&source_id=released-native-inventory&first_line=1&last_line=1&output_language=en", + "response_sha256": "5d36f55807538f00c360d21fe8dbb5c4d7121efbcd5fcf45568e501b873833a0", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/ad/native-observations.jsonl", + "first_line": 1, + "last_line": 1, + "text": "{\"submission_index\":1,\"result_id\":\"0f01f973558f\",\"candidate_ref\":\"agent-1\",\"valid\":true,\"value\":49550815819.0,\"submitted_at\":\"2026-10-11T12:53:51.989Z\",\"observed_at\":\"2026-10-11T12:56:06.845Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/submissions/agent-1/report.json\",\"report_sha256\":\"262bf849bc739f1aa0465ab33becd52256904745e87a3b3f88f62145619e9714\",\"observation_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-45-01a12aef-c280-7042-a2fc-add2d305a19e.jsonl\",\"observation_artifact_sha256\":\"3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4\",\"observation_line\":379}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06&analysis_key=b47bda9123569d535da8fe09002f817da8c8844bf6367e65fb2eecd51f4f9d1f&source_id=released-native-inventory&first_line=13&last_line=13&output_language=en", + "response_sha256": "bc8496f68021fe8373b80a686febc72ea0e3bf1b894bd0e7afbf51b1ea1454a6", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/ad/native-observations.jsonl", + "first_line": 13, + "last_line": 13, + "text": "{\"submission_index\":13,\"result_id\":\"80bee97892b5\",\"candidate_ref\":\"agent-13\",\"valid\":true,\"value\":49653326363.0,\"submitted_at\":\"2026-10-12T01:34:30.830Z\",\"observed_at\":\"2026-10-12T01:36:57.526Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/submissions/agent-13/report.json\",\"report_sha256\":\"1ff915107bfd3a25bdbb111ffa7eb44d0b78fd20a594dc34b773db99b25db3bb\",\"observation_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-45-01a12aef-c280-7042-a2fc-add2d305a19e.jsonl\",\"observation_artifact_sha256\":\"3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4\",\"observation_line\":10516}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-ad-analysis-20261011-04&pack_id=c0fbc1b0f66924887b50369bd496c8c31849814b66d9ec1480988918dd61db06&analysis_key=b47bda9123569d535da8fe09002f817da8c8844bf6367e65fb2eecd51f4f9d1f&source_id=released-native-inventory&first_line=16&last_line=16&output_language=en", + "response_sha256": "9cf78700c7d7691f64b4e69f9b389fec0dd0f622e165f386b3eaecf39eaf097c", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/ad/native-observations.jsonl", + "first_line": 16, + "last_line": 16, + "text": "{\"submission_index\":16,\"result_id\":\"9be1f24ecc8b\",\"candidate_ref\":\"agent-16\",\"valid\":true,\"value\":49650446168.0,\"submitted_at\":\"2026-10-12T05:36:53.759Z\",\"observed_at\":\"2026-10-12T05:39:34.830Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/submissions/agent-16/report.json\",\"report_sha256\":\"f8ad028fc2fe4d0441bb46fb0df6644d5d5eeacbf0b41d02c32c48ceae399a70\",\"observation_artifact\":\"native-runs/runs/ad-official-native3600-integration-20261011-04/ad_placement_optimization/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-45-01a12aef-c280-7042-a2fc-add2d305a19e.jsonl\",\"observation_artifact_sha256\":\"3d04ccf7de4d4b536d35bed34f91c27846280b4aab3419c58d174654bbec19c4\",\"observation_line\":14520}", + "applicability": "current" + } + } + ] +} diff --git a/apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json b/apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json index 0212c2acae..60321e9364 100644 --- a/apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json +++ b/apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json @@ -11,8 +11,8 @@ "Six purposively selected native-feedback episodes; population and causal effect remain unknown.", "Complete saved analysis responses, including original judgments, costs and language-specific cache misses, are unchanged.", "No Explore events were recorded in this selected analysis Goal. Empty history is not a reconstructed solver history.", - "Only the included native-report-4 citation range is portable; uncaptured citations are explicitly unavailable.", - "No authority store, execution state, raw session transcript, model process or solver is restored by this display fixture." + "No authority store, execution state, raw session transcript, model process or solver is restored by this display fixture.", + "The included native-report-4 range, baseline/retained/final inventory rows and terminal retention range are portable. Other citations remain explicitly unavailable." ] }, "analysis_readbacks": { @@ -3038,6 +3038,54 @@ "text": " \"pass_rate\": 1.0,\n \"score\": 0.8359291793199752,\n \"score_0_100\": 83.59291793199752,\n \"timed_out\": false,\n \"runtime_seconds\": 63.152724742889404,\n \"test_details\": [],\n \"valid\": true,\n \"summary\": \"valid=true final_score=0.835929 overall_score=0.865158 macro_f1=0.850742 micro_f1=0.851681 balanced_accuracy=0.852224 one_minus_ece=0.989116 avg_time=18.698s checks=4/4\",", "applicability": "current" } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-graph-analysis-20261011-04&pack_id=020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1&analysis_key=34f5c3e46e75c1a937b33c98ae27109af0865621a30792a0be1651fac24a23f2&source_id=released-native-inventory&first_line=16&last_line=16", + "response_sha256": "0651057126c012fae6e801246f944307c3f89fc62e2b436a5743f34e96135dcb", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/graph/native-observations.jsonl", + "first_line": 16, + "last_line": 16, + "text": "{\"submission_index\":16,\"result_id\":\"6e610c29e0ae\",\"candidate_ref\":\"agent-16\",\"valid\":true,\"value\":0.849544264387785,\"submitted_at\":\"2026-10-12T03:59:48.720Z\",\"observed_at\":\"2026-10-12T04:01:22.153Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/submissions/agent-16/report.json\",\"report_sha256\":\"1b6bf435d3ffee98cd9da360ba16842a432f24a236bb921c3048f2027da9d1d4\",\"observation_artifact\":\"native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl\",\"observation_artifact_sha256\":\"0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1\",\"observation_line\":10043}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-graph-analysis-20261011-04&pack_id=020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1&analysis_key=34f5c3e46e75c1a937b33c98ae27109af0865621a30792a0be1651fac24a23f2&source_id=released-native-inventory&first_line=18&last_line=18", + "response_sha256": "060ad2fc74e415851e23e49fa54cffc22d133cb31e857bf3c021e3c9d62e398b", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/graph/native-observations.jsonl", + "first_line": 18, + "last_line": 18, + "text": "{\"submission_index\":18,\"result_id\":\"ba86adc7653e\",\"candidate_ref\":\"agent-18\",\"valid\":true,\"value\":0.8485288027628158,\"submitted_at\":\"2026-10-12T06:00:45.288Z\",\"observed_at\":\"2026-10-12T06:02:40.679Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/submissions/agent-18/report.json\",\"report_sha256\":\"3de7073f0959ad2ce0e089782cdb23760fad92db6ab6af9024363b352e8c629c\",\"observation_artifact\":\"native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl\",\"observation_artifact_sha256\":\"0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1\",\"observation_line\":11739}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-graph-analysis-20261011-04&pack_id=020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1&analysis_key=34f5c3e46e75c1a937b33c98ae27109af0865621a30792a0be1651fac24a23f2&source_id=released-native-inventory&first_line=2&last_line=2", + "response_sha256": "714dc46c200e0eddaf4795e3deea6b193ced07a328bf615a03fcd557e56dca79", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/graph/native-observations.jsonl", + "first_line": 2, + "last_line": 2, + "text": "{\"submission_index\":2,\"result_id\":\"e00dc3959c91\",\"candidate_ref\":\"agent-2\",\"valid\":true,\"value\":0.8172464473565724,\"submitted_at\":\"2026-10-11T13:37:19.792Z\",\"observed_at\":\"2026-10-11T13:39:12.474Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/submissions/agent-2/report.json\",\"report_sha256\":\"b67702eaf09c908143cf59372697d041838f08bbcfdc13b151c0736ba33de94b\",\"observation_artifact\":\"native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-48-01a12aef-d388-7233-ac51-5462f173f3e2.jsonl\",\"observation_artifact_sha256\":\"0d23d4a7aec8d84732c9c797ee236e4201b693d725522e21f3a09f2f93cc14f1\",\"observation_line\":722}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-graph-analysis-20261011-04&pack_id=020a5c5ee69d6795192a5544255219b8f91e1d0e272fe6db355e99d226903db1&analysis_key=34f5c3e46e75c1a937b33c98ae27109af0865621a30792a0be1651fac24a23f2&source_id=original-terminal-result&first_line=7&last_line=9", + "response_sha256": "374d13e98fb31a93785d1f7a6a571ee4d28fb68eb328387f6eb2a05881f3ce32", + "response": { + "source_id": "original-terminal-result", + "artifact": "native-runs/runs/graph-official-native3600-integration-20261011-04/graph_node_classification/final_result.json", + "first_line": 7, + "last_line": 9, + "text": " \"best_pass_rate\": 1.0,\n \"best_round\": \"agent-16\",\n \"total_rounds\": 18,", + "applicability": "current" + } } ] } diff --git a/apps/presentation/dashboard/src/data/fixtures/edgebench-portfolio-native-results.json b/apps/presentation/dashboard/src/data/fixtures/edgebench-portfolio-native-results.json new file mode 100644 index 0000000000..0834cb0f5f --- /dev/null +++ b/apps/presentation/dashboard/src/data/fixtures/edgebench-portfolio-native-results.json @@ -0,0 +1,1217 @@ +{ + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "display_name": "Portfolio · captured native results", + "events": [], + "provenance": { + "kind": "captured_read_only_analysis_api", + "captured_at": "2026-11-04T08:46:21.828152+00:00", + "source_run_id": "portfolio-official-native3600-integration-20261011-04", + "limitations": [ + "Historical 11–12 October native run from an existing formally registered read-only analysis Goal; not the recent history fixture or a new solver run.", + "The original, Chinese and English analysis API responses and native citation responses are unchanged.", + "Six registered episodes have no saved analysis job or result. No model judgment or analysis usage is synthesized.", + "Initial, best and final native results are measured; retained result and adopted improvement remain unknown.", + "The selected analysis Goal has no Explore records. No solver history or relationship is reconstructed.", + "No authority store, execution state, raw session transcript, model process or solver is restored by this display fixture." + ] + }, + "analysis_readbacks": { + "original": { + "request_path": "/api/chat/research-analysis?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc", + "response_sha256": "b33ebfcc7c233f8cf87b70e23b3bc76e34d5b91b8fcf11393e532afa9597b717", + "response": { + "ok": true, + "catalog": { + "ok": true, + "availability": "available", + "reason_code": null, + "default_pack_id": "43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc", + "packs": [ + { + "pack_id": "43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc", + "title": "Portfolio Official · 12h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "e61bcb63c26f1f76e641c667cf52aa87654babb3ce661f4ca741697e0ce17ebf", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/portfolio/native-outcomes.json", + "artifact_sha256": "55c1c3489b17ea3e4b63e5d0b85fcab39ab25d64fe23e00b978cf4e8fc94c933", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:46.720Z", + "end": "2026-10-12T00:28:14.011Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 12, + "valid_count": 12, + "initial": { + "submission_index": 1, + "result_id": "c38948e32bda", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:41:35.201Z", + "observed_at": "2026-10-11T12:41:56.877Z", + "value": 0, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 7, + "result_id": "4309ac4b9da9", + "candidate_ref": "agent-7", + "submitted_at": "2026-10-11T18:49:30.402Z", + "observed_at": "2026-10-11T18:49:52.611Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 7, + "last_line": 7 + } + }, + "retained": null, + "final_valid": { + "submission_index": 12, + "result_id": "da425c4f9ced", + "candidate_ref": "agent-12", + "submitted_at": "2026-10-12T00:08:10.729Z", + "observed_at": "2026-10-12T00:08:33.391Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 12, + "last_line": 12 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + } + ] + }, + "pack": { + "ok": true, + "pack_id": "43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc", + "title": "Portfolio Official · 12h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "e61bcb63c26f1f76e641c667cf52aa87654babb3ce661f4ca741697e0ce17ebf", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/portfolio/native-outcomes.json", + "artifact_sha256": "55c1c3489b17ea3e4b63e5d0b85fcab39ab25d64fe23e00b978cf4e8fc94c933", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:46.720Z", + "end": "2026-10-12T00:28:14.011Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 12, + "valid_count": 12, + "initial": { + "submission_index": 1, + "result_id": "c38948e32bda", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:41:35.201Z", + "observed_at": "2026-10-11T12:41:56.877Z", + "value": 0, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 7, + "result_id": "4309ac4b9da9", + "candidate_ref": "agent-7", + "submitted_at": "2026-10-11T18:49:30.402Z", + "observed_at": "2026-10-11T18:49:52.611Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 7, + "last_line": 7 + } + }, + "retained": null, + "final_valid": { + "submission_index": 12, + "result_id": "da425c4f9ced", + "candidate_ref": "agent-12", + "submitted_at": "2026-10-12T00:08:10.729Z", + "observed_at": "2026-10-12T00:08:33.391Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 12, + "last_line": 12 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + }, + "objective": "分析原生 Official 研究如何推进原任务，区分事实、模型解释和独立验证。原始任务见 original-objective。此 Goal 是对真实原生轨迹的新分析生命周期；历史 Official 没有 LoopX 执行 Goal。保留历史 score_countable=false，只作诊断参考，不能和新恢复组作因果效果比较。原生结果及时间由原文件提供；未验证最终保留候选、贡献归因或货币成本，不可用 best 代替 retained。数值过程指标仅对声明样本适用。", + "episodes": [ + { + "id": "native-feedback-1", + "question": "原生提交 agent-1 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-1", + "original-feedback-1", + "original-message-143", + "original-message-191", + "original-message-223", + "original-message-275" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-3", + "question": "原生提交 agent-3 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-3", + "original-feedback-3", + "original-message-1108", + "original-message-1117", + "original-message-1144", + "original-message-1191" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-5", + "question": "原生提交 agent-5 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-5", + "original-feedback-5", + "original-message-2013", + "original-message-2041", + "original-message-2067", + "original-message-2108" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-8", + "question": "原生提交 agent-8 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-8", + "original-feedback-8", + "original-message-3665", + "original-message-3758", + "original-message-3802", + "original-message-3863" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-10", + "question": "原生提交 agent-10 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-10", + "original-feedback-10", + "original-message-4920", + "original-message-4944", + "original-message-4975", + "original-message-5007" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-12", + "question": "原生提交 agent-12 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-12", + "original-feedback-12", + "original-message-6390", + "original-message-6425", + "original-message-6463", + "original-message-6500" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + } + ], + "job": null, + "result": null, + "applicability": "current" + }, + "execution_enabled": false + } + }, + "zh": { + "request_path": "/api/chat/research-analysis?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc&output_language=zh", + "response_sha256": "cf7eb84be2cc7b349c3118c278a6957e84796033f7bedd207a2d5802206a45d5", + "response": { + "ok": true, + "catalog": { + "ok": true, + "availability": "available", + "reason_code": null, + "default_pack_id": "43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc", + "packs": [ + { + "pack_id": "43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc", + "title": "Portfolio Official · 12h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "13053fe75418e586dbc1f36c8ec94aa13a691eda84f0184e11c5071a59086982", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "zh" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/portfolio/native-outcomes.json", + "artifact_sha256": "55c1c3489b17ea3e4b63e5d0b85fcab39ab25d64fe23e00b978cf4e8fc94c933", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:46.720Z", + "end": "2026-10-12T00:28:14.011Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 12, + "valid_count": 12, + "initial": { + "submission_index": 1, + "result_id": "c38948e32bda", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:41:35.201Z", + "observed_at": "2026-10-11T12:41:56.877Z", + "value": 0, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 7, + "result_id": "4309ac4b9da9", + "candidate_ref": "agent-7", + "submitted_at": "2026-10-11T18:49:30.402Z", + "observed_at": "2026-10-11T18:49:52.611Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 7, + "last_line": 7 + } + }, + "retained": null, + "final_valid": { + "submission_index": 12, + "result_id": "da425c4f9ced", + "candidate_ref": "agent-12", + "submitted_at": "2026-10-12T00:08:10.729Z", + "observed_at": "2026-10-12T00:08:33.391Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 12, + "last_line": 12 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + } + ] + }, + "pack": { + "ok": true, + "pack_id": "43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc", + "title": "Portfolio Official · 12h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "13053fe75418e586dbc1f36c8ec94aa13a691eda84f0184e11c5071a59086982", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "zh" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/portfolio/native-outcomes.json", + "artifact_sha256": "55c1c3489b17ea3e4b63e5d0b85fcab39ab25d64fe23e00b978cf4e8fc94c933", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:46.720Z", + "end": "2026-10-12T00:28:14.011Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 12, + "valid_count": 12, + "initial": { + "submission_index": 1, + "result_id": "c38948e32bda", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:41:35.201Z", + "observed_at": "2026-10-11T12:41:56.877Z", + "value": 0, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 7, + "result_id": "4309ac4b9da9", + "candidate_ref": "agent-7", + "submitted_at": "2026-10-11T18:49:30.402Z", + "observed_at": "2026-10-11T18:49:52.611Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 7, + "last_line": 7 + } + }, + "retained": null, + "final_valid": { + "submission_index": 12, + "result_id": "da425c4f9ced", + "candidate_ref": "agent-12", + "submitted_at": "2026-10-12T00:08:10.729Z", + "observed_at": "2026-10-12T00:08:33.391Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 12, + "last_line": 12 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + }, + "objective": "分析原生 Official 研究如何推进原任务，区分事实、模型解释和独立验证。原始任务见 original-objective。此 Goal 是对真实原生轨迹的新分析生命周期；历史 Official 没有 LoopX 执行 Goal。保留历史 score_countable=false，只作诊断参考，不能和新恢复组作因果效果比较。原生结果及时间由原文件提供；未验证最终保留候选、贡献归因或货币成本，不可用 best 代替 retained。数值过程指标仅对声明样本适用。", + "episodes": [ + { + "id": "native-feedback-1", + "question": "原生提交 agent-1 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-1", + "original-feedback-1", + "original-message-143", + "original-message-191", + "original-message-223", + "original-message-275" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-3", + "question": "原生提交 agent-3 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-3", + "original-feedback-3", + "original-message-1108", + "original-message-1117", + "original-message-1144", + "original-message-1191" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-5", + "question": "原生提交 agent-5 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-5", + "original-feedback-5", + "original-message-2013", + "original-message-2041", + "original-message-2067", + "original-message-2108" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-8", + "question": "原生提交 agent-8 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-8", + "original-feedback-8", + "original-message-3665", + "original-message-3758", + "original-message-3802", + "original-message-3863" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-10", + "question": "原生提交 agent-10 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-10", + "original-feedback-10", + "original-message-4920", + "original-message-4944", + "original-message-4975", + "original-message-5007" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-12", + "question": "原生提交 agent-12 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-12", + "original-feedback-12", + "original-message-6390", + "original-message-6425", + "original-message-6463", + "original-message-6500" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + } + ], + "job": null, + "result": null, + "applicability": "current" + }, + "execution_enabled": false + } + }, + "en": { + "request_path": "/api/chat/research-analysis?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc&output_language=en", + "response_sha256": "ef0f5b90e9204839bccde669f8cbf321795c8e285d12ea87ecdd71cc9b02ba5a", + "response": { + "ok": true, + "catalog": { + "ok": true, + "availability": "available", + "reason_code": null, + "default_pack_id": "43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc", + "packs": [ + { + "pack_id": "43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc", + "title": "Portfolio Official · 12h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "2f0347d340b9258128556de94689fd69f8ebb19f4bf066674f0c1b63ee65c056", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "en" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/portfolio/native-outcomes.json", + "artifact_sha256": "55c1c3489b17ea3e4b63e5d0b85fcab39ab25d64fe23e00b978cf4e8fc94c933", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:46.720Z", + "end": "2026-10-12T00:28:14.011Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 12, + "valid_count": 12, + "initial": { + "submission_index": 1, + "result_id": "c38948e32bda", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:41:35.201Z", + "observed_at": "2026-10-11T12:41:56.877Z", + "value": 0, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 7, + "result_id": "4309ac4b9da9", + "candidate_ref": "agent-7", + "submitted_at": "2026-10-11T18:49:30.402Z", + "observed_at": "2026-10-11T18:49:52.611Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 7, + "last_line": 7 + } + }, + "retained": null, + "final_valid": { + "submission_index": 12, + "result_id": "da425c4f9ced", + "candidate_ref": "agent-12", + "submitted_at": "2026-10-12T00:08:10.729Z", + "observed_at": "2026-10-12T00:08:33.391Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 12, + "last_line": 12 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + } + } + ] + }, + "pack": { + "ok": true, + "pack_id": "43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc", + "title": "Portfolio Official · 12h · 原生反馈片段", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "selection": { + "rule": "Deterministic six equal submission-order strata among native results with exact captured feedback observation; two preceding and up to two following bounded assistant narrative messages. Purposive native-feedback episodes, not an exhaustive research population. Full released report inventory supplied separately; temporal adjacency alone is not evidence adoption.", + "population_episodes": null, + "exhaustive": false + }, + "episode_count": 6, + "source_count": 40, + "analysis_key": "2f0347d340b9258128556de94689fd69f8ebb19f4bf066674f0c1b63ee65c056", + "method_version": "research-episodes-v2", + "config": { + "model": "gpt-6.1-sol", + "effort": "max", + "service_tier": "fast", + "output_language": "en" + }, + "availability": "available", + "reason_code": null, + "native_outcomes": { + "artifact": "product-goal-sources/official/portfolio/native-outcomes.json", + "artifact_sha256": "55c1c3489b17ea3e4b63e5d0b85fcab39ab25d64fe23e00b978cf4e8fc94c933", + "projection": { + "method": "source_bound_native_outcomes_v1", + "scope": { + "goal_id": "edgebench-official-portfolio-analysis-20261011-04", + "case_id": "portfolio_risk_calibration", + "run_id": "portfolio-official-native3600-integration-20261011-04", + "revision": "b5023a765a39bdbfd01f8c7496378cfc84bcbbda:native-session-sha256:2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca", + "window": "Original captured native session 2026-10-11T12:28:46.720Z to 2026-10-12T00:28:14.011Z; declared solver budget 12.00h. Analysis lifecycle created later." + }, + "start": "2026-10-11T12:28:46.720Z", + "end": "2026-10-12T00:28:14.011Z", + "complete": true, + "metric": { + "name": "Native task score", + "unit": "raw", + "higher_is_better": true + }, + "sample_count": 12, + "valid_count": 12, + "initial": { + "submission_index": 1, + "result_id": "c38948e32bda", + "candidate_ref": "agent-1", + "submitted_at": "2026-10-11T12:41:35.201Z", + "observed_at": "2026-10-11T12:41:56.877Z", + "value": 0, + "citation": { + "source_id": "released-native-inventory", + "first_line": 1, + "last_line": 1 + } + }, + "best_valid": { + "submission_index": 7, + "result_id": "4309ac4b9da9", + "candidate_ref": "agent-7", + "submitted_at": "2026-10-11T18:49:30.402Z", + "observed_at": "2026-10-11T18:49:52.611Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 7, + "last_line": 7 + } + }, + "retained": null, + "final_valid": { + "submission_index": 12, + "result_id": "da425c4f9ced", + "candidate_ref": "agent-12", + "submitted_at": "2026-10-12T00:08:10.729Z", + "observed_at": "2026-10-12T00:08:33.391Z", + "value": 10.86, + "citation": { + "source_id": "released-native-inventory", + "first_line": 12, + "last_line": 12 + } + }, + "limitations": [ + "Provider-recorded measurements in the declared capture; not study countability or a causal effect.", + "Validity and retained selection belong to the native provider, not model judgments or Todo status.", + "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", + "Submission order determines initial/final; delayed feedback does not make an older candidate current.", + "Costs retain their own measured windows; no monetary, whole-study or per-result cost is inferred." + ] + }, + "applicability": "current", + "reason_code": null + }, + "objective": "分析原生 Official 研究如何推进原任务，区分事实、模型解释和独立验证。原始任务见 original-objective。此 Goal 是对真实原生轨迹的新分析生命周期；历史 Official 没有 LoopX 执行 Goal。保留历史 score_countable=false，只作诊断参考，不能和新恢复组作因果效果比较。原生结果及时间由原文件提供；未验证最终保留候选、贡献归因或货币成本，不可用 best 代替 retained。数值过程指标仅对声明样本适用。", + "episodes": [ + { + "id": "native-feedback-1", + "question": "原生提交 agent-1 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-1", + "original-feedback-1", + "original-message-143", + "original-message-191", + "original-message-223", + "original-message-275" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-3", + "question": "原生提交 agent-3 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-3", + "original-feedback-3", + "original-message-1108", + "original-message-1117", + "original-message-1144", + "original-message-1191" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-5", + "question": "原生提交 agent-5 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-5", + "original-feedback-5", + "original-message-2013", + "original-message-2041", + "original-message-2067", + "original-message-2108" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-8", + "question": "原生提交 agent-8 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-8", + "original-feedback-8", + "original-message-3665", + "original-message-3758", + "original-message-3802", + "original-message-3863" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-10", + "question": "原生提交 agent-10 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-10", + "original-feedback-10", + "original-message-4920", + "original-message-4944", + "original-message-4975", + "original-message-5007" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + }, + { + "id": "native-feedback-12", + "question": "原生提交 agent-12 的反馈实际检验了什么？前后原文支持哪些方法修改、保留或拒绝？未提供的实现、有效性或采用关系应标未知，不用相邻时间推断因果。", + "source_ids": [ + "original-objective", + "original-runtime", + "released-native-inventory", + "native-report-12", + "original-feedback-12", + "original-message-6390", + "original-message-6425", + "original-message-6463", + "original-message-6500" + ], + "eligible_metrics": [ + "goal_alignment", + "benefit_test", + "discrimination", + "information_gain", + "evidence_adoption", + "redundancy" + ] + } + ], + "job": null, + "result": null, + "applicability": "current" + }, + "execution_enabled": false + } + } + }, + "analysis_citations": [ + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc&analysis_key=e61bcb63c26f1f76e641c667cf52aa87654babb3ce661f4ca741697e0ce17ebf&source_id=released-native-inventory&first_line=1&last_line=1", + "response_sha256": "b71a5f6353312e9ab4eab1176beb21e8f26590d1e4dbfcb52d21f3e9b7401ef3", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/portfolio/native-observations.jsonl", + "first_line": 1, + "last_line": 1, + "text": "{\"submission_index\":1,\"result_id\":\"c38948e32bda\",\"candidate_ref\":\"agent-1\",\"valid\":true,\"value\":0.0,\"submitted_at\":\"2026-10-11T12:41:35.201Z\",\"observed_at\":\"2026-10-11T12:41:56.877Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/submissions/agent-1/report.json\",\"report_sha256\":\"f95bbb33203af509c4eebe9e7dcf214b269f8d7d6258526fdb57fe731df5be63\",\"observation_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-41-01a12aef-b2e3-7151-abf6-dd1590ccc65e.jsonl\",\"observation_artifact_sha256\":\"2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca\",\"observation_line\":209}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc&analysis_key=e61bcb63c26f1f76e641c667cf52aa87654babb3ce661f4ca741697e0ce17ebf&source_id=released-native-inventory&first_line=7&last_line=7", + "response_sha256": "a9eebe1d0f3b0e18c499a14df7646ed59a1b3a6b3def1a37a806d4374c79449f", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/portfolio/native-observations.jsonl", + "first_line": 7, + "last_line": 7, + "text": "{\"submission_index\":7,\"result_id\":\"4309ac4b9da9\",\"candidate_ref\":\"agent-7\",\"valid\":true,\"value\":10.86,\"submitted_at\":\"2026-10-11T18:49:30.402Z\",\"observed_at\":\"2026-10-11T18:49:52.611Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/submissions/agent-7/report.json\",\"report_sha256\":\"9c5fe89d3fad64db0ec4b993961f88248b0cacf49f52b37feec7e148e99b2924\",\"observation_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-41-01a12aef-b2e3-7151-abf6-dd1590ccc65e.jsonl\",\"observation_artifact_sha256\":\"2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca\",\"observation_line\":3197}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc&analysis_key=e61bcb63c26f1f76e641c667cf52aa87654babb3ce661f4ca741697e0ce17ebf&source_id=released-native-inventory&first_line=12&last_line=12", + "response_sha256": "0eebf43739b07ee7cc15fb17cf1c297f6100bddb83b358462502a9fc9c9e367a", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/portfolio/native-observations.jsonl", + "first_line": 12, + "last_line": 12, + "text": "{\"submission_index\":12,\"result_id\":\"da425c4f9ced\",\"candidate_ref\":\"agent-12\",\"valid\":true,\"value\":10.86,\"submitted_at\":\"2026-10-12T00:08:10.729Z\",\"observed_at\":\"2026-10-12T00:08:33.391Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/submissions/agent-12/report.json\",\"report_sha256\":\"20190047d95816bc8b2cc460b10f95541e64eb8d9f8844e0fdf955b228c60508\",\"observation_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-41-01a12aef-b2e3-7151-abf6-dd1590ccc65e.jsonl\",\"observation_artifact_sha256\":\"2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca\",\"observation_line\":6432}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc&analysis_key=13053fe75418e586dbc1f36c8ec94aa13a691eda84f0184e11c5071a59086982&source_id=released-native-inventory&first_line=1&last_line=1&output_language=zh", + "response_sha256": "b71a5f6353312e9ab4eab1176beb21e8f26590d1e4dbfcb52d21f3e9b7401ef3", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/portfolio/native-observations.jsonl", + "first_line": 1, + "last_line": 1, + "text": "{\"submission_index\":1,\"result_id\":\"c38948e32bda\",\"candidate_ref\":\"agent-1\",\"valid\":true,\"value\":0.0,\"submitted_at\":\"2026-10-11T12:41:35.201Z\",\"observed_at\":\"2026-10-11T12:41:56.877Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/submissions/agent-1/report.json\",\"report_sha256\":\"f95bbb33203af509c4eebe9e7dcf214b269f8d7d6258526fdb57fe731df5be63\",\"observation_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-41-01a12aef-b2e3-7151-abf6-dd1590ccc65e.jsonl\",\"observation_artifact_sha256\":\"2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca\",\"observation_line\":209}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc&analysis_key=13053fe75418e586dbc1f36c8ec94aa13a691eda84f0184e11c5071a59086982&source_id=released-native-inventory&first_line=7&last_line=7&output_language=zh", + "response_sha256": "a9eebe1d0f3b0e18c499a14df7646ed59a1b3a6b3def1a37a806d4374c79449f", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/portfolio/native-observations.jsonl", + "first_line": 7, + "last_line": 7, + "text": "{\"submission_index\":7,\"result_id\":\"4309ac4b9da9\",\"candidate_ref\":\"agent-7\",\"valid\":true,\"value\":10.86,\"submitted_at\":\"2026-10-11T18:49:30.402Z\",\"observed_at\":\"2026-10-11T18:49:52.611Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/submissions/agent-7/report.json\",\"report_sha256\":\"9c5fe89d3fad64db0ec4b993961f88248b0cacf49f52b37feec7e148e99b2924\",\"observation_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-41-01a12aef-b2e3-7151-abf6-dd1590ccc65e.jsonl\",\"observation_artifact_sha256\":\"2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca\",\"observation_line\":3197}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc&analysis_key=13053fe75418e586dbc1f36c8ec94aa13a691eda84f0184e11c5071a59086982&source_id=released-native-inventory&first_line=12&last_line=12&output_language=zh", + "response_sha256": "0eebf43739b07ee7cc15fb17cf1c297f6100bddb83b358462502a9fc9c9e367a", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/portfolio/native-observations.jsonl", + "first_line": 12, + "last_line": 12, + "text": "{\"submission_index\":12,\"result_id\":\"da425c4f9ced\",\"candidate_ref\":\"agent-12\",\"valid\":true,\"value\":10.86,\"submitted_at\":\"2026-10-12T00:08:10.729Z\",\"observed_at\":\"2026-10-12T00:08:33.391Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/submissions/agent-12/report.json\",\"report_sha256\":\"20190047d95816bc8b2cc460b10f95541e64eb8d9f8844e0fdf955b228c60508\",\"observation_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-41-01a12aef-b2e3-7151-abf6-dd1590ccc65e.jsonl\",\"observation_artifact_sha256\":\"2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca\",\"observation_line\":6432}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc&analysis_key=2f0347d340b9258128556de94689fd69f8ebb19f4bf066674f0c1b63ee65c056&source_id=released-native-inventory&first_line=1&last_line=1&output_language=en", + "response_sha256": "b71a5f6353312e9ab4eab1176beb21e8f26590d1e4dbfcb52d21f3e9b7401ef3", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/portfolio/native-observations.jsonl", + "first_line": 1, + "last_line": 1, + "text": "{\"submission_index\":1,\"result_id\":\"c38948e32bda\",\"candidate_ref\":\"agent-1\",\"valid\":true,\"value\":0.0,\"submitted_at\":\"2026-10-11T12:41:35.201Z\",\"observed_at\":\"2026-10-11T12:41:56.877Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/submissions/agent-1/report.json\",\"report_sha256\":\"f95bbb33203af509c4eebe9e7dcf214b269f8d7d6258526fdb57fe731df5be63\",\"observation_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-41-01a12aef-b2e3-7151-abf6-dd1590ccc65e.jsonl\",\"observation_artifact_sha256\":\"2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca\",\"observation_line\":209}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc&analysis_key=2f0347d340b9258128556de94689fd69f8ebb19f4bf066674f0c1b63ee65c056&source_id=released-native-inventory&first_line=7&last_line=7&output_language=en", + "response_sha256": "a9eebe1d0f3b0e18c499a14df7646ed59a1b3a6b3def1a37a806d4374c79449f", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/portfolio/native-observations.jsonl", + "first_line": 7, + "last_line": 7, + "text": "{\"submission_index\":7,\"result_id\":\"4309ac4b9da9\",\"candidate_ref\":\"agent-7\",\"valid\":true,\"value\":10.86,\"submitted_at\":\"2026-10-11T18:49:30.402Z\",\"observed_at\":\"2026-10-11T18:49:52.611Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/submissions/agent-7/report.json\",\"report_sha256\":\"9c5fe89d3fad64db0ec4b993961f88248b0cacf49f52b37feec7e148e99b2924\",\"observation_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-41-01a12aef-b2e3-7151-abf6-dd1590ccc65e.jsonl\",\"observation_artifact_sha256\":\"2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca\",\"observation_line\":3197}", + "applicability": "current" + } + }, + { + "request_path": "/api/chat/research-analysis/citation?goal_id=edgebench-official-portfolio-analysis-20261011-04&pack_id=43d7256976866d8d31487f9fd5e5b704ea6f641711f84d18c745e783b9d1f1fc&analysis_key=2f0347d340b9258128556de94689fd69f8ebb19f4bf066674f0c1b63ee65c056&source_id=released-native-inventory&first_line=12&last_line=12&output_language=en", + "response_sha256": "0eebf43739b07ee7cc15fb17cf1c297f6100bddb83b358462502a9fc9c9e367a", + "response": { + "source_id": "released-native-inventory", + "artifact": "product-goal-sources/official/portfolio/native-observations.jsonl", + "first_line": 12, + "last_line": 12, + "text": "{\"submission_index\":12,\"result_id\":\"da425c4f9ced\",\"candidate_ref\":\"agent-12\",\"valid\":true,\"value\":10.86,\"submitted_at\":\"2026-10-12T00:08:10.729Z\",\"observed_at\":\"2026-10-12T00:08:33.391Z\",\"observed_at_meaning\":\"first captured completed native-submit command containing this exact submission id and score; not inferred feedback-release time\",\"report_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/submissions/agent-12/report.json\",\"report_sha256\":\"20190047d95816bc8b2cc460b10f95541e64eb8d9f8844e0fdf955b228c60508\",\"observation_artifact\":\"native-runs/runs/portfolio-official-native3600-integration-20261011-04/portfolio_risk_calibration/collected/native-sessions/sessions/2026/10/11/rollout-2026-10-11T12-28-41-01a12aef-b2e3-7151-abf6-dd1590ccc65e.jsonl\",\"observation_artifact_sha256\":\"2a03d0a6e100f61a69aca900bfd2820a89dd5431a819c152b306a6bca33749ca\",\"observation_line\":6432}", + "applicability": "current" + } + } + ] +} diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx index 17220c65c9..40c995b3c2 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx @@ -108,12 +108,12 @@ export function episodeQuestionGroups(result: Tr // Keep headline values readable without rounding a nonzero measurement to zero. // Full native precision remains available in the existing evidence/details. -function formatNativeMeasurement(value: number, zh: boolean, signed = false) { +function formatNativeMeasurement(value: number, zh: boolean, signed = false, significantDigits = 4) { const magnitude = Math.abs(value); const scientific = magnitude > 0 && (magnitude < 0.0001 || magnitude >= 1_000_000); return new Intl.NumberFormat(zh ? "zh-CN" : "en", { maximumFractionDigits: 4, - ...(scientific ? {notation: "scientific", maximumSignificantDigits: 4} : {}), + ...(scientific ? {notation: "scientific", maximumSignificantDigits: significantDigits} : {}), signDisplay: signed ? "exceptZero" : "auto", }).format(value === 0 ? 0 : value); } @@ -147,13 +147,22 @@ function NativeResults({native, zh, onCitation}: {native: ResearchProgressPack[" if (!native || !outcomes) return

{zh ? "原生测量 · 未采集" : "Native measurements · not captured"}

; const historical = native.applicability === "historical"; const points = [["best_valid", "窗口内最佳有效值", "Best valid in capture"], ["initial", "首次提交", "Initial submission"], ["final_valid", "最后有效提交", "Final valid submission"], ["retained", "原生保留结果", "Native retained result"]] as const; + // The real Ad capture has distinct best/final values that both round to + // 4.965E10. Keep the compact notation, increasing precision only until the + // distinct scientific measurements remain distinguishable (up to double precision). + const scientificValues = [...new Set(points.flatMap(([key]) => { + const point = outcomes[key], magnitude = Math.abs(point?.value ?? 0); + return point && magnitude > 0 && (magnitude < 0.0001 || magnitude >= 1_000_000) ? [point.value] : []; + }))]; + let significantDigits = 4; + while (significantDigits < 17 && new Set(scientificValues.map(point => formatNativeMeasurement(point, zh, false, significantDigits))).size < scientificValues.length) significantDigits++; const period = new Intl.DateTimeFormat(zh ? "zh-CN" : "en", {timeZone: "UTC", month: "2-digit", day: "2-digit", hour: "2-digit", minute: "2-digit", hour12: false, ...(new Date(outcomes.start).getUTCFullYear() !== new Date(outcomes.end).getUTCFullYear() ? {year: "numeric"} : {})}); const value = ([key, cn, en]: typeof points[number]) => { const point = outcomes[key]; return
{zh ? cn : en}
{point ? : {zh ? "未知" : "Unknown"}{key === "best_valid" ? · {outcomes.metric.unit} : null}}
; }; return
diff --git a/examples/personal-workspace-browser/research-history.md b/examples/personal-workspace-browser/research-history.md index 9581fcde63..00b77d207e 100644 --- a/examples/personal-workspace-browser/research-history.md +++ b/examples/personal-workspace-browser/research-history.md @@ -3,7 +3,8 @@ The default read-only preview uses the complete saved Explore log from the 3 November Grid Turing Robot context-on run. The `research-real-benchmark` browser scenario reads both context arms, the Ad and Schema histories below, -and the saved Graph analysis through the real HTTP history owner. It verifies complete original text and append order, download +the saved Graph analysis, and Portfolio/Ad native results without saved analysis +through the real HTTP history owner. It verifies complete original text and append order, download readback, source preservation, no writes, and desktop/390px reading. These are the core populated-page references; synthetic fixtures cover boundaries such as races, artificial paging pressure and missing fields. @@ -100,6 +101,41 @@ tokens**. These are analysis costs; solver token usage and money remain unknown. The invalid first submission remains missing rather than zero, and the native best/retained result remains distinct from the final submitted candidate. +The core reader also uses the actual registered [Portfolio](../../apps/presentation/dashboard/src/data/fixtures/edgebench-portfolio-native-results.json) +and [Ad placement](../../apps/presentation/dashboard/src/data/fixtures/edgebench-ad-native-results.json) +analysis Goals. Their six-episode packs have **no saved model result or job**. +Native measurements remain readable without inventing judgments, analysis usage, +or an adoption decision: + +| Captured native run | First valid comparison | Best valid | Final valid | Retained | +| --- | --- | --- | --- | --- | +| Graph | agent-2: 0.8172464473565724 | agent-16: 0.849544264387785 | agent-18: 0.8485288027628158 | agent-16: 0.849544264387785 | +| Portfolio | agent-1: **0** | agent-7: 10.86 | agent-12: 10.86 | Unknown | +| Ad placement | agent-1: 49,550,815,819 | agent-13: 49,653,326,363 | agent-16: 49,650,446,168 | Unknown | + +Graph's source-bound retained improvement is **+0.032297817031212595 raw** +against its fixed agent-2 baseline. Its final candidate change is only ++0.03128235540624347. The terminal-result citation identifies agent-16 as retained; +the last submitted candidate is not silently substituted. This is a native +comparison, not LoopX's independently isolated causal contribution. Scores have +different task meanings and are not added across these runs. + +The Ad capture exposed a presentation failure: four significant digits displayed +both best and final as `4.965E10`. Scientific notation now adds only enough digits +to distinguish the captured values (`4.9653E10` and `4.965E10` here). Equal native +values still display equally, and full precision remains in the citation reader. + +The saved Graph analysis also preserves a negative result from round 11: a +reported rollback retained round 10 at 0.84213. Round 4 reports local experimental +follow-through, while rounds 8, 15 and 18 announce restoration or future work +without establishing completed adoption in their selected excerpts. The later +terminal retention evidence does not rewrite those earlier model judgments. +There are no positive discrimination judgments: two are no and four unknown; +bundled interventions, timing noise and missing controlled contrasts still limit +explanatory claims. Native evaluation and positive measured benefit remain +different questions. These statements describe the saved, uncalibrated analysis +of six purposive episodes, not a new whole-run or causal assessment. + This fixture freezes the real HTTP readbacks, including the Chinese/English cache misses and the saved unspecified-language original. The packaged frontend must recover that original without inventing a translation, changing its usage @@ -107,11 +143,22 @@ or invoking a model. The preview replays captured GET responses; it does **not** import the original analysis store or qualify the live execution backend. The existing synthetic process fixture separately protects that owner boundary. Tests pin the original HTTP byte digests and read all 36 explanations in both -interfaces at desktop/390px widths. - -Only one public-safe native-result citation range is bundled. Other cited ranges -return an explicit unavailable response; no transcript or failure stack is -substituted. The selected Goal has no Explore log: its empty history is preserved. +interfaces at desktop/390px widths. All three cases also open each available +native value's original citation; Graph opens its terminal retention evidence. +An uncaptured judgment citation fails visibly, followed by a successful captured +citation read; missing sources never substitute another excerpt. +The real zero in Portfolio stays zero, and missing retention or analysis usage +in Portfolio and Ad never borrows Graph's values. The three formally registered +Goals were additionally read through the production HTTP owner with analysis +execution disabled; this qualifies reading, not model dispatch or installation. + +The Graph fixture bundles its existing native-report-4 range plus baseline, +best/retained, final and terminal-retention citations. Portfolio and Ad bundle +their initial, best and final native citations with the exact language-specific +request identities. Other cited ranges return an explicit unavailable response; +no transcript or failure stack is substituted. These analysis Goals have no +Explore logs: their empty histories are preserved, separately from the recent +real-history cases above. Pass `--fixture apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json` to the same read-only preview to inspect this case. All mutation verbs are denied. diff --git a/examples/personal-workspace-browser/research-real-benchmark.mjs b/examples/personal-workspace-browser/research-real-benchmark.mjs index 0253a115d0..3474b76b34 100644 --- a/examples/personal-workspace-browser/research-real-benchmark.mjs +++ b/examples/personal-workspace-browser/research-real-benchmark.mjs @@ -4,6 +4,100 @@ import {resolve} from "node:path"; import {outputDir, repoRoot} from "./fixture.mjs"; import {startHistoryOwner} from "./research-history.mjs"; +async function readNativeResults(page, history, capture, language) { + const pack = capture.analysis_readbacks.original.response.pack; + const native = pack.native_outcomes.projection; + const progress = history.locator(".research-progress"); + await progress.locator(".research-native-results").waitFor(); + if (pack.scope.case_id === "ad_placement_optimization") { + const best = await progress.locator('[data-native-point="best_valid"] strong').innerText(); + const final = await progress.locator('[data-native-point="final_valid"] strong').innerText(); + assert.notEqual(best, final, "The real best and final results must remain distinguishable before opening evidence"); + } + for (const key of ["initial", "best_valid", "retained", "final_valid"]) { + const row = progress.locator(`[data-native-point="${key}"]`); + const point = native[key]; + if (!point) { + assert.match(await row.innerText(), language === "zh" ? /未知/ : /Unknown/); + assert.equal(await row.getByRole("button").count(), 0, "Missing retention is not a zero-valued result"); + continue; + } + if (point.value === 0) assert.equal(await row.locator("strong").innerText(), "0"); + await row.getByRole("button").click(); + const source = history.locator(".analysis-source"); + await source.locator("pre").waitFor(); + const citation = capture.analysis_citations.find(item => { + const query = new URL(item.request_path, "http://fixture").searchParams; + return query.get("source_id") === point.citation.source_id + && query.get("first_line") === String(point.citation.first_line) + && query.get("last_line") === String(point.citation.last_line); + }); + assert(citation, "Each native value has its captured original citation"); + assert.equal(await source.locator("pre").innerText(), citation.response.text); + assert.equal(JSON.parse(citation.response.text).value, point.value); + await source.getByRole("button", {name: language === "zh" ? "关闭原文" : "Close source", exact: true}).click(); + } + const retained = progress.locator(".research-retained-improvement"); + if (native.retained_improvement) { + assert.equal(native.retained.candidate_ref, "agent-16"); + assert.equal(native.final_valid.candidate_ref, "agent-18"); + assert.equal(native.retained_improvement.baseline.candidate_ref, "agent-2"); + assert.match(await retained.locator("[data-retained-improvement]").innerText(), /\+0\.0323/); + await retained.locator(":scope > details > summary").click(); + await retained.getByRole("button", {name: language === "zh" ? "查看原文" : "Read original evidence", exact: true}).click(); + const source = history.locator(".analysis-source"); + await source.locator("pre").waitFor(); + assert.match(await source.locator("pre").innerText(), /"best_round": "agent-16"/); + await source.getByRole("button", {name: language === "zh" ? "关闭原文" : "Close source", exact: true}).click(); + } else { + assert.equal(await retained.count(), 0, "A best/final result cannot supply missing adoption evidence"); + } + await progress.locator(".research-progress-basis > summary").click(); + const cost = progress.locator(".research-analysis-cost"); + if (!pack.job) { + assert.equal(pack.result, null); + assert.match(await cost.innerText(), language === "zh" ? /未采集/ : /Not captured/); + assert.equal(await cost.locator(".research-analysis-values").count(), 0); + assert.equal(await history.locator("[data-analysis-judgment]").count(), 0); + } + await progress.locator(".research-progress-basis > summary").click(); + for (const width of [1440, 390]) { + await page.setViewportSize({width, height: 900}); + await progress.scrollIntoViewIfNeeded(); + assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth), false); + const name = pack.scope.case_id.split("_")[0]; + await page.screenshot({path: resolve(outputDir, `research-real-native-${name}-${language}-${width}.png`), animations: "disabled"}); + } + await page.setViewportSize({width: 1440, height: 900}); +} + +async function readUnanalyzedNativeResults(browser, collectCoverage) { + const coverageEntries = []; + for (const name of ["portfolio", "ad"]) { + const path = resolve(repoRoot, `apps/presentation/dashboard/src/data/fixtures/edgebench-${name}-native-results.json`); + const capture = JSON.parse(await readFile(path, "utf8")); + const owner = await startHistoryOwner(path); + let page; + try { + page = await browser.newPage({viewport: {width: 1440, height: 900}}); + if (collectCoverage) await page.coverage.startJSCoverage({resetOnNavigation: false}); + for (const language of ["zh", "en"]) { + await page.addInitScript(value => localStorage.setItem("loopx-pw-locale", value), language); + await page.goto(`${owner.base}/chat/?goalId=${capture.goal_id}&view=research`); + await readNativeResults(page, page.locator(".research-history"), capture, language); + } + const readback = await (await page.request.get(`${owner.base}/fixture/readback`)).json(); + assert.equal(readback.source_unchanged, true); + assert.equal(readback.mutation_attempts, 0); + if (collectCoverage) coverageEntries.push(...await page.coverage.stopJSCoverage()); + } finally { + if (page) await page.close(); + await owner.close(); + } + } + return coverageEntries; +} + async function readSavedGraphAnalysis(browser, collectCoverage) { const path = resolve(repoRoot, "apps/presentation/dashboard/src/data/fixtures/edgebench-graph-saved-analysis.json"); const capture = JSON.parse(await readFile(path, "utf8")); @@ -18,6 +112,7 @@ async function readSavedGraphAnalysis(browser, collectCoverage) { await page.addInitScript(value => localStorage.setItem("loopx-pw-locale", value), language); await page.goto(`${owner.base}/chat/?goalId=${capture.goal_id}&view=research`); const history = page.locator(".research-history"); + await readNativeResults(page, history, capture, language); await history.getByRole("button", {name: language === "zh" ? "查看分析" : "View analysis", exact: true}).click(); const analysis = history.locator(".research-analysis[role=region]"); await analysis.locator(".analysis-episode").first().waitFor(); @@ -36,6 +131,15 @@ async function readSavedGraphAnalysis(browser, collectCoverage) { assert(text.includes(assessment.reason), `${episode.id}/${assessment.metric}: complete original reasoning remains readable`); } } + const missingEpisode = analysis.locator('[data-analysis-episode="native-feedback-1"]'); + await missingEpisode.locator(":scope > summary").click(); + for (const summary of await missingEpisode.locator(".analysis-judgments > details:not([open]) > summary").all()) await summary.click(); + await missingEpisode.getByRole("button", {name: /native-report-1/}).first().click(); + const unavailable = history.locator(".analysis-source"); + await unavailable.locator(".analysis-error").waitFor(); + assert.match(await unavailable.locator(".analysis-error").innerText(), /not captured/); + assert.equal(await unavailable.locator("pre").count(), 0, "Uncaptured citations do not substitute unrelated evidence"); + await unavailable.getByRole("button", {name: language === "zh" ? "关闭原文" : "Close source", exact: true}).click(); const citation = capture.analysis_citations[0].response; const cited = saved.result.assessments.find(row => row.citations.some(c => c.source_id === citation.source_id && c.first_line === citation.first_line && c.last_line === citation.last_line)); @@ -73,12 +177,13 @@ async function readSavedGraphAnalysis(browser, collectCoverage) { } } -// Core whole-page references: complete original display state from three tasks. +// Core whole-page references: original history and independently scoped native outcomes. // Synthetic scenarios remain separate boundary/race/overflow tests. export const researchRealBenchmarkScenario = { id: "research-real-benchmark", async run({browser, collectCoverage}) { const coverageEntries = []; + coverageEntries.push(...await readUnanalyzedNativeResults(browser, collectCoverage)); const observations = []; for (const [arm, fixture] of [ ["grid-on", "edgebench-grid-context-on-20261103-history.json"], diff --git a/tests/presentation/test_research_history_fixture.py b/tests/presentation/test_research_history_fixture.py index b7fc566274..77e414f3d8 100644 --- a/tests/presentation/test_research_history_fixture.py +++ b/tests/presentation/test_research_history_fixture.py @@ -72,7 +72,10 @@ def test_synthetic_provider_uses_selected_python_with_real_arguments_pipes_and_e assert json.loads(result.stdout) == [arguments, "Synthetic input", str(tmp_path), sys.executable] -@pytest.mark.parametrize("fixture_name", ["research-synthesis-readback.json", "edgebench-graph-saved-analysis.json"]) +@pytest.mark.parametrize("fixture_name", [ + "research-synthesis-readback.json", "edgebench-graph-saved-analysis.json", + "edgebench-portfolio-native-results.json", "edgebench-ad-native-results.json", +]) def test_captured_analysis_fixture_serves_real_saved_results_without_source_mutation(tmp_path, monkeypatch, fixture_name): fixture = ROOT / "apps/presentation/dashboard/src/data/fixtures" / fixture_name capture = json.loads(fixture.read_bytes()) @@ -98,23 +101,50 @@ def read(path): if "analysis_readbacks" in capture: # Pinned original HTTP bytes, not a self-updating digest. expected = { - "original": "0e4bb4b931784abe1faf8cba49922aeb36f6c96f9e1730fdc77d82e7ebd3da2f", - "zh": "cb8fdf1a61915f06168816c82367e9d037bdf49424028cd2793dfe61364def64", - "en": "d76b579672d87f8b071d871ecfd5797b986f295b8dc93bc967471c0e1aab0b2b", - } + "edgebench-graph-saved-analysis.json": { + "original": "0e4bb4b931784abe1faf8cba49922aeb36f6c96f9e1730fdc77d82e7ebd3da2f", + "zh": "cb8fdf1a61915f06168816c82367e9d037bdf49424028cd2793dfe61364def64", + "en": "d76b579672d87f8b071d871ecfd5797b986f295b8dc93bc967471c0e1aab0b2b", + }, + "edgebench-portfolio-native-results.json": { + "original": "b33ebfcc7c233f8cf87b70e23b3bc76e34d5b91b8fcf11393e532afa9597b717", + "zh": "cf7eb84be2cc7b349c3118c278a6957e84796033f7bedd207a2d5802206a45d5", + "en": "ef0f5b90e9204839bccde669f8cbf321795c8e285d12ea87ecdd71cc9b02ba5a", + }, + "edgebench-ad-native-results.json": { + "original": "56670c1183da44d227be9d15088698baca8bfe800a9bc87ac0e715f072f58c69", + "zh": "08d1f35b52f9b58a867e104591a55f885e9c1076a510b9ca92d3e022785e9b6f", + "en": "d145136ee17d870ffbbcf89e45516b74302f71883fd82be92462f62faad263e7", + }, + }[fixture_name] for language, record in capture["analysis_readbacks"].items(): with urlopen(base + record["request_path"], timeout=10) as response: raw = response.read() assert hashlib.sha256(raw).hexdigest() == expected[language] == record["response_sha256"] view = capture["analysis_readbacks"]["original"]["response"]["pack"] - assert view["episode_count"] == 6 and len(view["result"]["assessments"]) == 36 - assert {row["verdict"] for row in view["result"]["assessments"]} == {"yes", "no", "unknown"} - assert view["job"]["usage"]["input_tokens"] == 53768 - assert view["job"]["usage"]["output_tokens"] == 19278 - assert view["native_outcomes"]["projection"]["initial"] is None + assert view["episode_count"] == 6 + native = view["native_outcomes"]["projection"] + if fixture_name == "edgebench-graph-saved-analysis.json": + assert len(view["result"]["assessments"]) == 36 + assert {row["verdict"] for row in view["result"]["assessments"]} == {"yes", "no", "unknown"} + assert view["job"]["usage"]["input_tokens"] == 53768 + assert view["job"]["usage"]["output_tokens"] == 19278 + assert native["initial"] is None + assert native["retained"]["candidate_ref"] == "agent-16" + assert native["final_valid"]["candidate_ref"] == "agent-18" + assert native["retained_improvement"]["baseline"]["candidate_ref"] == "agent-2" + assert native["retained_improvement"]["value"] == 0.032297817031212595 + else: + assert view["job"] is None and view["result"] is None + assert native["retained"] is None and "retained_improvement" not in native + initial, best, final = ((0, 10.86, 10.86) if "portfolio" in fixture_name + else (49550815819, 49653326363, 49650446168)) + assert native["initial"]["value"] == initial + assert native["best_valid"]["value"] == best + assert native["final_valid"]["value"] == final assert "solver_usage" not in view and not events - citation = capture["analysis_citations"][0] - assert read(citation["request_path"]) == citation["response"] + for citation in capture["analysis_citations"]: + assert read(citation["request_path"]) == citation["response"] for query in [ {"output_language": "fr"}, {"pack_id": "wrong-pack"}, ]: @@ -122,7 +152,7 @@ def read(path): read("/api/chat/research-analysis?" + urlencode({"goal_id": capture["goal_id"], **query})) assert rejected.value.code == 404 with pytest.raises(HTTPError) as rejected: - read(citation["request_path"].replace("native-report-4", "uncaptured-source")) + read(citation["request_path"].replace(citation["response"]["source_id"], "uncaptured-source")) assert rejected.value.code == 404 # No analysis artifacts, provider output or job state are synthesized. assert not list(self.runtime_root.parent.glob("analysis-*"))