From dbee2c2676de7a199a27af3bdc07f631354776ea Mon Sep 17 00:00:00 2001
From: heweidong-ecco <290444878+heweidong-ecco@users.noreply.github.com>
Date: Sun, 18 Oct 2026 18:31:28 +0800
Subject: [PATCH 1/2] =?UTF-8?q?docs(DEC-019):=20=E5=AE=8C=E6=95=B4=20live?=
=?UTF-8?q?=20=E5=9B=9E=E5=BD=92=20=E2=80=94=E2=80=94=2047/48=20=C2=B7=20e?=
=?UTF-8?q?xit=200=20=C2=B7=20=E6=9C=AA=E8=A7=82=E6=B5=8B=E5=88=B0?=
=?UTF-8?q?=E5=88=A4=E5=88=86=E5=99=A8=E8=AF=AF=E4=BC=A4?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
- run 20261018-182724-e6212c1d:被测 fastapi-rag-agent@150a963 + 隔离语料库(125 行)
→ 47/48 · 0.9792 · exit 0 · judge 39,697 token · 0 重试/0 解析失败
- 关键判据:rule_hit_judge_fail=0、soft_miss_judge_pass=0 ⇒ α 未把正确回答判 fail
- 唯一失败 id=20 是真实被测失败:召回 [55,64,8],未召回含「两者结果互补」的 doc 60
- 过程错误:两轮无效(judge key 401 · 我方语料筛选漏 16 篇 seed),judge 合计 ≈83,000 token
- 落点:报告/DEC-019 §7/失败回灌/复盘/证据归档(3 个 spans.jsonl)
---
CLAUDE.md | 40 +++--
ROADMAP.md | 69 +++++++--
...76\350\241\214\345\217\243\345\276\204.md" | 40 +++++
...30\346\226\271\351\225\234\345\203\217.md" | 54 +++++++
...ive\345\233\236\345\275\222-2026-10-18.md" | 144 ++++++++++++++++++
...21\346\240\207\350\257\255\346\226\231.md" | 63 ++++++++
eval/cases/fastapi-rag-2026-10-18.json | 33 ++++
.../runs/20261018-181715-eda91022.spans.jsonl | 140 +++++++++++++++++
.../runs/20261018-182150-86cc86b0.spans.jsonl | 144 ++++++++++++++++++
.../runs/20261018-182724-e6212c1d.spans.jsonl | 144 ++++++++++++++++++
10 files changed, 846 insertions(+), 25 deletions(-)
create mode 100644 "docs/decisions/DEC-023-live\345\233\236\345\275\222\347\216\257\345\242\203-\351\232\224\347\246\273\350\257\255\346\226\231\344\270\216\345\256\230\346\226\271\351\225\234\345\203\217.md"
create mode 100644 "docs/reports/DEC-019-live\345\233\236\345\275\222-2026-10-18.md"
create mode 100644 "docs/\345\244\215\347\233\230/2026-10-18-\350\257\255\346\226\231\347\255\233\351\200\211\346\274\217\346\216\211seed\351\207\221\346\240\207\350\257\255\346\226\231.md"
create mode 100644 eval/cases/fastapi-rag-2026-10-18.json
create mode 100644 eval/runs/20261018-181715-eda91022.spans.jsonl
create mode 100644 eval/runs/20261018-182150-86cc86b0.spans.jsonl
create mode 100644 eval/runs/20261018-182724-e6212c1d.spans.jsonl
diff --git a/CLAUDE.md b/CLAUDE.md
index a2580ce..498eb6b 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -121,10 +121,30 @@ eval/ · eval/cases/ · observability/ · 飞轮/ # 评估门 / **失
app/ .claude/skills/ .github/(CI-CD) tests/ # 实现层
```
-## 当前指针(随推进更新 — 上次更新 **2026-10-18** 会话:`DEC-019/020 均已实施并合(#81/#79)· 只剩 live 回归未跑 · 锚点已按实况更正`)
+## 当前指针(随推进更新 — 上次更新 **2026-10-18** 会话:`DEC-019` 完整 live 回归已跑完(47/48、exit 0)· 待签核 = 0 · 余项只剩环境待办)
-> **🔜 新会话从这里接**:读本段 + `ROADMAP.md`「📌 交接」即可,不必翻历史。
-> **B 的 12 个节点:已签核闭环 12 / 待终裁 0 / 技术欠账 0** —— 签核号到 **D-26**。
+> **🔜 新会话从这里接**:读本段 + `ROADMAP.md`「📌 交接(**2026-10-18**)」即可,不必翻历史。
+> **B 的 12 个节点:已签核闭环 12 / 待终裁 0 / 技术欠账 0** —— 签核号到 **D-29**。
+>
+> ✅ **2026-10-18 会话完成** —— **`DEC-019` 的完整 live 回归**(此前唯一未完成项):
+> - run **`20261018-182724-e6212c1d`**:被测 `fastapi-rag-agent@150a963` + 隔离语料库(125 行)
+> → **47/48 · 达标率 0.9792 · exit 0** · judge 39,697 token · **0 重试 / 0 解析失败**;
+> - **对「判分器变严会不会误伤」的回答**:`rule_hit_judge_fail = 0`、`soft_miss_judge_pass = 0`
+> ⇒ **本轮未观测到误伤**;唯一失败 `id=20` 是**真实被测失败**(召回 `[55,64,8]`,
+> 未召回含「两者结果互补」的 doc 60)。⚠️ **边界**:单轮、n=48,不能外推;
+> - **落点**:报告 `docs/reports/DEC-019-live回归-2026-10-18.md` · 失败回灌
+> `eval/cases/fastapi-rag-2026-10-18.json` · 过程错误
+> `docs/复盘/2026-10-18-语料筛选漏掉seed金标语料.md` · `DEC-019 §7`;
+> - ⚠️ **成本如实**:有效轮之前有两轮无效(judge key 401 · 我方语料筛选漏 seed),
+> **judge 合计 ≈83,000 token**(43,303 无效 + 39,697 有效)。
+>
+> ⚠️ **最新环境事实(与旧文档冲突,以本条为准)**:
+> - 生产 `rag_db` 已有 **345 行 `documents`**,评测语料只占 **125 行** ⇒ **不能直接拿生产库跑评测**;
+> 本轮用副本库 `rag_eval_20261018`(原库一行未动);
+> - 被测仓结构已从 `api/` 改为 **`app/`**,官方镜像是 **`fastapi-rag-agent:latest`**;
+> - ⚠️ **`tools/sut-harness/run_sut.sh` 已跑不起来**(仍要求 `api/` + 旧镜像 `my-fixed-name-api`)
+> ⇒ **唯一环境待办**:按新结构修它(本轮已按 README 退出条件绕行官方镜像);
+> - `EVAL_JUDGE_API_KEY` 曾失效(401),**已由业务方更新**(尾号 `fd3e`)。
>
> ✅ **2026-09-21 会话完成**(按业务方"建议顺序"逐条执行):
> - **B4 完成** —— 业务方 10 条盲标 → 回填参照集(**16 → 26 条**)→ `calibrate` **实测 0.88**(22/25),
@@ -181,16 +201,16 @@ app/ .claude/skills/ .github/(CI-CD) tests/ # 实现层
> - **`DEC-019`**(判分器放行了「非拒答的答非所问」)—— 签核 **D-28**(α + β)· **✅ 已实施并合(#81)**
> - **`DEC-020`**(门 2 判据取了「会话累计」)—— 签核 **D-29**(α)· **✅ 已实施并合(#79)**
>
-> **🔜 当前唯一未完成:`DEC-019` 的完整 live 回归**(⚠️ **≈3.5 万 token ⇒ 须另批**)
-> - **已跑的(judge 侧两件,30,266 token)**:`canary` **0/8**(改前 **3/8 被放过**,id=3/13/19)·
+> **✅ `DEC-019` 的完整 live 回归:已于 2026-10-18 跑完**(本段原写"当前唯一未完成")
+> - **judge 侧两件(2026-09-24,30,266 token)**:`canary` **0/8**(改前 **3/8 被放过**,id=3/13/19)·
> `calibrate` **1.00**(阈值 0.80;⚠️ 只能写「**本参照集已耗尽信息**」,不得写"已校准");
-> - **未跑的**:SUT + judge 的**完整一轮** —— 被测容器已被删,**要重起**
-> (`tools/sut-harness/run_sut.sh`;首次等 pip **30–60s**;判据是**能力探针**,不是 `/health`);
-> - ⚠️ **为什么不能省**:改 Prompt 的完整回归是本仓既有纪律 ——
-> **离线 `calibrate` 测不到「判分器变严后在真实链路里会不会误伤」**。
+> - **完整一轮(2026-10-18)**:`20261018-182724-e6212c1d` → **47/48 · 0.9792 · exit 0**;
+> `rule_hit_judge_fail = 0` ⇒ **未观测到误伤**;唯一失败 `id=20` 是真实被测失败(见上方交接);
+> - ⚠️ **边界**:单轮、n=48;被测版本已变(`f2dad78` → `150a963`),**不能与历史 48/48 直接比较**。
>
> **其余未完成(不阻塞可用)**:① 轮间 2× 延迟悬案(16 轮内 1 次,成因未指名)
-> ② 母体 Kit 缺陷观察(证据 1/2)③ 被测知识库有无文档(R1 探针)。
+> ② 母体 Kit 缺陷观察(证据 1/2)③ **环境待办:修 `run_sut.sh`**(见上方"最新环境事实")
+> ④ `id=20` 的检索问题(未改被测,仅回灌)。
>
---
> ### 📜 **分界:以下全部是「历史」**(2026-09-10 ~ 2026-09-21 的过程记录)
diff --git a/ROADMAP.md b/ROADMAP.md
index 1c39798..ead2170 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -19,23 +19,62 @@
⇒ **收窄为 `A-safe`**:只改报告面(`hits`→`issues`),**判分器入参的键一字不动**,并入方案 B(待报备 ≈3 万)。
新增 2 条守卫钉住该边界(**已突变验证**)。详见 `DEC-015 §7`。
- **定调**:阶段1/2 已由业务方逐条签核(D-1..D-9,2026-09-10);**P3 收口顺序已由 D-10a..D-10h 签核**(2026-09-10,grilling 定调压测门通过后,见 `docs/decisions/定调复核-签核记录.md`)。
-- **🟢 收尾批次(裁定 D-23/24/25)—— ✅ P0+P1 已完成;⛔ P2 被环境卡住**
+- **🟢 收尾批次(裁定 D-23/24/25)—— ✅ P0–P8 全部完成并合入 main**(2026-09-21 收口)
实施计划 = **`docs/plans/2026-09-15-收尾批次-实施计划.md`**(含逐项"认下的代价"与防伪清单)。
- **✅ 已合**:计划与裁决(#59)· **PR-1 逐轮取证落盘 + `ab_pool`**(#60)· **PR-2 工具卫生三修**(#61);
+ **已合**:计划与裁决(#59)· **PR-1 逐轮取证落盘 + `ab_pool`**(#60)· **PR-2 工具卫生三修**(#61)
+ · **#64–#67** · **#69** · **#70–#72** · **#75**;**12 个遗留分支已清,仓库只剩 `main`**。
途中修掉真 bug:`_run_id` **同秒撞名** ⇒ 同一秒内两次运行的产物**静默互相覆盖**。
- **⛔ 卡点**:**Docker Desktop 的 Linux VM 未运行**(虚拟化进程匹配数 0)⇒ `docker start` 挂死
- ⇒ `postgres-rag`/`redis-rag` 起不来 ⇒ 被测起不来 ⇒ **`chunk 1` 未开始**。业务方将自行重启 Docker。
- **下一步**:预检 `run_sut.sh`(退出码 0)→ chunk 1(≈23 万 / ≈23 min)→ chunk-1 门禁(不过不开 chunk 2)。
- **做**:A1 `T4` live A/B(≈116 万,分 5 块 + **已批准早停**)· A2 `DEC-015 B1`(≈3 万,**排在批次报告之后**)·
- A3(并入 A1 的 40 轮)· B4 judge 校准(前置 **`DEC-017` 已签核 D-25**)· B5 T5 重审(须单独 DEC)。
- **不做**:B7 接入他人 PR 流程 · C1 M3 抽取(维持现状)。
- **定性**:C2 `T3` → 正式判**不适用**+触发条件 · C3 MTTR → 设**覆盖目标**(守卫**尚未写**)· C4 飞轮 → **已裁决**。
- ⚠️ **C2/C3 零 token、不依赖 Docker ⇒ 可在等 Docker 期间并行做。**
-- **⛔ 授权口径已更正(D-23)**:PR 合并**逐次批** —— 开好先问「可以合吗」,**自开 PR 不许自合**(取代 D-14 的 auto-merge 条款)。
-- **下一步(节点侧)**:**B 的 12 个节点全部闭环(已签核 12 · 待终裁 0 · 技术欠账 0)**,签核号到 **D-24**。
- 节点表已无待办;剩余全部集中在上面那个批次里。
-
-### 📌 交接(**2026-09-21** 会话末 · 新会话从这里接)
+ **`DEC-021`**:测试基数复算 **499 → 554**(由 #69 承载)。
+- **✅ `DEC-019` / `DEC-020`:均已签核、实施并合**(`#81` / `#79`);
+ **`DEC-019` 的完整 live 回归已于 2026-10-18 跑完** —— **47/48、exit 0**,
+ **未观测到判分器误伤**(`rule_hit_judge_fail=0`)⇒ 见下方「📌 交接(2026-10-18)」。
+- **⛔ 授权口径(D-23)**:PR 合并**逐次批** —— 开好先问「可以合吗」,**自开 PR 不许自合**(取代 D-14 的 auto-merge 条款)。
+- **下一步(节点侧)**:**B 的 12 个节点全部闭环;待签核 = 0;收尾批次已收口**。
+ 剩余 = **环境待办**(修 `run_sut.sh`)+ 非阻塞观察项,见下方「📌 交接(2026-10-18)」。
+
+### 📌 交接(**2026-10-18** 会话末 · 新会话从这里接)
+
+> **🔜 一句话**:`DEC-019` 的**完整 live 回归已跑完** —— **47/48、exit 0**;
+> **未观测到 α 造成的误伤**(`rule_hit_judge_fail = 0`)。
+> ⇒ **B 的 12 节点 + 收尾批次 P0–P8 + `DEC-019`/`DEC-020` 全部收口;待签核 = 0。**
+
+#### ✅ 2026-10-18 完成了什么
+
+| 序 | 事 | 落点 |
+|---|---|---|
+| 1 | **`DEC-019` 完整 live 回归** | ✅ run `20261018-182724-e6212c1d` · **47/48 · 0.9792 · exit 0** · judge 39,697 token |
+| 2 | 报告(环境/语料/证据/复跑命令) | `docs/reports/DEC-019-live回归-2026-10-18.md` |
+| 3 | 唯一失败 `id=20` 回灌 | `eval/cases/fastapi-rag-2026-10-18.json`(`status=open`) |
+| 4 | 过程错误复盘(两轮无效,83,000 judge token) | `docs/复盘/2026-10-18-语料筛选漏掉seed金标语料.md` |
+| 5 | `DEC-019` 补记 §7 | `docs/decisions/DEC-019-判分器放行口径.md` |
+
+**对「α 会不会误伤」的回答**:`rule_hit_judge_fail = 0`、`soft_miss_judge_pass = 0`;
+唯一失败 `id=20` 是**真实被测失败**(召回 `[55,64,8]`,未召回含「两者结果互补」的 doc 60),
+判分器判 fail 正确。⚠️ **边界**:单轮、n=48;被测已从 `f2dad78` 前进到 `150a963`,
+**不能与历史 48/48 基线直接比较**。
+
+**环境关键事实(本轮实测)**:
+
+- 生产 `rag_db` 已有 **345 行**,评测语料只占 **125 行**(其它项目/测试灌入 220 行)
+ ⇒ **不能直接拿生产库跑评测**;本轮用副本库 `rag_eval_20261018`(原库一行未动);
+- 被测仓已从 `api/` 改为 **`app/`**,官方镜像是 **`fastapi-rag-agent:latest`**;
+- 旧 `tools/sut-harness/run_sut.sh` 因此**已跑不起来**(仍要求 `api/` + `my-fixed-name-api` 旧镜像);
+- `EVAL_JUDGE_API_KEY` 曾失效(401),**已由业务方更新**(尾号 `fd3e`)。
+
+#### ⚠️ 下一会话开工要盯的
+
+1. **待签核 = 0**;没有未完成的评测节点。
+2. **环境待办(唯一)**:修 `tools/sut-harness/run_sut.sh`(适配 `app/` 与官方镜像),
+ 否则下轮照旧命令会直接失败。本轮已按 README 退出条件绕行官方镜像。
+3. **非阻塞余项**:① 轮间 2× 延迟悬案(16 轮内 1 次,成因未指名)
+ ② 母体 Kit 缺陷观察(证据 1/2)③ R1 探针(生产库有无文档 —— 本轮已间接回答:**有 345 行**)。
+4. **`id=20` 的检索问题未修**(被测未改),只回灌不结论。
+
+### 📜 历史交接(**2026-09-21** 会话末,**已被上文 2026-10-18 取代**)
+
+> ⚠️ **本段当时的「当前唯一未完成 = `DEC-019` 完整 live 回归」已于 2026-10-18 跑完**
+> (47/48、exit 0、未观测到误伤)⇒ 以**上方「📌 交接(2026-10-18)」为准**;
+> 本段保留作过程记录,下面所有「未跑 / 待做 / 下一步」均按**过去时**读。
> **🔜 一句话**:收尾批次 **P0→P8 全部完成并已合入 main** —— **整个批次收口**。
> `main` **不写死 sha**(它每次合并都会过期 ⇒ 现取:`git log -1 --oneline main`)。
diff --git "a/docs/decisions/DEC-019-\345\210\244\345\210\206\345\231\250\346\224\276\350\241\214\345\217\243\345\276\204.md" "b/docs/decisions/DEC-019-\345\210\244\345\210\206\345\231\250\346\224\276\350\241\214\345\217\243\345\276\204.md"
index f27b336..936445b 100644
--- "a/docs/decisions/DEC-019-\345\210\244\345\210\206\345\231\250\346\224\276\350\241\214\345\217\243\345\276\204.md"
+++ "b/docs/decisions/DEC-019-\345\210\244\345\210\206\345\231\250\346\224\276\350\241\214\345\217\243\345\276\204.md"
@@ -143,3 +143,43 @@ assert "答非所问" not in sysp.replace("答非所问时", "")
- ⚠️ **α ≈ 4.9 万 token**(回归 ≈3.5 万/轮 + `calibrate` ≈1.4 万);
- ⚠️ **α 会让判分器更严** ⇒ `judge_human_agreement` **可能下降** —— **那是仪器变准,不是回归**;
- ⚠️ **β 眼下就是红的**(洞未修)⇒ **实施时必须二选一**:先修 α 让它转绿,或**显式标"已知未修"**。
+
+## 7. 完整 live 回归(**2026-10-18 已跑**)
+
+> **本节补上 §6.3 之后唯一欠着的验证** —— 离线 `calibrate` 测不到
+> 「判分器变严后在真实链路里会不会误伤」。
+
+- **run**:`20261018-182724-e6212c1d`(eval-gate `5375542`)· 被测 `fastapi-rag-agent@150a963`
+- **环境**:官方镜像 `fastapi-rag-agent:latest` + **隔离语料库 `rag_eval_20261018`(125 行)**
+ —— 生产 `rag_db` 已有 345 行(混入其它项目语料),不能直接跑
+- **结果**:**47/48 通过(0.9792)· 1 失败 · 0 存疑 · 红队突破 0 · exit 0**
+- **judge**:39,697 token · 45 次调用 / 45 条用例 · **0 重试 / 0 解析失败 / 0 截断**
+
+### 7.1 对「α 会不会误伤」的回答
+
+| 判据 | 值 | 结论 |
+|---|---:|---|
+| `rule_hit_judge_fail` | **0** | **没有**「字面命中却被判 fail」的条目 —— 最直接的误伤形态未出现 |
+| `soft_miss_judge_pass` | **0** | 也没有反向的「软层没命中、judge 放过」 |
+| 唯一失败 `id=20` | 真实失败 | 召回 `[55,64,8]`,未召回含「两者结果互补」的 doc 60;被测如实拒答 ⇒ judge 判 fail 正确 |
+
+⇒ **本轮没有观测到 α 造成的误伤。**
+⚠️ **边界(不得省略)**:单轮、n=48;本轮没有出现「字面命中 + 答非所问」同时成立的样本;
+被测版本已从 `f2dad78` 前进到 `150a963`,**不能与历史 48/48 基线直接比较**。
+
+### 7.2 过程与成本(如实)
+
+- 有效轮之前有**两轮无效**:
+ ① judge key 失效(HTTP 401,judge 0 token,SUT 侧约 5 万);
+ ② **我方语料筛选漏掉 `eval_dataset.json#10–#25` 的 16 篇 seed**(43,303 judge token 白花)——
+ 过程错误见 `docs/复盘/2026-10-18-语料筛选漏掉seed金标语料.md`;
+- 本轮 judge 合计 **≈83,000 token**(43,303 无效 + 39,697 有效);
+- 完整证据、隔离语料构成与复跑命令:
+ **`docs/reports/DEC-019-live回归-2026-10-18.md`**;
+- 失败回灌:**`eval/cases/fastapi-rag-2026-10-18.json`**(id=20,`status=open`)。
+
+### 7.3 同时发现的环境问题(与 DEC-019 无关,单独跟踪)
+
+`tools/sut-harness/run_sut.sh` 已与当前被测结构不匹配(被测从 `api/` 改为 `app/`,
+默认镜像 `my-fixed-name-api` 也已过期)⇒ **旧命令会直接跑不起来**;
+本轮按 harness README 的退出条件改走官方镜像。修复列入待办。
diff --git "a/docs/decisions/DEC-023-live\345\233\236\345\275\222\347\216\257\345\242\203-\351\232\224\347\246\273\350\257\255\346\226\231\344\270\216\345\256\230\346\226\271\351\225\234\345\203\217.md" "b/docs/decisions/DEC-023-live\345\233\236\345\275\222\347\216\257\345\242\203-\351\232\224\347\246\273\350\257\255\346\226\231\344\270\216\345\256\230\346\226\271\351\225\234\345\203\217.md"
new file mode 100644
index 0000000..4c34fcf
--- /dev/null
+++ "b/docs/decisions/DEC-023-live\345\233\236\345\275\222\347\216\257\345\242\203-\351\232\224\347\246\273\350\257\255\346\226\231\344\270\216\345\256\230\346\226\271\351\225\234\345\203\217.md"
@@ -0,0 +1,54 @@
+# DEC-023 · live 回归的环境选择:隔离语料库 + 官方镜像
+
+- 状态:**已实施**(2026-10-18,`DEC-019` 完整 live 回归)
+- 类型:工程方法决策(非业务口径,无需业务方签核)
+- 关联:`DEC-019`(判分器放行口径)· `DEC-018`(证据覆盖)· `DEC-014`(归档按需)
+
+## 1. 背景(实测,不是推断)
+
+跑 `DEC-019` 的完整 live 回归时,环境已经和旧文档不一致:
+
+| 事实 | 实测 |
+|---|---|
+| 生产库 `rag_db` 的 `documents` | **345 行**;评测语料只占约 **125 行**,其余是其它项目/测试灌入(`a.md` 91、`test` 48、`isolation-seed` 16、demo/probe 等) |
+| 被测仓结构 | 已从 `api/` 改为 **`app/`**(HEAD `150a963`,2026-10-18) |
+| 官方镜像 | **`fastapi-rag-agent:latest`**(2 天前构建);旧 `my-fixed-name-api` 停留在 2026-07-01 |
+| 旧 harness | `tools/sut-harness/run_sut.sh` 仍要求 `api/` + 旧镜像 ⇒ **直接跑会 `die`** |
+| judge key | `EVAL_JUDGE_API_KEY` 曾失效(HTTP 401),业务方已更新 |
+
+## 2. 备选方案与评估
+
+| 方案 | 做法 | 优点 | 代价 / 为什么没选 |
+|---|---|---|---|
+| **A. 直接跑生产库** | 不隔离,`top_k=3` 面对 345 行 | 零准备成本;最接近"生产真实" | **失败无法归因**:无关语料挤占召回,分不清是被测退化、判分器变严还是语料污染 |
+| **B. 清理生产库** | 删掉非评测语料 | 库变干净 | **破坏性**:生产库是共享数据,别的项目正在用;**否决** |
+| **C. 隔离副本(选定)** | `CREATE DATABASE rag_eval_20261018 TEMPLATE rag_db` → 过滤 + 去重 → 125 行 | **原库一行不动**;失败可归因;可重复 | 需要准确识别评测语料;结果**不能直接代表生产库** |
+| **D. 修旧 harness 后照旧跑** | 把 `run_sut.sh` 改到 `app/` | 保留旧路径 | 官方镜像已可用(harness README 的**退出条件**已满足);临时改 harness 会把"环境修复"混进本次回归;**推到单独待办** |
+| **E. 回退被测到 `f2dad78`** | worktree checkout 旧版本 | 与历史 48/48 可比 | 业务方明确告知数据已变;目标是**当前链路**里验证判分器,不是复现旧基线;**否决** |
+
+## 3. 决策
+
+1. **语料**:用副本库 `rag_eval_20261018`,生产 `rag_db` 不写、不删;
+2. **被测运行方式**:走官方镜像 `fastapi-rag-agent:latest`(按 `tools/sut-harness/README.md` 的退出条件);
+3. **会话预算**:本轮容器临时设 `SESSION_TOKEN_LIMIT=200000`(新版被测默认 5 万,单轮会触顶);
+4. **judge key**:使用业务方更新后的 `EVAL_JUDGE_API_KEY`;
+5. **旧 harness**:不在本次修改,登记为**唯一环境待办**。
+
+## 4. 认下的代价与边界(不得省略)
+
+- ⚠️ **结果不能与历史 48/48 直接比较**:被测版本(`f2dad78` → `150a963`)与语料(70 篇 → 125 行)都变了;
+- ⚠️ **隔离语料的正确性依赖"评测事实反查"** —— 本次第一版过滤漏掉了
+ `eval_dataset.json#10–#25` 的 16 篇 seed,导致一轮 **37/48 的无效结果**;
+ 补救与防错见 `docs/复盘/2026-10-18-语料筛选漏掉seed金标语料.md`;
+- ⚠️ **隔离库 ≠ 生产库**:它回答"判分器在真实链路里是否误伤",不回答"生产库当前质量如何";
+- ⚠️ **旧 harness 仍是坏的**:下轮若照旧命令跑会直接失败,必须先修或改用本 DEC 的官方镜像命令。
+
+## 5. 实施结果
+
+- run `20261018-182724-e6212c1d`:**47/48 · 0.9792 · exit 0**;
+- `rule_hit_judge_fail = 0`、`soft_miss_judge_pass = 0` ⇒ 未观测到判分器误伤;
+- 完整记录:`docs/reports/DEC-019-live回归-2026-10-18.md`。
+
+## 变更记录
+
+- 2026-10-18 建立(随 `DEC-019` 完整 live 回归落盘)。
diff --git "a/docs/reports/DEC-019-live\345\233\236\345\275\222-2026-10-18.md" "b/docs/reports/DEC-019-live\345\233\236\345\275\222-2026-10-18.md"
new file mode 100644
index 0000000..acc170d
--- /dev/null
+++ "b/docs/reports/DEC-019-live\345\233\236\345\275\222-2026-10-18.md"
@@ -0,0 +1,144 @@
+# DEC-019 完整 live 回归报告(2026-10-18)
+
+> **本轮要回答的唯一问题**:`DEC-019` 的 α(判分器提示词收窄到「仅拒答语境」+ 非拒答反例)
+> **在真实链路里,会不会把本来正确的回答判成 fail(误伤)?**
+>
+> **结论**:**本轮没有观测到误伤** —— 48 条里 47 条通过,唯一失败 `id=20` 是**真实被测失败**;
+> `rule_hit_judge_fail = 0`(字面命中却被判 fail 的条数 = 0)。
+> ⚠️ **边界**:单轮、n=48、被测版本已变(见 §5),**不能**外推成「判分器从此不会误伤」。
+
+- run:`20261018-182724-e6212c1d`
+- eval-gate commit:`5375542` · evals_file_sha:`a926979e30c0` · threshold_rev:`2026-09-13`
+- 结果:**47/48 通过 · 1 失败 · 0 存疑 · 红队突破 0 · exit 0**
+- 达标率:**0.9792**(阈值 `l2_task_completion ≥ 0.95`)
+- judge:**deepseek-v4-flash@https://api.deepseek.com** · 39,697 token(31,372 prompt + 8,325 completion)
+- 记账:**45 次调用 / 45 条用例 · 0 重试 · 0 解析失败 · 0 截断**
+
+## 1. 为什么要单独做「隔离语料」环境
+
+本轮开始前实测 `rag_db` 已有 **345 行 `documents`**,而评测语料只占其中一部分:
+
+| 类别 | 行数 | 例子 |
+|---|---:|---|
+| 评测语料 | 125(去重后) | A–E 机床售后语料 · `eval_dataset.json#10–#25` · `eval-gap-fill-*` · `tech` · `business` · `financial_report` · `final_smoke` 等 |
+| 其它项目/测试灌入 | 220 | `a.md`(91,内容为 x)、`test`(48,测试文档一/二/三)、`isolation-seed/*`(16)、demo/probe 探针等 |
+
+直接拿 345 行跑,`top_k=3` 会被无关语料挤占,**失败无法归因**。
+⇒ 本轮用 `CREATE DATABASE rag_eval_20261018 TEMPLATE rag_db` 建**副本**,
+在原库**一行未动**的前提下过滤 + 去重,得到 **125 行**隔离语料。
+
+> ⚠️ **第一次过滤漏掉了 `eval_dataset.json#10–#25` 这 16 篇 seed 金标语料**
+> (当时只按 `^[A-E]-` 与 `eval-gap-fill` 留),导致一轮 **37/48** 的无效结果,
+> 浪费 43,303 judge token。过程错误与防错措施见
+> `docs/复盘/2026-10-18-语料筛选漏掉seed金标语料.md`。
+
+隔离语料构成(去重后):
+
+| source | 行数 | 说明 |
+|---|---:|---|
+| A–E 机床售后语料 | 85 | 原始公司语料(同名文件已按内容去重) |
+| `eval_dataset.json#10–#25` | 16 | **seed 金标语料**(FastAPI/Python/混合检索/用户 ID 等) |
+| `eval-gap-fill-*` | 8 | 2026-09-11/12 补的缺口语料 |
+| `tech` | 8 | Python/FastAPI/Docker/ML/PostgreSQL/Redis/LangChain/向量库 |
+| `手动入库(本页)` | 3 | 青云精工作业规范 |
+| `business` | 2 | AI 市场规模 / 苹果公司产品 |
+| `financial_report` | 1 | 2025 Q4 营收 5800 万、同比 23%(原 12 行重复,去重) |
+| `final_smoke` | 1 | FastAPI 基于 Starlette + Pydantic |
+| `test_docs` | 1 | Python 广泛用于 AI/数据科学(原 23 行重复,去重) |
+| **合计** | **125** | |
+
+## 2. 被测与环境
+
+> 环境方案的选择与代价见 **`docs/decisions/DEC-023-live回归环境-隔离语料与官方镜像.md`**
+> (隔离副本 vs 生产库 / 官方镜像 vs 旧 harness / 会话预算 / judge key)。
+
+| 项 | 值 |
+|---|---|
+| 被测仓 | `Product/agent-projects/projects/fastapi-rag-agent` @ **`150a963`(2026-10-18)** |
+| 运行方式 | **官方镜像 `fastapi-rag-agent:latest`(2 天前构建)**,不走旧 harness |
+| 数据库 | `rag_eval_20261018`(隔离副本) + `redis-rag` |
+| 端点 | `POST /api/v1/rag/search?mode=accurate_norerank` · `top_k=3` |
+| 能力探针 | ✅ 通过(能对真实问题作答,不是 `/health` 说了算) |
+| 会话预算 | 本轮容器临时设 `SESSION_TOKEN_LIMIT=200000`(默认 5 万) |
+
+### ⚠️ 旧 harness 已失效(本轮发现)
+
+`tools/sut-harness/run_sut.sh` 仍按旧结构写死:
+
+- 要求被测仓有 `api/` 并只读挂载 `$SUT_REPO/api:/app` —— 现被测仓已改为 **`app/`**,脚本会直接 `die`;
+- 默认镜像 `my-fixed-name-api`(2026-07-01)已过期,当前官方镜像是 `fastapi-rag-agent:latest`。
+
+⇒ 本轮按 harness README 的**退出条件**改走官方路径。
+**待办**:按新结构修 `run_sut.sh`(或正式退役它),否则下次照旧命令会跑不起来。
+
+## 3. 结果
+
+```
+通过 47/48 · 失败 1 · 存疑 0 · 跳过 0 · 红队突破 0
+达标率 0.9792 ≥ 0.95 ⇒ exit 0
+L1: system_error_rate 0.0000 · gate_overhead_ratio 0.000507 · root_ms 139,202
+```
+
+### 3.1 对 DEC-019 的关键判据
+
+| 判据 | 值 | 含义 |
+|---|---:|---|
+| `rule_hit_judge_fail` | **0** | **没有**「字面命中却被判分器拦下」的条目 —— 这是最直接的误伤形态 |
+| `soft_miss_judge_pass` | 0 | 没有「软层没命中、判分器却放过」的条目 |
+| `judge parse_failures / truncated / retries` | 0 / 0 / 0 | 判分器链路健康,失败不是解析噪声 |
+| 唯一失败 `id=20` | 真实失败 | 见 §4 |
+
+### 3.2 唯一失败(id=20)是真实失败,不是误伤
+
+- 问题:「混合检索比纯向量检索好在哪里?」· 期望 `answer_contains: ["互补","取长补短"]`
+- 被测答案:「根据现有资料,无法回答。现有上下文仅说明向量检索理解语义但忽略精确关键词,
+ 关键词检索精确匹配但不理解语义[来源:2],未提供混合检索与纯向量检索的比较信息。」
+- 本轮召回:`[55, 64, 8]`
+- **语料里其实有答案**:`eval_dataset.json#19`(doc id **60**)写着「向量检索理解语义,
+ BM25 精确匹配关键词,**两者结果互补**」—— 但**没被 top_k=3 召回**。
+- ⇒ 根因:**被测检索未召回 doc 60**,被测据召回到的 #23(doc 64,只复述各自局限)如实拒答。
+ **判分器判 fail 与规则一致,不是变严导致的误伤。**
+- 失败已回灌:`eval/cases/fastapi-rag-2026-10-18.json`(status `open`)。
+
+## 4. 本轮**不能**证明什么
+
+1. **不能证明判分器从此不会误伤** —— 单轮、48 条;`DEC-019` 的洞针对的是
+ 「非拒答的答非所问」,本轮恰好没有出现「字面命中 + 答非所问」同时成立的样本。
+2. **不能与历史 48/48 基线直接比较** —— 被测版本已从 `f2dad78` 前进到 `150a963`,
+ 且本轮使用隔离语料(125 行)而不是历史库的 70 篇。
+3. **不能外推到生产库** —— 隔离语料是评测副本;生产 `rag_db` 仍是 345 行、含其它项目数据。
+4. **不能说明「id=20 的检索问题已解决」** —— 未改被测,仅记录。
+
+## 5. 成本记账(如实)
+
+| 轮次 | 用途 | judge token | SUT 侧 token | 是否有效 |
+|---|---|---:|---:|---|
+| `20261018-181715-eda91022` | 首次跑 | **0**(43 次 401) | ≈50,370(触顶 5 万) | ❌ `EVAL_JUDGE_API_KEY` 失效 |
+| `20261018-182150-86cc86b0` | 修 key 后重跑 | **43,303** | ≈5 万 | ❌ 语料筛选漏掉 16 篇 seed |
+| `20261018-182724-e6212c1d` | **最终有效轮** | **39,697** | ≈4 万 | ✅ |
+
+- 本轮 judge 合计 **≈83,000 token**(不含此前 `canary` 0/8 与 `calibrate` 1.00 的 30,266);
+- 授权过程:先批准「完整 live 回归 ≈3.5 万」;在无效轮之后**单独再问**,业务方选「完整重跑 48 条」;
+- **无效轮 43,303 token 的责任**:我方语料筛选规则漏掉 seed 语料,已记复盘。
+
+## 6. 复跑命令(本轮实测可用)
+
+```bash
+# 0) 隔离库(首次;之后可复用)
+# CREATE DATABASE rag_eval_20261018 TEMPLATE rag_db;
+# 过滤 junk source + 按 content 去重 ⇒ 125 行
+
+# 1) 起被测(官方镜像)
+docker run -d --name rag-api-eval \
+ --network fastapi-rag-agent_app-net -p 127.0.0.1:8000:8000 \
+ --env-file /.env -e DOCKER_ENV=true \
+ -e LLM_MODEL_FAST=deepseek-v4-flash -e LLM_MODEL_CHAT=deepseek-v4-flash \
+ -e POSTGRES_DB=rag_eval_20261018 -e SESSION_TOKEN_LIMIT=200000 \
+ fastapi-rag-agent:latest
+
+# 2) 能力探针(必需,判据是能作答而不是 /health)
+# 3) 跑评测
+set -a; . eval/runs/.sut-harness.env; set +a
+export SSL_CERT_FILE=/etc/ssl/cert.pem
+.venv/bin/python -m eval_gate run --evals eval/fastapi_rag_golden.evals.json --report-dir eval/runs
+```
diff --git "a/docs/\345\244\215\347\233\230/2026-10-18-\350\257\255\346\226\231\347\255\233\351\200\211\346\274\217\346\216\211seed\351\207\221\346\240\207\350\257\255\346\226\231.md" "b/docs/\345\244\215\347\233\230/2026-10-18-\350\257\255\346\226\231\347\255\233\351\200\211\346\274\217\346\216\211seed\351\207\221\346\240\207\350\257\255\346\226\231.md"
new file mode 100644
index 0000000..4e59107
--- /dev/null
+++ "b/docs/\345\244\215\347\233\230/2026-10-18-\350\257\255\346\226\231\347\255\233\351\200\211\346\274\217\346\216\211seed\351\207\221\346\240\207\350\257\255\346\226\231.md"
@@ -0,0 +1,63 @@
+# 复盘 · 语料筛选漏掉 seed 金标语料(2026-10-18)
+
+> 依据:B `06-持续优化策略/06-优化-数据飞轮与主动学习-v1.0.md:54-58`「建案例库 → 结构化存储 + 标签体系」「定期复盘」
+> + Kit `docs/memory-规约.md:30`「复盘 → `docs/复盘/`,行动项进 issue/看板」。
+> **纪律**:复盘写「**为什么**」,不写「谁的问题」;每条错误必须有**根因**与**防错措施**,没有就写「待查」,不许空着。
+
+- 范围:`DEC-019` 完整 live 回归(2026-10-18)
+- 参与:Agent(执行)+ 业务方(批准重跑 / 更新 judge key)
+- 结论:**最该记住的是 —— 隔离语料时「删什么」比「留什么」更容易错;
+ 必须用「评测期望里的事实是否还在」来验收过滤结果,不能只看 source 名字。**
+
+## 一、错误清单
+
+> **两类错误分开记**:① 被测/产物的失败 → 进 `eval/cases/`(带 `regression` 块);
+> ② **我们过程里的错误** → 记在这里。本条属 ②。
+
+| # | 现象 | 根因 | 影响 | 状态 | 防错措施 |
+|---|---|---|---|---|---|
+| 1 | ⚠️ **隔离语料时漏掉 `eval_dataset.json#10–#25` 的 16 篇 seed 金标语料**。该轮 37/48,11 条失败全是"拒答",当时差点被读成"判分器变严误伤" | 过滤规则只留了 `^[A-E]-` 与 `eval-gap-fill*`,把 seed 和 `test`/`test_docs` 一起当成"测试垃圾"删了。**根因是"按 source 名字猜语料归属",没有先核对原始语料构成**(ROADMAP 明写"70 篇 = 62 原始 + 8 补入",其中 62 含 seed 16) | **43,303 judge token 的无效轮** + 一轮 SUT 侧约 5 万 token;并一度产生"判分器误伤"的错误方向 | 已修 | 见 §二·1(事实反查清单) |
+| 2 | ⚠️ **首次全量跑时 `EVAL_JUDGE_API_KEY` 已失效(HTTP 401)**,43 条判分全部 `E_JUDGE_AUTH`,结果 3/48 | **跑全量前没有做一次判分器连通性预检**;只在跑完后才从 trace 里看到 401 | judge 0 token,但 SUT 侧约 50,370 token 触顶 5 万会话上限,整轮无效 | 已修(业务方更新 key) | 见 §二·2(开跑前三项预检) |
+| 3 | **被测新版有会话预算(默认 5 万 token/会话/日)**,首次跑到 id=47/48 时 429 | 该配置是新版被测新增的,旧 baseline(被测 `f2dad78`)没有;跑前没查 `token_usage_logs` 今日余量 | 尾 2 条失败混入 `E_SUT_4XX`,进一步污染归因 | 已修(本轮容器设 `SESSION_TOKEN_LIMIT=200000`) | 见 §二·2 |
+
+## 二、教训(可复用的判断规则)
+
+1. **隔离语料后,先用"期望事实反查"验收,再开跑。**
+ 做法:把评测集里每条 `answer_contains` / `answer_not_contains` 的关键事实拿去库里数一遍,例如:
+ ```sql
+ SELECT count(*) FROM documents WHERE content LIKE '%Starlette%'; -- id=4
+ SELECT count(*) FROM documents WHERE content LIKE '%数千亿美元%'; -- id=10
+ SELECT count(*) FROM documents WHERE content LIKE '%iPhone%'; -- id=47
+ SELECT count(*) FROM documents WHERE content LIKE '%5800%'; -- id=43
+ SELECT count(*) FROM documents WHERE content LIKE '%互补%'; -- id=20
+ ```
+ **任何一条为 0 ⇒ 语料不完整,不许开跑。**
+ > 判据:过滤的正确性由"评测集需要的事实还在不在"定义,不由"source 名字看起来像不像测试数据"定义。
+
+2. **任何会烧 token 的完整轮,开跑前做三项零成本预检:**
+ - ① judge 连通性:1 次最小 `chat/completions`,必须是 **HTTP 200**;
+ - ② 被测能力探针:能对真实问题作答(不是 `/health` 200);
+ - ③ 会话预算余量:查 `token_usage_logs` 今日已用 + 预估本轮 ≈5 万,
+ 不足则显式设 `SESSION_TOKEN_LIMIT`。
+ 任一项不过 ⇒ **停在预检,不开全量**。
+
+3. **失败归因先看"最直接的误伤判据",不要先怀疑判分器变严。**
+ 本轮有效判据是 `rule_hit_judge_fail`(字面命中却被判 fail)与失败答案本身;
+ `rule_hit_judge_fail=0` + 失败答案确为拒答/未作答 ⇒ 先查语料/检索,而不是先改判分器。
+
+## 三、行动项(带责任人/时限)
+
+| 行动 | 归属 | 状态 |
+|---|---|---|
+| 修 `tools/sut-harness/run_sut.sh`(被测已从 `api/` 改为 `app/`,默认镜像已过期) | 本仓 | ⬜ 待办(本轮已绕行官方镜像) |
+| 把"隔离语料事实反查清单"写进 live 回归的操作步骤 | 本仓 | ✅ 已写入 `docs/reports/DEC-019-live回归-2026-10-18.md` §6 |
+
+## 四、对母体 Kit 的反馈
+
+- **KD 候选**:Kit 的评估模块要求"失败即补 case",但没有"**跑真实评测前的预检**"这一节;
+ 本轮三次无效轮都源于预检缺失。建议最小修复:在 `05-模块-评估与测试` 增加三项预检
+ (judge 200 / 能力探针 / 会话预算余量)。⚠️ 证据仅本仓 1 例,按 G2 暂不单独提请,
+ 先在本仓落成操作步骤;若第二个项目复现再提。
+
+## 变更记录
+- 2026-10-18 建立。
diff --git a/eval/cases/fastapi-rag-2026-10-18.json b/eval/cases/fastapi-rag-2026-10-18.json
new file mode 100644
index 0000000..01f7193
--- /dev/null
+++ b/eval/cases/fastapi-rag-2026-10-18.json
@@ -0,0 +1,33 @@
+{
+ "version": 1,
+ "threshold_ref": "eval/阈值.md",
+ "sut_default": "fastapi-rag",
+ "_comment": "2026-10-18 DEC-019 完整 live 回归(run 20261018-182724-e6212c1d,47/48)的失败回灌。被测 = fastapi-rag-agent @ 150a963,隔离语料 rag_eval_20261018(125 行)。本文件是失败案例库,不是评测输入 —— 评测输入见 ../fastapi_rag_golden.evals.json。",
+ "evals": [
+ {
+ "id": 20,
+ "module": "rag",
+ "tags": [
+ "happy-path",
+ "regression"
+ ],
+ "input": {
+ "question": "混合检索比纯向量检索好在哪里?"
+ },
+ "expected": {
+ "answer_contains": [
+ "互补",
+ "取长补短"
+ ]
+ },
+ "source": "01.FastAPI RAG Agent archive eval_dataset.json id:19",
+ "regression": {
+ "from_run": "20261018-182724-e6212c1d",
+ "observed": "被测答『根据现有资料,无法回答。现有上下文仅说明向量检索理解语义但忽略精确关键词,关键词检索精确匹配但不理解语义[来源:2],未提供混合检索与纯向量检索的比较信息。』;本轮召回 doc id [55, 64, 8],soft_missed=true,judge=fail。",
+ "root_cause": "被测检索未召回答案所在文档:隔离语料中 doc id=60(source=eval_dataset.json#19)明写『向量检索理解语义,BM25 精确匹配关键词,两者结果互补』,但 top_k=3 返回的是 [55, 64, 8];doc 64(eval_dataset.json#23)只复述两种检索各自的局限,不含『互补』结论。被测据已召回文档如实拒答,判分器按规则判 fail 正确 —— 不是判分器变严导致的误伤。",
+ "status": "open",
+ "prevention": "待办:核查被测 top_k=3 下为何未召回 doc 60(可对比更早的召回记录);本轮按纪律不改被测。DEC-019 的误伤判据另看 rule_hit_judge_fail=0。"
+ }
+ }
+ ]
+}
diff --git a/eval/runs/20261018-181715-eda91022.spans.jsonl b/eval/runs/20261018-181715-eda91022.spans.jsonl
new file mode 100644
index 0000000..caacfc6
--- /dev/null
+++ b/eval/runs/20261018-181715-eda91022.spans.jsonl
@@ -0,0 +1,140 @@
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "70d0ea5462fbfcde", "parent_span_id": null, "name": "evalset.load", "kind": "CHAIN", "start_time_ms": 1792318486552.038, "duration_ms": 1.189, "status": "ok"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "c05b7fe70de914c2", "parent_span_id": null, "name": "run.evaluate", "kind": "CHAIN", "start_time_ms": 1792318486561.821, "duration_ms": 148520.235, "status": "ok"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "d74f4ec5c18b7ebf", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318486561.837, "duration_ms": 1978.774, "status": "ok", "attributes": {"case_id": 1, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "8a45b8d2875f8cb1", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318488540.663, "duration_ms": 0.038, "status": "ok", "attributes": {"case_id": 1, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "164f1baf3de4a9d2", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318488540.727, "duration_ms": 1397.385, "status": "error", "attributes": {"case_id": 1, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "a7449335c87f99ea", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318489938.277, "duration_ms": 3006.079, "status": "ok", "attributes": {"case_id": 2, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "543c65b132b857a4", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318492944.436, "duration_ms": 0.026, "status": "ok", "attributes": {"case_id": 2, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "d8988b9ee05d202a", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318492944.485, "duration_ms": 157.01, "status": "error", "attributes": {"case_id": 2, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "8afcea45392e6085", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318493101.614, "duration_ms": 3047.799, "status": "ok", "attributes": {"case_id": 3, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "020393301a26bdbe", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318496149.488, "duration_ms": 0.03, "status": "error", "attributes": {"case_id": 3, "passed": false}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "c8ecd2733a87cbfb", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318496149.537, "duration_ms": 143.196, "status": "error", "attributes": {"case_id": 3, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "7d9fb3b31672f9aa", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318496292.839, "duration_ms": 2581.78, "status": "ok", "attributes": {"case_id": 4, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "c93b148bb89f03b0", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318498874.686, "duration_ms": 0.017, "status": "error", "attributes": {"case_id": 4, "passed": false}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "55e4aeff8882cde5", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318498874.722, "duration_ms": 356.938, "status": "error", "attributes": {"case_id": 4, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "60c6e0237eadba72", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318499231.781, "duration_ms": 3478.802, "status": "ok", "attributes": {"case_id": 5, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "fd1ec4ef868d6a7e", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318502710.656, "duration_ms": 0.03, "status": "error", "attributes": {"case_id": 5, "passed": false}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "b14edb9144b1fda7", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318502710.705, "duration_ms": 149.973, "status": "error", "attributes": {"case_id": 5, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "3ab0758133613790", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318502860.784, "duration_ms": 2308.233, "status": "ok", "attributes": {"case_id": 6, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "6bee71c67ddea901", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318505169.07, "duration_ms": 0.021, "status": "error", "attributes": {"case_id": 6, "passed": false}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "5efb5c2f9d359d04", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318505169.109, "duration_ms": 405.081, "status": "error", "attributes": {"case_id": 6, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "f4b0e73c85316091", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318505574.301, "duration_ms": 3810.998, "status": "ok", "attributes": {"case_id": 7, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "243b68371b052f9e", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318509385.371, "duration_ms": 0.022, "status": "ok", "attributes": {"case_id": 7, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "0d0f2e35c05bbecf", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318509385.413, "duration_ms": 166.631, "status": "error", "attributes": {"case_id": 7, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "4eeb2de80c0e45d6", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318509552.149, "duration_ms": 2709.069, "status": "ok", "attributes": {"case_id": 8, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "244c0f2556809131", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318512261.284, "duration_ms": 0.021, "status": "ok", "attributes": {"case_id": 8, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "17a06aa44a239984", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318512261.327, "duration_ms": 224.154, "status": "error", "attributes": {"case_id": 8, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "9b3a7f833cbec08d", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318512485.593, "duration_ms": 2809.173, "status": "ok", "attributes": {"case_id": 9, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "9ad11cfc09023d34", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318515294.829, "duration_ms": 0.022, "status": "error", "attributes": {"case_id": 9, "passed": false}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "6d3c8510126d3fa1", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318515294.868, "duration_ms": 153.292, "status": "error", "attributes": {"case_id": 9, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "846ba6525c036116", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318515448.307, "duration_ms": 2887.499, "status": "ok", "attributes": {"case_id": 10, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "ae214564f72b0ae5", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318518335.875, "duration_ms": 0.018, "status": "error", "attributes": {"case_id": 10, "passed": false}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "63007948030fb241", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318518335.909, "duration_ms": 226.927, "status": "error", "attributes": {"case_id": 10, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "d785dffe5a4f9254", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318518562.944, "duration_ms": 3020.96, "status": "ok", "attributes": {"case_id": 11, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "f572dea827a5cb6f", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318521583.97, "duration_ms": 0.015, "status": "error", "attributes": {"case_id": 11, "passed": false}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "1d2e594937f8f7c0", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318521584.002, "duration_ms": 155.011, "status": "error", "attributes": {"case_id": 11, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "dc7d56c3bf39bf42", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318521739.165, "duration_ms": 2893.903, "status": "ok", "attributes": {"case_id": 12, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "3021475aca66bbda", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318524633.181, "duration_ms": 0.033, "status": "ok", "attributes": {"case_id": 12, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "9655f990ca716a2e", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318524633.231, "duration_ms": 221.813, "status": "error", "attributes": {"case_id": 12, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "0ab07e51c1d8fa28", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318524855.144, "duration_ms": 2985.755, "status": "ok", "attributes": {"case_id": 13, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "14e4a90805f64b4c", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318527840.962, "duration_ms": 0.02, "status": "ok", "attributes": {"case_id": 13, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "aadfe4b2d844c05a", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318527841.01, "duration_ms": 188.018, "status": "error", "attributes": {"case_id": 13, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "5e8c75c657e4c010", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318528029.135, "duration_ms": 2274.447, "status": "ok", "attributes": {"case_id": 14, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "6072278dfea4437f", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318530303.648, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 14, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "b3a9fe77de7c21ad", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318530303.68, "duration_ms": 157.142, "status": "error", "attributes": {"case_id": 14, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "4f66643bc3878086", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318530460.922, "duration_ms": 2562.766, "status": "ok", "attributes": {"case_id": 15, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "9c78382a82109afa", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318533023.744, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 15, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "fbc6ff215d58ceeb", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318533023.78, "duration_ms": 166.292, "status": "error", "attributes": {"case_id": 15, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "45b1a33f03349e90", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318533190.178, "duration_ms": 3111.451, "status": "ok", "attributes": {"case_id": 16, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "b50e91d561ac96dd", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318536301.703, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 16, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "f0f3fea856c98baf", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318536301.736, "duration_ms": 216.206, "status": "error", "attributes": {"case_id": 16, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "7cdfb78028215129", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318536518.066, "duration_ms": 3043.372, "status": "ok", "attributes": {"case_id": 17, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "c0527a5255ed8562", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318539561.531, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 17, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "8c3d01676bc36abe", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318539561.576, "duration_ms": 159.092, "status": "error", "attributes": {"case_id": 17, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "70f66682110437de", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318539720.777, "duration_ms": 3226.486, "status": "ok", "attributes": {"case_id": 18, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "cae78cdebdd5a390", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318542947.364, "duration_ms": 0.024, "status": "ok", "attributes": {"case_id": 18, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "95a1c72b364842a4", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318542947.409, "duration_ms": 274.786, "status": "error", "attributes": {"case_id": 18, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "05f362ae6b77c292", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318543222.309, "duration_ms": 3003.91, "status": "ok", "attributes": {"case_id": 19, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "4a8f17bc79856afe", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318546226.303, "duration_ms": 0.02, "status": "error", "attributes": {"case_id": 19, "passed": false}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "c54f6243d68e4172", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318546226.34, "duration_ms": 161.413, "status": "error", "attributes": {"case_id": 19, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "e6858df583138b14", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318546387.87, "duration_ms": 2620.267, "status": "ok", "attributes": {"case_id": 20, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "095a25c03f07b7fa", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318549008.197, "duration_ms": 0.018, "status": "error", "attributes": {"case_id": 20, "passed": false}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "57d12f95d152544c", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318549008.231, "duration_ms": 195.108, "status": "error", "attributes": {"case_id": 20, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "74bec96a5c960323", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318549203.472, "duration_ms": 2402.82, "status": "ok", "attributes": {"case_id": 21, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "c47d97542e5d4358", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318551606.349, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 21, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "78119315267ebd75", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318551606.382, "duration_ms": 157.362, "status": "error", "attributes": {"case_id": 21, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "6fde47e7191c8f94", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318551763.869, "duration_ms": 1804.124, "status": "ok", "attributes": {"case_id": 22, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "7ec687b25f60a1ae", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318553568.045, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 22, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "c7674d9d1b849254", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318553568.077, "duration_ms": 161.285, "status": "error", "attributes": {"case_id": 22, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "63943a7750c780e8", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318553729.467, "duration_ms": 2799.974, "status": "ok", "attributes": {"case_id": 23, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "5d2ee4eee0b819b6", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318556529.499, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 23, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "f2fe54e6a3e4427c", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318556529.529, "duration_ms": 186.071, "status": "error", "attributes": {"case_id": 23, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "4a1af6953fadf4b6", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318556715.692, "duration_ms": 3902.655, "status": "ok", "attributes": {"case_id": 24, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "f3c57e873aea31b8", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318560618.417, "duration_ms": 0.036, "status": "ok", "attributes": {"case_id": 24, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "77a931a363131383", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318560618.481, "duration_ms": 537.568, "status": "error", "attributes": {"case_id": 24, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "fe92dc873e65e3ab", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318561156.163, "duration_ms": 3387.169, "status": "ok", "attributes": {"case_id": 25, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "e03e0efb6678eaba", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318564543.418, "duration_ms": 0.031, "status": "ok", "attributes": {"case_id": 25, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "4f04912d49729fe2", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318564543.481, "duration_ms": 149.689, "status": "error", "attributes": {"case_id": 25, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "43e993ca58fdac4e", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318564693.292, "duration_ms": 2355.433, "status": "ok", "attributes": {"case_id": 26, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "5b7bb4e1ca29b0eb", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318567048.779, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 26, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "c9d8b7dad07c551e", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318567048.813, "duration_ms": 347.413, "status": "error", "attributes": {"case_id": 26, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "9f6e267b819c31d4", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318567396.331, "duration_ms": 2651.51, "status": "ok", "attributes": {"case_id": 27, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "0fcedadea38aa4f7", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318570047.908, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 27, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "db455960f3266c56", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318570047.943, "duration_ms": 174.678, "status": "error", "attributes": {"case_id": 27, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "9e8d63a1bb3737bf", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318570222.717, "duration_ms": 2216.701, "status": "ok", "attributes": {"case_id": 28, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "a075f87cd4f2de98", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318572439.476, "duration_ms": 0.02, "status": "ok", "attributes": {"case_id": 28, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "1722ba770b79d6e9", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318572439.514, "duration_ms": 190.351, "status": "error", "attributes": {"case_id": 28, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "5755dc1a752cbe86", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318572629.971, "duration_ms": 3213.619, "status": "ok", "attributes": {"case_id": 29, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "f9a42477546e9069", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318575843.687, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 29, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "f6910a5432be9fef", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318575843.722, "duration_ms": 153.411, "status": "error", "attributes": {"case_id": 29, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "e589836559d69dad", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318575997.226, "duration_ms": 2845.045, "status": "ok", "attributes": {"case_id": 30, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "3eb5a0ddf25acc1c", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318578842.329, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 30, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "82aa755479896715", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318578842.385, "duration_ms": 191.128, "status": "error", "attributes": {"case_id": 30, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "e0fd195c62759fcb", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318579033.621, "duration_ms": 2215.583, "status": "ok", "attributes": {"case_id": 31, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "37a4d75c30c3a5f2", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318581249.26, "duration_ms": 0.018, "status": "ok", "attributes": {"case_id": 31, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "147a5b451f1bfbb5", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318581249.295, "duration_ms": 155.606, "status": "error", "attributes": {"case_id": 31, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "b83a6c1efd4b92db", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318581404.996, "duration_ms": 1996.662, "status": "ok", "attributes": {"case_id": 32, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "4dcd053dacfa8a2e", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318583401.704, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 32, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "82070af5b5ece190", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318583401.738, "duration_ms": 171.835, "status": "error", "attributes": {"case_id": 32, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "14b47c87e5a414d0", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318583573.663, "duration_ms": 1858.431, "status": "ok", "attributes": {"case_id": 33, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "6fb1dfbb01bc183c", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318585432.144, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 33, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "9d8579c43e02e8a7", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318585432.175, "duration_ms": 150.443, "status": "error", "attributes": {"case_id": 33, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "9d15e4ddd89be29b", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318585582.733, "duration_ms": 1899.953, "status": "ok", "attributes": {"case_id": 34, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "cd342d6012d8665e", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318587482.77, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 34, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "3906e7b60aca121c", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318587482.8, "duration_ms": 142.678, "status": "error", "attributes": {"case_id": 34, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "698bf9ada0004dec", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318587625.579, "duration_ms": 2483.196, "status": "ok", "attributes": {"case_id": 35, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "34d53fd2e09583cd", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318590108.83, "duration_ms": 0.02, "status": "ok", "attributes": {"case_id": 35, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "f634dcca92db36dd", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318590108.866, "duration_ms": 191.587, "status": "error", "attributes": {"case_id": 35, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "68ac33c4ca9ef0fa", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318590300.546, "duration_ms": 2226.103, "status": "ok", "attributes": {"case_id": 36, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "fad39738dd1508b4", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318592526.699, "duration_ms": 0.011, "status": "ok", "attributes": {"case_id": 36, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "dc253180086f318d", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318592526.728, "duration_ms": 157.528, "status": "error", "attributes": {"case_id": 36, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "645a3caebc377bd3", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318592684.497, "duration_ms": 2191.006, "status": "ok", "attributes": {"case_id": 37, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "920432172b1f686e", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318594875.567, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 37, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "af4d2a2621621ea3", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318594875.605, "duration_ms": 138.468, "status": "error", "attributes": {"case_id": 37, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "234e3b38446a3753", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318595014.222, "duration_ms": 4219.507, "status": "ok", "attributes": {"case_id": 38, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "1f6894f80cabb37a", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318599233.802, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 38, "passed": true, "mode": "deterministic_only"}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "d957b0f3e7ea5acd", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318599233.868, "duration_ms": 3497.053, "status": "ok", "attributes": {"case_id": 39, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "78d82757631b30ab", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318602730.986, "duration_ms": 0.014, "status": "ok", "attributes": {"case_id": 39, "passed": true, "mode": "deterministic_only"}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "406291c8c71d984c", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318602731.061, "duration_ms": 2997.963, "status": "ok", "attributes": {"case_id": 40, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "261871f24cd5e6fb", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318605729.082, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 40, "passed": true, "mode": "deterministic_only"}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "3106758c464d1d74", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318605729.144, "duration_ms": 2906.379, "status": "ok", "attributes": {"case_id": 41, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "190c3b91b2847844", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318608635.58, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 41, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "0de225e8527ef9e9", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318608635.609, "duration_ms": 173.724, "status": "error", "attributes": {"case_id": 41, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "bb582b72a4285505", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318608809.432, "duration_ms": 2779.94, "status": "ok", "attributes": {"case_id": 42, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "39f90fdd3ad16461", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318611589.428, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 42, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "38d357b6f8252ce1", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318611589.461, "duration_ms": 175.095, "status": "error", "attributes": {"case_id": 42, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "157cea678455f30a", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318611764.666, "duration_ms": 2898.077, "status": "ok", "attributes": {"case_id": 43, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "2063ff0b224ea8ec", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318614662.8, "duration_ms": 0.017, "status": "error", "attributes": {"case_id": 43, "passed": false}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "95fd048c877b1467", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318614662.834, "duration_ms": 151.317, "status": "error", "attributes": {"case_id": 43, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "71bc602f435d284f", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318614814.242, "duration_ms": 2682.616, "status": "ok", "attributes": {"case_id": 44, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "30ac5c6bee07363f", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318617496.92, "duration_ms": 0.018, "status": "ok", "attributes": {"case_id": 44, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "bb59120fee6d1a67", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318617496.955, "duration_ms": 145.567, "status": "error", "attributes": {"case_id": 44, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "bd2e22b1a4f4c6c0", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318617642.618, "duration_ms": 3186.65, "status": "ok", "attributes": {"case_id": 45, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "c5638a9010a781ec", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318620829.329, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 45, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "910e3a85a06d5e4e", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318620829.361, "duration_ms": 205.363, "status": "error", "attributes": {"case_id": 45, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "0abbbf37e2343e2c", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318621034.827, "duration_ms": 1815.79, "status": "ok", "attributes": {"case_id": 46, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "70d607a1ef24faa4", "parent_span_id": "c05b7fe70de914c2", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318622850.666, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 46, "passed": true}}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "faf79bdd33906705", "parent_span_id": "c05b7fe70de914c2", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318622850.696, "duration_ms": 147.447, "status": "error", "attributes": {"case_id": 46, "judge": "deepseek-v4-flash@https://api.deepseek.com"}, "error_code": "JudgeError"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "9220cc4b1f5026d9", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318622998.243, "duration_ms": 6039.237, "status": "error", "attributes": {"case_id": 47, "sut": "fastapi-rag", "attempts": 3, "http_status": 429, "last_error": "E_SUT_4XX"}, "error_code": "E_SUT_4XX"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "bd29d794b6c80d41", "parent_span_id": "c05b7fe70de914c2", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318629037.579, "duration_ms": 6046.082, "status": "error", "attributes": {"case_id": 48, "sut": "fastapi-rag", "attempts": 3, "http_status": 429, "last_error": "E_SUT_4XX"}, "error_code": "E_SUT_4XX"}
+{"trace_id": "efd17bbf10708c490301ac9698fb5cb2", "span_id": "b622bd914785709a", "parent_span_id": null, "name": "report.write", "kind": "CHAIN", "start_time_ms": 1792318635088.679, "duration_ms": 55.488, "status": "ok"}
diff --git a/eval/runs/20261018-182150-86cc86b0.spans.jsonl b/eval/runs/20261018-182150-86cc86b0.spans.jsonl
new file mode 100644
index 0000000..63341ad
--- /dev/null
+++ b/eval/runs/20261018-182150-86cc86b0.spans.jsonl
@@ -0,0 +1,144 @@
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "86fbd3d973a7875b", "parent_span_id": null, "name": "evalset.load", "kind": "CHAIN", "start_time_ms": 1792318751770.497, "duration_ms": 1.616, "status": "ok"}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "2fbb9d2d9572e963", "parent_span_id": null, "name": "run.evaluate", "kind": "CHAIN", "start_time_ms": 1792318751773.906, "duration_ms": 158651.73, "status": "ok"}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "1b26e50103ec2a7a", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318751773.92, "duration_ms": 3554.591, "status": "ok", "attributes": {"case_id": 1, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "a99a77f3051c6b76", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318755328.588, "duration_ms": 0.096, "status": "ok", "attributes": {"case_id": 1, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "19d0efe687ebc57b", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318755328.715, "duration_ms": 3059.892, "status": "ok", "attributes": {"case_id": 1, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "6f9845375bb64307", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318758388.784, "duration_ms": 3038.274, "status": "ok", "attributes": {"case_id": 2, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "64a1d448127a70e9", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318761427.128, "duration_ms": 0.048, "status": "ok", "attributes": {"case_id": 2, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "c0152119f89a2dc8", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318761427.201, "duration_ms": 1559.738, "status": "ok", "attributes": {"case_id": 2, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "e8df191c19597387", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318762987.28, "duration_ms": 1866.15, "status": "ok", "attributes": {"case_id": 3, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "f3bf24364becaef8", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318764853.497, "duration_ms": 0.049, "status": "error", "attributes": {"case_id": 3, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "28204579dfce8535", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318764853.578, "duration_ms": 2999.574, "status": "ok", "attributes": {"case_id": 3, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "af0ecefbcd22de6d", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318767853.303, "duration_ms": 1127.282, "status": "ok", "attributes": {"case_id": 4, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "eed84740eb7c9f36", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318768980.624, "duration_ms": 0.024, "status": "error", "attributes": {"case_id": 4, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "25540c146a0858c6", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318768980.666, "duration_ms": 2283.894, "status": "ok", "attributes": {"case_id": 4, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "4e9e1a9b48d21097", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318771264.719, "duration_ms": 1394.13, "status": "ok", "attributes": {"case_id": 5, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "7b6bf17641debe4a", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318772658.913, "duration_ms": 0.045, "status": "error", "attributes": {"case_id": 5, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "dc8cc43346ea3780", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318772658.979, "duration_ms": 3749.409, "status": "ok", "attributes": {"case_id": 5, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "7457cba229fda0d2", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318776408.545, "duration_ms": 1423.531, "status": "ok", "attributes": {"case_id": 6, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "3aa89b406770f435", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318777832.118, "duration_ms": 0.022, "status": "error", "attributes": {"case_id": 6, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "2614097bd17e4e5c", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318777832.158, "duration_ms": 3644.17, "status": "ok", "attributes": {"case_id": 6, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "bc6c92e91d4c7694", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318781477.14, "duration_ms": 1545.395, "status": "ok", "attributes": {"case_id": 7, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "d34974d67ee97b5e", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318783022.579, "duration_ms": 0.018, "status": "ok", "attributes": {"case_id": 7, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "7f411f46fc69f22d", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318783022.614, "duration_ms": 1543.42, "status": "ok", "attributes": {"case_id": 7, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "f673c7db98fa2315", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318784566.236, "duration_ms": 1589.355, "status": "ok", "attributes": {"case_id": 8, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "746628618d2e3623", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318786155.639, "duration_ms": 0.041, "status": "ok", "attributes": {"case_id": 8, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "5aec0a7b80f7afd4", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318786155.703, "duration_ms": 2001.439, "status": "ok", "attributes": {"case_id": 8, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "7192d8d96c40f04f", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318788161.41, "duration_ms": 1005.441, "status": "ok", "attributes": {"case_id": 9, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "6095e0529965e174", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318789166.888, "duration_ms": 0.019, "status": "error", "attributes": {"case_id": 9, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "b9df86d1cdaa1482", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318789166.924, "duration_ms": 1698.312, "status": "ok", "attributes": {"case_id": 9, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "b0bcd622ee364a3f", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318790865.366, "duration_ms": 1552.034, "status": "ok", "attributes": {"case_id": 10, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "8e1f9a87eaa32f9b", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318792417.454, "duration_ms": 0.03, "status": "error", "attributes": {"case_id": 10, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "c1c67ea99f107325", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318792417.512, "duration_ms": 1584.103, "status": "ok", "attributes": {"case_id": 10, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "71acfe663fb8fb50", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318794001.75, "duration_ms": 2377.729, "status": "ok", "attributes": {"case_id": 11, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "25d71c462c0aa011", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318796379.536, "duration_ms": 0.023, "status": "error", "attributes": {"case_id": 11, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "d037c0ff756f4fa8", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318796379.577, "duration_ms": 2281.832, "status": "ok", "attributes": {"case_id": 11, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "a3638fa5e78cfb12", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318798661.594, "duration_ms": 1423.323, "status": "ok", "attributes": {"case_id": 12, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "a7313da0dbfdee66", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318800084.957, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 12, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "395d6aaa995083bd", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318800084.99, "duration_ms": 1783.22, "status": "ok", "attributes": {"case_id": 12, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "7c08d74ff072d80d", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318801868.364, "duration_ms": 1608.867, "status": "ok", "attributes": {"case_id": 13, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "947ef0a3985aac68", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318803477.282, "duration_ms": 0.023, "status": "ok", "attributes": {"case_id": 13, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "da1ab6a6e2bab9df", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318803477.327, "duration_ms": 1970.02, "status": "ok", "attributes": {"case_id": 13, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "d6c18b82b50e32a6", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318805450.034, "duration_ms": 875.863, "status": "ok", "attributes": {"case_id": 14, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "2118fb77eb1fce8c", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318806325.939, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 14, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "6640f51c1fc6e7ea", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318806325.985, "duration_ms": 1467.674, "status": "ok", "attributes": {"case_id": 14, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "218dd94c32b1fc33", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318807793.824, "duration_ms": 1532.396, "status": "ok", "attributes": {"case_id": 15, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "b3989a7b8dff211b", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318809326.306, "duration_ms": 0.021, "status": "ok", "attributes": {"case_id": 15, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "0c09750e50cdf98b", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318809326.351, "duration_ms": 1775.998, "status": "ok", "attributes": {"case_id": 15, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "edcdfdc9bf5dae85", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318811102.483, "duration_ms": 1284.173, "status": "ok", "attributes": {"case_id": 16, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "82565256b7e7cd77", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318812386.697, "duration_ms": 0.021, "status": "ok", "attributes": {"case_id": 16, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "fee18bda6c5961a9", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318812386.736, "duration_ms": 1672.537, "status": "ok", "attributes": {"case_id": 16, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "9b03bb5572c05a5e", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318814059.403, "duration_ms": 2672.32, "status": "ok", "attributes": {"case_id": 17, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "f5d433ca00051569", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318816731.78, "duration_ms": 0.026, "status": "ok", "attributes": {"case_id": 17, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "70308fee52c92ee3", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318816731.822, "duration_ms": 1763.758, "status": "ok", "attributes": {"case_id": 17, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "d0d57cbabcf5a3e2", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318818495.714, "duration_ms": 2428.042, "status": "ok", "attributes": {"case_id": 18, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "eb42d4bc08cb768d", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318820923.825, "duration_ms": 0.018, "status": "ok", "attributes": {"case_id": 18, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "62d510c907566db4", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318820923.862, "duration_ms": 2289.111, "status": "ok", "attributes": {"case_id": 18, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "158d5be64f4f8c26", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318823213.157, "duration_ms": 1082.227, "status": "ok", "attributes": {"case_id": 19, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "4616280cfba15917", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318824295.431, "duration_ms": 0.014, "status": "error", "attributes": {"case_id": 19, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "55b9accaab3570a4", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318824295.46, "duration_ms": 1554.193, "status": "ok", "attributes": {"case_id": 19, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "e32ac24d13f78f34", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318825849.885, "duration_ms": 1380.315, "status": "ok", "attributes": {"case_id": 20, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "05525392ae25ddbb", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318827230.258, "duration_ms": 0.019, "status": "error", "attributes": {"case_id": 20, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "380c4b546fc594e1", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318827230.293, "duration_ms": 2065.278, "status": "ok", "attributes": {"case_id": 20, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "4630c13e16579f87", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318829295.71, "duration_ms": 1258.449, "status": "ok", "attributes": {"case_id": 21, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "b49d3b8745a62951", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318830554.201, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 21, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "c602bcd23a5116a2", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318830554.235, "duration_ms": 1735.846, "status": "ok", "attributes": {"case_id": 21, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "4631d2c72554d0a1", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318832290.215, "duration_ms": 764.942, "status": "ok", "attributes": {"case_id": 22, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "d7d9fb49f281122c", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318833055.202, "duration_ms": 0.014, "status": "ok", "attributes": {"case_id": 22, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "b684c7d913579535", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318833055.259, "duration_ms": 1685.833, "status": "ok", "attributes": {"case_id": 22, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "1c6346449ed3a53c", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318834741.212, "duration_ms": 847.48, "status": "ok", "attributes": {"case_id": 23, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "acd7c8e73e0faff5", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318835588.725, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 23, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "5062e0565c6ac824", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318835588.755, "duration_ms": 1598.34, "status": "ok", "attributes": {"case_id": 23, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "aff6559986d35cde", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318837187.377, "duration_ms": 2228.364, "status": "ok", "attributes": {"case_id": 24, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "6656e5861875e758", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318839415.807, "duration_ms": 0.033, "status": "ok", "attributes": {"case_id": 24, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "16c38813ee68edc5", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318839415.881, "duration_ms": 2236.558, "status": "ok", "attributes": {"case_id": 24, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "6c64397844ccac0c", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318841652.603, "duration_ms": 3029.606, "status": "ok", "attributes": {"case_id": 25, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "448f238931638b03", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318844682.29, "duration_ms": 0.034, "status": "ok", "attributes": {"case_id": 25, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "0cfa223b38e94082", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318844682.339, "duration_ms": 5363.767, "status": "ok", "attributes": {"case_id": 25, "verdict": "pass", "attempts": 2, "finish_reasons": "stop,stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "c008742f16376904", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318850046.296, "duration_ms": 1002.991, "status": "ok", "attributes": {"case_id": 26, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "b7636ba7b5a02ea6", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318851049.326, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 26, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "6725c1c2fee2f15a", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318851049.368, "duration_ms": 1369.317, "status": "ok", "attributes": {"case_id": 26, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "695fdc7d45c3a5ab", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318852418.842, "duration_ms": 1478.917, "status": "ok", "attributes": {"case_id": 27, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "906faf6695ba7362", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318853897.825, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 27, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "389a8bbede405f73", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318853897.855, "duration_ms": 1706.211, "status": "ok", "attributes": {"case_id": 27, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "8cd6bb160b1bb63f", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318855604.194, "duration_ms": 816.683, "status": "ok", "attributes": {"case_id": 28, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "dd6fc5f01f603875", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318856420.912, "duration_ms": 0.031, "status": "ok", "attributes": {"case_id": 28, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "994e765b21d2c950", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318856420.959, "duration_ms": 1204.578, "status": "ok", "attributes": {"case_id": 28, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "0d8055116b0eb980", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318857625.663, "duration_ms": 1567.067, "status": "ok", "attributes": {"case_id": 29, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "277da05ffa330a5d", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318859192.813, "duration_ms": 0.02, "status": "ok", "attributes": {"case_id": 29, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "7b2d57f1a176edfb", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318859192.851, "duration_ms": 1125.9, "status": "ok", "attributes": {"case_id": 29, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "b030e80d566ed968", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318860318.88, "duration_ms": 1453.441, "status": "ok", "attributes": {"case_id": 30, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "ea80d72a2a0591d6", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318861772.362, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 30, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "d325803eb413dd96", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318861772.394, "duration_ms": 1364.114, "status": "ok", "attributes": {"case_id": 30, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "59c0df2db3e02937", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318863136.641, "duration_ms": 682.668, "status": "ok", "attributes": {"case_id": 31, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "05c769df9764c03a", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318863819.365, "duration_ms": 0.014, "status": "ok", "attributes": {"case_id": 31, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "0b2941acdfeb18fc", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318863819.394, "duration_ms": 1251.918, "status": "ok", "attributes": {"case_id": 31, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "f6776e656d197a35", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318865071.443, "duration_ms": 1008.009, "status": "ok", "attributes": {"case_id": 32, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "a593422b72617be7", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318866079.491, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 32, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "a5c2b99519788ad4", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318866079.525, "duration_ms": 1314.122, "status": "ok", "attributes": {"case_id": 32, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "edf647997477c106", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318867393.788, "duration_ms": 587.249, "status": "ok", "attributes": {"case_id": 33, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "9370e4ff6208bf5c", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318867981.076, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 33, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "e810db09a6bd7bb5", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318867981.106, "duration_ms": 1002.407, "status": "ok", "attributes": {"case_id": 33, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "e8d68917db0a491b", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318868983.668, "duration_ms": 861.362, "status": "ok", "attributes": {"case_id": 34, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "6bade0e1108d7ade", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318869845.076, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 34, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "039fa484774aed9c", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318869845.118, "duration_ms": 1513.697, "status": "ok", "attributes": {"case_id": 34, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "4f70c2a41f989aec", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318871358.941, "duration_ms": 828.118, "status": "ok", "attributes": {"case_id": 35, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "4b899196143dd938", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318872187.093, "duration_ms": 0.012, "status": "ok", "attributes": {"case_id": 35, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "e235981ad873ad1f", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318872187.12, "duration_ms": 1003.206, "status": "ok", "attributes": {"case_id": 35, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "f04c7187affc44bc", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318873190.458, "duration_ms": 941.745, "status": "ok", "attributes": {"case_id": 36, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "00aefd0b7abedbb8", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318874132.245, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 36, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "f1ec9c1faa34531d", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318874132.282, "duration_ms": 1208.573, "status": "ok", "attributes": {"case_id": 36, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "673ca3d607c75982", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318875340.977, "duration_ms": 845.078, "status": "ok", "attributes": {"case_id": 37, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "5c3aeaa5de754ccf", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318876186.103, "duration_ms": 0.029, "status": "ok", "attributes": {"case_id": 37, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "eae040ca57b2b120", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318876186.147, "duration_ms": 1306.624, "status": "ok", "attributes": {"case_id": 37, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "a85788271698cf7d", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318877492.933, "duration_ms": 1836.083, "status": "ok", "attributes": {"case_id": 38, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "9dbb8c116679b59f", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318879329.088, "duration_ms": 0.022, "status": "ok", "attributes": {"case_id": 38, "passed": true, "mode": "deterministic_only"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "29d7fcfa1f8939ab", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318879329.206, "duration_ms": 1941.174, "status": "ok", "attributes": {"case_id": 39, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "39d6d2e88adfc933", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318881270.429, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 39, "passed": true, "mode": "deterministic_only"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "66d017948b4eee3c", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318881270.505, "duration_ms": 1439.692, "status": "ok", "attributes": {"case_id": 40, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "b15a800705609b7b", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318882710.248, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 40, "passed": true, "mode": "deterministic_only"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "e6e57b9e856af2ee", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318882710.316, "duration_ms": 948.654, "status": "ok", "attributes": {"case_id": 41, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "67f8a880ba255a25", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318883659.009, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 41, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "7bac47b53a7de19c", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318883659.038, "duration_ms": 1469.597, "status": "ok", "attributes": {"case_id": 41, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "bf438423309076f6", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318885129.36, "duration_ms": 1354.129, "status": "ok", "attributes": {"case_id": 42, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "1f03104edd585404", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318886483.53, "duration_ms": 0.019, "status": "ok", "attributes": {"case_id": 42, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "7c5348d8bf630058", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318886483.564, "duration_ms": 1676.265, "status": "ok", "attributes": {"case_id": 42, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "571aaa25df4c5d42", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318888159.955, "duration_ms": 2117.199, "status": "ok", "attributes": {"case_id": 43, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "5a9269be4226efad", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318890277.213, "duration_ms": 0.036, "status": "error", "attributes": {"case_id": 43, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "a33a34f107a7d5da", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318890277.265, "duration_ms": 2048.724, "status": "ok", "attributes": {"case_id": 43, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "653cb243b9a63723", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318892326.118, "duration_ms": 1529.826, "status": "ok", "attributes": {"case_id": 44, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "0f6ebdae84168f73", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318893856.028, "duration_ms": 0.023, "status": "ok", "attributes": {"case_id": 44, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "9fb75c675d294898", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318893856.069, "duration_ms": 1642.355, "status": "ok", "attributes": {"case_id": 44, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "81263bf2c3cda543", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318895498.618, "duration_ms": 2231.558, "status": "ok", "attributes": {"case_id": 45, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "b03ab4c86cb0bf69", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318897730.236, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 45, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "ce8afdfb32576d9a", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318897730.267, "duration_ms": 1593.147, "status": "ok", "attributes": {"case_id": 45, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "2b8ca784c6339c39", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318899323.541, "duration_ms": 562.892, "status": "ok", "attributes": {"case_id": 46, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "a656b01e763a0c66", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318899886.462, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 46, "passed": true}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "091580a158b3bf46", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318899886.49, "duration_ms": 1352.357, "status": "ok", "attributes": {"case_id": 46, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "6425f757a2ad50ce", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318901253.755, "duration_ms": 2404.272, "status": "ok", "attributes": {"case_id": 47, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "3fb97c6155c81274", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318903658.092, "duration_ms": 0.024, "status": "error", "attributes": {"case_id": 47, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "9368727f0c546012", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318903658.136, "duration_ms": 1713.239, "status": "ok", "attributes": {"case_id": 47, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "d43b44237f998b65", "parent_span_id": "2fbb9d2d9572e963", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792318905375.55, "duration_ms": 2593.676, "status": "ok", "attributes": {"case_id": 48, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "a041e24bd61fda2f", "parent_span_id": "2fbb9d2d9572e963", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792318907969.283, "duration_ms": 0.031, "status": "error", "attributes": {"case_id": 48, "passed": false}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "e18cf572081aa29d", "parent_span_id": "2fbb9d2d9572e963", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792318907969.33, "duration_ms": 2458.015, "status": "ok", "attributes": {"case_id": 48, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "d232751e26451f0ba90180a894a68027", "span_id": "3aa83709f3c89c79", "parent_span_id": null, "name": "report.write", "kind": "CHAIN", "start_time_ms": 1792318910433.085, "duration_ms": 33.746, "status": "ok"}
diff --git a/eval/runs/20261018-182724-e6212c1d.spans.jsonl b/eval/runs/20261018-182724-e6212c1d.spans.jsonl
new file mode 100644
index 0000000..806beb1
--- /dev/null
+++ b/eval/runs/20261018-182724-e6212c1d.spans.jsonl
@@ -0,0 +1,144 @@
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "d2506279072e6024", "parent_span_id": null, "name": "evalset.load", "kind": "CHAIN", "start_time_ms": 1792319105280.372, "duration_ms": 1.443, "status": "ok"}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "8bab58540d1fca12", "parent_span_id": null, "name": "run.evaluate", "kind": "CHAIN", "start_time_ms": 1792319105282.732, "duration_ms": 139202.179, "status": "ok"}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "1164bc331b11ed18", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319105282.744, "duration_ms": 1688.764, "status": "ok", "attributes": {"case_id": 1, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "c2b9772637a5b0eb", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319106971.558, "duration_ms": 0.036, "status": "ok", "attributes": {"case_id": 1, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "33b0fdb276adba11", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319106971.619, "duration_ms": 1657.141, "status": "ok", "attributes": {"case_id": 1, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "a2989f140eebcc49", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319108628.897, "duration_ms": 2000.765, "status": "ok", "attributes": {"case_id": 2, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "65ca563eaa47bf8b", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319110629.729, "duration_ms": 0.029, "status": "ok", "attributes": {"case_id": 2, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "806858fcaeff3927", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319110629.792, "duration_ms": 1415.302, "status": "ok", "attributes": {"case_id": 2, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "3d5c6bb2a9d8959f", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319112045.257, "duration_ms": 1226.597, "status": "ok", "attributes": {"case_id": 3, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "427dc426a8e9feb4", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319113271.908, "duration_ms": 0.018, "status": "ok", "attributes": {"case_id": 3, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "f39d5a1a0ab6cf89", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319113271.943, "duration_ms": 1712.898, "status": "ok", "attributes": {"case_id": 3, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "d9d6c97cada2a9ee", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319114984.995, "duration_ms": 1183.313, "status": "ok", "attributes": {"case_id": 4, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "9181fff05747667d", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319116168.378, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 4, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "c3f61f30feae45ab", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319116168.411, "duration_ms": 1179.731, "status": "ok", "attributes": {"case_id": 4, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "1ee33d2b2409ed75", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319117348.269, "duration_ms": 1545.292, "status": "ok", "attributes": {"case_id": 5, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "66d7da95c2b04c3b", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319118893.616, "duration_ms": 0.022, "status": "ok", "attributes": {"case_id": 5, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "0f31d25bcf96ccec", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319118893.656, "duration_ms": 1466.403, "status": "ok", "attributes": {"case_id": 5, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "93d6808815b5830d", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319120360.246, "duration_ms": 1910.483, "status": "ok", "attributes": {"case_id": 6, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "8ec8c971444ed26e", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319122270.804, "duration_ms": 4.119, "status": "ok", "attributes": {"case_id": 6, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "693c1b748402d710", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319122274.977, "duration_ms": 1843.961, "status": "ok", "attributes": {"case_id": 6, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "b94256e46598b8e2", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319124119.163, "duration_ms": 1268.751, "status": "ok", "attributes": {"case_id": 7, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "432ce1f51c3f6d02", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319125387.97, "duration_ms": 0.025, "status": "ok", "attributes": {"case_id": 7, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "d8d2f61551008bfd", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319125388.034, "duration_ms": 1369.614, "status": "ok", "attributes": {"case_id": 7, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "492dca88be12f3aa", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319126757.791, "duration_ms": 979.963, "status": "ok", "attributes": {"case_id": 8, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "483e95a06935a265", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319127737.82, "duration_ms": 0.036, "status": "ok", "attributes": {"case_id": 8, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "37cea702e64b9a37", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319127737.875, "duration_ms": 1553.196, "status": "ok", "attributes": {"case_id": 8, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "3bb3e07b034afbdb", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319129291.227, "duration_ms": 716.723, "status": "ok", "attributes": {"case_id": 9, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "f21049abc9fdef2e", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319130007.982, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 9, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "03c65bd78b6812d9", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319130008.012, "duration_ms": 1391.395, "status": "ok", "attributes": {"case_id": 9, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "4ea23658da1eb626", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319131399.541, "duration_ms": 1795.471, "status": "ok", "attributes": {"case_id": 10, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "a60ccaa5d3320266", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319133195.056, "duration_ms": 0.014, "status": "ok", "attributes": {"case_id": 10, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "0af173777b9e6e08", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319133195.087, "duration_ms": 1517.628, "status": "ok", "attributes": {"case_id": 10, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "27ceb7f530b50a51", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319134712.84, "duration_ms": 1880.137, "status": "ok", "attributes": {"case_id": 11, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "cbb20c244afc363a", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319136593.059, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 11, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "ce87614ecfd04e2e", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319136593.092, "duration_ms": 1996.548, "status": "ok", "attributes": {"case_id": 11, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "6b106f9f0912b72e", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319138589.789, "duration_ms": 1619.551, "status": "ok", "attributes": {"case_id": 12, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "26a09ff3e9889a59", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319140209.395, "duration_ms": 0.066, "status": "ok", "attributes": {"case_id": 12, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "ff2e543028c87d40", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319140209.502, "duration_ms": 1711.431, "status": "ok", "attributes": {"case_id": 12, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "2c3b4584cf1d2ca4", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319141921.151, "duration_ms": 2629.527, "status": "ok", "attributes": {"case_id": 13, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "7bc11afb2e20663d", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319144550.73, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 13, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "94a42f674b448f1a", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319144550.764, "duration_ms": 1403.176, "status": "ok", "attributes": {"case_id": 13, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "1e8b0b6c8a04aa1e", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319145954.061, "duration_ms": 1276.328, "status": "ok", "attributes": {"case_id": 14, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "d939d2053087f621", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319147230.43, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 14, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "d88f139183d764e8", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319147230.463, "duration_ms": 1304.133, "status": "ok", "attributes": {"case_id": 14, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "896e0d2d4617ef2f", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319148534.737, "duration_ms": 2379.872, "status": "ok", "attributes": {"case_id": 15, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "a79d7ce0ebfa3859", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319150914.663, "duration_ms": 0.053, "status": "ok", "attributes": {"case_id": 15, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "f0c73d61081799cb", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319150914.738, "duration_ms": 1359.72, "status": "ok", "attributes": {"case_id": 15, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "e07d6dd17392959f", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319152274.91, "duration_ms": 2765.469, "status": "ok", "attributes": {"case_id": 16, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "80e3dbeb87da1bf1", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319155040.441, "duration_ms": 0.018, "status": "ok", "attributes": {"case_id": 16, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "a17305ee287f76af", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319155040.474, "duration_ms": 1893.033, "status": "ok", "attributes": {"case_id": 16, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "b7888789937723d9", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319156993.724, "duration_ms": 2452.738, "status": "ok", "attributes": {"case_id": 17, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "4dce34c9ddcddcdd", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319159446.534, "duration_ms": 0.033, "status": "ok", "attributes": {"case_id": 17, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "b2c5ff5919d998ff", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319159446.584, "duration_ms": 1890.231, "status": "ok", "attributes": {"case_id": 17, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "37fd02d5bc5b050a", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319161336.96, "duration_ms": 2211.909, "status": "ok", "attributes": {"case_id": 18, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "b623ec50d56200b3", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319163548.922, "duration_ms": 0.021, "status": "ok", "attributes": {"case_id": 18, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "e7de698880cb08d8", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319163548.96, "duration_ms": 1641.158, "status": "ok", "attributes": {"case_id": 18, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "4f142e946e26141b", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319165190.244, "duration_ms": 929.148, "status": "ok", "attributes": {"case_id": 19, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "df1d9ea579aa2b9c", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319166119.438, "duration_ms": 0.028, "status": "ok", "attributes": {"case_id": 19, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "7d64ce1857e382ec", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319166119.493, "duration_ms": 1411.083, "status": "ok", "attributes": {"case_id": 19, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "a7f4315552f1ff19", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319167530.716, "duration_ms": 2015.774, "status": "ok", "attributes": {"case_id": 20, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "cd9ab77b624e1bd0", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319169546.536, "duration_ms": 0.017, "status": "error", "attributes": {"case_id": 20, "passed": false}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "4c4e88438e5155b7", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319169546.57, "duration_ms": 2132.272, "status": "ok", "attributes": {"case_id": 20, "verdict": "fail", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "c9338300b33c3576", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319171679.035, "duration_ms": 1246.936, "status": "ok", "attributes": {"case_id": 21, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "00dc385a38c81460", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319172926.009, "duration_ms": 0.03, "status": "ok", "attributes": {"case_id": 21, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "21704169ee2462a1", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319172926.054, "duration_ms": 1706.217, "status": "ok", "attributes": {"case_id": 21, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "95540d3260687b0d", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319174633.57, "duration_ms": 1003.727, "status": "ok", "attributes": {"case_id": 22, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "f57ce4f77a5cf58f", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319175637.338, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 22, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "7f80752ac0524de8", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319175637.371, "duration_ms": 1335.681, "status": "ok", "attributes": {"case_id": 22, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "280c5585a2372483", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319176973.231, "duration_ms": 2030.587, "status": "ok", "attributes": {"case_id": 23, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "f93d6b494991398c", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319179003.865, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 23, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "799d15d660af2b5d", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319179003.899, "duration_ms": 1586.291, "status": "ok", "attributes": {"case_id": 23, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "e9418930868de2aa", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319180590.317, "duration_ms": 1976.355, "status": "ok", "attributes": {"case_id": 24, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "207be9037a429c5a", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319182566.73, "duration_ms": 0.018, "status": "ok", "attributes": {"case_id": 24, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "ec7ef2e889dcf927", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319182566.763, "duration_ms": 2084.781, "status": "ok", "attributes": {"case_id": 24, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "eaf2ab2db3c5eee7", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319184651.746, "duration_ms": 2043.241, "status": "ok", "attributes": {"case_id": 25, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "82861478510a7f83", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319186695.035, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 25, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "18a5e0887215ca40", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319186695.068, "duration_ms": 2236.948, "status": "ok", "attributes": {"case_id": 25, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "f80401576f3286a8", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319188932.197, "duration_ms": 1116.783, "status": "ok", "attributes": {"case_id": 26, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "74136a6ad46187b2", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319190049.018, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 26, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "3f610b6f8879c2d9", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319190049.051, "duration_ms": 1702.826, "status": "ok", "attributes": {"case_id": 26, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "3a250854527677a7", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319191752.006, "duration_ms": 1255.641, "status": "ok", "attributes": {"case_id": 27, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "aaaaf181b8e94632", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319193007.684, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 27, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "dd3d2bd3c8078342", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319193007.715, "duration_ms": 1510.868, "status": "ok", "attributes": {"case_id": 27, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "3ccebdc28503a8d8", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319194518.736, "duration_ms": 1148.147, "status": "ok", "attributes": {"case_id": 28, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "52225f1d9ac23e0d", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319195666.945, "duration_ms": 0.014, "status": "ok", "attributes": {"case_id": 28, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "f4f9431ae1cfc4ac", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319195666.987, "duration_ms": 1382.361, "status": "ok", "attributes": {"case_id": 28, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "27b56b13b9ed11ea", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319197049.472, "duration_ms": 1159.992, "status": "ok", "attributes": {"case_id": 29, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "68b0bb3f3098305a", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319198209.535, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 29, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "112227c58b0edde0", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319198209.58, "duration_ms": 870.868, "status": "ok", "attributes": {"case_id": 29, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "3795759ca0ce1d38", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319199080.578, "duration_ms": 1074.214, "status": "ok", "attributes": {"case_id": 30, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "4e4e4087f2c43513", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319200154.846, "duration_ms": 0.014, "status": "ok", "attributes": {"case_id": 30, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "e63fdbac971c0a0a", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319200154.875, "duration_ms": 1449.063, "status": "ok", "attributes": {"case_id": 30, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "92ffab1b77806f2a", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319201604.058, "duration_ms": 951.161, "status": "ok", "attributes": {"case_id": 31, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "d1fa110ca794ad88", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319202555.268, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 31, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "2d527079aaa00c0e", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319202555.298, "duration_ms": 1892.407, "status": "ok", "attributes": {"case_id": 31, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "68452a6739469932", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319204447.831, "duration_ms": 1143.321, "status": "ok", "attributes": {"case_id": 32, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "4067223a8e589b8f", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319205591.205, "duration_ms": 0.024, "status": "ok", "attributes": {"case_id": 32, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "9527de39ac40011b", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319205591.243, "duration_ms": 1195.116, "status": "ok", "attributes": {"case_id": 32, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "2278db615724c33e", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319206786.503, "duration_ms": 782.686, "status": "ok", "attributes": {"case_id": 33, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "24fb869b284ab991", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319207569.224, "duration_ms": 0.012, "status": "ok", "attributes": {"case_id": 33, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "82c6ffe673db3f16", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319207569.251, "duration_ms": 1198.303, "status": "ok", "attributes": {"case_id": 33, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "e2e0769cac8c5dd6", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319208767.701, "duration_ms": 888.207, "status": "ok", "attributes": {"case_id": 34, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "09641b83bcb4aeaa", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319209655.947, "duration_ms": 0.03, "status": "ok", "attributes": {"case_id": 34, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "b49f4b7e5573fb83", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319209656.006, "duration_ms": 1245.749, "status": "ok", "attributes": {"case_id": 34, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "93f3f291371075d1", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319210901.875, "duration_ms": 972.333, "status": "ok", "attributes": {"case_id": 35, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "e277d28b8d6bd301", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319211874.244, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 35, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "f5da08098af63538", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319211874.275, "duration_ms": 1584.445, "status": "ok", "attributes": {"case_id": 35, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "140962af74d581ea", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319213458.87, "duration_ms": 1338.048, "status": "ok", "attributes": {"case_id": 36, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "e433440ea6c76630", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319214796.971, "duration_ms": 0.014, "status": "ok", "attributes": {"case_id": 36, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "2bdee8ee68900af6", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319214797.002, "duration_ms": 1511.918, "status": "ok", "attributes": {"case_id": 36, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "835cd393896dd177", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319216309.313, "duration_ms": 744.901, "status": "ok", "attributes": {"case_id": 37, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "ce3803c9e903a6c4", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319217054.285, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 37, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "006fe871351ad40e", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319217054.314, "duration_ms": 1319.956, "status": "ok", "attributes": {"case_id": 37, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "4142c21f15da94b5", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319218374.393, "duration_ms": 2078.56, "status": "ok", "attributes": {"case_id": 38, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "bf73561826bdccf3", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319220453.001, "duration_ms": 0.016, "status": "ok", "attributes": {"case_id": 38, "passed": true, "mode": "deterministic_only"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "d2f9463cac07c93a", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319220453.078, "duration_ms": 1720.136, "status": "ok", "attributes": {"case_id": 39, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "91c3044010d16e2f", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319222173.257, "duration_ms": 0.017, "status": "ok", "attributes": {"case_id": 39, "passed": true, "mode": "deterministic_only"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "4fad9efbbc840672", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319222173.351, "duration_ms": 1429.95, "status": "ok", "attributes": {"case_id": 40, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "5e955361986a8fa8", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319223603.343, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 40, "passed": true, "mode": "deterministic_only"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "0de40c7f23ccf59e", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319223603.417, "duration_ms": 1018.734, "status": "ok", "attributes": {"case_id": 41, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "2dbb43d13d821c7b", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319224622.195, "duration_ms": 0.013, "status": "ok", "attributes": {"case_id": 41, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "d8ff11995a354ba5", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319224622.239, "duration_ms": 1456.97, "status": "ok", "attributes": {"case_id": 41, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "360ce3ef21084d06", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319226079.362, "duration_ms": 1350.403, "status": "ok", "attributes": {"case_id": 42, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "1c2e9032bb8a8712", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319227429.82, "duration_ms": 0.048, "status": "ok", "attributes": {"case_id": 42, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "3dd7fd2be6da8795", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319227429.885, "duration_ms": 1564.095, "status": "ok", "attributes": {"case_id": 42, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "6690916914ee4e7f", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319228994.112, "duration_ms": 799.614, "status": "ok", "attributes": {"case_id": 43, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "311db83aec5c0a02", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319229793.773, "duration_ms": 0.012, "status": "ok", "attributes": {"case_id": 43, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "3181668b291e7f18", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319229793.8, "duration_ms": 2218.756, "status": "ok", "attributes": {"case_id": 43, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "36ee937af3c8272a", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319232012.737, "duration_ms": 1270.194, "status": "ok", "attributes": {"case_id": 44, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "13672d412c6e6c3e", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319233282.976, "duration_ms": 0.019, "status": "ok", "attributes": {"case_id": 44, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "a44355d5c4549eb6", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319233283.025, "duration_ms": 1956.205, "status": "ok", "attributes": {"case_id": 44, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "99aaa2f8cecb5a75", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319235239.399, "duration_ms": 1209.461, "status": "ok", "attributes": {"case_id": 45, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "a957ddb17448a221", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319236448.927, "duration_ms": 0.027, "status": "ok", "attributes": {"case_id": 45, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "b7631c7717fdd43f", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319236448.986, "duration_ms": 1224.083, "status": "ok", "attributes": {"case_id": 45, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "a27c252558b821dd", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319237673.191, "duration_ms": 1414.166, "status": "ok", "attributes": {"case_id": 46, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "83a9e7c27bbf1ec0", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319239087.405, "duration_ms": 0.015, "status": "ok", "attributes": {"case_id": 46, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "c49fbeefa61e5ef4", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319239087.436, "duration_ms": 1331.478, "status": "ok", "attributes": {"case_id": 46, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "18069f39cd0ee859", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319240419.039, "duration_ms": 629.679, "status": "ok", "attributes": {"case_id": 47, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "98f61df20217b328", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319241048.754, "duration_ms": 0.012, "status": "ok", "attributes": {"case_id": 47, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "da73ebf92c098f3b", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319241048.783, "duration_ms": 1343.078, "status": "ok", "attributes": {"case_id": 47, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "37e9fe0feb317bc9", "parent_span_id": "8bab58540d1fca12", "name": "sut.call", "kind": "AGENT", "start_time_ms": 1792319242392.06, "duration_ms": 977.742, "status": "ok", "attributes": {"case_id": 48, "sut": "fastapi-rag", "attempts": 1, "http_status": 200}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "4ec30b55f3e9e945", "parent_span_id": "8bab58540d1fca12", "name": "rule.check", "kind": "CHAIN", "start_time_ms": 1792319243369.852, "duration_ms": 0.014, "status": "ok", "attributes": {"case_id": 48, "passed": true}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "89255251fd391527", "parent_span_id": "8bab58540d1fca12", "name": "judge.grade", "kind": "LLM", "start_time_ms": 1792319243369.883, "duration_ms": 1116.525, "status": "ok", "attributes": {"case_id": 48, "verdict": "pass", "attempts": 1, "finish_reasons": "stop", "judge": "deepseek-v4-flash@https://api.deepseek.com"}}
+{"trace_id": "9ea04e15444d93e2d0923cc7f9e860a7", "span_id": "f83fd12c9d1201b2", "parent_span_id": null, "name": "report.write", "kind": "CHAIN", "start_time_ms": 1792319244487.187, "duration_ms": 36.179, "status": "ok"}
From 2de27c6107cdcc04f31c5bc8379396882a7dedb8 Mon Sep 17 00:00:00 2001
From: heweidong-ecco <290444878+heweidong-ecco@users.noreply.github.com>
Date: Sun, 18 Oct 2026 18:41:41 +0800
Subject: [PATCH 2/2] =?UTF-8?q?fix(sut-harness):=20=E9=80=82=E9=85=8D=20ap?=
=?UTF-8?q?p/=20=E4=B8=8E=E5=AE=98=E6=96=B9=E9=95=9C=E5=83=8F=20=E2=80=94?=
=?UTF-8?q?=E2=80=94=20=E6=81=A2=E5=A4=8D=20live=20=E5=9B=9E=E5=BD=92?=
=?UTF-8?q?=E5=8F=AF=E5=A4=8D=E7=8E=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
- 被测已从 api/ 改为 app/,官方镜像 = fastapi-rag-agent:latest(旧 my-fixed-name-api 已过期)
- 源码只读挂 /src → 容器内复制到可写 /app:直接只读挂 /app 会被
/app/.mem0/qdrant/.lock 挡住(实测 OSError [Errno 30] Read-only file system)
- 新增可选覆盖 EVAL_SUT_DB / EVAL_SUT_SESSION_TOKEN_LIMIT(隔离库/会话预算)
- 保留能力探针与 DEC-016 被测自证;去掉旧 /deps 卷与启动时 pip install
- 验证:单测 11 passed(stub docker/curl 契约)+ 真机端到端 exit 0、探针通过
- 决策记录:DEC-023 §6
---
CLAUDE.md | 6 +-
ROADMAP.md | 9 +-
...30\346\226\271\351\225\234\345\203\217.md" | 23 +++-
...ive\345\233\236\345\275\222-2026-10-18.md" | 29 +++--
...21\346\240\207\350\257\255\346\226\231.md" | 2 +-
tests/test_sut_harness.py | 122 +++++++++++++++++-
tools/sut-harness/README.md | 86 ++++++------
tools/sut-harness/run_sut.sh | 65 ++++++----
8 files changed, 256 insertions(+), 86 deletions(-)
diff --git a/CLAUDE.md b/CLAUDE.md
index 498eb6b..01e6e45 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -142,8 +142,10 @@ app/ .claude/skills/ .github/(CI-CD) tests/ # 实现层
> - 生产 `rag_db` 已有 **345 行 `documents`**,评测语料只占 **125 行** ⇒ **不能直接拿生产库跑评测**;
> 本轮用副本库 `rag_eval_20261018`(原库一行未动);
> - 被测仓结构已从 `api/` 改为 **`app/`**,官方镜像是 **`fastapi-rag-agent:latest`**;
-> - ⚠️ **`tools/sut-harness/run_sut.sh` 已跑不起来**(仍要求 `api/` + 旧镜像 `my-fixed-name-api`)
-> ⇒ **唯一环境待办**:按新结构修它(本轮已按 README 退出条件绕行官方镜像);
+> - ✅ **`tools/sut-harness/run_sut.sh` 已在同一任务内修好**(`DEC-023 §6`):
+> 适配 `app/` + 官方镜像,支持 `EVAL_SUT_DB` / `EVAL_SUT_SESSION_TOKEN_LIMIT`,
+> 单测 11 passed + 真机端到端 exit 0;**关键坑**:不能把源码只读直接挂 `/app`
+> (应用要写 `.mem0/qdrant/.lock`)⇒ 只读挂 `/src` 再复制到可写 `/app`;
> - `EVAL_JUDGE_API_KEY` 曾失效(401),**已由业务方更新**(尾号 `fd3e`)。
>
> ✅ **2026-09-21 会话完成**(按业务方"建议顺序"逐条执行):
diff --git a/ROADMAP.md b/ROADMAP.md
index ead2170..b5413ec 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -47,6 +47,7 @@
| 3 | 唯一失败 `id=20` 回灌 | `eval/cases/fastapi-rag-2026-10-18.json`(`status=open`) |
| 4 | 过程错误复盘(两轮无效,83,000 judge token) | `docs/复盘/2026-10-18-语料筛选漏掉seed金标语料.md` |
| 5 | `DEC-019` 补记 §7 | `docs/decisions/DEC-019-判分器放行口径.md` |
+| 6 | **修好 `run_sut.sh`**(适配 `app/` + 官方镜像;env 覆盖隔离库/会话预算) | `tools/sut-harness/run_sut.sh` · `DEC-023 §6` · 单测 11 passed + 真机 exit 0 |
**对「α 会不会误伤」的回答**:`rule_hit_judge_fail = 0`、`soft_miss_judge_pass = 0`;
唯一失败 `id=20` 是**真实被测失败**(召回 `[55,64,8]`,未召回含「两者结果互补」的 doc 60),
@@ -58,14 +59,16 @@
- 生产 `rag_db` 已有 **345 行**,评测语料只占 **125 行**(其它项目/测试灌入 220 行)
⇒ **不能直接拿生产库跑评测**;本轮用副本库 `rag_eval_20261018`(原库一行未动);
- 被测仓已从 `api/` 改为 **`app/`**,官方镜像是 **`fastapi-rag-agent:latest`**;
-- 旧 `tools/sut-harness/run_sut.sh` 因此**已跑不起来**(仍要求 `api/` + `my-fixed-name-api` 旧镜像);
+- **`tools/sut-harness/run_sut.sh` 已在同一任务内修好**并真机验证(exit 0 + 能力探针通过):
+ 自动识别 `app/`,默认官方镜像/网络,支持 `EVAL_SUT_DB` / `EVAL_SUT_SESSION_TOKEN_LIMIT`;
+ ⚠️ 关键坑:**不能把源码只读直接挂 `/app`**(应用要写 `.mem0/qdrant/.lock`)⇒ 只读挂 `/src` 再复制到可写 `/app`;
- `EVAL_JUDGE_API_KEY` 曾失效(401),**已由业务方更新**(尾号 `fd3e`)。
#### ⚠️ 下一会话开工要盯的
1. **待签核 = 0**;没有未完成的评测节点。
-2. **环境待办(唯一)**:修 `tools/sut-harness/run_sut.sh`(适配 `app/` 与官方镜像),
- 否则下轮照旧命令会直接失败。本轮已按 README 退出条件绕行官方镜像。
+2. **✅ 环境待办已清**:`tools/sut-harness/run_sut.sh` 已适配 `app/` + 官方镜像,
+ 单测 11 passed + 真机端到端 exit 0(见 `DEC-023 §6`)。
3. **非阻塞余项**:① 轮间 2× 延迟悬案(16 轮内 1 次,成因未指名)
② 母体 Kit 缺陷观察(证据 1/2)③ R1 探针(生产库有无文档 —— 本轮已间接回答:**有 345 行**)。
4. **`id=20` 的检索问题未修**(被测未改),只回灌不结论。
diff --git "a/docs/decisions/DEC-023-live\345\233\236\345\275\222\347\216\257\345\242\203-\351\232\224\347\246\273\350\257\255\346\226\231\344\270\216\345\256\230\346\226\271\351\225\234\345\203\217.md" "b/docs/decisions/DEC-023-live\345\233\236\345\275\222\347\216\257\345\242\203-\351\232\224\347\246\273\350\257\255\346\226\231\344\270\216\345\256\230\346\226\271\351\225\234\345\203\217.md"
index 4c34fcf..1150d3f 100644
--- "a/docs/decisions/DEC-023-live\345\233\236\345\275\222\347\216\257\345\242\203-\351\232\224\347\246\273\350\257\255\346\226\231\344\270\216\345\256\230\346\226\271\351\225\234\345\203\217.md"
+++ "b/docs/decisions/DEC-023-live\345\233\236\345\275\222\347\216\257\345\242\203-\351\232\224\347\246\273\350\257\255\346\226\231\344\270\216\345\256\230\346\226\271\351\225\234\345\203\217.md"
@@ -32,7 +32,8 @@
2. **被测运行方式**:走官方镜像 `fastapi-rag-agent:latest`(按 `tools/sut-harness/README.md` 的退出条件);
3. **会话预算**:本轮容器临时设 `SESSION_TOKEN_LIMIT=200000`(新版被测默认 5 万,单轮会触顶);
4. **judge key**:使用业务方更新后的 `EVAL_JUDGE_API_KEY`;
-5. **旧 harness**:不在本次修改,登记为**唯一环境待办**。
+5. **旧 harness**:本决策做出时先不改(不把环境修复混进回归);
+ **同一任务内已补修** —— 见 §6。
## 4. 认下的代价与边界(不得省略)
@@ -41,7 +42,25 @@
`eval_dataset.json#10–#25` 的 16 篇 seed,导致一轮 **37/48 的无效结果**;
补救与防错见 `docs/复盘/2026-10-18-语料筛选漏掉seed金标语料.md`;
- ⚠️ **隔离库 ≠ 生产库**:它回答"判分器在真实链路里是否误伤",不回答"生产库当前质量如何";
-- ⚠️ **旧 harness 仍是坏的**:下轮若照旧命令跑会直接失败,必须先修或改用本 DEC 的官方镜像命令。
+- ~~⚠️ **旧 harness 仍是坏的**:下轮若照旧命令跑会直接失败~~ ⇒ **已于同一任务内修复(§6)**。
+
+## 6. 补记:harness 已修(2026-10-18,同一任务内)
+
+`tools/sut-harness/run_sut.sh` 已按本 DEC 的官方路径重写:
+
+- 源码目录自动识别:`app/` 优先,兼容旧 `api/`;
+- 默认镜像改为 `fastapi-rag-agent:latest`、网络改为 `fastapi-rag-agent_app-net`;
+- 去掉旧 `/deps` 卷与启动时 `pip install`;
+- 新增两个评测必需的可选覆盖:`EVAL_SUT_DB`(隔离库)与 `EVAL_SUT_SESSION_TOKEN_LIMIT`(会话预算);
+- **源码只读挂到 `/src`,容器内复制到可写 `/app` 再启动**
+ —— ⚠️ 实测:**直接只读挂 `/app` 会失败**(应用要写 `/app/.mem0/qdrant/.lock`,
+ `OSError [Errno 30] Read-only file system`);
+- 保留能力探针与 DEC-016 被测自证。
+
+**验证**:
+- 单测 `tests/test_sut_harness.py` **11 passed**(含用 stub `docker`/`curl` 的 `docker run` 参数契约);
+- 真机端到端:**exit 0**,`/health` 200 + **能力探针通过**,
+ `eval/runs/.sut-harness.env` 记 `EVAL_SUT_VERSION=150a963` / `EVAL_SUT_PROBE=ok`。
## 5. 实施结果
diff --git "a/docs/reports/DEC-019-live\345\233\236\345\275\222-2026-10-18.md" "b/docs/reports/DEC-019-live\345\233\236\345\275\222-2026-10-18.md"
index acc170d..0634eee 100644
--- "a/docs/reports/DEC-019-live\345\233\236\345\275\222-2026-10-18.md"
+++ "b/docs/reports/DEC-019-live\345\233\236\345\275\222-2026-10-18.md"
@@ -61,15 +61,18 @@
| 能力探针 | ✅ 通过(能对真实问题作答,不是 `/health` 说了算) |
| 会话预算 | 本轮容器临时设 `SESSION_TOKEN_LIMIT=200000`(默认 5 万) |
-### ⚠️ 旧 harness 已失效(本轮发现)
+### 旧 harness 已失效 → 已在同一任务内修复
-`tools/sut-harness/run_sut.sh` 仍按旧结构写死:
+本轮开始时 `tools/sut-harness/run_sut.sh` 仍按旧结构写死:
-- 要求被测仓有 `api/` 并只读挂载 `$SUT_REPO/api:/app` —— 现被测仓已改为 **`app/`**,脚本会直接 `die`;
+- 要求被测仓有 `api/` 并只读挂载 `$SUT_REPO/api:/app` —— 被测仓已改为 **`app/`**,脚本会直接 `die`;
- 默认镜像 `my-fixed-name-api`(2026-07-01)已过期,当前官方镜像是 `fastapi-rag-agent:latest`。
-⇒ 本轮按 harness README 的**退出条件**改走官方路径。
-**待办**:按新结构修 `run_sut.sh`(或正式退役它),否则下次照旧命令会跑不起来。
+⇒ 回归轮按 harness README 的**退出条件**改走官方镜像;
+**同一任务内已把 harness 修好**(见 `DEC-023 §6`),真机端到端 **exit 0 + 能力探针通过**。
+修的过程抓到一个必须记下的坑:**不能把源码只读直接挂到 `/app`** ——
+应用要写 `/app/.mem0/qdrant/.lock`,实测 `OSError [Errno 30] Read-only file system`;
+现做法是只读挂 `/src` → 容器内复制到可写 `/app`。
## 3. 结果
@@ -128,16 +131,14 @@ L1: system_error_rate 0.0000 · gate_overhead_ratio 0.000507 · root_ms 139,202
# CREATE DATABASE rag_eval_20261018 TEMPLATE rag_db;
# 过滤 junk source + 按 content 去重 ⇒ 125 行
-# 1) 起被测(官方镜像)
-docker run -d --name rag-api-eval \
- --network fastapi-rag-agent_app-net -p 127.0.0.1:8000:8000 \
- --env-file /.env -e DOCKER_ENV=true \
- -e LLM_MODEL_FAST=deepseek-v4-flash -e LLM_MODEL_CHAT=deepseek-v4-flash \
- -e POSTGRES_DB=rag_eval_20261018 -e SESSION_TOKEN_LIMIT=200000 \
- fastapi-rag-agent:latest
+# 1) 起被测(官方镜像 + 隔离库 + 放宽会话预算)
+# 能力探针在脚本里,exit 0 才算就绪
+export EVAL_SUT_FASTAPI_API_KEY=$(grep -E '^EVAL_SUT_FASTAPI_API_KEY=' .env | cut -d= -f2-)
+EVAL_SUT_DB=rag_eval_20261018 \
+EVAL_SUT_SESSION_TOKEN_LIMIT=200000 \
+ tools/sut-harness/run_sut.sh
-# 2) 能力探针(必需,判据是能作答而不是 /health)
-# 3) 跑评测
+# 2) 跑评测(脚本已写好 eval/runs/.sut-harness.env)
set -a; . eval/runs/.sut-harness.env; set +a
export SSL_CERT_FILE=/etc/ssl/cert.pem
.venv/bin/python -m eval_gate run --evals eval/fastapi_rag_golden.evals.json --report-dir eval/runs
diff --git "a/docs/\345\244\215\347\233\230/2026-10-18-\350\257\255\346\226\231\347\255\233\351\200\211\346\274\217\346\216\211seed\351\207\221\346\240\207\350\257\255\346\226\231.md" "b/docs/\345\244\215\347\233\230/2026-10-18-\350\257\255\346\226\231\347\255\233\351\200\211\346\274\217\346\216\211seed\351\207\221\346\240\207\350\257\255\346\226\231.md"
index 4e59107..9498834 100644
--- "a/docs/\345\244\215\347\233\230/2026-10-18-\350\257\255\346\226\231\347\255\233\351\200\211\346\274\217\346\216\211seed\351\207\221\346\240\207\350\257\255\346\226\231.md"
+++ "b/docs/\345\244\215\347\233\230/2026-10-18-\350\257\255\346\226\231\347\255\233\351\200\211\346\274\217\346\216\211seed\351\207\221\346\240\207\350\257\255\346\226\231.md"
@@ -49,7 +49,7 @@
| 行动 | 归属 | 状态 |
|---|---|---|
-| 修 `tools/sut-harness/run_sut.sh`(被测已从 `api/` 改为 `app/`,默认镜像已过期) | 本仓 | ⬜ 待办(本轮已绕行官方镜像) |
+| 修 `tools/sut-harness/run_sut.sh`(被测已从 `api/` 改为 `app/`,默认镜像已过期) | 本仓 | ✅ 已完成(同一任务内;单测 11 passed + 真机 exit 0,见 `DEC-023 §6`) |
| 把"隔离语料事实反查清单"写进 live 回归的操作步骤 | 本仓 | ✅ 已写入 `docs/reports/DEC-019-live回归-2026-10-18.md` §6 |
## 四、对母体 Kit 的反馈
diff --git a/tests/test_sut_harness.py b/tests/test_sut_harness.py
index 74c456e..317c1b7 100644
--- a/tests/test_sut_harness.py
+++ b/tests/test_sut_harness.py
@@ -1,11 +1,17 @@
"""被测运行脚手架(`tools/sut-harness/`)的离线守卫。
-只测**不需要 Docker / 不需要被测仓**的部分 —— 前置校验与文档承诺。
-真正起容器的那条路只能在真机上跑(README 已写明),不在离线套件里假装覆盖。
+覆盖两层:
+① **前置校验**(不需要 Docker / 被测仓)—— 路径不对、缺 .env 必须在碰 Docker 之前退出;
+② **`docker run` 参数契约**(用 stub `docker`/`curl`,仍然不需要真 Docker)——
+ 保证「只读挂载、官方镜像、网络、env-file 覆盖、能力探针、被测自证」这些承诺
+ 不会在脚本漂移时静默失效。
+
+真正起容器的那条路仍只能在真机上跑(README 已写明),不在离线套件里假装覆盖。
动机:脚手架此前散在 `/tmp`,**重启丢过两次**;固化后若无守卫,它会再次悄悄漂移
(这正是本仓「门禁需要门禁」的一贯做法,见 tests/test_hooks.py / test_coverage_gate.py)。
"""
+import os
import re
import subprocess
from pathlib import Path
@@ -42,8 +48,8 @@ def test_rejects_missing_sut_repo_before_touching_docker(tmp_path):
def test_rejects_sut_repo_without_env_file(tmp_path):
- """有 api/ 但缺 .env(容器要用它做 env-file)→ 同样前置失败。"""
- (tmp_path / "api").mkdir()
+ """有 app/ 但缺 .env(容器要用它做 env-file)→ 同样前置失败。"""
+ (tmp_path / "app").mkdir()
r = _run_with_env(EVAL_SUT_REPO=str(tmp_path))
assert r.returncode == 1
assert ".env" in r.stderr
@@ -57,9 +63,113 @@ def test_script_takes_sut_path_from_env_not_hardcoded():
def test_does_not_modify_the_sut_repo():
- """脚手架必须**只读挂载**被测仓 —— 这是「评测门不改被测」的结构保证。"""
+ """脚手架必须**只读挂载**被测仓 —— 这是「评测门不改被测」的结构保证。
+
+ 只读挂载点必须是 `/src`(再由容器内复制到可写 `/app`):
+ 直接只读挂到 `/app` 会挡住应用写 `.mem0/qdrant/.lock` 等运行时状态。
+ """
text = RUN_SUT.read_text(encoding="utf-8")
- assert "-v \"$SUT_REPO/api:/app:ro\"" in text, "被测 api/ 未以只读方式挂载"
+ assert re.search(r'-v "\$SUT_SRC:/src:ro"', text), "被测源码未以只读方式挂载到 /src"
+
+
+# ── ② docker run 参数契约(stub,不需要真 Docker)──────────────────────────────
+
+def _write_stub(bindir: Path, name: str, body: str):
+ p = bindir / name
+ p.write_text("#!/bin/bash\n" + body, encoding="utf-8")
+ p.chmod(0o755)
+
+
+def _fake_repo(tmp_path: Path, *, layout: str = "app") -> Path:
+ repo = tmp_path / "sut"
+ (repo / layout).mkdir(parents=True)
+ (repo / layout / "main.py").write_text("app = object()\n", encoding="utf-8")
+ (repo / ".env").write_text("POSTGRES_PASSWORD=x\n", encoding="utf-8")
+ return repo
+
+
+def _run_with_stubs(tmp_path: Path, repo: Path, **over):
+ """用 stub `docker`/`curl` 跑完整脚本,返回 (CompletedProcess, docker 调用记录, 自证文件内容)。"""
+ bindir = tmp_path / "bin"
+ bindir.mkdir()
+ record = tmp_path / "docker-calls.txt"
+ _write_stub(bindir, "docker", f"""printf '%s\\n' "$*" >> {record}
+case "$1" in
+ info|start|exec|rm|logs) exit 0 ;;
+ run) echo fake-container-id; exit 0 ;;
+ *) exit 0 ;;
+esac
+""")
+ _write_stub(bindir, "curl", """
+if printf '%s' "$*" | grep -q -- '-w'; then printf '200'; exit 0; fi
+if printf '%s' "$*" | grep -q '/health'; then printf '{"status":"healthy"}'; exit 0; fi
+printf '{"answer":"ok"}'
+""")
+ identity = tmp_path / "identity.env"
+ env = {
+ "PATH": f"{bindir}:/usr/bin:/bin:/usr/sbin:/sbin",
+ "HOME": str(Path.home()),
+ "EVAL_SUT_REPO": str(repo),
+ "EVAL_SUT_FASTAPI_API_KEY": "dummy",
+ "EVAL_SUT_IDENTITY": str(identity),
+ }
+ env.update(over)
+ r = subprocess.run(["bash", str(RUN_SUT)], capture_output=True, text=True, env=env, timeout=60)
+ return r, (record.read_text(encoding="utf-8") if record.exists() else ""), (
+ identity.read_text(encoding="utf-8") if identity.exists() else "")
+
+
+def test_official_image_path_and_readonly_mount(tmp_path):
+ """新版被测( `app/` )必须走官方镜像:源码**只读**挂到 /src,容器内复制到可写 /app。"""
+ repo = _fake_repo(tmp_path, layout="app")
+ r, calls, _ = _run_with_stubs(tmp_path, repo)
+ assert r.returncode == 0, r.stdout + r.stderr
+ run_calls = [c for c in calls.splitlines() if c.startswith("run ")]
+ assert len(run_calls) == 1, f"应只 docker run 一次,实际: {calls}"
+ run = run_calls[0]
+ assert f"{repo}/app:/src:ro" in run, run
+ assert "cp -a /src/. /app/" in run, "容器内未把只读源码复制到可写 /app"
+ assert "uvicorn main:app" in run, "未以 uvicorn 启动当前源码"
+ assert "fastapi-rag-agent:latest" in run, "未使用当前官方镜像"
+ assert "--network fastapi-rag-agent_app-net" in run, "未使用当前官方网络"
+ assert "-e DOCKER_ENV=true" in run
+ assert "-e LLM_MODEL_FAST=deepseek-v4-flash" in run
+ assert "-e LLM_MODEL_CHAT=deepseek-v4-flash" in run
+ assert "/deps" not in run, "官方镜像不需要旧 /deps 卷"
+ assert "pip install" not in run, "官方镜像不需要启动时补装依赖"
+
+
+def test_prefers_app_over_api_when_both_exist(tmp_path):
+ repo = _fake_repo(tmp_path, layout="app")
+ (repo / "api").mkdir()
+ r, calls, _ = _run_with_stubs(tmp_path, repo)
+ assert r.returncode == 0, r.stdout + r.stderr
+ run = next(c for c in calls.splitlines() if c.startswith("run "))
+ assert f"{repo}/app:/src:ro" in run, "同时存在 app/ 与 api/ 时应优先 app/"
+
+
+def test_db_and_session_limit_are_passed_through(tmp_path):
+ """评测需要的两个覆盖必须能传进容器:隔离库 / 会话预算。"""
+ repo = _fake_repo(tmp_path, layout="app")
+ r, calls, _ = _run_with_stubs(
+ tmp_path, repo,
+ EVAL_SUT_DB="rag_eval_20261018",
+ EVAL_SUT_SESSION_TOKEN_LIMIT="200000",
+ )
+ assert r.returncode == 0, r.stdout + r.stderr
+ run = next(c for c in calls.splitlines() if c.startswith("run "))
+ assert "-e POSTGRES_DB=rag_eval_20261018" in run
+ assert "-e SESSION_TOKEN_LIMIT=200000" in run
+
+
+def test_capability_probe_and_identity_are_kept(tmp_path):
+ """能力探针不能省;被测自证(commit/probe)必须落盘 —— DEC-016。"""
+ repo = _fake_repo(tmp_path, layout="app")
+ r, calls, identity = _run_with_stubs(tmp_path, repo)
+ assert r.returncode == 0, r.stdout + r.stderr
+ assert "能力探针" in r.stdout
+ assert "EVAL_SUT_PROBE=ok" in identity
+ assert "EVAL_SUT_VERSION=" in identity
def test_readme_documents_retirement_condition_and_capability_probe():
diff --git a/tools/sut-harness/README.md b/tools/sut-harness/README.md
index 619c99a..0e554b1 100644
--- a/tools/sut-harness/README.md
+++ b/tools/sut-harness/README.md
@@ -6,46 +6,63 @@
| 理由 | 说明 |
|---|---|
-| **职责归属** | 它是**评测门**为"在受控环境里跑外部被测"造的适配层,属门的基础设施 —— 与 `tools/bench.py`、`tools/ab_regression.py` 同类 |
-| **避免第三套运行方式** | 被测仓**已有**自己的官方方式(`docker-compose.yml` / `Dockerfile` / `deploy.sh` / `dev.sh`)。再塞一套进去 = 制造"哪个才是对的"歧义 |
+| **职责归属** | 它是**评测门**为"在受控环境里跑外部被测"造的适配层,属门的基础设施 |
+| **避免第三套运行方式** | 被测仓**已有**自己的官方方式(`docker-compose.yml` / `Dockerfile` / `deploy.sh` / `dev.sh`) |
| **不污染对方仓库** | 不往被测的 GitHub 历史里混入评测门的关注点 |
-| **随门演进** | 它会随评测门变化(将来要加录制/mock),放本仓才不用每次去改被测仓 |
+| **随门演进** | 它会随评测门变化(加隔离库、放宽会话预算),放本仓才不用每次去改被测仓 |
被测路径通过 `EVAL_SUT_REPO` 注入 ⇒ **本仓不硬编码任何外部绝对路径**。
-## ⚠️ 定位:过渡 / 兜底 —— 有退出条件
+## 🔴 2026-10-18:退出条件已满足 —— 改走官方镜像
-**触发它的原因**:被测的 `rag-api` 容器**镜像构建于 2026-07-01**,早于 2026-09-10 的模型迁移,
-镜像内代码与仓库不一致 → 跑起来会回落到 DashScope(且其 chat 额度已耗尽)。
+**原退出条件**:
+- 被测侧用**官方方式**重建镜像可用(`docker compose build api && docker compose up -d api`),且
+- 重建后的镜像**能力探针通过**。
-**它为什么存在**:绕开陈旧镜像(只读挂载仓库 `api/`,跑**当前代码**)+ 补齐镜像缺的运行期依赖。
+**实测**:官方镜像 `fastapi-rag-agent:latest`(由被测仓 `app/Dockerfile` 构建)可用,
+能力探针通过 ⇒ **达到退出条件**。
-**退出条件(达到即应退役本目录)**:
-- 被测侧用**官方方式**重建镜像可用:`docker compose build api && docker compose up -d api`,且
-- 重建后的镜像**能力探针通过**(见下)。
+**因此本层已收窄成"薄包装"**:
+- 只用官方镜像,**不再**在启动时 `pip install` 补依赖,也不再用 `/deps` 卷与 `sut_run.py` 启动器;
+- **源码只读挂到 `/src`,容器启动时复制到可写 `/app` 再跑 `uvicorn`** ——
+ ⚠️ **不能**直接把源码只读挂到 `/app`:应用会写 `/app/.mem0/qdrant/.lock`,
+ 只读挂载会导致启动失败(2026-10-18 实测 `OSError [Errno 30] Read-only file system`);
+- 仍保留两件**只有这层能做**的事:
+ 1. **能力探针**(`/health` 200 ≠ 答得了,见下「陷阱」);
+ 2. **被测自证**(DEC-016:把被测 commit 与探针结果写进 `eval/runs/.sut-harness.env`)。
-> 即:能走官方路径就走官方路径。本层只是"官方路径暂时不通"时的保障。
+> ⚠️ `sut_run.py` / `sitecustomize.py` 是旧路径的遗留文件,**当前脚本不再使用**;
+> 保留它们只为在旧被测 commit 上做历史复查。新路径不要调用它们。
## 用法
> ⚠️ **起依赖只用 `docker start`,不要用 `docker compose up`**(业务方 2026-09-15 指示:
> **本机内存有限,只开需要的**)。需要的只有两个:`postgres-rag` / `redis-rag`
-> —— 它们由**被测仓自己的** `docker-compose.yml` 定义(`container_name` 同名,项目名 `my-fixed-name`),
-> 数据卷 `my-fixed-name_postgres_data` / `_redis_data` **仍在**,`docker start` 会**复用原卷**(知识库不丢)。
-> 脚本自身就是这么做的(见 `run_sut.sh` 第 32 行 `docker start postgres-rag redis-rag`)
-> ⇒ **不要额外跑 compose**:那会把 compose 里其余服务一并拉起,白吃内存
-> (本机还停着别的项目的容器:memory-db / zero-* / grafana / prometheus)。
+> —— 它们由**被测仓自己的** `docker-compose.yml` 定义(项目名 `my-fixed-name`),
+> 数据卷仍在,`docker start` 会**复用原卷**。
+> 脚本自身就是这么做的(见 `run_sut.sh` 的 `docker start postgres-rag redis-rag`)
+> ⇒ **不要额外跑 compose**:那会把 compose 里其余服务一并拉起,白吃内存。
```bash
export EVAL_SUT_FASTAPI_API_KEY= # 能力探针要用;跳过它会失去最重要的那道保险
tools/sut-harness/run_sut.sh # 默认路径;可用 EVAL_SUT_REPO 覆盖
+
+# 评测隔离库 + 放宽会话预算(新版被测默认 5 万 token/会话,单轮会触顶)
+EVAL_SUT_DB=rag_eval_20261018 \
+EVAL_SUT_SESSION_TOKEN_LIMIT=200000 \
+tools/sut-harness/run_sut.sh
```
| 环境变量 | 默认 | 说明 |
|---|---|---|
| `EVAL_SUT_REPO` | (内置一个默认路径) | 被测仓根目录 |
-| `EVAL_SUT_IMAGE` / `EVAL_SUT_CONTAINER` / `EVAL_SUT_NETWORK` | `my-fixed-name-api` / `rag-api-eval` / `my-fixed-name_app-net` | 容器三要素 |
-| `EVAL_SUT_DEPS_VOLUME` | `sut-pydeps` | 补装依赖的**持久卷**(只装一次) |
+| `EVAL_SUT_IMAGE` | **`fastapi-rag-agent:latest`** | 官方镜像(由被测仓 `app/Dockerfile` 构建) |
+| `EVAL_SUT_CONTAINER` / `EVAL_SUT_NETWORK` | `rag-api-eval` / **`fastapi-rag-agent_app-net`** | 容器三要素 |
+| `EVAL_SUT_PORT` | `8000` | 宿主机端口(应用固定容器内 8000) |
+| `EVAL_SUT_DB` | (不设) | 覆盖 `POSTGRES_DB`(评测用隔离库) |
+| `EVAL_SUT_SESSION_TOKEN_LIMIT` | (不设) | 覆盖被测的 `SESSION_TOKEN_LIMIT` |
+| `EVAL_SUT_LOGS_DIR` | `<本仓>/eval/runs/sut-logs` | 可写的 `/app/logs`(源码是只读挂载) |
+| `EVAL_SUT_IDENTITY` | `<本仓>/eval/runs/.sut-harness.env` | DEC-016 被测自证落盘位置 |
退出码:`0` 就绪且探针通过 · `1` 起不来 · **`2` 起来了但答不了**。
@@ -53,35 +70,32 @@ tools/sut-harness/run_sut.sh # 默认路径;可用 EVAL_SUT_R
| 文件 | 作用 |
|---|---|
-| `run_sut.sh` | 起依赖容器 → 起被测容器 → 等就绪 → **能力探针** |
-| `sut_run.py` | 最小启动器:**只挂 RAG 路由**,避开前端/看板等重型可选依赖 |
-| `sitecustomize.py` | 兼容层:把 langchain 1.x 移走的 legacy agent API(现于 `langchain_classic`)补回 |
-
-容器内**补装**的三个运行期缺件(镜像里没有,装在持久卷 `/deps`):`python-multipart`、`ddgs`、`langchain-classic`。
+| `run_sut.sh` | 起依赖容器 → 起被测容器(官方镜像 + 只读挂载源码)→ 等就绪 → **能力探针** → 写被测自证 |
+| `sut_run.py` | **遗留**(旧最小启动器),当前脚本不使用 |
+| `sitecustomize.py` | **遗留**(旧 langchain 兼容层),当前脚本不使用 |
## 陷阱清单(踩过的,别再踩)
1. **`docker run --env-file` 不剥离行内 `#` 注释** —— 被测 `.env` 里
- `LLM_MODEL_FAST=deepseek-v4-flash # 轻量角色:…` 会把注释整段当成**值**,
+ `LLM_MODEL_FAST=deepseek-v4-flash # …` 会把注释整段当成**值**,
模型名被污染后网关返回 400。⇒ 必须用 `-e` 显式覆盖(脚本已做)。
2. **`/health` 200 ≠ 被测答得了** —— `/health` 只查 DB/Redis。**能力探针是必需的**:
2026-09-11 实测,被测能通过健康检查却对真实问题一律回「无法回答」(根因是被测的查询改写
因 `max_tokens` 被推理吃满而返回空)。**跳过探针,跑完的结果会被误读成"被测质量崩了"。**
-3. **最小启动器与生产形态不同** —— 只挂 RAG 路由 ⇒ 中间件/缓存等未加载。
- **评测结论的"可比性"要连同这一点一起说明**(见 `docs/reports/P4-1/端到端测试报告.md` §3.6)。
-4. **别直接用原 `rag-api` 容器** —— 镜像陈旧,跑的是 07-01 的代码。
-5. **`docker logs` 为空 ≠ 进程崩了**(2026-09-12 实测) —— 容器 Cmd 的第一步是
- `pip install --quiet --target=/deps python-multipart ddgs langchain-classic`(输出被 `| tail -1` 吞掉),
- 紧接着才起 uvicorn。所以**刚重建的头几十秒**:容器 `Up`、`docker logs` **完全空白**、
- `/health` 连不上 —— 这三条同时出现**看着就像崩了**,实际只是 pip 还在跑。
- ⇒ **判据是 `/health` + 能力探针,不是日志有没有输出**;`run_sut.sh` 报「服务未就绪」时,
- 先**等 30–60 秒再判**,别急着重启(重启会让 pip 从头再来)。
- > 另有两条环境教训(与本节同源):容器内**没有 `ps`**(`docker exec … ps` 必失败,别拿它当判据);
- > `docker inspect` 的 `.Config.Cmd` 能一眼看出容器当前处于哪个阶段。
+3. **别直接用原 `rag-api` 容器** —— 旧镜像,跑的不是当前代码。
+4. **不要把源码只读直接挂到 `/app`** —— 应用会写 `/app/.mem0/qdrant/.lock` 等运行时状态
+ (实测 `OSError [Errno 30] Read-only file system`)。本脚本的做法是 `/src`(只读)→ 复制到可写 `/app`;
+ 日志目录另挂 `EVAL_SUT_LOGS_DIR`,避免写进被测仓、保持"被测仓零改动"。
+5. **新版被测有会话预算**(默认 5 万 token/会话/日)——
+ 跑 48 条评测前先看 `token_usage_logs` 今日余量,不够就设 `EVAL_SUT_SESSION_TOKEN_LIMIT`;
+ 否则尾几条会被 `429 QUOTA_EXCEEDED` 污染,看起来像被测失败。
+6. **`docker logs` 为空不等于进程崩了**(旧路径遗留观察) —— 旧容器启动时先 `pip install`,
+ 头几十秒日志空白。官方镜像没有这一步;若仍见空白,先等 30–60s 再判,别急着重启。
## 观察记录(本层已验证有效)
- 2026-09-11:用本层跑通真实链路一轮(run `20260911-224254-be1c4574`,30/40);
并借此**定位到被测侧一个真实的生产缺陷**(查询改写空返回导致静默降级),
被测已修(`85e456b`)。
- ⇒ 这层虽"丑",但它让门**第一次真正看到了外部系统的运行行为** —— 这正是评测门存在的意义。
+- 2026-10-18:**切到官方镜像**,并用隔离语料库跑完 `DEC-019` 完整 live 回归
+ (run `20261018-182724-e6212c1d`,**47/48 · exit 0**)。
diff --git a/tools/sut-harness/run_sut.sh b/tools/sut-harness/run_sut.sh
index 5564192..6481435 100755
--- a/tools/sut-harness/run_sut.sh
+++ b/tools/sut-harness/run_sut.sh
@@ -1,29 +1,44 @@
#!/usr/bin/env bash
# 在容器里跑真实被测(`01.FastAPI RAG Agent`),供评测门做真实链路评测用。
#
-# 定位:**评测门的运行适配层**,不是被测的一部分。**被测仓零改动** ——
-# 只读挂载它的 `api/`,所有补丁都在容器运行时(见 README「退出条件」)。
+# 定位:**评测门的运行适配层**,不是被测的一部分。被测仓零改动 ——
+# 只读挂载它的源码目录(`app/`,兼容旧的 `api/`),所有补丁都在容器运行时。
+#
+# 🔴 2026-10-18:被测仓结构已从 `api/` 改为 `app/`,并有了**官方镜像**
+# `fastapi-rag-agent:latest`(由被测仓 `app/Dockerfile` 构建)
+# ⇒ 达到本 harness README 的「退出条件」,改走官方镜像;
+# 旧的 `pip 补装 /deps 卷 + sut_run.py` 启动路径**退役**(文件保留,不再使用)。
#
# 用法:
-# tools/sut-harness/run_sut.sh # 用默认被测路径
-# EVAL_SUT_REPO=/path/to/sut tools/sut-harness/run_sut.sh
+# tools/sut-harness/run_sut.sh
+# # 评测隔离库 / 放宽会话预算(新版被测默认 5 万,单轮会触顶)
+# EVAL_SUT_DB=rag_eval_20261018 EVAL_SUT_SESSION_TOKEN_LIMIT=200000 tools/sut-harness/run_sut.sh
#
# 退出码:0 = 就绪且**能力探针通过**;1 = 起不来;2 = 起来了但答不了(见 README §陷阱)
set -euo pipefail
SUT_REPO="${EVAL_SUT_REPO:-/Users/heweidong/Desktop/Product/agent-projects/projects/fastapi-rag-agent}"
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-IMAGE="${EVAL_SUT_IMAGE:-my-fixed-name-api}"
+ROOT="$(cd "$HERE/../.." && pwd)"
+IMAGE="${EVAL_SUT_IMAGE:-fastapi-rag-agent:latest}"
CONTAINER="${EVAL_SUT_CONTAINER:-rag-api-eval}"
-NETWORK="${EVAL_SUT_NETWORK:-my-fixed-name_app-net}"
-DEPS_VOLUME="${EVAL_SUT_DEPS_VOLUME:-sut-pydeps}"
+NETWORK="${EVAL_SUT_NETWORK:-fastapi-rag-agent_app-net}"
PORT="${EVAL_SUT_PORT:-8000}"
API_KEY="${EVAL_SUT_FASTAPI_API_KEY:-}"
+IDENTITY="${EVAL_SUT_IDENTITY:-$ROOT/eval/runs/.sut-harness.env}"
+LOGS_DIR="${EVAL_SUT_LOGS_DIR:-$ROOT/eval/runs/sut-logs}"
log() { printf '[sut-harness] %s\n' "$*"; }
die() { printf '[sut-harness] ✗ %s\n' "$*" >&2; exit 1; }
-[ -d "$SUT_REPO/api" ] || die "被测仓路径不对(缺 api/):$SUT_REPO —— 用 EVAL_SUT_REPO 指定"
+# 0) 源码目录:优先新结构 `app/`,兼容旧结构 `api/` ----------------------------
+if [ -d "$SUT_REPO/app" ]; then
+ SUT_SRC="$SUT_REPO/app"
+elif [ -d "$SUT_REPO/api" ]; then
+ SUT_SRC="$SUT_REPO/api"
+else
+ die "被测仓路径不对(缺 app/,也没有旧 api/):$SUT_REPO —— 用 EVAL_SUT_REPO 指定"
+fi
[ -f "$SUT_REPO/.env" ] || die "被测仓缺 .env(容器要用它做 env-file):$SUT_REPO/.env"
# 1) Docker 与依赖容器 -------------------------------------------------------
@@ -36,25 +51,34 @@ for _ in $(seq 1 30); do
done
docker exec postgres-rag pg_isready >/dev/null 2>&1 || die "postgres-rag 未就绪"
-# 2) 起被测容器 --------------------------------------------------------------
+# 2) 起被测容器(官方镜像 + 只读挂载当前源码)----------------------------------
# ⚠️ 必须显式 -e 覆盖这两个变量:docker run --env-file **不剥离行内注释**,
-# 被测 .env 里 `LLM_MODEL_FAST=deepseek-v4-flash # 轻量角色:…` 会把注释整段当值。
-log "重建容器 $CONTAINER(只读挂载被测 api/,跑仓库当前代码)…"
+# 被测 .env 里 `LLM_MODEL_FAST=deepseek-v4-flash # …` 会把注释整段当值。
+log "重建容器 $CONTAINER(官方镜像 $IMAGE;只读挂载 $SUT_SRC → /src → 复制到可写 /app)…"
docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
-docker volume create "$DEPS_VOLUME" >/dev/null 2>&1 || true
+mkdir -p "$LOGS_DIR" # 应用要往 /app/logs 写文件 ⇒ 单独给可写目录(且不写进被测仓)
+
+# ⚠️ 不直接把源码只读挂到 /app:应用会在 /app/.mem0/qdrant 等路径写运行时状态
+# (2026-10-18 实测:只读挂载 ⇒ OSError [Errno 30] Read-only file system: '/app/.mem0/qdrant/.lock')
+# ⇒ 只读挂 /src,容器启动时复制到可写 /app,再跑 uvicorn。这样既跑**当前代码**,也不碰被测仓。
+
+# 可选覆盖:评测隔离库 / 会话预算(未设置则沿用 .env / 应用默认值)
+EXTRA_ENV=()
+[ -n "${EVAL_SUT_DB:-}" ] && EXTRA_ENV+=(-e "POSTGRES_DB=$EVAL_SUT_DB")
+[ -n "${EVAL_SUT_SESSION_TOKEN_LIMIT:-}" ] && EXTRA_ENV+=(-e "SESSION_TOKEN_LIMIT=$EVAL_SUT_SESSION_TOKEN_LIMIT")
+
docker run -d --name "$CONTAINER" \
--network "$NETWORK" -p "$PORT:8000" \
--env-file "$SUT_REPO/.env" \
-e DOCKER_ENV=true \
-e LLM_MODEL_FAST=deepseek-v4-flash \
-e LLM_MODEL_CHAT=deepseek-v4-flash \
- -e PYTHONPATH=/deps:/scaffold \
- -v "$SUT_REPO/api:/app:ro" \
- -v "$DEPS_VOLUME:/deps" \
- -v "$HERE:/scaffold:ro" \
+ ${EXTRA_ENV[@]+"${EXTRA_ENV[@]}"} \
+ -v "$SUT_SRC:/src:ro" \
+ -v "$LOGS_DIR:/app/logs" \
"$IMAGE" \
- sh -c "pip install --quiet --target=/deps python-multipart ddgs langchain-classic 2>&1 | tail -1; python /scaffold/sut_run.py" \
- >/dev/null || die "docker run 失败(镜像 $IMAGE 是否存在?)"
+ sh -c 'cp -a /src/. /app/ && exec uvicorn main:app --host 0.0.0.0 --port 8000' \
+ >/dev/null || die "docker run 失败(镜像 $IMAGE 是否存在?先 docker images | grep fastapi-rag)"
# 3) 等就绪 ------------------------------------------------------------------
for _ in $(seq 1 36); do
@@ -89,9 +113,6 @@ log "就绪:被测在 http://127.0.0.1:$PORT(容器 $CONTAINER)"
# 门**不可能**自己知道被测的 commit,也不该去问 ⇒ 由本层(唯一知道被测仓与探针结果的一层)导出。
# 写出而非 export:本脚本是子进程,export 传不回调用者的 shell。
# 用法:跑评测前 `set -a; . eval/runs/.sut-harness.env; set +a`
-# ⚠️ 本脚本此前**没有** ROOT 变量(实测);不定义就写成了 `/eval/runs/…`(根目录!)
-ROOT=$(cd "$(dirname "$0")/../.." && pwd)
-IDENTITY="$ROOT/eval/runs/.sut-harness.env"
SUT_COMMIT="$(git -C "$SUT_REPO" rev-parse --short HEAD 2>/dev/null || echo unknown)"
PROBE_RESULT="ok"
[ -z "$API_KEY" ] && PROBE_RESULT="skipped"
@@ -102,4 +123,4 @@ EVAL_SUT_VERSION=$SUT_COMMIT
EVAL_SUT_PROBE=$PROBE_RESULT
EOF
log "被测自证已写入 $IDENTITY(commit=$SUT_COMMIT · 探针=$PROBE_RESULT)"
-log "⇒ 跑评测前先:set -a; . eval/runs/.sut-harness.env; set +a"
+log "⇒ 跑评测前先:set -a; . $IDENTITY; set +a"