From 39a31389d0b066f0b0d5a04b3f528c1245cecf42 Mon Sep 17 00:00:00 2001 From: zhaozhaozz <6450679+zhaozhaozz@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:54:18 +0000 Subject: [PATCH 1/2] Refresh completed reruns by ID after coalesced notifications --- README.md | 1 + collect_with_oidc.py | 44 ++++++++- docs/actions-collection.md | 3 + docs/ci-history-lanes.md | 1 + docs/incremental-history.md | 10 ++- gsb/run_changes.py | 42 +++++++++ gsb/sync.py | 47 ++++++++-- publish.py | 6 +- test/journey-rerun-collection.py | 147 +++++++++++++++++++++++++++++++ tests/test_oidc_collection.py | 63 ++++++++++++- tests/test_run_changes.py | 107 ++++++++++++++++++++++ 11 files changed, 452 insertions(+), 19 deletions(-) create mode 100644 gsb/run_changes.py create mode 100644 test/journey-rerun-collection.py create mode 100644 tests/test_run_changes.py diff --git a/README.md b/README.md index 23aa10e99..0e9b7dc26 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Coverage 页从源仓 GitHub Actions 产物中读取 `node-coverage-summary-*` ```bash PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -v +PYTHONDONTWRITEBYTECODE=1 python3 test/journey-rerun-collection.py node test/sync-e2e.mjs node .e2e/node_modules/playwright/cli.js test --config test/playwright.config.cjs ``` diff --git a/collect_with_oidc.py b/collect_with_oidc.py index 1c4d0dd52..90b1dcc9f 100644 --- a/collect_with_oidc.py +++ b/collect_with_oidc.py @@ -5,6 +5,7 @@ from datetime import datetime, timezone import json import os +import re from pathlib import Path import subprocess import sys @@ -17,6 +18,8 @@ ROOT = Path(__file__).resolve().parent # GitCode sync records fetched from the bot for publish.py; not uploaded anywhere by itself. SYNC_RECORDS = ROOT / ".tmp" / "gitcode-sync.json" +RUN_CHANGES = ROOT / ".tmp" / "run-changes.json" +RUN_RECEIPT = ROOT / ".tmp" / "run-changes-receipt.json" class CredentialError(RuntimeError): @@ -79,6 +82,35 @@ def sync_records(url, identity): return {"ok": False, "error": "bot unreachable"} +def changed_runs(url, identity): + try: + value = request_json(url, identity, {}, "POST") + return value if isinstance(value, dict) else {"ok": False} + except CredentialError: + print("::warning::Rerun hints unavailable; pending IDs remain at the bot.", flush=True) + return {"ok": False} + + +def acknowledge_runs(url, audience, receipt, requested, source): + # The file only exists after publish.py committed both progress and public data. + # Read a fresh GitHub identity: collection can outlive the original OIDC JWT. + try: + value = json.loads(receipt.read_text()) + from gsb.run_changes import valid_change + rows = value.get("runs") + if (value.get("source") != source or not isinstance(value.get("commit"), str) + or not re.fullmatch('[0-9a-f]{40}', value['commit']) + or not isinstance(rows, list) or not rows or len(rows) > 20 + or not all(valid_change(row) and row in requested.get("runs", []) for row in rows)): + raise ValueError("invalid rerun receipt") + reply = request_json(url, oidc_identity(audience), {"runs": rows, "commit": value["commit"]}, "POST") + if not isinstance(reply, dict) or reply.get("ok") is not True: + raise ValueError("rerun acknowledgement unavailable") + except Exception: + # Publication succeeded; lost acknowledgements are safe to retry next collection. + print("::warning::Rerun acknowledgement failed; pending IDs remain at the bot.", flush=True) + + def main(): grants = [] stage = "configuration" @@ -109,10 +141,18 @@ def main(): stage = "GitCode sync records" SYNC_RECORDS.parent.mkdir(parents=True, exist_ok=True) SYNC_RECORDS.write_text(json.dumps(sync_records(urllib.parse.urlunsplit(endpoint._replace(path="/actions/gitcode-sync")), identity)), encoding="utf-8") + stage = "rerun hints" + requested = changed_runs(urllib.parse.urlunsplit(endpoint._replace(path="/actions/reruns")), identity) + RUN_CHANGES.write_text(json.dumps(requested), encoding="utf-8") + RUN_RECEIPT.unlink(missing_ok=True) # The collector receives neither OIDC credentials nor any App private key. child_env = {k: v for k, v in os.environ.items() if k not in {"ACTIONS_ID_TOKEN_REQUEST_TOKEN", "ACTIONS_ID_TOKEN_REQUEST_URL", "GH_TOKEN", "GITHUB_TOKEN", "GSB_PUBLISH_TOKEN"} and "PRIVATE_KEY" not in k} - child_env.update(GITHUB_TOKEN=grants[0], GSB_PUBLISH_TOKEN=grants[1], GSB_GITCODE_SYNC_FILE=str(SYNC_RECORDS)) - return subprocess.run([sys.executable, str(ROOT / "publish.py"), "--repo", context["source"], "--publish-repo", context["target"], "--output", ".tmp/collected-site", "--incremental"], env=child_env, cwd=ROOT, check=False).returncode + child_env.update(GITHUB_TOKEN=grants[0], GSB_PUBLISH_TOKEN=grants[1], GSB_GITCODE_SYNC_FILE=str(SYNC_RECORDS), + GSB_RUN_CHANGES_FILE=str(RUN_CHANGES), GSB_RUN_CHANGES_RECEIPT=str(RUN_RECEIPT)) + code = subprocess.run([sys.executable, str(ROOT / "publish.py"), "--repo", context["source"], "--publish-repo", context["target"], "--output", ".tmp/collected-site", "--incremental"], env=child_env, cwd=ROOT, check=False).returncode + if code == 0 and RUN_RECEIPT.exists(): + acknowledge_runs(urllib.parse.urlunsplit(endpoint._replace(path="/actions/reruns/ack")), audience, RUN_RECEIPT, requested, context["source"]) + return code except Exception as error: status = getattr(error, "status", None) detail = f" (HTTP {status})" if isinstance(status, int) else "" diff --git a/docs/actions-collection.md b/docs/actions-collection.md index 0e63ac93e..6e0ac2279 100644 --- a/docs/actions-collection.md +++ b/docs/actions-collection.md @@ -26,6 +26,8 @@ App 的 Actions 写权限在 App 注册页 **Permissions & events → Repository 兑换令牌后，客户端用同一 OIDC 身份向同一 Worker 的 `/actions/gitcode-sync` 读取 GitCode 同步记录，经 `GSB_GITCODE_SYNC_FILE` 交给发布器；读取失败只把页面标为过期，不影响采集。见 [GitCode 同步记录](gitcode-sync.md)。 +客户端还向同一 Worker `POST /actions/reruns`（正文 `{}`）领取最多 20 条完成重跑通知，经非秘密的 `GSB_RUN_CHANGES_FILE` 交给采集器。Worker 仅合并 run ID／attempt／revision，不为每个完成事件调度采集；仍随既有 Issue／PR 合并刷新、8 小时空闲刷新、每日兜底或手动采集处理。发布器按 ID 优先读取最新 attempt、状态与必要报告，不回扫全部 Actions 历史。成功原子提交后才生成 `GSB_RUN_CHANGES_RECEIPT`，父进程重新领取短期 OIDC 身份，向 `/actions/reruns/ack` 确认成功项；失败项和确认丢失均保留，旧确认不删除后来的新 attempt。两个接口沿用固定看板仓 ID／所有者 ID、`main`、`collect.yml` 和 `schedule`／`workflow_dispatch` 的校验，不新增信任或安装令牌权限。建议先发布 Bot 接口再更新采集器；旧 Bot 不提供接口时给出警告并继续普通采集。请求、下载和时间预算保持不变。见 [增量同步与历史](incremental-history.md)。 + 采集步骤从环境变量读取源仓 `GITHUB_TOKEN` 和目标仓 `GSB_PUBLISH_TOKEN`，运行 `publish.py --repo ... --publish-repo ... --output .tmp/collected-site --incremental`。临时目录不上传，不写回原始测试 artifact。发布器基于本轮 checkout 的准确 HEAD，将 `.sync/` 进度及变化的 `site/` 分片放入一个 Git 提交。保留源码和工作流；并发冲突直接失败，下轮从新 main 续跑。只有 site 变化才触发 Pages。详情见 [增量同步](incremental-history.md)。 E2E 执行记录的 HTML 报告不进入提交：采集成功后，`collect.yml` 把新读取的报告上传为本次运行的 `e2e-html` 产物，保留到对应源产物的 `expires_at`；`pages.yml` 以 `actions: read` 读取这些产物并在部署前挂载，取不到时页面改为 GitHub 产物链接。见 [E2E 执行记录](e2e-run-records.md)。 @@ -42,6 +44,7 @@ Bot 的“已触发采集”只表示 GitHub 接受调度。实际采集结果 ```bash PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -v +PYTHONDONTWRITEBYTECODE=1 python3 test/journey-rerun-collection.py ``` 测试覆盖正式／测试映射、拒绝跨站和恶意 inputs、读写凭据隔离、公开内容边界、非强制提交和测试报告统计。Bot 仓 `npm run test:worker-adapter` 覆盖 Worker 到模拟 Actions API 的签名凭据与触发链路。两者不代表真实 GitHub runner 或 Pages 已执行，上线前仍需验证。 diff --git a/docs/ci-history-lanes.md b/docs/ci-history-lanes.md index 82d87ef22..c3e41caa1 100644 --- a/docs/ci-history-lanes.md +++ b/docs/ci-history-lanes.md @@ -42,6 +42,7 @@ CI 页的「CI 分层历史」按触发方式展示该分支线窗口内的全 - 只展示已采集的 Actions run，不触发、不执行测试，也不修改源仓工作流。 - 窗口天数与时区是代码常量（`WINDOW_DAYS`、`DISPLAY_ZONE`）。 - 位置按 run 创建时间；重跑后颜色更新，但不会移到重跑当天。 +- 重跑完成通知随现有刷新节奏合并处理，每轮最多按 ID 补查 20 条最新 attempt 与必要报告，不为此扫描全部 Actions 历史或单独触发采集；发布成功后确认，失败保留到下次刷新。旧通知未送达时仍由原有对账兜底。 - 构建报告页的三张摘要卡与本图独立。 ## 验证 diff --git a/docs/incremental-history.md b/docs/incremental-history.md index 7d0229896..825acf4fa 100644 --- a/docs/incremental-history.md +++ b/docs/incremental-history.md @@ -2,7 +2,7 @@ ## 功能与数据范围 -每个看板仓单独保存同步状态和公开数据。Bot 只提交刷新信号，Actions 使用 API 采集；漏 Webhook、重复触发或 Bot 停机都不改变数据事实来源。定时工作流从仓内断点继续，全部 Webhook 存档仍由 Bot 按原规则保存。 +每个看板仓单独保存同步状态和公开数据。Bot 提交刷新信号及合并后的重跑 ID，Actions 使用 API 采集；漏 Webhook、重复触发或 Bot 停机都不改变数据事实来源。定时工作流从仓内断点继续，全部 Webhook 存档仍由 Bot 按原规则保存。 Issue / PR 的开放及关闭记录不设保留条数上限。Actions 保存 run / attempt 的状态、工作流、触发事件、门禁／每日／版本分类、分支、head 仓库、SHA、GitHub 链接与可核验测试指标。版本发布也逐页回填。历史尚未补齐时，页面显示“历史回填中”和已采集数量；列表回填完成不代表所有报告都有可读指标。 @@ -27,7 +27,11 @@ Issue / PR 的开放及关闭记录不设保留条数上限。Actions 保存 run 主采集的产物下载仍使用原有 160 MiB 预算。[E2E 执行记录](e2e-run-records.md)在其后使用独立的有界预算，一次列表、最多两份 ZIP、每份最多 80 MiB，避免历史详情耗尽额度后执行记录一直无法升级。升级失败保留原有步骤和 HTML bundle，不以空记录覆盖已发布内容。 -Issue 按 updated 的 since 水位增量读取；PR 按 updated 倒序读到上一水位，均重叠一小时。历史按创建顺序逐页补齐。每周开始一轮小批量全表对账，修复翻页期间顺序变化、旧 PR 更新及旧构建重跑；对账也有持久游标。旧已完成 run 的重新运行最迟由对账发现；新建运行与已知在途运行优先处理。 +Issue 按 updated 的 since 水位增量读取；PR 按 updated 倒序读到上一水位，均重叠一小时。历史按创建顺序逐页补齐。每周开始一轮小批量全表对账，修复翻页期间顺序变化、旧 PR 更新及漏通知的旧构建重跑；对账也有持久游标，新建运行与已知在途运行优先处理。 + +Worker 接收 `workflow_run.completed` 中 attempt 大于 1 的重跑完成通知，同一 run 只留最新 attempt 与 revision；普通第一次完成不入这条队列。不依赖 `requested`。下一次原有采集最多读取 20 条待办，优先调用 `/repos/{source}/actions/runs/{id}` 获取当前 attempt、状态，再按现有缓存读取对应 job 和必要报告；即使创建时间已在增量窗口之外也能更新，不为重跑新增 Actions 历史列表扫描。批次轮转，失败 ID 不挡住其他待办；刷新间隔、合并窗口、请求／时间和产物预算均不增加。 + +只有最新详情已读完、且 `.sync/` 与 `site/` 原子提交成功，采集器才发送对应 ID、attempt、revision 的确认。源端错误、预算用尽、报告暂不可读、提交冲突或确认失败均保留 Bot 待办，下一次原有刷新再试；旧确认不会删除更晚的新 attempt。本地预览不确认。未部署此接口的 Bot 不影响原有增量采集；历史对账仍作为漏通知的原有兜底。 Actions 历史按固定 created 时间窗遍历，遇到 GitHub 每次筛选查询 1,000 条的限制，将窗口递归切成不重叠秒区间。极端情况下同一秒仍超过 1,000 条会显式记录 `time_window_saturated` 并保持未完成，不静默丢弃；需再细分到工作流后重试。 @@ -59,4 +63,4 @@ Actions 产物上传后不再变化，看板已经读过的内容不再下载： Pages 每次部署仍整站上传 site，并非文件级增量部署。GitHub Pages 有站点容量边界；本实现不承诺无限字节容量。数据增长后需观测仓库与站点体积，再决定外置历史存储，不能悄悄改回业务条数截断。 -验证入口：`tests/test_incremental.py`。覆盖超旧上限回填、预算重启、权限失败、重开撤销计数、缓存和过期、attempt 隔离及原子提交冲突。 +验证入口：`tests/test_incremental.py`、`tests/test_run_changes.py` 与 `tests/test_oidc_collection.py`。覆盖超旧上限回填、预算重启、权限失败、重开撤销计数、缓存和过期、attempt 隔离、按 ID 补采、详情失败保留待办及发布后确认。`PYTHONDONTWRITEBYTECODE=1 python3 test/journey-rerun-collection.py` 通过本地 HTTP 替身运行真实采集 CLI，验证窗口外的失败 run 重跑变绿、源端失败不覆盖旧结论、原子发布冲突不确认；不访问真实 GitHub，也不派发采集。 diff --git a/gsb/run_changes.py b/gsb/run_changes.py new file mode 100644 index 000000000..0e1de8073 --- /dev/null +++ b/gsb/run_changes.py @@ -0,0 +1,42 @@ +"""Bounded, nonsecret rerun hints fetched from the collector's trusted bot.""" +from __future__ import annotations + +import json +from pathlib import Path +import re + +BATCH_SIZE = 20 + + +def valid_change(row): + return (isinstance(row, dict) and set(row) == {'id', 'attempt', 'revision'} + and type(row['id']) is int and 0 < row['id'] <= 2**53 - 1 + and type(row['attempt']) is int and 1 < row['attempt'] <= 2**53 - 1 + and isinstance(row['revision'], str) and re.fullmatch('[0-9a-f]{32}', row['revision']) is not None) + + +def load_changes(path, source): + if not path: + return [] + value = json.loads(Path(path).read_text()) + if not isinstance(value, dict) or value.get('ok') is not True: + return [] # An unavailable/older bot keeps its hints; ordinary collection can continue. + if not isinstance(value.get('source'), str) or value['source'].lower() != source.lower(): + raise ValueError('rerun source mismatch') + rows = value.get('runs') + if not isinstance(rows, list) or len(rows) > BATCH_SIZE or not all(valid_change(row) for row in rows): + raise ValueError('invalid rerun hints') + if len({row['id'] for row in rows}) != len(rows): + raise ValueError('duplicate rerun hints') + return rows + + +def write_receipt(path, source, changes, commit): + """Only publish.py calls this after its atomic Git update succeeded.""" + if path and changes: + value = {'source': source, 'runs': changes, 'commit': commit} + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + temp = target.with_suffix('.tmp') + temp.write_text(json.dumps(value, separators=(',', ':'))) + temp.replace(target) diff --git a/gsb/sync.py b/gsb/sync.py index 5b1390de5..0ac5b433d 100644 --- a/gsb/sync.py +++ b/gsb/sync.py @@ -147,6 +147,29 @@ def save_item(self, kind, raw): def queue(self, key, due=None): self.state["pending"].setdefault(key, {"due": due or stamp(self.now), "tries": 0}) + def refresh_changed_runs(self, changes): + """Get only notified IDs, including reruns outside the created-time window.""" + for hint in changes: + try: + raw = self.gh.get(self.base + f"/actions/runs/{hint['id']}") + attempt = raw.get("run_attempt") + if raw.get("id") != hint['id'] or type(attempt) is not int or attempt < hint['attempt']: + raise GitHubError("rerun metadata not ready", kind="rerun_not_ready") + self.save_run(raw) + key = History.key("runs", slim_run(raw, self.settings.get("workflows", {}))) + self.state["pending"]["run:" + key]["due"] = stamp(self.now) + self.changed_jobs.append((hint, key)) + self.state["errors"].pop("rerun:" + str(hint['id']), None) + except GitHubError as err: + self.state["errors"]["rerun:" + str(hint['id'])] = {"kind": err.kind, "at": stamp(self.now)} + + def completed_changes(self): + # Reports may need another poll or exceed this run's budget. Until settled, + # keep the bot hint as well as the existing durable detail job in Pages. + return [hint for hint, key in self.changed_jobs + if "run:" + key not in self.state["pending"] + and (self.history.get("runs", key) or {}).get("status") == "completed"] + def objects(self, kind, mode, pages=4): store = self.state[mode] cursor = store.setdefault(kind, {"page": 1, "done": False}) @@ -273,13 +296,16 @@ def graphql(self, *_): _enrich_prs_graphql(Context(OnePR(), self.cfg, self.now, self.meta), [row]) self.history.put("prs", row) - def pending(self, count=30): - ready = [(key, value) for key, value in self.state["pending"].items() if value["due"] <= stamp(self.now)] - latest = sorted(ready, key=lambda item: (not item[0].startswith("pr:"), -int(item[0].split(":")[1].split("-")[0])))[:max(1, count // 2)] - selected = {key for key, _ in latest} - # Reserve half the batch for the oldest backlog so fresh activity cannot - # permanently starve historical reports. - due = latest + sorted((item for item in ready if item[0] not in selected), key=lambda item: (item[1]["due"], item[0]))[:count-len(latest)] + def pending(self, count=30, priority=(), only_priority=False): + ready = [(key, value) for key, value in self.state["pending"].items() if value["due"] <= stamp(self.now) and (not only_priority or key in priority)] + urgent = [item for item in ready if item[0] in priority][:count] + urgent_keys = {key for key, _ in urgent} + remaining = count - len(urgent) + rest = [item for item in ready if item[0] not in urgent_keys] + latest = sorted(rest, key=lambda item: (not item[0].startswith("pr:"), -int(item[0].split(":")[1].split("-")[0])))[:(remaining + 1) // 2] + selected = urgent_keys | {key for key, _ in latest} + # Keep half the remaining slots for old backlog after the bounded hints. + due = urgent + latest + sorted((item for item in ready if item[0] not in selected), key=lambda item: (item[1]["due"], item[0]))[:remaining-len(latest)] for key, job in due: kind, ident = key.split(":", 1) try: @@ -346,10 +372,13 @@ def pending(self, count=30): job["tries"] += 1 job.update(error=err.kind, due=stamp(self.now + timedelta(seconds=min(86400, 60 * 2 ** min(job["tries"], 10))))) - def collect(self): + def collect(self, changed_runs=()): + self.changed_jobs = [] self.meta = self.gh.get(self.base) if self.meta.get("private") or self.meta.get("visibility", "public") != "public": raise ValueError("public dashboard requires a public source") + self.guarded("reruns", lambda: self.refresh_changed_runs(changed_runs)) + self.guarded("rerun_details", lambda: self.pending(count=20, priority={"run:" + key for _, key in self.changed_jobs}, only_priority=True)) # Fair lane budgets prevent a large initial backfill starving live data. for mode in ("incremental", "backfill"): for kind in ("issues", "prs"): @@ -367,7 +396,7 @@ def collect(self): self.guarded("reconcile:" + kind, lambda k=kind: self.objects(k, "reconcile", pages=1)) self.guarded("reconcile:runs", lambda: self.runs("reconcile", pages=1)) self.guarded("reconcile:releases", lambda: self.releases("reconcile", pages=1)) - self.guarded("details", self.pending) + self.guarded("details", lambda: self.pending(priority={"run:" + key for _, key in self.changed_jobs})) self.guarded("tagged", lambda: self.tagged.refresh_schema(self.gh.get_text_file, self.repo)) self.state["last_poll"] = stamp(self.now) return self diff --git a/publish.py b/publish.py index 6e47a4fb3..d4d548ebd 100644 --- a/publish.py +++ b/publish.py @@ -11,7 +11,7 @@ import subprocess import sys -from gsb import gitcode_sync +from gsb import gitcode_sync, run_changes from gsb.github import GitHub, GitHubError, discover_token from gsb.project import REPO_RE, build_project @@ -150,7 +150,8 @@ def main(): if args.incremental: from gsb.sync import Sync from gsb.incremental_project import build_snapshot - sync = Sync(gh, ROOT, args.repo, settings, requests=args.request_budget).collect() + sync = Sync(gh, ROOT, args.repo, settings, requests=args.request_budget).collect( + changed_runs=run_changes.load_changes(os.environ.get("GSB_RUN_CHANGES_FILE"), args.repo)) phase = "snapshot" snapshot = build_snapshot(sync) records = e2e_run_records(sync, snapshot) @@ -189,6 +190,7 @@ def main(): result["commit"] = publish_batch(publisher, args.publish_repo, args.branch, base, files, source_token=token) result["sync"] = sync.progress() result["changed_files"] = len(files) + run_changes.write_receipt(os.environ.get("GSB_RUN_CHANGES_RECEIPT"), args.repo, sync.completed_changes(), result["commit"] or base) else: result["commit"] = publish(publisher, args.output, args.publish_repo, args.branch, source_token=token) result["requests"] = {"source": getattr(gh, "calls", None), "publish": getattr(publisher, "calls", None)} diff --git a/test/journey-rerun-collection.py b/test/journey-rerun-collection.py new file mode 100644 index 000000000..acc2b6ef2 --- /dev/null +++ b/test/journey-rerun-collection.py @@ -0,0 +1,147 @@ +"""CLI journey: an old failed run becomes green through one notified ID and an atomic publication.""" +from datetime import datetime, timedelta, timezone +import io +import json +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +from urllib.parse import urlsplit +import zipfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from gsb.history import History +from gsb.project import slim_run + + +class RerunCLIJourney(unittest.TestCase): + def journey(self, fail=False, conflict=False): + (ROOT / '.tmp').mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=ROOT / '.tmp') as folder: + root = Path(folder) + for name in ['gsb', 'static']: + shutil.copytree(ROOT / name, root / name, ignore=shutil.ignore_patterns('__pycache__')) + shutil.copyfile(ROOT / 'publish.py', root / 'publish.py') + source, target = 'example/source', 'example/board' + now = datetime.now(timezone.utc).replace(microsecond=0) + stamp = lambda dt: dt.isoformat().replace('+00:00', 'Z') + old = dict(id=42, run_attempt=1, name='CI', workflow_id=7, event='push', status='completed', conclusion='failure', + head_branch='main', head_sha='a' * 40, html_url=f'https://github.com/{source}/actions/runs/42', + created_at=stamp(now - timedelta(days=5)), updated_at=stamp(now - timedelta(days=5)), + run_started_at=stamp(now - timedelta(days=5))) + latest = dict(old, run_attempt=2, conclusion='success', run_started_at=stamp(now - timedelta(minutes=2)), updated_at=stamp(now)) + history = History(root); history.put('runs', slim_run(old, {})) + for name, data in history.files().items(): + path = root / name; path.parent.mkdir(parents=True, exist_ok=True); path.write_text(data) + (root / '.sync').mkdir(exist_ok=True) + done = {kind: {'done': True, 'page': 1, 'until': stamp(now), **({'windows': []} if kind == 'runs' else {})} + for kind in ['issues', 'prs', 'runs', 'releases']} + (root / '.sync/state.json').write_text(json.dumps({'schema': 1, 'repository': source, 'started_at': stamp(now), 'migrated': True, + 'backfill': done, 'incremental': {}, 'pending': {}, 'errors': {}, 'reconcile': {**done, 'next_at': stamp(now + timedelta(days=7))}})) + (root / 'board-config.json').write_text(json.dumps({'deployments': {source: {'repository': target, 'environment': 'test', 'label': 'fixture'}}})) + change = {'id': 42, 'attempt': 2, 'revision': '1' * 32} + hints, receipt = root / 'hints.json', root / 'receipt.json' + hints.write_text(json.dumps({'ok': True, 'source': source, 'runs': [change]})) + archive = io.BytesIO() + with zipfile.ZipFile(archive, 'w') as bundle: + bundle.writestr('junit.xml', '') + base = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() + calls, updates = [], [] + class API(BaseHTTPRequestHandler): + def log_message(self, *_): pass + def reply(self, value, status=200, binary=False): + raw = value if binary else json.dumps(value).encode() + self.send_response(status); self.send_header('Content-Type', 'application/zip' if binary else 'application/json') + self.send_header('Content-Length', str(len(raw))); self.end_headers(); self.wfile.write(raw) + def do_GET(self): + parsed = urlsplit(self.path); path = parsed.path; calls.append(('GET', path, parsed.query)) + if path == f'/repos/{source}': + return self.reply({'full_name': source, 'html_url': 'https://github.com/' + source, 'default_branch': 'main', + 'private': False, 'visibility': 'public', 'created_at': old['created_at']}) + if path == f'/repos/{target}': return self.reply({'private': False}) + if path == f'/repos/{target}/git/ref/heads/main': return self.reply({'object': {'sha': 'e' * 40 if conflict else base}}) + if path == f'/repos/{target}/git/commits/{base}': return self.reply({'tree': {'sha': 'b' * 40}}) + if path == f'/repos/{source}/actions/runs/42': + return self.reply({'message': 'temporary fixture failure'}, 503) if fail else self.reply(latest) + if path.endswith('/attempts/2/jobs'): return self.reply({'jobs': []}) + if path.endswith('/runs/42/artifacts'): + return self.reply({'artifacts': [{'id': 7, 'name': 'ut-results', 'size_in_bytes': len(archive.getvalue()), + 'created_at': stamp(now), 'expired': False}]}) + if path == f'/repos/{source}/actions/artifacts/7/zip': + self.send_response(302) + self.send_header('Location', f'http://127.0.0.1:{self.server.server_port}/blob/7') + self.end_headers(); return + if path == '/blob/7': + assert not self.headers.get('Authorization'), 'artifact blob must be anonymous' + return self.reply(archive.getvalue(), binary=True) + if path == f'/repos/{source}/actions/runs': return self.reply({'total_count': 0, 'workflow_runs': []}) + if path == f'/repos/{source}/actions/artifacts': return self.reply({'total_count': 0, 'artifacts': []}) + if path.endswith(('/issues', '/pulls', '/releases')): return self.reply([]) + return self.reply({'message': 'fixture endpoint unavailable'}, 404) + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + calls.append(('POST', self.path, '')); updates.append((self.path, body)) + if self.path.endswith('/git/trees'): return self.reply({'sha': 'c' * 40}) + if self.path.endswith('/git/commits'): return self.reply({'sha': 'd' * 40}) + return self.reply({'message': 'fixture endpoint unavailable'}, 404) + def do_PATCH(self): + body = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + calls.append(('PATCH', self.path, '')); updates.append((self.path, body)); self.reply({}) + server = ThreadingHTTPServer(('127.0.0.1', 0), API) + thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() + bootstrap = '''import runpy,sys,urllib.request,urllib.error,urllib.parse +from gsb import github +github.API_ROOT=sys.argv.pop(1) +original=urllib.request.OpenerDirector.open +def isolated(self,req,*args,**kw): + url=req if isinstance(req,str) else req.full_url + if urllib.parse.urlsplit(url).hostname!='127.0.0.1':raise urllib.error.URLError('network isolated') + return original(self,req,*args,**kw) +urllib.request.OpenerDirector.open=isolated +sys.argv[0]='publish.py' +runpy.run_path('publish.py',run_name='__main__')''' + import os + env = {**os.environ, 'GITHUB_TOKEN': 'fixture-source', 'GSB_PUBLISH_TOKEN': 'fixture-target', + 'GSB_RUN_CHANGES_FILE': str(hints), 'GSB_RUN_CHANGES_RECEIPT': str(receipt), 'PYTHONDONTWRITEBYTECODE': '1', + 'TMPDIR': str(root), 'GSB_GITCODE_SYNC_FILE': ''} + try: + result = subprocess.run([sys.executable, '-c', bootstrap, f'http://127.0.0.1:{server.server_port}', '--repo', source, + '--publish-repo', target, '--output', str(root / 'preview'), '--incremental'], cwd=root, env=env, + capture_output=True, text=True, timeout=45) + finally: + server.shutdown(); server.server_close(); thread.join() + if conflict: + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertFalse(receipt.exists(), 'failed publication must not acknowledge the bot hint') + self.assertFalse(any(method == 'PATCH' for method, _, _ in calls)) + return + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + exact = [call for call in calls if call[1] == f'/repos/{source}/actions/runs/42'] + self.assertEqual(len(exact), 1) + # Normal recent polling remains; there is no historical fallback query for the notified run. + from urllib.parse import parse_qs + for _, path, query in calls: + if path == f'/repos/{source}/actions/runs': + self.assertGreater(parse_qs(query)['created'][0].split('..')[0], old['created_at']) + tree = next(body['tree'] for path, body in updates if path.endswith('/git/trees')) + snapshot = json.loads(next(entry['content'] for entry in tree if entry['path'] == 'site/data/snapshot.json')) + lane = next(lane for lane in snapshot['sections']['ci']['data']['lanes']['lanes'] if lane['key'] == 'main') + points = [cell for column in lane['days'] for cell in column] + self.assertEqual([(point['attempt'], point['outcome']) for point in points], [(1, 'failure')] if fail else [(2, 'success')]) + self.assertTrue(any(entry['path'] == '.sync/state.json' for entry in tree)) + self.assertEqual(updates[-1], (f'/repos/{target}/git/refs/heads/main', {'sha': 'd' * 40, 'force': False})) + if fail: + self.assertFalse(receipt.exists(), 'failed source lookup must remain pending at the bot') + else: + self.assertEqual(json.loads(receipt.read_text()), {'source': source, 'commit': 'd' * 40, 'runs': [change]}) + def test_old_failed_run_becomes_green_and_receipt_follows_publication(self): self.journey() + def test_failed_lookup_preserves_red_and_does_not_ack(self): self.journey(fail=True) + def test_publication_conflict_does_not_ack(self): self.journey(conflict=True) + + +if __name__ == '__main__': unittest.main() diff --git a/tests/test_oidc_collection.py b/tests/test_oidc_collection.py index e2ca70627..8fc1a2197 100644 --- a/tests/test_oidc_collection.py +++ b/tests/test_oidc_collection.py @@ -19,10 +19,12 @@ def setUp(self): 'SDBOT_TOKEN_AUDIENCE': 'test-audience', 'ACTIONS_ID_TOKEN_REQUEST_URL': 'https://issuer.example/oidc?x=1', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN': 'runner-auth', 'SDBOT_GITHUB_APP_PRIVATE_KEY': 'must-not-inherit'} self.calls = [] - temp = tempfile.TemporaryDirectory(dir=client.ROOT / '.tmp' if (client.ROOT / '.tmp').is_dir() else None) + (client.ROOT / '.tmp').mkdir(exist_ok=True) + temp = tempfile.TemporaryDirectory(dir=client.ROOT / '.tmp') self.addCleanup(temp.cleanup) - patcher = patch.object(client, 'SYNC_RECORDS', Path(temp.name) / 'gitcode-sync.json') - patcher.start(); self.addCleanup(patcher.stop) + for name, file in [('SYNC_RECORDS', 'gitcode-sync.json'), ('RUN_CHANGES', 'run-changes.json'), ('RUN_RECEIPT', 'receipt.json')]: + patcher = patch.object(client, name, Path(temp.name) / file) + patcher.start(); self.addCleanup(patcher.stop) def request(self, url, bearer, body=None, method='GET', **kwargs): self.calls.append((url, bearer, body, method)) @@ -36,6 +38,10 @@ def request(self, url, bearer, body=None, method='GET', **kwargs): if url == 'https://broker.example/actions/gitcode-sync': self.assertEqual((body, method), ({}, 'POST')) return {'ok': True, 'enabled': True, 'source': self.source, 'target': 'openJiuwen/sciencediscovery', 'records': [], 'pulls': []} + if url == 'https://broker.example/actions/reruns': + return {'ok': True, 'source': self.source, 'runs': []} + if url == 'https://broker.example/actions/reruns/ack': + return {'ok': True} purpose = body['purpose'] return {'token': purpose + '-token', 'repository': self.source if purpose == 'source' else self.target, 'purpose': purpose, 'expires_at': (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat()} @@ -117,3 +123,54 @@ def request(url, bearer, body=None, method='GET', **kwargs): written = json.loads(client.SYNC_RECORDS.read_text()) self.assertEqual(written, {'ok': False, 'error': 'bot returned HTTP 503'}) + + def test_rerun_ack_only_follows_a_successful_publish_receipt_with_a_fresh_identity(self): + change = {'id': 42, 'attempt': 2, 'revision': 'a' * 32} + def request(url, bearer, body=None, method='GET', **kwargs): + if url.endswith('/actions/reruns'): + self.calls.append((url, bearer, body, method)) + return {'ok': True, 'source': self.source, 'runs': [change]} + return self.request(url, bearer, body, method, **kwargs) + def publish(*_args, **kwargs): + self.assertFalse(client.RUN_RECEIPT.exists(), 'stale receipts are removed before starting') + self.assertEqual(json.loads(Path(kwargs['env']['GSB_RUN_CHANGES_FILE']).read_text())['runs'], [change]) + client.RUN_RECEIPT.write_text(json.dumps({'source': self.source, 'runs': [change], 'commit': 'b' * 40})) + return SimpleNamespace(returncode=0) + client.RUN_RECEIPT.write_text('stale') + with patch.dict(os.environ, self.env, clear=True), patch.object(client, 'request_json', side_effect=request), \ + patch.object(client, 'mask'), patch.object(client.subprocess, 'run', side_effect=publish): + self.assertEqual(client.main(), 0) + ack = [call for call in self.calls if call[0].endswith('/reruns/ack')] + self.assertEqual(ack[0][2], {'commit': 'b' * 40, 'runs': [change]}) + self.assertEqual(sum(call[0].startswith('https://issuer.example/') for call in self.calls), 2) + + def test_failed_publish_stale_receipt_or_hint_fetch_never_loses_pending_ids(self): + for code in [1, 7]: + self.calls.clear(); client.RUN_RECEIPT.write_text('{"commit":"stale"}') + with patch.dict(os.environ, self.env, clear=True), patch.object(client, 'request_json', side_effect=self.request), \ + patch.object(client, 'mask'), patch.object(client.subprocess, 'run', return_value=SimpleNamespace(returncode=code)): + self.assertEqual(client.main(), code) + self.assertFalse(any(call[0].endswith('/reruns/ack') for call in self.calls)) + self.assertFalse(client.RUN_RECEIPT.exists()) + def request(url, *args, **kwargs): + if url.endswith('/actions/reruns'): + raise client.CredentialError('private response') + return self.request(url, *args, **kwargs) + out = io.StringIO() + with patch.dict(os.environ, self.env, clear=True), patch.object(client, 'request_json', side_effect=request), \ + patch.object(client, 'mask'), patch.object(client.subprocess, 'run', return_value=SimpleNamespace(returncode=0)), patch('sys.stdout', out): + self.assertEqual(client.main(), 0) + self.assertEqual(json.loads(client.RUN_CHANGES.read_text()), {'ok': False}) + self.assertNotIn('private response', out.getvalue()) + + def test_lost_ack_does_not_fail_successful_publication_or_expose_credentials(self): + receipt = {'source': self.source, 'runs': [{'id': 42, 'attempt': 2, 'revision': 'a' * 32}], 'commit': 'b' * 40} + client.RUN_RECEIPT.write_text(json.dumps(receipt)) + out = io.StringIO() + for response in [client.CredentialError('secret'), {'ok': False}]: + with patch.object(client, 'oidc_identity', return_value='fresh-secret'), \ + patch.object(client, 'request_json', **({'side_effect': response} if isinstance(response, Exception) else {'return_value': response})), \ + patch('sys.stdout', out): + client.acknowledge_runs('https://broker.example/actions/reruns/ack', 'test-audience', client.RUN_RECEIPT, {'runs': receipt['runs']}, self.source) + self.assertIn('pending IDs remain', out.getvalue()) + self.assertNotIn('secret', out.getvalue()) diff --git a/tests/test_run_changes.py b/tests/test_run_changes.py new file mode 100644 index 000000000..6c595c89e --- /dev/null +++ b/tests/test_run_changes.py @@ -0,0 +1,107 @@ +from copy import deepcopy +from datetime import timedelta +import json +import re +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from gsb.run_changes import BATCH_SIZE, load_changes, write_receipt +from gsb.sync import Sync, stamp +from gsb.github import GitHubError +from test_incremental import Source, NOW, REPO, run + +ROOT = Path(__file__).resolve().parents[1] + + +def hint(ident=1, attempt=2): + return {'id': ident, 'attempt': attempt, 'revision': f'{ident:032x}'} + + +class RerunSource(Source): + def __init__(self, rows): + super().__init__() + self.exact = {row['id']: row for row in rows} + self.fail = set() + def get(self, path, params=None): + match = re.fullmatch(r'/repos/[^/]+/[^/]+/actions/runs/(\d+)(?:/attempts/(\d+))?', path) + if match: + ident = int(match[1]); self.calls.append((path, {})) + if ident in self.fail: + raise GitHubError('not logged', kind='server', status=503) + row = deepcopy(self.exact[ident]) + if match[2]: row['run_attempt'] = int(match[2]) + return row + return super().get(path, params) + + +class RunChangeTests(unittest.TestCase): + def setUp(self): + (ROOT / '.tmp').mkdir(exist_ok=True) + temp = tempfile.TemporaryDirectory(dir=ROOT / '.tmp'); self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + def report(self, _gh, _cfg, row, **kwargs): + row.update(reports_status='available', jobs_status='available', tests=[]) + return row + def test_old_run_refreshes_by_id_before_lists_without_replaying_old_attempts(self): + old = run(1); old['conclusion'] = 'failure'; old['created_at'] = stamp(NOW - timedelta(days=20)) + newer = dict(old, run_attempt=2, conclusion='success', updated_at=stamp(NOW)) + gh = RerunSource([newer]) + sync = Sync(gh, self.root, REPO, now=NOW) + sync.save_run(old) + with patch('gsb.sync.run_details', side_effect=self.report): sync.collect([hint()]) + self.assertEqual(sync.history.get('runs', '1-2')['conclusion'], 'success') + self.assertEqual(sync.history.get('runs', '1-1')['conclusion'], 'failure') + self.assertEqual(sync.completed_changes(), [hint()]) + self.assertEqual(gh.calls[1][0], '/repos/' + REPO + '/actions/runs/1') + # No special historical listing is introduced to discover this old rerun. + self.assertTrue(all(set(params) == {'created', 'page', 'per_page'} for path, params in gh.calls if path.endswith('/actions/runs'))) + def test_failed_metadata_and_exhausted_budget_do_not_acknowledge(self): + gh = RerunSource([run(1, 2)]); gh.fail.add(1) + sync = Sync(gh, self.root, REPO, now=NOW).collect([hint()]) + self.assertEqual(sync.completed_changes(), []) + self.assertEqual(sync.state['errors']['rerun:1']['kind'], 'server') + sync = Sync(gh, self.root, REPO, now=NOW, requests=1).collect([hint()]) + self.assertEqual(sync.completed_changes(), []) + def test_failed_details_keep_the_hint_and_page_checkpoint_for_next_collection(self): + gh = RerunSource([run(1, 2)]) + old = run(1); sync = Sync(gh, self.root, REPO, now=NOW); sync.save_run(old) + with patch('gsb.sync.run_details', side_effect=GitHubError('private', kind='server', status=503)): + sync.collect([hint()]) + self.assertEqual(sync.completed_changes(), []) + self.assertIn('run:1-2', sync.state['pending']) + for name, content in sync.files().items(): + path = self.root / name; path.parent.mkdir(parents=True, exist_ok=True); path.write_text(content) + recovered = Sync(gh, self.root, REPO, now=NOW + timedelta(hours=8)) + with patch('gsb.sync.run_details', side_effect=self.report): recovered.collect([hint()]) + self.assertEqual(recovered.completed_changes(), [hint()]) + def test_notified_details_get_a_turn_before_bulk_backfill(self): + row = run(1, 2); gh = RerunSource([row]) + sync = Sync(gh, self.root, REPO, now=NOW, requests=2); sync.save_run(run(1)) + inspected = [] + def inspect(_gh, _cfg, record, **_kw): + inspected.append((record['id'], record['attempt'])); return self.report(_gh, _cfg, record) + with patch('gsb.sync.run_details', side_effect=inspect): sync.collect([hint()]) + self.assertEqual(inspected[0], (1, 2)) + self.assertEqual(inspected.count((1, 2)), 1) + self.assertEqual(sync.completed_changes(), [hint()]) + def test_api_behind_notification_or_run_still_in_progress_stays_pending(self): + for row in [run(1, 1), dict(run(1, 2), status='in_progress', conclusion=None)]: + with self.subTest(row=row): + sync = Sync(RerunSource([row]), self.root, REPO, now=NOW); sync.save_run(run(1)) + with patch('gsb.sync.run_details', side_effect=self.report): sync.collect([hint()]) + self.assertEqual(sync.completed_changes(), []) + def test_receipts_and_input_are_nonsecret_bounded_and_source_scoped(self): + path = self.root / 'input.json' + value = {'ok': True, 'source': REPO, 'runs': [hint()]}; path.write_text(json.dumps(value)) + self.assertEqual(load_changes(path, REPO.upper()), [hint()]) + for bad in [{**value, 'source': 'other/source'}, {**value, 'runs': [hint()] * (BATCH_SIZE + 1)}, + {**value, 'runs': [{**hint(), 'id': '../evil'}]}, {**value, 'runs': [{**hint(), 'id': True}]}, + {**value, 'runs': [{**hint(), 'attempt': 1}]}, {**value, 'runs': [hint(), hint()]}]: + path.write_text(json.dumps(bad)) + with self.assertRaises(ValueError): load_changes(path, REPO) + receipt = self.root / 'receipt.json' + write_receipt(receipt, REPO, [], 'a' * 40); self.assertFalse(receipt.exists()) + write_receipt(receipt, REPO, [hint()], 'b' * 40) + self.assertEqual(json.loads(receipt.read_text()), {'source': REPO, 'runs': [hint()], 'commit': 'b' * 40}) From 88b87055a24688a720f3558fd490193de65ec381 Mon Sep 17 00:00:00 2001 From: zhaozhaozz <6450679+zhaozhaozz@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:40:32 +0000 Subject: [PATCH 2/2] Align Real E2E curve regression checks with missing-score markers --- docs/dashboard-pages.md | 2 +- test/journey-board.spec.cjs | 54 ++++++++++++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/docs/dashboard-pages.md b/docs/dashboard-pages.md index 033c4706d..f1024b4a1 100644 --- a/docs/dashboard-pages.md +++ b/docs/dashboard-pages.md @@ -58,7 +58,7 @@ CI、PR、Issue 数据不可读取时显示“状态未知”，不会显示为 E2E 页展示： - 测试文件与最近一次 UT / E2E 用例数； -- Real E2E 质量评分：顶部是最近两天的 Nightly、最近一次 `real-e2e-results` 的出处和四个汇总数字；下面与 E2E 执行记录同一种排版，按评分体系（DeepResearchBench、BiomniBench、Research Team、PUCT Compression）分类，类别可收起，每个用例一排 ✓／×，按时间从左到右表示每次记录的交付结果，悬停或聚焦显示日期、交付、评分状态、各项分数、耗时和 run，点击打开该 run。每个用例下方直接画出每项分数和运行耗时的历史曲线，并标出最新值；交付失败的运行在分数曲线上画 ×，用虚线沿用上次实测（无历史按 0），缺失的分数留空不补零。历史为该分支线每个用例最近 30 个运行日（每天取一次运行）。质量分数仅供观察，不改变 CI 结论；点击用例名打开该用例的旅程报告页 `real-e2e.html`，列出场景、前置条件、评分表和执行历史。 +- Real E2E 质量评分：顶部是最近两天的 Nightly、最近一次 `real-e2e-results` 的出处和四个汇总数字；下面与 E2E 执行记录同一种排版，按评分体系（DeepResearchBench、BiomniBench、Research Team、PUCT Compression）分类，类别可收起，每个用例一排 ✓／×，按时间从左到右表示每次记录的交付结果，悬停或聚焦显示日期、交付、评分状态、各项分数、耗时和 run，点击打开该 run。每个用例下方直接画出每项分数和运行耗时的历史曲线，并标出最新值；分数曲线从首次成功实测开始；此前的失败仍保留在状态轴上，不在曲线虚构零分。首次实测后的交付失败画红色 × 并用虚线沿用上次实测；后续缺失评分的 × 与上次有分数的运行平齐，虚线经过缺失运行连接到下次实测，提示注明缺失原因和它不是本次测量值，不补零。历史为该分支线每个用例最近 30 个运行日（每天取一次运行）。质量分数仅供观察，不改变 CI 结论；点击用例名打开该用例的旅程报告页 `real-e2e.html`，列出场景、前置条件、评分表和执行历史。 - 保留期内 E2E 运行的[执行记录](e2e-run-records.md)：按类别和用例汇总的执行状态时间轴、各次步骤与错误摘要，及历史点直接跳转的 HTML 附件； - [标签化测试](tagged-tests.md)； - 按层 / 包的测试文件分布，以及按包的测试结构。 diff --git a/test/journey-board.spec.cjs b/test/journey-board.spec.cjs index 79d20c614..bf3e2eaa9 100644 --- a/test/journey-board.spec.cjs +++ b/test/journey-board.spec.cjs @@ -601,20 +601,60 @@ test('Real E2E cases show run marks and score curves inline like the E2E run rec await expect(trends.locator('figcaption > span:first-child')).toHaveText(['RACE', 'Citation accuracy', 'Verification coverage', '运行耗时']); await expect(trends.first().locator('figcaption')).toContainText('（沿用上次实测）'); await expect(trends.first().locator('.score-dot')).toHaveCount(11); - await expect(trends.first().locator('.score-failed')).toHaveCount(1); + await expect(trends.first().locator('.score-x')).toHaveCount(1); await expect(trends.first().locator('.score-line.carried')).toHaveCount(1); + await expect(trends.first().locator('.score-x title')).toContainText('交付失败，沿用上次实测'); + const carried = await trends.first().evaluate((figure) => { + const measured = [...figure.querySelectorAll('.score-dot')].at(-1); + const { e: x, f: y } = figure.querySelector('.score-x').transform.baseVal.consolidate().matrix; + const line = figure.querySelector('.score-line.carried'); + const start = line.getPointAtLength(0), end = line.getPointAtLength(line.getTotalLength()); + return { previous: { x: Number(measured.getAttribute('cx')), y: Number(measured.getAttribute('cy')) }, + cross: { x, y }, start: { x: start.x, y: start.y }, end: { x: end.x, y: end.y }, dash: getComputedStyle(line).strokeDasharray }; + }); + expect(carried.cross.y).toBeCloseTo(carried.previous.y, 1); + expect(carried.cross.x).toBeGreaterThan(carried.previous.x); + expect(carried.start.x).toBeCloseTo(carried.previous.x, 1); + expect(carried.start.y).toBeCloseTo(carried.previous.y, 1); + expect(carried.end.x).toBeCloseTo(carried.cross.x, 1); + expect(carried.end.y).toBeCloseTo(carried.cross.y, 1); + expect(carried.dash).not.toBe('none'); await expect(trends.last().locator('.score-dot')).toHaveCount(12); - await expect(trends.last().locator('.score-failed')).toHaveCount(0); + await expect(trends.last().locator('.score-x')).toHaveCount(0); - // A failed first run has no score to carry: it is drawn at 0 and says so. + // Score curves start at the first measurement; the earlier failure remains on the status axis. const team = realCase('TC-E2E-01'); await expect(team.locator('.e2e-history-point').first()).toHaveClass(/failed/); - await expect(team.locator('.real-trend').first().locator('.score-failed title')).toContainText('无历史按 0'); - // A missing score leaves a gap instead of a zero. + const teamScore = team.locator('.real-trend').first(); + await expect(teamScore.locator('.score-dot')).toHaveCount(11); + await expect(teamScore.locator('.score-x')).toHaveCount(0); + await expect(teamScore.locator('.score-line')).toHaveCount(10); + await expect(teamScore.locator('.score-dot').first().locator('title')).toContainText('run 806'); + await expect(teamScore.locator('title').filter({ hasText: 'run 804' })).toHaveCount(0); + await expect(team.locator('.real-trend').last().locator('.score-dot')).toHaveCount(12); + // Later missing scores use a cross at the last measured height and a dashed bridge, never zero. const puct = realCase('PUCT-COMPRESS'); await expect(puct.locator('.real-trend')).toHaveCount(3); - await expect(puct.locator('.real-trend').nth(1).locator('.score-dot')).toHaveCount(11); - await expect(puct.locator('.real-trend').nth(1).locator('.score-line')).toHaveCount(9); + const missingScore = puct.locator('.real-trend').nth(1); + await expect(missingScore.locator('.score-dot')).toHaveCount(11); + await expect(missingScore.locator('.score-x')).toHaveCount(1); + await expect(missingScore.locator('.score-line')).toHaveCount(10); + await expect(missingScore.locator('.score-line.gap')).toHaveCount(1); + const gap = await missingScore.evaluate((figure) => { + const { e: x, f: y } = figure.querySelector('.score-x').transform.baseVal.consolidate().matrix; + const measured = [...figure.querySelectorAll('.score-dot')].map((dot) => [Number(dot.getAttribute('cx')), Number(dot.getAttribute('cy'))]); + const previous = measured.findLast((point) => point[0] < x), next = measured.find((point) => point[0] > x); + const line = figure.querySelector('.score-line.gap'); + return { x, y, previous, next, path: line.getAttribute('d').match(/-?\d+(?:\.\d+)?/g).map(Number), dash: getComputedStyle(line).strokeDasharray }; + }); + expect(gap.y).toBeCloseTo(gap.previous[1], 1); + const expectedGap = [...gap.previous, gap.x, gap.previous[1], ...gap.next]; + expect(gap.path).toHaveLength(expectedGap.length); + expectedGap.forEach((value, index) => expect(gap.path[index]).toBeCloseTo(value, 1)); + expect(gap.dash).not.toBe('none'); + await missingScore.locator('.score-x').hover(); + await expect(page.locator('#tooltip')).toContainText('评分缺失'); + await expect(page.locator('#tooltip')).toContainText('不是本次测量值'); // Scores stay inside each chart's own scale. const rubric = realCase('BiomniBench-da-14-1').locator('.real-trend').first().locator('svg');