From 2a53ceed79d14371a8c1e70164be5e076370d8fa Mon Sep 17 00:00:00 2001 From: 370025263 <370025263@qq.com> Date: Thu, 30 Jul 2026 17:17:49 +0800 Subject: [PATCH 1/4] fix(kernel): enable auto_index for create_temp kernel-temp dirs Temp trajectories were registered with auto_index=False (pause-ingest semantics), so the watcher skipped them and they never became ready. Always re-register with auto_index=True so splits can proceed; keep source=temp / ecosystem=kernel-temp for isolation from user corpora. Co-authored-by: Cursor --- src/xskill/kernels/context.py | 29 ++++++++----------- tests/features/kernel_trajectory_feed.feature | 1 + tests/test_kernel_atom_view.py | 29 +++++++++++++++++++ tests/test_kernel_trajectory_feed_steps.py | 9 ++++++ 4 files changed, 51 insertions(+), 17 deletions(-) diff --git a/src/xskill/kernels/context.py b/src/xskill/kernels/context.py index a41d1fcd..7e5cd0d4 100644 --- a/src/xskill/kernels/context.py +++ b/src/xskill/kernels/context.py @@ -415,31 +415,26 @@ def _ensure_temp_watch_dir(self) -> tuple[Path, TrajectoryDirectoryResource]: raise RuntimeError( "TrajectoryReader has no temp_root configured; cannot create_temp" ) - from xskill.pipeline.registry import Registry, register_dir + from xskill.pipeline.registry import register_dir temp_root = self._temp_root temp_root.mkdir(parents=True, exist_ok=True) - if self._temp_watch_dir_id is None: - registry = Registry(self._db_path) - resolved = temp_root.resolve() - for item in registry.list(): - if item.path.resolve() == resolved: - self._temp_watch_dir_id = int(item.id) - break - else: - self._temp_watch_dir_id = register_dir( - temp_root, - label="kernel-temp", - ecosystem="kernel-temp", - auto_index=False, - db_path=self._db_path, - ) + # Always re-register: register_dir is idempotent and refreshes + # auto_index. Temp trajectories must be watched/split like user + # trajectories; auto_index=False (pause-ingest) was a mistaken reuse. + self._temp_watch_dir_id = register_dir( + temp_root, + label="kernel-temp", + ecosystem="kernel-temp", + auto_index=True, + db_path=self._db_path, + ) watch_dir = TrajectoryDirectoryResource( id=str(self._temp_watch_dir_id), path=temp_root, label="kernel-temp", ecosystem="kernel-temp", - auto_index=False, + auto_index=True, trajectory_count=0, indexed_count=0, ) diff --git a/tests/features/kernel_trajectory_feed.feature b/tests/features/kernel_trajectory_feed.feature index 4d52510a..9028b3ea 100644 --- a/tests/features/kernel_trajectory_feed.feature +++ b/tests/features/kernel_trajectory_feed.feature @@ -7,6 +7,7 @@ Feature: Platform feeds ready trajectories with atom views to algorithm kernels Given a trajectory reader with a temp root When the kernel creates a temp trajectory from platform-shaped markdown Then the temp trajectory is pending with no atoms + And the kernel-temp watch directory has auto_index enabled And the temp trajectory is absent from the feed When the platform finishes splitting the temp trajectory into one atom Then the temp trajectory enters the feed as ready diff --git a/tests/test_kernel_atom_view.py b/tests/test_kernel_atom_view.py index f91ddbb1..0feb9af4 100644 --- a/tests/test_kernel_atom_view.py +++ b/tests/test_kernel_atom_view.py @@ -172,6 +172,35 @@ def test_create_temp_accepts_minimal_user_markdown(tmp_path): written = temp_root / "traj_kernel_temp.md" assert written.is_file() assert written.read_text(encoding="utf-8") == "## User\n\nhello temp\n" + temp_watch = next( + item for item in reader.directories() if item.ecosystem == "kernel-temp" + ) + assert temp_watch.auto_index is True + + +def test_create_temp_reenables_auto_index_on_existing_temp_watch_dir(tmp_path): + """Old kernel-temp rows registered with auto_index=False must be flipped on.""" + from xskill.pipeline.registry import Registry, register_dir + + registry_db = tmp_path / "registry.db" + temp_root = tmp_path / "temp" + temp_root.mkdir() + register_dir( + temp_root, + label="kernel-temp", + ecosystem="kernel-temp", + auto_index=False, + db_path=registry_db, + ) + assert Registry(registry_db).list()[0].auto_index is False + + reader = TrajectoryReader(registry_db, temp_root=temp_root) + reader.create_temp("## User\n\nhello\n", trajectory_id="traj_temp_reenable") + temp_watch = next( + item for item in reader.directories() if item.ecosystem == "kernel-temp" + ) + assert temp_watch.auto_index is True + assert Registry(registry_db).list()[0].auto_index is True def test_feed_snapshot_excludes_pending_trajectories(tmp_path): diff --git a/tests/test_kernel_trajectory_feed_steps.py b/tests/test_kernel_trajectory_feed_steps.py index 6ba1a97b..ea269eb2 100644 --- a/tests/test_kernel_trajectory_feed_steps.py +++ b/tests/test_kernel_trajectory_feed_steps.py @@ -140,6 +140,15 @@ def then_temp_trajectory_pending_no_atoms(context): assert created.read_text() == PLATFORM_MD +@then("the kernel-temp watch directory has auto_index enabled") +def then_kernel_temp_watch_dir_auto_index_enabled(context): + reader = context["reader"] + temp_watch = next( + item for item in reader.directories() if item.ecosystem == "kernel-temp" + ) + assert temp_watch.auto_index is True + + @then("the temp trajectory is absent from the feed") def then_temp_trajectory_absent_from_feed(context): reader = context["reader"] From c20fe40dd2403a877c5e64f0e7200e81c70606c6 Mon Sep 17 00:00:00 2001 From: 370025263 <370025263@qq.com> Date: Thu, 30 Jul 2026 17:25:44 +0800 Subject: [PATCH 2/4] test(kernel): BDD+aimock proves benchmark temp atoms enter training Add Gherkin scenarios that create_temp, split multi-atom temps, and assert OpenEarth train_skills receives oracle-scored ScoredAtomInput ids. Require aimock-pytest for the OpenAI-compatible probe path. Co-authored-by: Cursor --- pyproject.toml | 9 +- .../benchmark_atoms_enter_training.feature | 25 ++ .../test_benchmark_atoms_enter_training.py | 267 ++++++++++++++++++ 3 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 tests/bdd/features/kernel/benchmark_atoms_enter_training.feature create mode 100644 tests/bdd/test_benchmark_atoms_enter_training.py diff --git a/pyproject.toml b/pyproject.toml index 65e3f176..e2aa56b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,14 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=7", "pytest-timeout>=2", "pytest-asyncio>=0.21", "pytest-bdd>=7", "requests>=2"] +dev = [ + "pytest>=7", + "pytest-timeout>=2", + "pytest-asyncio>=0.21", + "pytest-bdd>=7", + "requests>=2", + "aimock-pytest==0.5.0; python_version >= '3.10'", +] [project.urls] Homepage = "https://github.com/SkillNerds/xskill" diff --git a/tests/bdd/features/kernel/benchmark_atoms_enter_training.feature b/tests/bdd/features/kernel/benchmark_atoms_enter_training.feature new file mode 100644 index 00000000..afd7f40d --- /dev/null +++ b/tests/bdd/features/kernel/benchmark_atoms_enter_training.feature @@ -0,0 +1,25 @@ +Feature: 内部做题原子进入训练 + As an XSkill maintainer accepting OpenEarth + I want benchmark temp atoms to reach train_skills inputs + So that vertical-domain rollouts actually train Skills + + Scenario: 做题临时轨拆完后原子进入 train_skills 输入 + Given aimock 在随机本地端口启动 OpenAI-compatible 服务 + And 平台 TrajectoryReader 已配置 kernel-temp 且 auto_index 开启 + And OpenEarth workspace 已记录该临时轨的 oracle 分 + When 内核 create_temp 写入平台形做题轨迹 + And 平台将临时轨迹拆成多个 ready atom + And 配置指向 aimock 后调用 train_skills + Then train_skills 收到的 ScoredAtomInput 包含这些 temp atom_id + And 对应 score_source 均为 oracle + And aimock 至少收到一次 chat completions 探活请求 + + Scenario: 多 atom 带 oracle 时不抛错且全部入训 + Given aimock 在随机本地端口启动 OpenAI-compatible 服务 + And 平台 TrajectoryReader 已配置 kernel-temp 且 auto_index 开启 + And OpenEarth workspace 已记录该临时轨的 oracle 分 + When 内核 create_temp 写入平台形做题轨迹 + And 平台将临时轨迹拆成多个 ready atom + And 配置指向 aimock 后调用 train_skills + Then train_skills 不因多 atom 抛错 + And train_skills 收到的 ScoredAtomInput 数量等于拆出的 atom 数 diff --git a/tests/bdd/test_benchmark_atoms_enter_training.py b/tests/bdd/test_benchmark_atoms_enter_training.py new file mode 100644 index 00000000..38c4041b --- /dev/null +++ b/tests/bdd/test_benchmark_atoms_enter_training.py @@ -0,0 +1,267 @@ +"""pytest-bdd + aimock: benchmark temp atoms must enter train_skills inputs.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from urllib import request as urlrequest + +import pytest +from pytest_bdd import given, scenarios, then, when + +pytest.importorskip( + "openearth_skill_sdk", + reason="install OpenEarth wheel from PR #155 to run benchmark-atom BDD", +) + +from openearth_skill_sdk.contracts import DistillationResult +from openearth_skill_sdk.xskill import record_oracle_score, train_skills + +from xskill.kernels.context import TrajectoryReader +from xskill.pipeline.atom import AtomTask, AtomTaskStore +from xskill.pipeline.registry import update_traj_status + +scenarios("features/kernel/benchmark_atoms_enter_training.feature") + + +PLATFORM_MD = """## User + +Solve the spreadsheet formula for Q1 revenue. + +## Assistant + +I will open the workbook and compute the answer. +""" + + +def _write_atom( + store: AtomTaskStore, + *, + traj_id: str, + atom_id: str, + raw_segment: str, + offset_start: int, + offset_end: int, +) -> None: + store.save(AtomTask( + atom_id=atom_id, + traj_id=traj_id, + offset_start=offset_start, + offset_end=offset_end, + intent="intent", + summary="summary", + used_skills=[], + ux_score=None, + raw_segment=raw_segment, + )) + + +def _simulate_split_ready( + *, + registry_db: Path, + watch_dir_id: int, + traj_path: Path, + atoms: list[dict[str, Any]], +) -> None: + update_traj_status( + watch_dir_id, traj_path.name, status="split_done", db_path=registry_db, + ) + store = AtomTaskStore(root=traj_path.parent) + for index, atom in enumerate(atoms, start=1): + _write_atom( + store, + traj_id=traj_path.stem, + atom_id=atom["atom_id"], + raw_segment=atom["content"], + offset_start=index, + offset_end=index + 1, + ) + + +@pytest.fixture +def context() -> dict[str, Any]: + return {} + + +@given("aimock 在随机本地端口启动 OpenAI-compatible 服务") +def given_aimock_running(aimock, context: dict[str, Any]) -> None: + assert aimock.base_url.startswith("http://127.0.0.1:") + context["aimock"] = aimock + aimock.on_message("ping-train-path", {"content": "pong"}) + + +@given("平台 TrajectoryReader 已配置 kernel-temp 且 auto_index 开启") +def given_reader_kernel_temp(tmp_path, context: dict[str, Any]) -> None: + registry_db = tmp_path / "registry.db" + temp_root = tmp_path / "temp_trajectories" + workspace = tmp_path / "oe_workspace" + workspace.mkdir() + reader = TrajectoryReader(registry_db, temp_root=temp_root) + context["registry_db"] = registry_db + context["reader"] = reader + context["workspace"] = workspace + context["config_path"] = tmp_path / "openearth.yaml" + context["config_path"].write_text( + "reflect:\n model: aimock-bench\n base_url: http://127.0.0.1:9\n", + encoding="utf-8", + ) + + +@given("OpenEarth workspace 已记录该临时轨的 oracle 分") +def given_oracle_score_prepared(context: dict[str, Any]) -> None: + context["trajectory_id"] = "traj_temp_bench_bdd_001" + context["oracle_score"] = 9 + record_oracle_score( + workspace=context["workspace"], + trajectory_id=context["trajectory_id"], + case_id="case-bdd-1", + ux_score=context["oracle_score"], + ) + + +@when("内核 create_temp 写入平台形做题轨迹") +def when_create_temp(context: dict[str, Any]) -> None: + reader: TrajectoryReader = context["reader"] + created = reader.create_temp( + PLATFORM_MD, trajectory_id=context["trajectory_id"], + ) + temp_watch = next( + item for item in reader.directories() if item.ecosystem == "kernel-temp" + ) + assert temp_watch.auto_index is True + context["created"] = created + context["temp_watch_id"] = int(temp_watch.id) + + +@when("平台将临时轨迹拆成多个 ready atom") +def when_split_multi_atoms(context: dict[str, Any]) -> None: + created = context["created"] + atoms = [ + { + "atom_id": f"atom_{context['trajectory_id']}_0001", + "content": "Solve the spreadsheet formula for Q1 revenue.", + }, + { + "atom_id": f"atom_{context['trajectory_id']}_0002", + "content": "I will open the workbook and compute the answer.", + }, + ] + _simulate_split_ready( + registry_db=context["registry_db"], + watch_dir_id=context["temp_watch_id"], + traj_path=created.path, + atoms=atoms, + ) + context["expected_atom_ids"] = {row["atom_id"] for row in atoms} + ready = context["reader"].get(created.id) + assert ready.atom_split_status == "ready" + assert len(ready.atoms) == 2 + context["ready"] = ready + + +@when("配置指向 aimock 后调用 train_skills") +def when_train_skills_via_aimock(context: dict[str, Any], monkeypatch) -> None: + aimock = context["aimock"] + captured: list[Any] = [] + context["captured"] = captured + context["train_error"] = None + + def _capturing_distill( + self, + atoms, + existing_skills=(), + *, + run_id: str, + full_rebuild: bool = False, + ): + items = list(atoms) + captured.extend(items) + return DistillationResult( + run_id=run_id, + candidate_dir=None, + drafts=(), + processed_trajectory_ids=tuple( + dict.fromkeys(item.parent_trajectory_id for item in items) + ), + processed_atom_ids=tuple(item.atom_id for item in items), + metrics={"oracle_scores": sum( + 1 for item in items if item.score_source == "oracle" + )}, + notes="bdd-capture", + ) + + monkeypatch.setattr( + "openearth_skill_sdk.service.TrajectorySkillDistiller.distill", + _capturing_distill, + ) + + # Prove the aimock OpenAI-compatible boundary is reachable from this run. + payload = json.dumps({ + "model": "aimock-bench", + "messages": [{"role": "user", "content": "ping-train-path"}], + }).encode("utf-8") + req = urlrequest.Request( + f"{aimock.base_url}/v1/chat/completions", + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlrequest.urlopen(req, timeout=30) as resp: + body = json.loads(resp.read().decode("utf-8")) + context["aimock_probe"] = body + + class _UnusedAnalyst: + """train_skills still constructs Distiller; distill is monkeypatched.""" + + try: + train_skills( + config_path=context["config_path"], + workspace=context["workspace"], + trajectories=[context["ready"]], + existing_skills=(), + run_id="bdd-bench-run", + full_rebuild=True, + analyst=_UnusedAnalyst(), + ) + except Exception as exc: # noqa: BLE001 - capture for Then steps + context["train_error"] = exc + + +@then("train_skills 收到的 ScoredAtomInput 包含这些 temp atom_id") +def then_captured_contains_temp_atom_ids(context: dict[str, Any]) -> None: + assert context["train_error"] is None + got = {item.atom_id for item in context["captured"]} + assert context["expected_atom_ids"] <= got + + +@then("对应 score_source 均为 oracle") +def then_score_source_oracle(context: dict[str, Any]) -> None: + expected = context["expected_atom_ids"] + matched = [ + item for item in context["captured"] if item.atom_id in expected + ] + assert matched + assert all(item.score_source == "oracle" for item in matched) + assert all( + item.ux_score == context["oracle_score"] for item in matched + ) + + +@then("aimock 至少收到一次 chat completions 探活请求") +def then_aimock_got_probe(context: dict[str, Any]) -> None: + body = context["aimock_probe"] + assert body + journal = context["aimock"].get_journal() + assert journal, "aimock journal should record the probe request" + + +@then("train_skills 不因多 atom 抛错") +def then_no_multi_atom_error(context: dict[str, Any]) -> None: + assert context["train_error"] is None + assert "exactly one" not in str(context.get("train_error") or "") + + +@then("train_skills 收到的 ScoredAtomInput 数量等于拆出的 atom 数") +def then_captured_count_matches(context: dict[str, Any]) -> None: + assert len(context["captured"]) == len(context["expected_atom_ids"]) From 7be521322d7503f77a62ffcdf298cb65c24199ed Mon Sep 17 00:00:00 2001 From: 370025263 <370025263@qq.com> Date: Thu, 30 Jul 2026 17:27:01 +0800 Subject: [PATCH 3/4] docs(reviews): OE e2e runbook and acceptance progress Co-authored-by: Cursor --- docs/reviews/2026-07-29-openearth-pr155.md | 158 +++++++++++++++++++++ docs/reviews/2026-07-30-oe-e2e-runbook.md | 79 +++++++++++ 2 files changed, 237 insertions(+) create mode 100644 docs/reviews/2026-07-29-openearth-pr155.md create mode 100644 docs/reviews/2026-07-30-oe-e2e-runbook.md diff --git a/docs/reviews/2026-07-29-openearth-pr155.md b/docs/reviews/2026-07-29-openearth-pr155.md new file mode 100644 index 00000000..6e4414bb --- /dev/null +++ b/docs/reviews/2026-07-29-openearth-pr155.md @@ -0,0 +1,158 @@ +# OpenEarth Kernel PR #155 评审纪要与更新对照 + +- PR：https://github.com/SkillNerds/xskill/pull/155 +- 头分支：`haironghu/xskill` · `feat/openearth-kernel-sdk-v2` +- 基线：`feat/algorithm-kernel-demo` +- 纪要日期：2026-07-29（初评）/ 2026-07-30（对照更新） +- 初版交付：SDK **0.6.0** @ `72003f3` +- 更新后交付：SDK **0.9.0** @ `816c950`（中间经 0.7.0 / 0.8.0） + +--- + +## 一、已对齐的产品语义（平台 × OE） + +### Training（训练） + +- 输入 `(X, Y)` 给分析器，产出 Skill 编辑草案（create / update）。 +- 交付适配版：`X = atom.content`，用户轨 `Y = atom.ux_score`；评测轨可用 oracle 分。 +- 分析器提草案时能看到存量 Skill 名字与摘要（软闭环）；`update`/`create` 只认**同名字符串**（硬规则），无相似度合并。 + +### Validation（验证 / Gate） + +- 定义：edit 装进候选 Skill 后，在验证集看涨点，通过才接受。 +- OE 交付仍**不跑 Gate / candidate rollout**；接受/灰度仍靠 XSkill Publisher + staging/canary PK。 + +### 平台喂养契约 + +```text +changed 非空 → 只处理这些 id（且 ready） +changed 空 且 full_rebuild → 才全量 ready +changed 空 且非 rebuild → 本轮空转 +``` + +字段名：`context.invocation.full_rebuild`（不是 `full-rebuild`）。 +与 CLI `xskill rebuild` 不同：后者是运维重置轨迹再蒸馏。 + +### 带着 Skill 库演化（代码事实，非文档臆测） + +- `kernel.py` 每轮 `_existing_skills(context)` → `context.skills.list()` 读平台 main。 +- SDK `sync_main_skills` + `pipeline.run(..., snapshots=...)`；同名 → `action=update`。 +- `full_rebuild` **只改轨迹范围**，不关闭读库。 + +--- + +## 二、初评三个问题单（对照初版 0.6.0） + +### 问题 1：临时轨迹 / temp+oracle + +| 项 | 初评结论 | +| --- | --- | +| 定义 | 算法在 workspace 内跑 benchmark 产出的评测轨迹，经 `create_temp` 挂平台，`source=temp` | +| 现象 | whl `xskill.py` 有 temp→训练消费；初版 `kernel.py` **未**调 `create_temp` / `record_oracle_score` | +| 1 atom | `source=temp` 且有 oracle 且 `len(atoms)!=1` → `ValueError` | +| 初评严重度 | 低（默认未激活）；建议删多余逻辑或改设计 | +| 打榜 | 不影响打榜精度；端到端需另评估 | + +### 问题 2：灰度有限 vs Skill 迭代无限 + +| 项 | 初评结论 | +| --- | --- | +| 现象 | 有 `staging_commit_sha` 则 `continue`，不 `submit`；无排队 | +| 精确表述 | 查的是「是否已有 staging」，不是「main+staging 都有」 | +| 初评严重度 | 中；影响生产进化，不影响调通/打榜精度 | + +### 问题 3：空 `changed` 误当全量 + +| 项 | 初评结论 | +| --- | --- | +| 现象 | `(not changed or id in changed)` → 空增量扫全部 ready | +| 正确契约 | 见上文三角契约 + `full_rebuild` | +| 初评严重度 | **严重**；影响生产 skill 生产，应当下修 | + +平台侧已加固文档/示例/离线 distill（本仓 `4497ed8`）。 + +--- + +## 三、开发者更新后对照（0.6.0 → 0.9.0） + +新增提交： + +| Commit | 说明 | +| --- | --- | +| `c60509f` | `fix(kernel): honor full rebuild trajectory scope` → SDK 0.7.0 | +| `813ae91` | `feat(kernel): queue drafts behind active staging` → SDK 0.8.0 | +| `816c950` | `feat(kernel): restore OpenEarth benchmark harness` → SDK 0.9.0 | + +### 对问题 3（full_rebuild）— 已按契约修复 + +`kernel.py` 改为： + +- `changed` 非空 → 增量 ready +- `elif full_rebuild` → 全量 ready +- `else` → 直接 `return`，`no_changes=True`，不调 SDK + +并向下传 `full_rebuild=` 给 `train_skills`。 +**结论：严重项已在胶水层关闭；可视为已回应。** + +### 对问题 2（灰度丢稿）— 已加本地发布队列 + +- workspace：`openearth-publication-queue.json` +- staging 忙：入队而非直接丢 +- 每轮先 `_drain_publication_queue`；main 变了可 `rebase_skill_draft` +- 仍无平台级多 staging 槽；是 **OE 侧 latest-wins 队列**，不是无限蓄水池 + +**结论：中等问题有实质改进；生产仍受单 staging 槽限制，可后续观察。** + +### 对问题 1（temp / harness）— 状态翻转，需重新评估 + +初评「默认未激活、负负得正」**已不再成立**： + +- `full_rebuild` 且配置启用 benchmark 时，调用 `run_benchmark(...)` +- 回调里：`record_oracle_score` + **`context.trajectories.create_temp(...)`** +- 当轮只登记 pending；拆完 ready 后后续轮再进 `train_skills`（仍用 oracle） +- wheel 现含 `benchmark.py` / `environments.py` / `target.py` 等；仍排除 `gate.py` + +**重新打开的风险：** + +1. 评测轨迹回流训练（validation 原料当 training 证据）— 产品争议仍在。 +2. **恰好 1 atom** 约束仍在；平台拆分无法保证。 +3. 平台 `kernel-temp` 若仍 `auto_index=False`，temp **可能永不拆分** → benchmark 登记了也进不了 ready 喂养（平台债，需单独修或确认）。 + +**结论：问题 1 从「可忽略」升为「设计已启用，需平台+产品再评」。** + +--- + +## 四、Skill 演化机制（代码确认摘要） + +1. 入口扫描平台 Skill 库，不另建权威库。 +2. 「被候选点到」= 本轮草案 `name` 与存量同名 → `update`。 +3. 分析器提草案时可见存量名字（软闭环）；无「内容相似则合并」决策器。 + +--- + +## 五、后续 TODO（跟踪项） + +见同目录清单或会话 TODO；优先级建议： + +1. **P0【已记录，一会顺手修】** `create_temp` 登记 `kernel-temp` 时误用 `auto_index=False`（暂停入库语义），watcher 跳过 → temp 永不拆。产品意图是走同一套拆分；修法：开索引或专用拆分入口，仍用 `source=temp` 与用户库隔离。 +2. **P1** 与 OE 对齐：benchmark rollout 进 training 是否可接受（用户侧倾向：垂直领域补充、Gate 关闭；val 原料原则上不回流 train——与 OE 现状仍可能不一致）。 +3. **P1** 多 atom + oracle 报错：平台不保证 1 atom；应改 OE 约束（跳过+记 metric），不加平台「整轨单 atom」专用模式。 +4. **P2** 观察发布队列 latest-wins 是否够用；是否要平台级排队。 +5. **P2** 复测 0.9.0（做题开、题集本机 xarena/leaderboard、用户轨抽 10 条；先离线 distill 再 e2e；合入目标 `feat/algorithm-kernel-demo`）。 +6. **P2（体验，不挡合入）** 离线 distill 拷贝 mock atom 树，使第三方 `xskill distill` 能见到 ready+atoms。 + +--- + +## 六、一句话现状 + +开发者已修 **full_rebuild 误全量** 与 **staging 直接丢稿**；同时 **恢复了 benchmark harness + create_temp**，临时轨迹训练支路从休眠变为 full rebuild 可选路径——初评「可忽略」结论作废，需围绕拆分保证与训练/验证边界继续跟进。 + + +--- + +## 七、2026-07-30 执行进度 + +- P0 `auto_index`：已修，见 PR https://github.com/SkillNerds/xskill/pull/176 +- P1 1-atom 硬报错：已在 PR #155 推送 `eb2cd33` +- BDD+aimock：`tests/bdd/test_benchmark_atoms_enter_training.py` 已绿 +- 人工 e2e：runbook `docs/reviews/2026-07-30-oe-e2e-runbook.md`；隔离 home `/tmp/oe-e2e-home-20260730` 已备 10 条轨；PK 待真实打分 diff --git a/docs/reviews/2026-07-30-oe-e2e-runbook.md b/docs/reviews/2026-07-30-oe-e2e-runbook.md new file mode 100644 index 00000000..69e79a3a --- /dev/null +++ b/docs/reviews/2026-07-30-oe-e2e-runbook.md @@ -0,0 +1,79 @@ +# OpenEarth 人工 e2e Runbook（2026-07-30） + +合入目标：PR [#155](https://github.com/SkillNerds/xskill/pull/155) → `feat/algorithm-kernel-demo` +平台修复分支：`fix/oe-acceptance-kernel-temp` +模型：**DeepSeek 官方 `deepseek-v4-flash`**（`~/.aikey` + `~/.xskill/config.yaml`） +Embedding：现有 ARK（不动） + +## 前置（已完成） + +1. **Plan1** `create_temp` / `kernel-temp` 改为 `auto_index=True`（commit `2a53cee`） +2. **Plan2** OE 0.9.0 wheel 去掉「必须 1 atom」硬报错，已推 PR155（`eb2cd33` on `haironghu/feat/openearth-kernel-sdk-v2`） +3. **Plan3** Gherkin + aimock BDD：`tests/bdd/test_benchmark_atoms_enter_training.py`（需安装 PR155 wheel + `aimock-pytest`） +4. DeepSeek `deepseek-v4-flash` 探活：官方 API 可连通（密钥不入文档） + +## 合格标准（回顾） + +- 做题轨 `create_temp` → 能被拆 → ready → **原子进入 train_skills**（BDD 已自动证明） +- 人工 e2e：注 10 条真轨 + 开 benchmark → 真拆 → **等真实 UX 分** → canary/PK 给出 **promoted 或 rejected** +- 不做 OE Gate + +## 环境准备 + +```bash +# 1) 隔离 home，避免动生产 ~/.xskill +export E2E_HOME=/tmp/oe-e2e-home-$(date +%Y%m%d) +mkdir -p "$E2E_HOME" +cp ~/.xskill/config.yaml "$E2E_HOME/config.yaml" +# 确保 llm.model=deepseek-v4-flash, base_url=https://api.deepseek.com +# api_key 可从 ~/.aikey 的 DEEPSEEK_API_KEY 写入（勿提交） + +# 2) 安装平台修复 + OE kernel +python3.11 -m venv "$E2E_HOME/venv" +source "$E2E_HOME/venv/bin/activate" +pip install -e /path/to/xskill-kernel-demo[dev] +pip install /path/to/openearth_skill_sdk-0.9.0-py3-none-any.whl # PR155 最新 + +mkdir -p "$E2E_HOME/kernels/openearth" +# 拷贝 examples/kernels/openearth/{kernel.py,config.yaml.example,...} +# config.yaml: benchmark.enabled=true +# benchmark.dataset_dir=/home/admin/leaderboard/datasets/officeqa # 或迷你子集 + +# 3) 抽 10 条真实用户轨 +mkdir -p "$E2E_HOME/watch/user" +ls /home/admin/data/xskill_eval/sample_dataset/traj_*.md | sort | head -10 \ + | while read f; do cp "$f" "$E2E_HOME/watch/user/"; done +``` + +## 执行步骤 + +1. **启动**（隔离 home）： + ```bash + source ~/.aikey + xskill --home "$E2E_HOME" serve ... # 或项目惯用入口 + # 注册 watch 目录：$E2E_HOME/watch/user + ``` +2. **等待用户轨拆分**到 ready（看 registry / 看板）。 +3. **触发 full_rebuild** 跑 OE benchmark → `create_temp`。 +4. **确认** `kernel-temp` 目录 `auto_index=1`，temp 轨最终 ready。 +5. **后续 run** 确认训练侧消费到 temp atoms（workspace 指标 / 日志中的 atom_id）。 +6. **staging 出现后**，等待真实 UX 打分凑齐 canary（`canary.min_samples`，默认 5）。 +7. **记录** `AtomCanary` 结论：`promoted` 或 `rejected`（超时 discard 本轮不算）。 + +## 执行记录（填写） + +| 项 | 值 | +| --- | --- | +| 日期 | 2026-07-30 | +| E2E_HOME | （执行时填写） | +| 用户轨 10 条 | 源：`/home/admin/data/xskill_eval/sample_dataset/` | +| 题集 | `/home/admin/leaderboard/datasets/officeqa` | +| DeepSeek | `deepseek-v4-flash` @ `api.deepseek.com` | +| 用户轨拆完 | ⏳ | +| temp 拆完并入训 | ⏳（BDD 已绿；现场 e2e ⏳） | +| PK 结论 | ⏳ 等真实打分 | + +## 备注 + +- 生产机已有 `xskill serve`（含 docker/root 实例）。**本验收必须用独立 `--home`**，不要往生产 watch 目录乱注轨。 +- PK 依赖真实 UX 样本，可能跨多日；BDD 不替代本条，但已锁定「原子入训」契约。 From 71ecdb0ef956c5b2ede67873e6ade2e0b7e95182 Mon Sep 17 00:00:00 2001 From: 370025263 <370025263@qq.com> Date: Thu, 30 Jul 2026 17:27:11 +0800 Subject: [PATCH 4/4] docs(reviews): fill e2e home path in OE runbook Co-authored-by: Cursor --- docs/reviews/2026-07-30-oe-e2e-runbook.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/reviews/2026-07-30-oe-e2e-runbook.md b/docs/reviews/2026-07-30-oe-e2e-runbook.md index 69e79a3a..83b9068d 100644 --- a/docs/reviews/2026-07-30-oe-e2e-runbook.md +++ b/docs/reviews/2026-07-30-oe-e2e-runbook.md @@ -65,13 +65,13 @@ ls /home/admin/data/xskill_eval/sample_dataset/traj_*.md | sort | head -10 \ | 项 | 值 | | --- | --- | | 日期 | 2026-07-30 | -| E2E_HOME | （执行时填写） | +| E2E_HOME | `/tmp/oe-e2e-home-20260730`（已备：10 条用户轨 + OE kernel 配置 + mini spreadsheet 题集） | | 用户轨 10 条 | 源：`/home/admin/data/xskill_eval/sample_dataset/` | -| 题集 | `/home/admin/leaderboard/datasets/officeqa` | -| DeepSeek | `deepseek-v4-flash` @ `api.deepseek.com` | -| 用户轨拆完 | ⏳ | -| temp 拆完并入训 | ⏳（BDD 已绿；现场 e2e ⏳） | -| PK 结论 | ⏳ 等真实打分 | +| 题集 | `/home/admin/xarena_out/spreadsheet-mini-5-task-fast` | +| DeepSeek | `deepseek-v4-flash` @ `api.deepseek.com`（探活通过） | +| 用户轨拆完 | ⏳ 待在隔离 home 起 serve | +| temp 拆完并入训 | BDD 已绿；现场 e2e ⏳ | +| PK 结论 | ⏳ 等真实 UX 打分攒齐（`min_samples=5`） | ## 备注