diff --git a/adapters/vllm_omni/manifest.yaml b/adapters/vllm_omni/manifest.yaml index 4f1aaa65..33f4dde7 100644 --- a/adapters/vllm_omni/manifest.yaml +++ b/adapters/vllm_omni/manifest.yaml @@ -331,6 +331,10 @@ rebase: # The wrapper refuses a TEST_TIMEOUT_SEC above this cap; it must not be # lower than the planned budget above or every required command is refused. module_max_timeout_sec: 21600 + # Ready's CPU "Simple" steps budget 20 minutes for CI agents. On the shared + # rebase host the same serial commands passed 53-88% of their items with + # zero failures before that budget expired (run-20261028-152707-a60091). + local_timeout_multiplier: 3 # Phase-2 module agents run on the CUDA wheel/runtime above. Broad pytest # paths must retain CUDA-capable and platform-neutral items while leaving # NPU/ROCm/XPU/MUSA-only items to their native CI lanes. diff --git a/doc/architecture/SPEC/app/core.md b/doc/architecture/SPEC/app/core.md index e9a9f21b..493f228b 100644 --- a/doc/architecture/SPEC/app/core.md +++ b/doc/architecture/SPEC/app/core.md @@ -1,6 +1,6 @@ # app/core.py —— 规范 - + `Copilot` is the headless workflow application. It resolves a TaskSpec, delegates reviewer verdict policy to `review.plan_gate`, confirms when needed, @@ -40,6 +40,10 @@ run(`_contained_run_dir` 拒绝越界路径);`--repo/--playbook/--task-par 发布 launcher 因此能把 resume 绑定到不可变的 campaign,而不会误入更新的 无关任务。损坏的 task.json 同样 BLOCKED 而非崩溃。 +`rerun_from`(`--rerun-from `,2026-10-29)是唯一会改变 resume 行为的 +参数:原样传给 `_execute`,由 executor 让该 step 及其后各 step 重新执行;保存的 +任务身份仍不改。 + ## 被信号终止的预约 run(2026-10-28) `_execute_reserved_locked` 的终态写入(`0 → done`、`BLOCKED_EXIT → blocked`、 diff --git a/doc/architecture/SPEC/app/workflow_execution.md b/doc/architecture/SPEC/app/workflow_execution.md index 9892e88a..b56ee041 100644 --- a/doc/architecture/SPEC/app/workflow_execution.md +++ b/doc/architecture/SPEC/app/workflow_execution.md @@ -1,6 +1,6 @@ # app/workflow_execution.py — workflow execution boundary - + `WorkflowExecution` runs an already selected playbook with the caller's current LLM dependency. It rechecks the repository context before starting, @@ -23,6 +23,9 @@ shutdown request exists, otherwise it propagates as before). Finalizers have already run inside `run_guarded` by then: GPU locks are released and the upstream scratch is kept, so the run is resumable with `--resume`. +A non-empty `rerun_from` is seeded as `state["rerun_from"]`; the executor +re-executes that step and every later one. + A losing run lock leaves active run artifacts untouched. A failed knowledge lock releases an acquired run lock. A caller-provided lock stays owned by the caller. Finalizers and locks run on every executor outcome, including failure. diff --git a/doc/architecture/SPEC/cli.md b/doc/architecture/SPEC/cli.md index 30ca547c..6a6115b4 100644 --- a/doc/architecture/SPEC/cli.md +++ b/doc/architecture/SPEC/cli.md @@ -1,6 +1,6 @@ # cli/ —— 规范 - + `LOC ~1420(6 个文件) · 接口 + 编排门面 · refactor-status: ok` @@ -114,3 +114,4 @@ RFC 子命令原样转发与共享入口:`test_rfc_integration.py`;服务契 `--resume` 现在接受可选 run id;`--repo`、`--playbook`、`--task-param` 与 `--resume` 同用时是对保存任务的断言(见 `app/core`)。 +`--rerun-from STEP_ID`(2026-10-29)只能与 `--resume` 同用,否则 argparse 报错。 diff --git a/doc/architecture/SPEC/engine/executor.md b/doc/architecture/SPEC/engine/executor.md index d5b8a0c1..2e0902e6 100644 --- a/doc/architecture/SPEC/engine/executor.md +++ b/doc/architecture/SPEC/engine/executor.md @@ -1,6 +1,6 @@ # engine/executor.py —— 规范 - + `LOC ~293 · 引擎底座(那个循环) · refactor-status: ok` @@ -34,6 +34,9 @@ helper:`_eval_when`、`_merge`。 step 自己的 params **之下**,所以 `--task-param limit=5` 不会被静默丢弃。 **playbook 自己的 params 在合并中获胜** —— 其中好几个是安全攸关的 (`force_push`、`pre_push`),因此被撰写下来的不变量**不得**被命令行覆盖。 +- **`state["rerun_from"]`**(操作者 `--resume --rerun-from `):开跑前从 + progress.json 删掉该 step 及其后所有 step 的检查点并落盘,记 `rerun_from` trace; + 未知 step id → BLOCKED,检查点不动。之前的 step 照常重放。 - step 的 span 会记录 playbook 的 step id 和 foreach 条目键:同一个 spec 可能在一个 playbook 里跑两次、还会再扇出,所以**光靠 `step` 无法标识这份工作**。 diff --git a/doc/architecture/SPEC/engine/steps/rebase_v3.md b/doc/architecture/SPEC/engine/steps/rebase_v3.md index 27a27ba0..4946c2c3 100644 --- a/doc/architecture/SPEC/engine/steps/rebase_v3.md +++ b/doc/architecture/SPEC/engine/steps/rebase_v3.md @@ -1,6 +1,6 @@ # engine/steps/rebase_v3.py —— 规范 - + `LOC ~2204 · step 库(v3 rebase 装配层) · refactor-status: oversized` @@ -134,6 +134,18 @@ harness 会话通过 MCP tool bridge 拿到同一套工具面,plan gate 在派 baseline commit 的 wheel,让"omni main 对上次对齐的 wheel 是否也失败"可问; 任何失败返回 None,保留保守的回归判定。 +## 操作者重跑与本地超时预算(2026-10-29) + +- `--resume --rerun-from wave1`:`_v3_module_rebase` 短路条件不变(done/skipped), + 所以 failed 模块重跑、done 模块直接返回;`_v3_wave_gate` 重新裁决 wave 2。 +- `_v3_test_loop` 在 `state["rerun_from"]` 非空时以 `retry_infra=True` 调 + `run_test_loop`:记录过的 timeout/watchdog infra 再跑一次,已通过的 job 不重跑。 + 普通 resume 下 infra 仍然粘滞。 +- `rebase.testing.local_timeout_multiplier`(默认 1,须为 >= 1 的数,否则 BLOCKED) + 只放大本地 `TestJob.timeout_sec`;构建出的 manifest 保留 CI 的 + `timeout_in_minutes`(shell golden 不变)。vllm-omni 取 3:Ready 的 20 分钟 CPU + Simple step 在共享 rebase 主机上通过 53–88% 用例、零失败时就被杀。 + ## 信号中断的模块:BLOCKED 收口、resume 重跑(2026-10-28) `_v3_module_rebase` 在 `rebase_module` 返回 `status == "interrupted"` diff --git a/doc/architecture/SPEC/rebase_engine.md b/doc/architecture/SPEC/rebase_engine.md index 0e668c3a..d329f167 100644 --- a/doc/architecture/SPEC/rebase_engine.md +++ b/doc/architecture/SPEC/rebase_engine.md @@ -1,6 +1,6 @@ # rebase_engine/ —— 规范 - + `LOC ~7500(26 个模块) · repo-rebase-v3 的原生 rebase 引擎 · refactor-status: ok` @@ -193,7 +193,9 @@ run-20261024 里这些链在 wave 1 完成后又占着 run 的 GPU 锁约 7 小 `host_env={CUDA_VISIBLE_DEVICES, HF_HOME}`(见 `tool_bridge`)。 - `module_pytest.py`:`rebase.testing.module_max_timeout_sec` 是 `TEST_TIMEOUT_SEC` 的上限(vllm-omni 与计划预算同为 21600),`module_test_platform` 过滤非本平台 - 的 pytest 项。 + 的 pytest 项。未显式给 `-m` 的模块跑同时带上与 Buildkite 相同的 run level: + 默认 `--run-level core_model`,标记为 `( or 平台中性) and core_model`。 + 调用方已写 `--run-level` 时,标记改用那一档,不再另加 core_model。 - `wheel.py`:`release_venv_holders`(仅 venv 结构存在时、排除调用者祖先链)、 EBUSY/ETXTBSY 重试、baseline rev-parse 前置校验、分支重置前 `git clean -ffd`。 diff --git a/src/infermatrix_copilot/app/core.py b/src/infermatrix_copilot/app/core.py index 4d71f742..844eb34c 100644 --- a/src/infermatrix_copilot/app/core.py +++ b/src/infermatrix_copilot/app/core.py @@ -264,7 +264,8 @@ def run_playbook(self, name: str, *, params: dict | None = None, def _execute(self, playbook, spec: TaskSpec, run_dir: Path, *, resolution_mode: str = "resume", tier: str = "?", resuming: bool = False, held_lock: RunLock | None = None, - planned_context: RepositoryContext | None = None) -> int: + planned_context: RepositoryContext | None = None, + rerun_from: str = "") -> int: """Compatibility entry point; application keeps the last-run summary.""" self.last_run_dir = run_dir self.last_blocked_reason = "" @@ -272,6 +273,7 @@ def _execute(self, playbook, spec: TaskSpec, run_dir: Path, *, playbook, spec, run_dir, resolution_mode=resolution_mode, tier=tier, resuming=resuming, held_lock=held_lock, planned_context=planned_context, llm=self.llm, + rerun_from=rerun_from, ) self.last_blocked_reason = result.blocked_reason return result.exit_code @@ -294,12 +296,14 @@ def run_queue(self, specs: list[TaskSpec], *, assume_yes: bool = False, return 0 def resume_last(self, run_id: str = "", *, repo: str = "", - playbook: str = "", params: dict | None = None) -> int: + playbook: str = "", params: dict | None = None, + rerun_from: str = "") -> int: """Resume a selected run, or the latest run when no id is supplied. Target arguments are assertions against the saved task, never edits to it. Release launchers can therefore bind resume to an immutable campaign - without accidentally entering a newer, unrelated task. + without accidentally entering a newer, unrelated task. `rerun_from` + re-executes that playbook step and everything after it. """ if run_id: try: @@ -348,7 +352,8 @@ def resume_last(self, run_id: str = "", *, repo: str = "", print(style("✋ cannot resume: ", "red", "bold") + str(exc)) return BLOCKED_EXIT print(f"↻ resuming {run_dir.name}: {spec.describe()}") - return self._execute(saved_playbook, spec, run_dir, resuming=True) + return self._execute(saved_playbook, spec, run_dir, resuming=True, + rerun_from=rerun_from) print("no resumable run found") return 1 diff --git a/src/infermatrix_copilot/app/workflow_execution.py b/src/infermatrix_copilot/app/workflow_execution.py index 0650d35a..b8b8c033 100644 --- a/src/infermatrix_copilot/app/workflow_execution.py +++ b/src/infermatrix_copilot/app/workflow_execution.py @@ -44,6 +44,7 @@ def run( resolution_mode: str = "resume", tier: str = "?", resuming: bool = False, held_lock: RunLock | None = None, planned_context: RepositoryContext | None = None, + rerun_from: str = "", ) -> ExecutionResult: """Run a resolved `playbook` to completion in `run_dir`: init tracing + notifier, seed the shared state (repo path, push policy, protected @@ -119,6 +120,8 @@ def run( "protected_branches": list(repo_context.protected_branches), "resuming": resuming, } + if rerun_from: + state["rerun_from"] = rerun_from if repo_context.high_risk_modules: state["high_risk_modules"] = list(repo_context.high_risk_modules) executor = Executor(self.registry, self.settings, run_dir=run_dir, diff --git a/src/infermatrix_copilot/cli/entry.py b/src/infermatrix_copilot/cli/entry.py index 390a1d36..90e1e0f9 100644 --- a/src/infermatrix_copilot/cli/entry.py +++ b/src/infermatrix_copilot/cli/entry.py @@ -120,6 +120,10 @@ def main(argv: list[str] | None = None) -> int: metavar="RUN_ID", help="resume RUN_ID, or the most recent run when omitted; " "repo/playbook/task params assert its saved identity") + parser.add_argument("--rerun-from", metavar="STEP_ID", + help="with --resume: execute this playbook step and " + "every later step again instead of replaying " + "their checkpoints") parser.add_argument("--playbook", help="run a specific playbook by name " "(incl. planner-invisible candidates); " @@ -215,7 +219,10 @@ def main(argv: list[str] | None = None) -> int: if args.resume is not None: return copilot.resume_last(args.resume, repo=args.repo, playbook=args.playbook or "", - params=parse_task_params(args.task_param)) + params=parse_task_params(args.task_param), + rerun_from=args.rerun_from or "") + if args.rerun_from: + parser.error("--rerun-from requires --resume") if args.playbook: params = parse_task_params(args.task_param) return copilot.run_playbook(args.playbook, params=params, diff --git a/src/infermatrix_copilot/engine/executor.py b/src/infermatrix_copilot/engine/executor.py index 7a3841b1..993a35a9 100644 --- a/src/infermatrix_copilot/engine/executor.py +++ b/src/infermatrix_copilot/engine/executor.py @@ -134,9 +134,31 @@ async def run(self, playbook: "Playbook", state: dict) -> RunOutcome: stack.enter_context(governed(governor_for(self.settings, ledger_dir_for(self.settings)))) return await self._run_steps(playbook, state) + def _drop_from(self, playbook: "Playbook", progress: dict, + step_id: str) -> str: + """Operator `--rerun-from`: forget `step_id` and every later step so + they execute again. Returns an error string for an unknown id.""" + ids = [s.id for s in playbook.steps] + if step_id not in ids: + return (f"--rerun-from: step '{step_id}' is not in playbook " + f"'{playbook.name}'") + later = ids[ids.index(step_id):] + dropped = [i for i in later if i in progress["completed"]] + for i in dropped: + del progress["completed"][i] + self._save_progress(progress) + self.trace.record("rerun_from", step=step_id, dropped=dropped) + return "" + async def _run_steps(self, playbook: "Playbook", state: dict) -> RunOutcome: progress = self._load_progress() outcome = RunOutcome(status="done") + if state.get("rerun_from"): + error = self._drop_from(playbook, progress, state["rerun_from"]) + if error: + outcome.status = "blocked" + outcome.blocked_reason = error + return outcome state.setdefault("playbook", playbook.name) # Run-scoped params (CLI `--task-param`, or intent-derived) reach every # step. Without this a step's `ctx.params.get(...)` only ever saw the diff --git a/src/infermatrix_copilot/engine/steps/rebase_v3.py b/src/infermatrix_copilot/engine/steps/rebase_v3.py index b1ebe31e..f29fa06a 100644 --- a/src/infermatrix_copilot/engine/steps/rebase_v3.py +++ b/src/infermatrix_copilot/engine/steps/rebase_v3.py @@ -10,6 +10,7 @@ from __future__ import annotations from dataclasses import asdict as _dc_asdict +from dataclasses import replace as _dc_replace import asyncio import re @@ -86,6 +87,21 @@ def manifest_job_to_test_job(j: Mapping): setup=j.get("setup", "")) +def _local_timeout_multiplier(testing: Mapping) -> float | StepResult: + """`rebase.testing.local_timeout_multiplier`: scales CI `timeout_in_minutes` + for the local loop only. CI budgets are sized for CI agents; a shared host + running the same serial command can pass every item and still be killed, + which records a structural infra failure the push gate blocks on. The + built manifest keeps the CI value (shell-golden parity).""" + factor = testing.get("local_timeout_multiplier", 1) + if isinstance(factor, bool) or not isinstance(factor, (int, float)) \ + or factor < 1: + return StepResult(False, FailureKind.BLOCKED, + "rebase.testing.local_timeout_multiplier must be a " + "number >= 1") + return factor + + def _tier_client(ctx: StepContext): """The run's agent backend via `Settings.tier_target` — the ONLY place a tier's model may pair with an endpoint and credential. Returns @@ -1522,9 +1538,17 @@ def notify_download(key: str, repo_id: str) -> None: record_fn=record_fn, review_fn=review_fn, report_fn=report_fn, notify_download=notify_download) jobs_by_slug = {j["slug"]: j for j in local_jobs} + timeout_multiplier = _local_timeout_multiplier(rb.get("testing") or {}) + if isinstance(timeout_multiplier, StepResult): + return timeout_multiplier + if timeout_multiplier != 1: + ctx.trace.record("local_timeout_multiplier", factor=timeout_multiplier) def _job(slug: str) -> TestJob: - return manifest_job_to_test_job(jobs_by_slug[slug]) + job = manifest_job_to_test_job(jobs_by_slug[slug]) + if timeout_multiplier != 1: + job = _dc_replace(job, timeout_sec=job.timeout_sec * timeout_multiplier) + return job def _to_result(outcome) -> tl.TestRunResult: infra = "timeout" if outcome.timed_out else \ @@ -1677,7 +1701,8 @@ async def debug_fn(slug: str, label: str, rc: int, local_jobs, substate=sub, run_fn=run_fn, baseline_fn=baseline_fn, debug_fn=debug_fn, visible_gpus=len([d for d in runner.cuda.split(",") if d.strip() - and not d.strip().startswith("-")])) + and not d.strip().startswith("-")]), + retry_infra=bool(ctx.state.get("rerun_from"))) tl.remove_main_worktree(Path(repo), worktree_path) infra = result["infra_failures"] + dropped_infra sub.update({"tests": { diff --git a/src/infermatrix_copilot/rebase_engine/module_pytest.py b/src/infermatrix_copilot/rebase_engine/module_pytest.py index 1389e56f..cc442a6f 100644 --- a/src/infermatrix_copilot/rebase_engine/module_pytest.py +++ b/src/infermatrix_copilot/rebase_engine/module_pytest.py @@ -193,6 +193,15 @@ def _job_key(args: list[str]) -> str: return "module_pytest" +def _explicit_option(args: list[str], name: str) -> str | None: + for index, arg in enumerate(args): + if arg == name and index + 1 < len(args): + return args[index + 1] + if arg.startswith(name + "="): + return arg.split("=", 1)[1] + return None + + def _platform_filtered_args(args: list[str], testing: dict) -> tuple[list[str], str | None]: """Add the adapter's platform selector to otherwise unfiltered pytest. @@ -201,6 +210,14 @@ def _platform_filtered_args(args: list[str], testing: dict) -> tuple[list[str], Keep platform-neutral and current-platform items, including tests marked for several platforms. A caller-provided ``-m`` expression is explicit and therefore remains authoritative. + + Buildkite pairs the mark with ``--run-level`` (Ready uses + ``-m 'core_model and cuda' --run-level core_model``). A hardware-only + expression collects ``advanced_model`` / ``full_model`` items and then + the default ``core_model`` run level loads dummy weights, so real-weight + fixtures error. An unmarked module run therefore also selects + ``core_model``. An explicit ``--run-level`` is paired with that same + level instead. """ platform = testing.get("module_test_platform") if platform is None or args[0] == "python" or args[0].startswith("python3"): @@ -210,9 +227,17 @@ def _platform_filtered_args(args: list[str], testing: dict) -> tuple[list[str], raise ValueError(f"rebase.testing.module_test_platform must be one of: {choices}") if any(arg == "-m" or arg.startswith("-m=") for arg in args): return args, None + level = _explicit_option(args, "--run-level") or "core_model" + if level not in {"core_model", "advanced_model", "full_model"}: + raise ValueError( + "--run-level must be core_model, advanced_model, or full_model") others = sorted(_ACCELERATOR_PLATFORMS - {platform}) - expression = f"{platform} or (" + " and ".join(f"not {name}" for name in others) + ")" - return [*args, "-m", expression], platform + expression = ( + f"({platform} or (" + " and ".join(f"not {name}" for name in others) + + f")) and {level}" + ) + extra = [] if _explicit_option(args, "--run-level") else ["--run-level", level] + return [*args, *extra, "-m", expression], platform def _log_hash(log_file: str) -> tuple[str | None, str]: diff --git a/src/infermatrix_copilot/rebase_engine/test_loop.py b/src/infermatrix_copilot/rebase_engine/test_loop.py index 4d639dd7..f130b2b5 100644 --- a/src/infermatrix_copilot/rebase_engine/test_loop.py +++ b/src/infermatrix_copilot/rebase_engine/test_loop.py @@ -107,18 +107,25 @@ async def run_test_loop( debug_fn: DebugFn, visible_gpus: int = 0, phase_label: str = "Phase 3", + retry_infra: bool = False, ) -> dict: """Returns the parent's shape plus the structural split: ``{"passed", "failed", "failed_tests", "skipped_tests", "completed_slugs", "infra_failures"}`` — `failed_tests` holds ONLY assertion failures; infrastructure failures live in `infra_failures`. Progress lives under the substate's ``phase3_progress`` (run-stamped, so a phase-4 fallback in - the same run resumes past everything phase 3 already ran).""" + the same run resumes past everything phase 3 already ran). + `retry_infra` (an operator-requested rerun only) runs recorded + infrastructure failures again instead of keeping them structural.""" prev = substate.get("phase3_progress", {}) or {} prev_completed = set(prev.get("completed", [])) prev_failed = set(prev.get("failed", [])) prev_skipped = set(prev.get("skipped", [])) prev_infra = list(prev.get("infra", [])) + if retry_infra and prev_infra: + log.info("%s rerun: retrying %d infrastructure failure(s): %s", + phase_label, len(prev_infra), "; ".join(prev_infra[:5])) + prev_infra = [] if prev_completed or prev_failed or prev_skipped: log.info("%s resume: %d passed, %d failed, %d skipped from previous " "attempt", phase_label, len(prev_completed), diff --git a/test/test_assembly.py b/test/test_assembly.py index 26337780..cf913922 100644 --- a/test/test_assembly.py +++ b/test/test_assembly.py @@ -294,6 +294,15 @@ def baseline_fn(slug): baseline_fn=baseline_fn, debug_fn=debug_fn)) assert "timeout" not in reran assert result2["infra_failures"] == ["timeout: timeout"] + # an operator-requested rerun retries it; a pass clears the record + runs["timeout"] = tl.TestRunResult(0) + reran.clear() + result3 = asyncio.run(tl.run_test_loop( + jobs, substate=sub, run_fn=lambda s: reran.append(s) or runs[s], + baseline_fn=baseline_fn, debug_fn=debug_fn, retry_infra=True)) + assert reran == ["timeout", "assertion"] + assert result3["infra_failures"] == [] + assert "timeout" in result3["completed_slugs"] def test_infra_failure_surfaces_assertions_it_already_reported(tmp_path): @@ -2789,3 +2798,20 @@ def test_runner_model_download_notify(tmp_path): setup="echo huggingface-cli download org/model x") assert runner.run(job2, env).rc == 0 assert [n for n in notified if n[0] == "dl2"] == [] + + +@pytest.mark.parametrize("testing,expected", [ + ({}, 1), ({"local_timeout_multiplier": 3}, 3), + ({"local_timeout_multiplier": 1.5}, 1.5)]) +def test_local_timeout_multiplier_scales_only_valid_factors(testing, expected): + from infermatrix_copilot.engine.steps.rebase_v3 import _local_timeout_multiplier + assert _local_timeout_multiplier(testing) == expected + + +@pytest.mark.parametrize("factor", [0, 0.5, -2, True, "3"]) +def test_local_timeout_multiplier_rejects_invalid_factor(factor): + from infermatrix_copilot.engine.steps.rebase_v3 import _local_timeout_multiplier + from infermatrix_copilot.engine.step import FailureKind, StepResult + result = _local_timeout_multiplier({"local_timeout_multiplier": factor}) + assert isinstance(result, StepResult) + assert result.failure == FailureKind.BLOCKED diff --git a/test/test_engine.py b/test/test_engine.py index 23f60662..fb6e6cae 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -127,6 +127,38 @@ def flaky(ctx): assert counter["n"] == 2 # flaky ran twice, stable once +def test_rerun_from_reexecutes_that_step_and_later_ones(env): + registry, executor, _ = env + calls = [] + for name in ("one", "two", "three"): + registry.register(make_step(f"s.{name}", lambda ctx, n=name: ( + calls.append(n), StepResult(True, summary=n))[1])) + pb = playbook([PlaybookStep("a", "s.one"), PlaybookStep("b", "s.two"), + PlaybookStep("c", "s.three")]) + assert asyncio.run(executor.run(pb, {})).status == "done" + calls.clear() + + outcome = asyncio.run(executor.run(pb, {"rerun_from": "b"})) + assert outcome.status == "done" + assert calls == ["two", "three"] + calls.clear() + assert asyncio.run(executor.run(pb, {})).status == "done" + assert calls == [] + + +def test_rerun_from_unknown_step_blocks_without_touching_checkpoint(env): + registry, executor, _ = env + registry.register(make_step("s.one", lambda ctx: StepResult(True, summary="one"))) + pb = playbook([PlaybookStep("a", "s.one")]) + assert asyncio.run(executor.run(pb, {})).status == "done" + before = executor.progress_file.read_text(encoding="utf-8") + + outcome = asyncio.run(executor.run(pb, {"rerun_from": "missing"})) + assert outcome.status == "blocked" + assert "missing" in outcome.blocked_reason + assert executor.progress_file.read_text(encoding="utf-8") == before + + def test_retryable_is_retried(env): registry, executor, _ = env attempts = {"n": 0} diff --git a/test/test_module_pytest_outcomes.py b/test/test_module_pytest_outcomes.py index 15daa992..2d49fbd8 100644 --- a/test/test_module_pytest_outcomes.py +++ b/test/test_module_pytest_outcomes.py @@ -164,19 +164,47 @@ def run(self, job, env): monkeypatch.setattr(module_pytest, "TestRunner", CapturingRunner) args = ["-vv", "-s", "tests/diffusion/"] assert module_pytest.main(args) == 0 - marker = "cuda or (not musa and not npu and not rocm and not xpu)" + marker = "(cuda or (not musa and not npu and not rocm and not xpu)) and core_model" assert observed["job"].command == ( - "python -m pytest -vv -s tests/diffusion/ -m " + shlex.quote(marker) + "python -m pytest -vv -s tests/diffusion/ --run-level core_model -m " + + shlex.quote(marker) ) receipt = json.loads(receipts(wrapper_env)[0].read_text()) requested = json.dumps(args, ensure_ascii=False, separators=(",", ":")) - effective = json.dumps([*args, "-m", marker], ensure_ascii=False, - separators=(",", ":")) + effective = json.dumps( + [*args, "--run-level", "core_model", "-m", marker], + ensure_ascii=False, separators=(",", ":")) assert receipt["requested_argv_sha256"] == hashlib.sha256(requested.encode()).hexdigest() assert receipt["effective_argv_sha256"] == hashlib.sha256(effective.encode()).hexdigest() assert receipt["platform_marker_injected"] == "cuda" +def test_explicit_run_level_is_paired_and_not_forced_to_core( + tmp_path, monkeypatch, wrapper_env +): + configure_adapter( + tmp_path, wrapper_env, monkeypatch, {"module_test_platform": "cuda"} + ) + observed = {} + + class CapturingRunner: + def __init__(self, **kwargs): + pass + + def run(self, job, env): + observed["job"] = job + return Outcome(rc=0) + + monkeypatch.setattr(module_pytest, "TestRunner", CapturingRunner) + args = ["-vv", "--run-level", "advanced_model", "tests/entrypoints/"] + assert module_pytest.main(args) == 0 + marker = "(cuda or (not musa and not npu and not rocm and not xpu)) and advanced_model" + assert observed["job"].command == ( + "python -m pytest -vv --run-level advanced_model tests/entrypoints/ -m " + + shlex.quote(marker) + ) + + def test_explicit_marker_and_command_mode_are_not_rewritten( tmp_path, monkeypatch, wrapper_env ): diff --git a/test/test_rebase_resume_selection.py b/test/test_rebase_resume_selection.py index 1a4ff859..61271999 100644 --- a/test/test_rebase_resume_selection.py +++ b/test/test_rebase_resume_selection.py @@ -58,7 +58,7 @@ def test_explicit_resume_selects_release_despite_unrelated_newer_run(resume_cont assert len(calls) == 1 assert calls[0][2] == release assert calls[0][1].params["force_upstream_commit"] == RELEASE_COMMIT - assert calls[0][3] == {"resuming": True} + assert calls[0][3] == {"resuming": True, "rerun_from": ""} def test_bare_resume_still_selects_latest(resume_context): @@ -180,4 +180,28 @@ def test_cli_forwards_run_selection_and_identity_assertions( assert calls == [((selected,), {"repo": "vllm-omni", "playbook": "repo-rebase-v3", "params": {"force_upstream_commit": RELEASE_COMMIT, - "retry_limit": 1}})] + "retry_limit": 1}, + "rerun_from": ""})] + + +def test_cli_forwards_rerun_from_only_with_resume(tmp_path, monkeypatch, capsys): + calls = [] + fake = SimpleNamespace(resume_last=lambda *args, **kwargs: calls.append( + (args, kwargs)) or 0) + monkeypatch.setattr(entry, "Copilot", lambda: fake) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("IMX_INVOCATION_ID", raising=False) + + assert entry.main(["--resume", RELEASE_RUN, "--rerun-from", "wave1"]) == 0 + assert calls[0][1]["rerun_from"] == "wave1" + with pytest.raises(SystemExit): + entry.main(["--playbook", "repo-rebase-v3", "--rerun-from", "wave1"]) + assert "--rerun-from requires --resume" in capsys.readouterr().err + + +def test_resume_passes_rerun_from_to_execution(resume_context): + copilot, calls = resume_context + _save_run(copilot.settings.run_root, RELEASE_RUN) + + assert copilot.resume_last(RELEASE_RUN, rerun_from="wave1") == 0 + assert calls[0][3]["rerun_from"] == "wave1"