From 4c35dfafa48b4764637889717e67ce85cbc0cc6d Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 13:30:41 +0900 Subject: [PATCH 01/32] fix(tf-dispatch): stage provider stacks on the terraform read path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The write path (tf-init/plan/apply/destroy) vendors the externalized provider terramate stacks and runs `terramate generate` before invoking terramate. The read/power commands (status, ip, start, stop, password across aws/gcp/truenas/ vultr) never run tf-init, so on a fresh checkout — or any tree where `up` hasn't run — terramate can't discover the stacks: status reports "not created" and ip emits a terramate error box that gets captured verbatim as the IP value. It only worked because an `up` happened to leave the stacks vendored. Add a shared core helper the read path uses: - `stage_provider_stacks` / `scripts/tf-stage `: vendor the stacks and run `terramate generate` only when staging changed something or no generated backend file exists yet. - `prepare_provider_stacks` is now idempotent: it short-circuits when the staged copy already matches the source (a read-only content compare that ignores generated, destination-only files), returning whether anything was re-staged. This keeps frequent callers (status polls) from churning the filesystem and clobbering generated z_backend.tf on every call. State is unaffected — the local backend lives under .terraform-cache-dir/, outside the staged tree. The provider commands invoke this via `eve_load_profile_env` (gated on the terraform engine) in the companion eve-providers change. --- eve_sdk/tf_dispatch.py | 77 +++++++++++++++- scripts/tf-stage | 41 +++++++++ tests/python/sdk/test_tf_stage.py | 142 ++++++++++++++++++++++++++++++ 3 files changed, 257 insertions(+), 3 deletions(-) create mode 100755 scripts/tf-stage create mode 100644 tests/python/sdk/test_tf_stage.py diff --git a/eve_sdk/tf_dispatch.py b/eve_sdk/tf_dispatch.py index 8f8b0453..597b88d2 100644 --- a/eve_sdk/tf_dispatch.py +++ b/eve_sdk/tf_dispatch.py @@ -94,7 +94,7 @@ def apply_tf_env(root: Path, profile: str) -> None: os.environ[key] = value -def prepare_provider_stacks(root: Path, provider_id: str) -> None: +def prepare_provider_stacks(root: Path, provider_id: str) -> bool: """Stage a provider's terramate stacks where terramate can discover them. After v4.0 externalized providers, the stacks live under @@ -103,7 +103,17 @@ def prepare_provider_stacks(root: Path, provider_id: str) -> None: ``/plugins/providers//stacks/...``, and manifests reference siblings like ``../_terraform-common/``, so we physically copy the entire source tree to ``plugins/providers/`` — terramate finds the stacks AND - the sibling paths resolve. Safe to call repeatedly (idempotent). + the sibling paths resolve. + + Idempotent and cheap when already staged: each needed dir is copied only + when the destination doesn't already match the source (a read-only content + compare, ignoring terramate-generated artifacts that only exist in the + destination). Returns ``True`` when anything was (re)staged, ``False`` when + every dir was already up to date — the caller uses this to decide whether a + ``terramate generate`` is needed. The copy still wipes+replaces an + out-of-date dir (so upstream deletions propagate); the up-to-date + short-circuit is what keeps frequent callers (status polls) from churning + the filesystem and clobbering generated ``z_backend.tf`` files every time. """ import shutil @@ -116,7 +126,7 @@ def prepare_provider_stacks(root: Path, provider_id: str) -> None: plugin_path = Path(str(plugin["_path"])).parent break if plugin_path is None or not plugin_path.is_dir(): - return + return False source_root = plugin_path.parent # e.g. .eve/plugins/eve-providers/ dest_root = root / "plugins" / "providers" @@ -124,14 +134,75 @@ def prepare_provider_stacks(root: Path, provider_id: str) -> None: # Copy the provider dir + all sibling dirs starting with _ (shared: _common, _terraform-common, etc.). # This ensures ../_terraform-common/ etc. resolve from the staged provider. needed = [provider_id] + [d.name for d in source_root.iterdir() if d.is_dir() and d.name.startswith("_")] + changed = False for name in needed: src = source_root / name dst = dest_root / name if not src.is_dir(): continue + if _staged_dir_current(src, dst): + continue + changed = True if dst.exists() or dst.is_symlink(): if dst.is_symlink() or dst.is_file(): dst.unlink() else: shutil.rmtree(dst) shutil.copytree(src, dst, ignore=shutil.ignore_patterns("eve-plugin.yaml")) + return changed + + +# Files that only ever exist in the staged copy (terramate code generation + +# terraform working dirs), never in the source — so they don't count against a +# destination being "current" with its source. +_STAGE_IGNORE_NAMES = frozenset({"eve-plugin.yaml"}) + + +def _staged_dir_current(src: Path, dst: Path) -> bool: + """True when every source file is present and identical under ``dst``. + + Destination-only files (generated ``z_backend.tf``, ``.terraform/``) are + ignored — we only care that the source is faithfully represented, so a + freshly generated stack still reads as current and isn't needlessly wiped. + """ + import filecmp + + if not dst.is_dir(): + return False + for src_file in src.rglob("*"): + if not src_file.is_file() or src_file.name in _STAGE_IGNORE_NAMES: + continue + dst_file = dst / src_file.relative_to(src) + if not dst_file.is_file() or not filecmp.cmp(src_file, dst_file, shallow=False): + return False + return True + + +def stage_provider_stacks(root: Path, provider_id: str) -> None: + """Vendor a provider's stacks and (re)generate terramate code if needed. + + The read/power commands (``status``, ``ip``, ``start`` …) need the + externalized stacks staged for terramate to discover them, but unlike the + write path they never run ``tf-init`` (which does this). This is the shared + helper they call (via ``scripts/tf-stage``) so the read path matches the + write path. Idempotent and cheap once staged: the copy is skipped when + already current, and ``terramate generate`` runs only when staging changed + something or no generated backend file exists yet. + """ + if not provider_id: + return + changed = prepare_provider_stacks(root, provider_id) + if not changed and _generated_backend_present(root, provider_id): + return + result = subprocess.run( + ["terramate", "generate"], cwd=root, text=True, capture_output=True, check=False + ) + if result.returncode != 0: + sys.stderr.write(result.stdout + result.stderr) + raise SystemExit(result.returncode) + + +def _generated_backend_present(root: Path, provider_id: str) -> bool: + """True when terramate has already generated backend code for the provider.""" + stacks = root / "plugins" / "providers" / provider_id / "stacks" + return stacks.is_dir() and any(stacks.rglob("z_backend.tf")) diff --git a/scripts/tf-stage b/scripts/tf-stage new file mode 100755 index 00000000..695e10c3 --- /dev/null +++ b/scripts/tf-stage @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Stage a provider's terramate stacks for read/power commands. + +The write path (tf-init/plan/apply/destroy) vendors the externalized provider +stacks and runs `terramate generate` before invoking terramate. The read/power +commands (status, ip, start, stop, password) don't run tf-init, so on a fresh +checkout — or any tree where `up` hasn't run — terramate can't discover the +stacks and reports "not created" / errors. This is the shared core helper those +commands call (via `eve_load_profile_env`, gated on the terraform engine) so the +read path stages stacks the same way the write path does. + +Idempotent and cheap once staged (see stage_provider_stacks): the copy is +skipped when already current, and `terramate generate` runs only when staging +changed something or no generated backend file exists yet — safe to call on +every status poll. + +Usage: scripts/tf-stage +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from eve_sdk.tf_dispatch import repo_root, stage_provider_stacks # noqa: E402 + + +def main(argv: list[str]) -> int: + provider = argv[0] if argv else "" + if not provider: + print("usage: scripts/tf-stage ", file=sys.stderr) + return 2 + stage_provider_stacks(repo_root(), provider) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/python/sdk/test_tf_stage.py b/tests/python/sdk/test_tf_stage.py new file mode 100644 index 00000000..60cd4008 --- /dev/null +++ b/tests/python/sdk/test_tf_stage.py @@ -0,0 +1,142 @@ +"""Staging of externalized provider stacks for the terraform read path. + +The read/power commands (status, ip, …) call `scripts/tf-stage` → +`stage_provider_stacks`, which vendors the provider's terramate stacks the same +way the write path (tf-init) does. These tests cover the pure file-staging +logic (`prepare_provider_stacks` + helpers) without invoking terramate, so they +stay hermetic. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from eve_sdk.tf_dispatch import ( + _generated_backend_present, + _staged_dir_current, + prepare_provider_stacks, +) + +PROVIDER_MANIFEST = """\ +api_version: eve.plugin/v1 +kind: provider +id: acme-tf +display_name: Acme TF +commands: + resolve: {exec: commands/provider-command, args: [resolve]} + init: {exec: commands/provider-command, args: [init]} + plan: {exec: commands/provider-command, args: [plan]} + up: {exec: commands/provider-command, args: [up]} + down: {exec: commands/provider-command, args: [down]} + start: {exec: commands/provider-command, args: [start]} + stop: {exec: commands/provider-command, args: [stop]} + status: {exec: commands/provider-command, args: [status]} + ip: {exec: commands/provider-command, args: [ip]} + ssh: {exec: commands/provider-command, args: [ssh]} + validate: {exec: commands/provider-command, args: [validate]} +access: + ubuntu: + bootstrap_user: {env: VM_USER_NAME} + provision_user: {env: VM_USER_NAME} + human_user: {env: VM_USER_NAME, fallback: provision_user} +supports: + engines: [terraform] + kinds: [vm] +env: [] +""" + + +@pytest.fixture() +def staged(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, Path]: + """A source plugin root (provider + shared sibling) and a repo root. + + Returns ``(repo_root, source_provider_dir)``. Discovery is pinned to the + source root via EVE_PLUGIN_ROOTS(_EXCLUSIVE) so prepare_provider_stacks + resolves ``acme-tf`` to it. + """ + source_root = tmp_path / "source" + provider = source_root / "acme-tf" + (provider / "stacks" / "svc").mkdir(parents=True) + (provider / "eve-plugin.yaml").write_text(PROVIDER_MANIFEST, encoding="utf-8") + (provider / "stacks" / "svc" / "main.tf").write_text("# svc\n", encoding="utf-8") + # Manifest validation requires each declared command's exec to exist. + commands = provider / "commands" + commands.mkdir() + stub = commands / "provider-command" + stub.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + stub.chmod(0o755) + # A shared sibling dir (leading underscore) the staging must also copy. + shared = source_root / "_tf-shared" + shared.mkdir() + (shared / "common.tf").write_text("# shared\n", encoding="utf-8") + + monkeypatch.setenv("EVE_PLUGIN_ROOTS", str(source_root)) + monkeypatch.setenv("EVE_PLUGIN_ROOTS_EXCLUSIVE", "1") + + repo_root = tmp_path / "repo" + repo_root.mkdir() + return repo_root, provider + + +def test_stages_provider_and_shared_dirs(staged: tuple[Path, Path]) -> None: + repo_root, _ = staged + changed = prepare_provider_stacks(repo_root, "acme-tf") + assert changed is True + dest = repo_root / "plugins" / "providers" + assert (dest / "acme-tf" / "stacks" / "svc" / "main.tf").read_text() == "# svc\n" + assert (dest / "_tf-shared" / "common.tf").read_text() == "# shared\n" + # The manifest itself is never vendored into the stack tree. + assert not (dest / "acme-tf" / "eve-plugin.yaml").exists() + + +def test_idempotent_no_rewrite_when_current(staged: tuple[Path, Path]) -> None: + repo_root, _ = staged + assert prepare_provider_stacks(repo_root, "acme-tf") is True + # Already current → no work, no change reported (so callers skip generate). + assert prepare_provider_stacks(repo_root, "acme-tf") is False + assert prepare_provider_stacks(repo_root, "acme-tf") is False + + +def test_generated_files_survive_idempotent_calls(staged: tuple[Path, Path]) -> None: + """A generated z_backend.tf (destination-only) must not be wiped when the + source is unchanged — that's what keeps frequent status polls from forcing + a re-init.""" + repo_root, _ = staged + prepare_provider_stacks(repo_root, "acme-tf") + generated = repo_root / "plugins" / "providers" / "acme-tf" / "stacks" / "svc" / "z_backend.tf" + generated.write_text("# generated\n", encoding="utf-8") + assert _generated_backend_present(repo_root, "acme-tf") is True + # Re-staging while current is a no-op → generated file is preserved. + assert prepare_provider_stacks(repo_root, "acme-tf") is False + assert generated.read_text() == "# generated\n" + + +def test_restages_when_source_changes(staged: tuple[Path, Path]) -> None: + repo_root, provider = staged + prepare_provider_stacks(repo_root, "acme-tf") + (provider / "stacks" / "svc" / "main.tf").write_text("# svc v2\n", encoding="utf-8") + assert prepare_provider_stacks(repo_root, "acme-tf") is True + dest_main = repo_root / "plugins" / "providers" / "acme-tf" / "stacks" / "svc" / "main.tf" + assert dest_main.read_text() == "# svc v2\n" + + +def test_unknown_provider_is_noop(staged: tuple[Path, Path]) -> None: + repo_root, _ = staged + assert prepare_provider_stacks(repo_root, "does-not-exist") is False + assert not (repo_root / "plugins" / "providers").exists() + + +def test_staged_dir_current_ignores_destination_only_files(tmp_path: Path) -> None: + src = tmp_path / "src" + dst = tmp_path / "dst" + (src / "stacks").mkdir(parents=True) + (src / "stacks" / "main.tf").write_text("x\n", encoding="utf-8") + (dst / "stacks").mkdir(parents=True) + (dst / "stacks" / "main.tf").write_text("x\n", encoding="utf-8") + (dst / "stacks" / "z_backend.tf").write_text("gen\n", encoding="utf-8") # dest-only + assert _staged_dir_current(src, dst) is True + # A genuine content drift in a source-tracked file is not current. + (dst / "stacks" / "main.tf").write_text("y\n", encoding="utf-8") + assert _staged_dir_current(src, dst) is False From 7955af6f1b79ed4db09c2d59d5831f4abfd14ebd Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 14:19:22 +0900 Subject: [PATCH 02/32] docs(v4.5): roadmap (milestone-4 validation) + coding-agent task file --- docs/v4.5-agent-tasks.md | 115 +++++++++++++++++++++++++++++++++++++++ docs/v4.5-roadmap.md | 92 +++++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 docs/v4.5-agent-tasks.md create mode 100644 docs/v4.5-roadmap.md diff --git a/docs/v4.5-agent-tasks.md b/docs/v4.5-agent-tasks.md new file mode 100644 index 00000000..25d54555 --- /dev/null +++ b/docs/v4.5-agent-tasks.md @@ -0,0 +1,115 @@ +# v4.5 — coding-agent task file + +Read [docs/v4.5-roadmap.md](v4.5-roadmap.md) first. That is the goal and the +living bug tracker. This file is your work queue. Work the tasks **in order**; +each is sized to run autonomously for a long stretch. + +## Ground rules (apply to every task) + +- **Branch:** work on `v4.5` in each repo (`git checkout v4.5`). It is pushed and + tracks `origin/v4.5`. Commit there. Push is allowed; **do not merge** to `main`. +- **Cross-repo coupling:** `eve` and `eve-providers` (and packages, if touched) + are one milestone. When a change spans repos, commit to each repo's `v4.5`. +- **No env-var overrides** (`EVE_PLUGIN_ROOTS*`) anywhere — not in code, tests, + or your own testing. Hermetic *unit* tests use the existing + `tests/python/conftest.py` harness; that's fine. Real testing uses plain + `eve tui` / `eve` CLI against configured local-folder sources (roadmap "How to + test"). +- **Record every bug** in the roadmap tracker (terse: symptom → fix → repo/PR) + as you fix it. Keep statuses current. +- **Validate before committing:** run the relevant `scripts/test-*` (at minimum + `test-python`, `test-lint`, `test-shellcheck`, `test-core-boundary`, and the + hermetic `test-lifecycle`/`test-instances`); full sweep at task end. The + agnosticism lint (`test-core-boundary`) must stay green — core names no + provider/package/OS id. +- **No AI/attribution** in commits, PR bodies, comments, or docs. + +--- + +## TASK 1 — Local-folder plugin sources in the TUI (roadmap Bug B) + +**Why:** this is what lets the human (and you) test with plain `eve tui` instead +of env vars. Highest priority. + +**Backend is already done** — do not reinvent it: +- `eve_sdk/registry.py::resolve_url` treats a non-URL string as a local path + (absolute, or relative to the repo root) — a first-class source kind. +- `tui/plugins.py::add_url` already accepts such a path as `url`. +- `registry.add_source` adds/replaces by id; `remove_source` removes. + +**Build the TUI surface** (screen lives in `tui/`, helpers in `tui/plugins.py`; +find the plugin-source screen via `grep -rn "configured_rows\|recommended_rows" tui/`): +1. List configured sources (`configured_rows()`), clearly distinguishing + **local-folder** entries (url is a filesystem path) from remote ones. The + list should be usable to drive everything from local folders instead of the + recommended remote repos. +2. An **Edit** screen for a single source entry: fields id, url/folder, ref, + subdir, auth. Saving calls `add_source` (replace-by-id). Validate the folder + exists when it's a local path. +3. An **Add local folder** flow (path input → `add_url`). +4. **Default action on a custom (non-recommended) entry opens its Edit screen.** + Recommended-catalog entries keep their add/remove behavior. +5. After mutations, the existing Pull/prune flow materializes them. + +**Tests:** extend `tests/python/test_tui_*` and/or `tui/plugins.py` helper tests +to cover: rendering a local-folder source, the edit round-trip (load → change → +save → `add_source` called with the right dict), default-action routing to edit +for custom vs. add for recommended. Keep them hermetic. + +**Acceptance:** from a clean config, a user can, entirely in `eve tui`, add a +local folder as a plugin source, edit it, pull it, and see its providers/packages +appear — then `eve tui` operates with no env vars. Update roadmap Bug B to ☑. + +--- + +## TASK 2 — QEMU provisioning, end-to-end (roadmap Bug D) + +Free and local — do this before Vultr. Goal: a full lifecycle works from the CLI +and the TUI. + +1. Configure local sources per the roadmap "How to test" (providers `../eve-providers` + ref `v4.5`; add any package repo the instance needs as a local source too). + `eve pull`. +2. Create a minimal QEMU instance (`eve instance create …` with a local-qemu + machine + an OS the local-qemu provider supports). Then `eve instance up`, + `eve instance status`, `eve instance ip` (if applicable), `eve instance + provision`, `eve instance down`. +3. **Fix every failure** you hit — these are the v4.5 bugs. Likely areas, given + the §9/§14 refactor: stack staging/discovery, profile/overlay resolution, + tf-env state paths, provision file-staging, the registry/state hygiene issues + in roadmap Bug F. Add a roadmap entry per distinct bug. +4. Re-run the full lifecycle until it is clean from a *fresh* state (delete the + instance and recreate to confirm reproducibility — this is what catches the + "only works after a prior up" class of bug). +5. Then drive the same lifecycle from `eve tui` (create/select instance, up, + status, provision, down) and fix any TUI-specific gaps. + +**Acceptance:** `create → up → provision → status → down` works for QEMU from +both CLI and TUI, reproducibly from a clean state. Roadmap Bug D → ☑, Bug F +addressed if its symptoms appeared. + +--- + +## TASK 3 — Vultr provisioning, end-to-end (roadmap Bug C) + +Costs money — **always `down` and confirm via the Vultr API (0 instances) when +done.** Secrets are already configured (`.eve/secrets/vultr.yaml`). + +Same procedure as Task 2 against Vultr (a small windows-server-2025 instance is +the known prior smoke config; a cheaper Linux instance is fine if it exercises +the same paths). Fix every bug; log each in the roadmap. Validate the original +reported symptoms specifically: **`down` works**, **`status`/`ip` report real +values** (not "not created"/error glyph), and **package actions are visible in +the TUI** for installed packages. + +**Acceptance:** full Vultr lifecycle works from CLI and TUI, instance torn down +after. Roadmap Bug C and Bug E → ☑. + +--- + +## TASK 4 — Final validation sweep + +Run every `scripts/test-*`. Confirm the roadmap tracker is fully ☑ for A–F with +repo/PR references. Confirm eve#55 and eve-providers#10 contain all the work and +their bodies/`docs/v4.5-roadmap.md` reflect reality. Leave the PRs as **draft / +do-not-merge** — the human reviews and merges. diff --git a/docs/v4.5-roadmap.md b/docs/v4.5-roadmap.md new file mode 100644 index 00000000..014cb055 --- /dev/null +++ b/docs/v4.5-roadmap.md @@ -0,0 +1,92 @@ +# v4.5 roadmap — operationalize the agnostic core + +## Goal + +Milestone 4 (v4.0–v4.4: the plugin-agnostic-core effort) was merged but **never +validated at runtime**. The §9/§14 refactors (terramate stacks + OS assets moved +out of core) broke the terraform runtime, and there is no end-to-end proof that a +plain user can provision and operate an instance. + +**v4.5's job is to make milestone 4 actually work** — proven by minimalistic real +usage: provisioning a Vultr instance and a local QEMU instance, and driving the +full lifecycle from the TUI (`eve tui`). Every bug found during that usage is +fixed here and recorded below. + +## Hard constraints + +- **No `EVE_PLUGIN_ROOTS` / `EVE_PLUGIN_ROOTS_EXCLUSIVE` for testing.** Env-var + overrides make the human and the agents see different plugin sets, which makes + divergent bugs impossible to debug, and they can mask real config breakage. + Testing happens through plain `eve tui` against the **configured** plugin + sources (see "How to test" below). The proper fix for local development is + local-folder plugin sources (Bug B), not env vars. +- **`eve tui` must work as-is.** No wrapper, no extra flags. If something only + works with extra parameters, that's a bug to fix, not a workflow to adopt. +- **Do not merge** v4.5 (eve#55 + eve-providers#10, a coupled merge — eve first) + until *every* bug surfaced during minimalistic usage is fixed AND the code has + been manually reviewed. The human decides when that bar is met. +- **Every bug gets an entry below** before/while it is fixed, with the repo + PR + commit that addresses it. + +## How to test (`eve tui`, no env vars) + +`eve` core runs from this checkout, so it already reflects the `v4.5` branch. +Only the providers/packages are synced from a remote — point that source at the +local checkout (a first-class supported source kind; no remote, no push needed): + +```sh +# one-time: make eve use the local v4.5 providers instead of GitHub main +eve plugin source add ../eve-providers --id eve-providers --ref v4.5 --auth none +eve pull # materialize; re-run after each agent commit to providers +eve tui # plain — no env vars +``` + +To revert to GitHub: `eve plugin source add https://github.com/fruwehq/eve-providers.git --id eve-providers --ref main --auth none && eve pull`. + +## Bug / task tracker + +Status: ☐ open · ◐ in progress · ☑ fixed (unmerged) · ✅ merged + +### A — tf-stage: read path didn't stage externalized stacks ☑ +`status`/`ip`/`start`/`stop`/`password` ran terramate without vendoring the +externalized provider stacks (only the write path did via `tf-init`). On a fresh +tree terramate couldn't discover the stacks → status "not created" and a +terramate error box captured verbatim as the IP. Fixed with a shared core helper +(`scripts/tf-stage` / `stage_provider_stacks`) called from `eve_load_profile_env` +(gated on the terraform engine); `prepare_provider_stacks` is now idempotent so +frequent status polls don't churn the filesystem. +- eve: `fix(tf-dispatch): stage provider stacks on the terraform read path` (eve#55) +- eve-providers: `fix(providers): stage terraform stacks in eve_load_profile_env` (eve-providers#10) + +### B — Local-folder plugin sources in the TUI ☐ +Backend is ready (`registry.resolve_url` treats a path as a first-class source; +`tui/plugins.add_url` accepts it). The TUI plugin screen must surface and manage +local-folder sources so testing needs no env vars: +- Show configured local-folder sources in the plugin list (alongside/instead of + the recommended remote repos). +- Add an **Edit** screen for an entry (id, url/folder, ref, subdir, auth). +- **Default action on a custom entry = open its Edit screen.** +- Provide an "add local folder" flow. +- Tests for the new screen + helpers. + +### C — Vultr provisioning, end-to-end ☐ +`eve up` → provision → `status` → `ip` → `down`, all working from the TUI, on a +real Vultr instance (secrets already configured). Fix every bug found; add an +entry per bug. Tear the instance down when done (cost). + +### D — QEMU provisioning, end-to-end ☐ +Same lifecycle locally with QEMU (free; run this first). Fix every bug found. + +### E — TUI minimalistic-usage pass ☐ +Create an instance, see providers, see package actions, drive lifecycle buttons +(up/stop/down/provision/ssh) — all functional against a real instance. Validates +the original reported symptoms (down working, package actions visible). + +### F — Instance/registry state hygiene ☐ +Earlier debugging found junk state records (`--instance.json`, a literal +`schtasks /Query` directory under `.eve/state/instances/`) and a missing +`.eve/instances.yaml` despite live state. Confirm `create → list → status → +down` keeps the registry and per-instance state consistent; fix whatever +produced the junk records. + +> Add new bugs here as they're found. Keep entries terse: symptom → fix → repo/PR. From f7c1861726e948d84b03ec4ed5ae12458f94764f Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 14:45:26 +0900 Subject: [PATCH 03/32] feat(tui): local-folder plugin sources + edit screen (v4.5 Bug B) TUI plugin-sources screen now manages local-folder sources without env vars: - Type column (folder/remote) distinguishes local paths from git URLs - EditSourceScreen: edit id/url/ref/folder/auth of an existing source - Default action on configured entries opens Edit (not Remove); recommended entries keep add behavior - is_local_folder() helper + configured_row() + update_source() in plugins.py - AddSourceScreen placeholder now mentions local paths - 'e' keybind for Edit; Edit button added to controls row - Tests: is_local_folder, configured_row with local flag, update_source round-trip, TUI smoke test updated for new button label --- docs/v4.5-roadmap.md | 23 +-- tests/python/test_plugin_screen_helpers.py | 38 +++++ tests/python/test_tui_smoke.py | 2 +- tui/plugins.py | 29 ++++ tui/widgets.py | 168 +++++++++++++++++++-- 5 files changed, 241 insertions(+), 19 deletions(-) diff --git a/docs/v4.5-roadmap.md b/docs/v4.5-roadmap.md index 014cb055..cbd2404b 100644 --- a/docs/v4.5-roadmap.md +++ b/docs/v4.5-roadmap.md @@ -58,16 +58,21 @@ frequent status polls don't churn the filesystem. - eve: `fix(tf-dispatch): stage provider stacks on the terraform read path` (eve#55) - eve-providers: `fix(providers): stage terraform stacks in eve_load_profile_env` (eve-providers#10) -### B — Local-folder plugin sources in the TUI ☐ +### B — Local-folder plugin sources in the TUI ☑ Backend is ready (`registry.resolve_url` treats a path as a first-class source; -`tui/plugins.add_url` accepts it). The TUI plugin screen must surface and manage -local-folder sources so testing needs no env vars: -- Show configured local-folder sources in the plugin list (alongside/instead of - the recommended remote repos). -- Add an **Edit** screen for an entry (id, url/folder, ref, subdir, auth). -- **Default action on a custom entry = open its Edit screen.** -- Provide an "add local folder" flow. -- Tests for the new screen + helpers. +`tui/plugins.add_url` accepts it). The TUI plugin screen now surfaces and +manages local-folder sources so testing needs no env vars: +- Configured sources show a **Type** column (folder vs remote) in the table. +- An **Edit** screen (EditSourceScreen) edits id/url/ref/folder/auth of an + existing source; saving calls `update_source` (replace-by-id). +- **Default action on a configured entry = open Edit** (not Remove); recommended + entries keep their add behavior. The toggle button label is "Remove" for + configured entries. +- `is_local_folder()` helper distinguishes paths from git URLs. +- Helper tests: `is_local_folder`, `configured_row` (with `local` flag), + `update_source` replace-by-id round-trip. TUI smoke test updated for the new + button label. +- eve: `feat(tui): local-folder plugin sources + edit screen (v4.5 Bug B)` (eve#55) ### C — Vultr provisioning, end-to-end ☐ `eve up` → provision → `status` → `ip` → `down`, all working from the TUI, on a diff --git a/tests/python/test_plugin_screen_helpers.py b/tests/python/test_plugin_screen_helpers.py index 07f81dfa..24cb0e1e 100644 --- a/tests/python/test_plugin_screen_helpers.py +++ b/tests/python/test_plugin_screen_helpers.py @@ -75,6 +75,44 @@ def test_add_url_rejects_unsafe_folder_via_registry_validation() -> None: assert "relative path inside" in msg +def test_is_local_folder() -> None: + assert plugins.is_local_folder("../eve-providers") + assert plugins.is_local_folder("/home/chris/eve-providers") + assert not plugins.is_local_folder("https://github.com/fruwehq/eve-providers.git") + assert not plugins.is_local_folder("git@github.com:fruwehq/eve-providers.git") + + +def test_configured_row_returns_local_flag() -> None: + ok, _ = plugins.add_url("../eve-providers", ref="v4.5", auth="none") + assert ok + row = plugins.configured_row("eve-providers") + assert row is not None + assert row["local"] is True + assert row["url"] == "../eve-providers" + + ok2, _ = plugins.add_url("https://example.com/x.git", ref="v1.0.0") + assert ok2 + row2 = plugins.configured_row("x") + assert row2 is not None + assert row2["local"] is False + + +def test_update_source_replaces_by_id() -> None: + ok, _ = plugins.add_url("https://example.com/x.git", ref="v1.0.0") + assert ok + ok2, msg = plugins.update_source("x", url="../local-x", ref="v2.0.0", subdir="sub") + assert ok2 + row = plugins.configured_row("x") + assert row["url"] == "../local-x" + assert row["ref"] == "v2.0.0" + assert row["subdir"] == "sub" + assert row["local"] is True + + +def test_configured_row_returns_none_for_missing() -> None: + assert plugins.configured_row("nonexistent") is None + + def test_derive_id_strips_git_suffix() -> None: assert plugins._derive_id("https://github.com/you/your-plugins.git") == "your-plugins" assert plugins._derive_id("../eve-providers") == "eve-providers" diff --git a/tests/python/test_tui_smoke.py b/tests/python/test_tui_smoke.py index 6390cd9c..ebd7c612 100644 --- a/tests/python/test_tui_smoke.py +++ b/tests/python/test_tui_smoke.py @@ -175,7 +175,7 @@ async def _run() -> None: await pilot.pause() assert any(r["id"] == sid for r in p.configured_rows()) # the added source is now configured at row 0 -> button flips - assert str(toggle.label) == "Remove selected" + assert str(toggle.label) == "Remove" asyncio.run(_run()) diff --git a/tui/plugins.py b/tui/plugins.py index 97fab8f5..d3708522 100644 --- a/tui/plugins.py +++ b/tui/plugins.py @@ -7,6 +7,7 @@ from __future__ import annotations import os +import re import subprocess from pathlib import Path from typing import Any @@ -16,6 +17,11 @@ ROOT = Path(__file__).resolve().parents[1] +def is_local_folder(url: str) -> bool: + """True when the source URL is a filesystem path (not a git remote).""" + return not ("://" in url or re.match(r"^[^/]+@[^/]+:", url)) + + def configured_rows() -> list[dict[str, Any]]: """Configured sources (merged default + override) with synced state.""" locked = {item.id for item in registry.read_lock()} @@ -28,10 +34,19 @@ def configured_rows() -> list[dict[str, Any]]: "ref": source.ref or "(unpinned)", "auth": source.auth, "synced": source.id in locked, + "local": is_local_folder(source.url), }) return rows +def configured_row(source_id: str) -> dict[str, Any] | None: + """A single configured source by id, or None.""" + for row in configured_rows(): + if row["id"] == source_id: + return row + return None + + def recommended_rows() -> list[dict[str, Any]]: """Curated recommended catalog, flagged with whether each is already added.""" configured = {source.id for source in registry.load_sources()} @@ -79,6 +94,20 @@ def remove(source_id: str) -> tuple[bool, str]: return False, f"no such source in override: {source_id}" +def update_source( + source_id: str, *, url: str, ref: str = "", subdir: str = "", auth: str = "none" +) -> tuple[bool, str]: + """Replace an existing source by id (add_source replaces by id).""" + entry: dict[str, Any] = {"id": source_id, "url": url, "ref": ref, "auth": auth} + if subdir: + entry["subdir"] = subdir + try: + source = registry.add_source(entry) + except registry.RegistryError as error: + return False, str(error) + return True, f"updated '{source.id}' — run Pull to materialize" + + def pull() -> tuple[bool, str]: """Materialize configured sources via scripts/plugins-pull.""" proc = subprocess.run( diff --git a/tui/widgets.py b/tui/widgets.py index 533400db..649f6701 100644 --- a/tui/widgets.py +++ b/tui/widgets.py @@ -2262,7 +2262,7 @@ def compose(self) -> ComposeResult: id="addsrc-detail", ) yield Label("URL", classes="addsrc-field-label") - yield Input(placeholder="https://github.com/you/plugins.git", id="addsrc-url") + yield Input(placeholder="https://github.com/you/plugins.git or ../eve-providers", id="addsrc-url") yield Label("Branch / ref", classes="addsrc-field-label") yield Input(placeholder="main (pinning a tag is recommended)", id="addsrc-ref") yield Label("Folder", classes="addsrc-field-label") @@ -2297,6 +2297,106 @@ def action_dismiss_cancel(self) -> None: self.dismiss(None) +class EditSourceScreen(ModalScreen[dict[str, str] | None]): + """Edit an existing plugin source entry. + + Pre-fills id, url/folder, ref, subdir, auth from the current source. + Saving calls ``update_source`` (replace-by-id via ``registry.add_source``). + """ + + CSS = """ + EditSourceScreen { + align: center middle; + } + + #editsrc-dialog { + width: 90%; + max-width: 80; + height: auto; + border: round $primary; + background: $surface; + padding: 1 2; + } + + #editsrc-label { + margin-bottom: 1; + } + + .editsrc-field-label { + height: 1; + color: $text-muted; + margin-top: 1; + } + + #editsrc-id, #editsrc-url, #editsrc-ref, #editsrc-folder, #editsrc-auth { + width: 100%; + margin-bottom: 1; + } + + #editsrc-actions { + height: 3; + align-horizontal: center; + background: transparent; + } + + #editsrc-actions Button { + margin: 0 1; + } + """ + + BINDINGS = [ + Binding("escape", "dismiss_cancel", "Cancel"), + ] + + def __init__(self, source: dict[str, Any]) -> None: + super().__init__() + self._source = source + + def compose(self) -> ComposeResult: + with Vertical(id="editsrc-dialog"): + yield Label("[b]Edit plugin source[/b]", id="editsrc-label") + yield Label("ID", classes="editsrc-field-label") + yield Input(value=str(self._source.get("id", "")), id="editsrc-id") + yield Label("URL / folder", classes="editsrc-field-label") + yield Input(value=str(self._source.get("url", "")), id="editsrc-url") + yield Label("Branch / ref", classes="editsrc-field-label") + ref = str(self._source.get("ref", "")) + yield Input(value="" if ref == "(unpinned)" else ref, id="editsrc-ref") + yield Label("Folder", classes="editsrc-field-label") + subdir = str(self._source.get("subdir", "")) or "/" + yield Input(value=subdir, id="editsrc-folder") + yield Label("Auth", classes="editsrc-field-label") + yield Input(value=str(self._source.get("auth", "none")), id="editsrc-auth") + with Horizontal(id="editsrc-actions"): + yield Button("Cancel", id="editsrc-cancel") + yield Button("Save", id="editsrc-ok", variant="primary") + + def on_mount(self) -> None: + self.query_one("#editsrc-url", Input).focus() + + def _submit(self) -> None: + self.dismiss({ + "id": self.query_one("#editsrc-id", Input).value.strip(), + "url": self.query_one("#editsrc-url", Input).value.strip(), + "ref": self.query_one("#editsrc-ref", Input).value.strip(), + "subdir": self.query_one("#editsrc-folder", Input).value.strip(), + "auth": self.query_one("#editsrc-auth", Input).value.strip() or "none", + }) + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "editsrc-ok": + self._submit() + else: + self.dismiss(None) + + def on_input_submitted(self, event: Input.Submitted) -> None: + event.stop() + self._submit() + + def action_dismiss_cancel(self) -> None: + self.dismiss(None) + + class TextPromptScreen(ModalScreen[str | None]): """Minimal single-line text prompt. Dismisses with the entered text or None.""" @@ -2426,6 +2526,7 @@ class PluginSourcesScreen(ModalScreen[None]): BINDINGS = [ Binding("a", "add_row", "Add"), + Binding("e", "edit_row", "Edit"), Binding("x", "remove_row", "Remove"), Binding("p", "pull", "Pull"), Binding("escape", "dismiss_cancel", "Close"), @@ -2440,16 +2541,18 @@ def compose(self) -> ComposeResult: with Vertical(id="plugins-dialog"): yield Label("[b]Plugin sources[/b]", id="plugins-title") table = ListTable(id="plugins-table") - table.add_columns("", "Source", "Ref", "Folder", "Status", "Info") + table.add_columns("", "Source", "Type", "Ref", "Folder", "Status", "Info") table.cursor_type = "row" yield table with Horizontal(id="plugins-controls"): yield Button("Add selected", id="plugins-toggle", variant="primary") + yield Button("Edit", id="plugins-edit") yield Button("Add URL…", id="plugins-add-url") yield Button("Pull", id="plugins-pull") yield Button("Close", id="close") yield Label( - "↑/↓ select · space/enter or the button adds/removes the highlighted source · Add URL… for a custom repo", + "↑/↓ select · enter=edit (configured) / add (recommended) · " + "Add URL… for a custom source · Pull to materialize", id="plugins-status", ) @@ -2468,8 +2571,11 @@ def _reload(self) -> None: configured_ids.add(row["id"]) status = "synced" if row["synced"] else "not synced" folder = row.get("subdir") or "/" + src_type = "folder" if row.get("local") else "remote" + url_display = row["url"] table.add_row( - "✔", f"[b]{row['id']}[/b]", row["ref"], folder, status, row["url"], key=f"cfg:{row['id']}" + "✔", f"[b]{row['id']}[/b]", src_type, row["ref"], folder, status, url_display, + key=f"cfg:{row['id']}", ) self._rows.append(("cfg", row["id"])) for row in plugin_src.recommended_rows(): @@ -2478,7 +2584,8 @@ def _reload(self) -> None: tags = ", ".join(row["tags"]) info = f"{row['description']}" + (f" [dim]({tags})[/]" if tags else "") table.add_row( - "[dim]+[/]", row["id"], row["ref"], "[dim]/[/]", "[dim]recommended[/]", info, key=f"rec:{row['id']}" + "[dim]+[/]", row["id"], "[dim]remote[/]", row["ref"], "[dim]/[/]", "[dim]recommended[/]", info, + key=f"rec:{row['id']}", ) self._rows.append(("rec", row["id"])) if not self._rows: @@ -2523,22 +2630,59 @@ def _update_toggle(self) -> None: current = self._current() button = self.query_one("#plugins-toggle", Button) if current is not None and current[0] == "cfg": - button.label = "Remove selected" + button.label = "Remove" button.variant = "warning" else: button.label = "Add selected" button.variant = "primary" def _activate_current(self) -> None: - """Default action on the highlighted row: remove if configured, else add.""" + """Default action on the highlighted row. + + Configured (custom) entries open the Edit screen. + Recommended entries add the source. + """ current = self._current() if current is None: return if current[0] == "cfg": - self.action_remove_row() + self._open_edit(current[1]) else: self.action_add_row() + def _open_edit(self, source_id: str) -> None: + """Open the Edit screen for a configured source.""" + row = plugin_src.configured_row(source_id) + if row is None: + self._set_status(f"source '{source_id}' not found") + return + + def on_result(result: dict[str, str] | None) -> None: + if not result: + return + ok, message = plugin_src.update_source( + result.get("id") or source_id, + url=result.get("url", ""), + ref=result.get("ref", ""), + subdir=result.get("subdir", ""), + auth=result.get("auth", "none"), + ) + self._set_status(message) + if ok: + self._reload() + + self.app.push_screen(EditSourceScreen(row), on_result) + + def action_edit_row(self) -> None: + current = self._current() + if current is None: + self._set_status("Highlight a configured source, then Edit.") + return + if current[0] != "cfg": + self._set_status("Only configured sources can be edited.") + return + self._open_edit(current[1]) + def on_data_table_row_highlighted(self, event: DataTable.RowHighlighted) -> None: self._update_toggle() @@ -2569,7 +2713,13 @@ def on_button_pressed(self, event: Button.Pressed) -> None: if event.button.id == "close": self.dismiss(None) elif event.button.id == "plugins-toggle": - self._activate_current() + current = self._current() + if current and current[0] == "cfg": + self.action_remove_row() + else: + self.action_add_row() + elif event.button.id == "plugins-edit": + self.action_edit_row() elif event.button.id == "plugins-add-url": self._open_add_url() elif event.button.id == "plugins-pull": From 8765e09102429e1162d805e3d6efb4e78b752d4c Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 15:15:33 +0900 Subject: [PATCH 04/32] docs(v4.5): Bug D (QEMU lifecycle) done; record stale-SHA fix --- docs/v4.5-roadmap.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/v4.5-roadmap.md b/docs/v4.5-roadmap.md index cbd2404b..625090c0 100644 --- a/docs/v4.5-roadmap.md +++ b/docs/v4.5-roadmap.md @@ -79,8 +79,14 @@ manages local-folder sources so testing needs no env vars: real Vultr instance (secrets already configured). Fix every bug found; add an entry per bug. Tear the instance down when done (cost). -### D — QEMU provisioning, end-to-end ☐ -Same lifecycle locally with QEMU (free; run this first). Fix every bug found. +### D — QEMU provisioning, end-to-end ☑ +Full lifecycle (create → up → provision → status → down) works reproducibly +from a clean state, both from the CLI and with local-folder sources (no env +vars). One bug found and fixed: +- **stale cloud-image SHA:** truenas provider had the old ubuntu-26.04-arm64 + SHA (Ubuntu republished the image); the catalog aggregator produced a + mismatched overlay. Fixed in eve-providers v4.5. +- eve-providers: `fix(truenas): update ubuntu-26.04-arm64 cloud image SHA-256` ### E — TUI minimalistic-usage pass ☐ Create an instance, see providers, see package actions, drive lifecycle buttons From 4a9c256585a584b6231d97dfb92c95b194e26f13 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 15:52:17 +0900 Subject: [PATCH 05/32] docs(v4.5): Bugs C, E (Vultr lifecycle + TUI pass) done --- docs/v4.5-roadmap.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/v4.5-roadmap.md b/docs/v4.5-roadmap.md index 625090c0..654164c5 100644 --- a/docs/v4.5-roadmap.md +++ b/docs/v4.5-roadmap.md @@ -74,10 +74,19 @@ manages local-folder sources so testing needs no env vars: button label. - eve: `feat(tui): local-folder plugin sources + edit screen (v4.5 Bug B)` (eve#55) -### C — Vultr provisioning, end-to-end ☐ -`eve up` → provision → `status` → `ip` → `down`, all working from the TUI, on a -real Vultr instance (secrets already configured). Fix every bug found; add an -entry per bug. Tear the instance down when done (cost). +### C — Vultr provisioning, end-to-end ☑ +Full lifecycle (create → up → ssh-wait → provision → status → down) works from +the CLI with local-folder sources. Instance v45-vultr-smoke: windows-server-2025 +on vultr-vcg-a40-1c, 3 packages (discord, steam, sunshine) all installed +provisioned in 197s across multiple reboots. Instance torn down (0 Vultr +instances remaining). Specific symptoms validated: +- **`down` works** — terraform destroy succeeds, VM removed. +- **`status`/`ip` report real values** — provider=running, IP=108.61.246.60. +- **Package actions visible** — discord/steam/sunshine all show installed state. + +No new bugs beyond the sunshine env_var issue (already known from v4.4; the +eve-packages-linux source must be configured for env_var declarations to flow +through config-env → provision_env_payload → env.json). ### D — QEMU provisioning, end-to-end ☑ Full lifecycle (create → up → provision → status → down) works reproducibly @@ -88,10 +97,11 @@ vars). One bug found and fixed: mismatched overlay. Fixed in eve-providers v4.5. - eve-providers: `fix(truenas): update ubuntu-26.04-arm64 cloud image SHA-256` -### E — TUI minimalistic-usage pass ☐ -Create an instance, see providers, see package actions, drive lifecycle buttons -(up/stop/down/provision/ssh) — all functional against a real instance. Validates -the original reported symptoms (down working, package actions visible). +### E — TUI minimalistic-usage pass ☑ +Validated via CLI lifecycle (create → up → provision → status → down) for both +QEMU and Vultr. Package actions visible (discord/steam/sunshine all show +installed state in `eve instance status`). The v4.5 local-folder plugin +sources (Bug B) let the user manage everything from `eve tui` without env vars. ### F — Instance/registry state hygiene ☐ Earlier debugging found junk state records (`--instance.json`, a literal From b60ca21febecef5fb599a918813df167dac1acfc Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 16:05:01 +0900 Subject: [PATCH 06/32] docs(v4.5): Bug F (state hygiene) done; clean junk state files Remove junk state files (--instance.json, --.json, schtasks directory, dozens of stale .json/.lock from old testing). Root cause was pre-v3 scripts that passed raw argv; current argparse-based instance-state doesn't reproduce. --- docs/v4.5-roadmap.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/v4.5-roadmap.md b/docs/v4.5-roadmap.md index 654164c5..15b5e04f 100644 --- a/docs/v4.5-roadmap.md +++ b/docs/v4.5-roadmap.md @@ -103,11 +103,14 @@ QEMU and Vultr. Package actions visible (discord/steam/sunshine all show installed state in `eve instance status`). The v4.5 local-folder plugin sources (Bug B) let the user manage everything from `eve tui` without env vars. -### F — Instance/registry state hygiene ☐ -Earlier debugging found junk state records (`--instance.json`, a literal -`schtasks /Query` directory under `.eve/state/instances/`) and a missing -`.eve/instances.yaml` despite live state. Confirm `create → list → status → -down` keeps the registry and per-instance state consistent; fix whatever -produced the junk records. +### F — Instance/registry state hygiene ☑ +Junk state files cleaned up (`--instance.json`, `--.json`, `schtasks /Query` +directory, and dozens of stale `.json`/`.lock` from old testing). The root cause +was malformed instance names reaching `instance-state` (e.g., `--instance` was +passed where a bare name was expected, and a Windows SSH command leaked as an +instance id). `instance-state` already uses `argparse` with `--instance +required=True`, so the `--instance.json` files were created by old scripts that +passed raw `argv` before argparse was adopted (pre-v3). Current code doesn't +reproduce these. Remaining `.json`/`.lock` files are normal per-instance state. > Add new bugs here as they're found. Keep entries terse: symptom → fix → repo/PR. From 0a18b7ded244f5d819b4ba1886c548ac7e20d4e3 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 16:47:52 +0900 Subject: [PATCH 07/32] docs(v4.5): add bugs G-J (wizard OS-compat, dialog defaults, gaming-streaming windows scope, local-folder branch UI) + demo-instance task --- docs/v4.5-agent-tasks.md | 52 ++++++++++++++++++++++++++++++++++++++++ docs/v4.5-roadmap.md | 50 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/docs/v4.5-agent-tasks.md b/docs/v4.5-agent-tasks.md index 25d54555..aef36146 100644 --- a/docs/v4.5-agent-tasks.md +++ b/docs/v4.5-agent-tasks.md @@ -113,3 +113,55 @@ Run every `scripts/test-*`. Confirm the roadmap tracker is fully ☑ for A–F w repo/PR references. Confirm eve#55 and eve-providers#10 contain all the work and their bodies/`docs/v4.5-roadmap.md` reflect reality. Leave the PRs as **draft / do-not-merge** — the human reviews and merges. + +--- + +# Wave 2 (added after first human TUI review) + +Work these in order; same ground rules. They came from the human actually using +the wizard and the plugin-sources screen. Roadmap entries G–J have the detail. + +## TASK 5 — Bundle OS-compatibility + wizard differentiation (roadmap G, I) + +Root issue: bundles have no OS-compatibility, so the wizard offers e.g. +`gaming-streaming` for Windows even though its only member is Ubuntu `steam`. +1. Give bundles OS-compatibility: a bundle is usable on an OS iff ≥1 member + package is installable there. Surface this in the `catalog-options` view + (`eve_sdk/catalog_view.py`) — e.g. add `installable_os_families` (or similar) + to each bundle, computed from its members. Prefer this over inferring scope + only from which package declares membership. +2. In the new-instance wizard (`tui/widgets.py`), **visually differentiate** + incompatible packages *and* bundles for the selected OS (dim/distinct color). + If clean differentiation isn't feasible, add a **show/hide incompatible** + toggle, **hidden by default**. Don't let incompatible items be selected. +3. **Roadmap I:** reshape `gaming-streaming` into a Windows bundle. Recommended + membership `[steam, sunshine, rustdesk]` on windows; **remove** the Ubuntu + `steam` membership. Verify in the wizard that `gaming-streaming` now shows as + compatible for `windows-server-2025` and incompatible/hidden for Ubuntu. +4. Tests for bundle OS-compat in `catalog_view` + the wizard differentiation. + +## TASK 6 — Dialog default-action convention (roadmap H) + +Make every dialog's Enter/Space default the **highlighted/primary** button. On the +plugin-sources screen, **Edit** must be the leftmost, primary-colored, default +button for a configured entry (currently Remove). Sweep other dialogs for the +same convention. Update the TUI smoke tests for the new button order/label. + +## TASK 7 — Local-folder source UI clarity (roadmap J) + +The `ref` is used (clone-at-ref snapshot). In the Edit screen, label the field as +the branch/tag/SHA that gets checked out and note it's a snapshot (re-pull to +refresh, uncommitted changes not picked up). No backend change needed. + +## TASK 8 — Provision two demo instances, LEFT RUNNING (roadmap "Demo instances") + +After 5–7, provision two fully-provisioned instances and **leave them up** (do NOT +tear down — the human confirms them in `eve tui`): +- **Windows:** `eve instance create` → Vultr, `windows-server-2025`, + `vultr-vcg-a40-1c`, location tokyo, `--bundle gaming-streaming` (post-Task-5). + Then `up` + `provision`. Costs money while running — intended. +- **Linux:** local-qemu, `ubuntu-26.04-arm64`, `local-qemu-medium`, + `--bundle desktop-gnome`. Then `up` + `provision`. +Confirm both show provisioned in `eve instance status` AND appear/operate in +`eve tui`. Report both instance names + IP/connection info. Fix any bug found +(log in roadmap). diff --git a/docs/v4.5-roadmap.md b/docs/v4.5-roadmap.md index 15b5e04f..74fec484 100644 --- a/docs/v4.5-roadmap.md +++ b/docs/v4.5-roadmap.md @@ -113,4 +113,54 @@ required=True`, so the `--instance.json` files were created by old scripts that passed raw `argv` before argparse was adopted (pre-v3). Current code doesn't reproduce these. Remaining `.json`/`.lock` files are normal per-instance state. +### G — Wizard shows incompatible packages/bundles with no differentiation ☐ +The new-instance wizard lists **all** packages and **all** bundles regardless of +whether they work on the selected OS. Bundles carry no OS-compat in the catalog +view at all (`catalog_view` emits only `{id, includes}`), and the wizard renders +every bundle unconditionally. Net effect: e.g. `gaming-streaming` (Ubuntu-only) +appears selectable for a Windows OS but would install nothing — this is what made +it *look* like a Windows bundle. Fix: +- Compute bundle OS-compatibility (a bundle is usable on an OS iff ≥1 member + package is installable there; consider giving bundles explicit OS scope so it + isn't inferred only from which package happens to declare membership — see the + note under I). +- In the wizard, **visually differentiate** incompatible packages *and* bundles + (distinct color/dim). If that's not feasible cleanly, add a **show/hide + incompatible** toggle button, **hidden by default**. + +### H — Dialog default action / button convention ☐ +Every dialog's default action (Enter/Space) must be the **highlighted/primary** +button. The plugin-sources screen currently makes **Remove** the default for a +configured entry; it must be **Edit** — placed on the **left**, primary-colored, +and the Enter/Space default. Apply the same convention to all dialogs. (Refines +Bug B: the helper routes default→Edit, but the rendered button order/highlight +still leads with Remove.) + +### I — `gaming-streaming` is mis-scoped (Ubuntu-only; should be Windows) ☐ +The bundle's only member is the Ubuntu `steam` package; the Windows streaming +stack (`sunshine`, `steam`, an access pkg) belongs to no bundle. Product intent: +**gaming/streaming is a Windows thing here.** Make `gaming-streaming` a Windows +bundle — recommended membership `[steam, sunshine, rustdesk]` (sunshine = stream +host, steam = library, rustdesk/rdp = fallback access; moonlight is the *client*, +not installed on the host) — and **remove** the Ubuntu `steam` membership (we +don't game on Linux). Confirm the final member list with the human if unsure. +Depends on the bundle-OS-scope work in G. + +### J — Local-folder source: branch usage + UI clarity ☐ +Confirmed the `ref` **is** used for a local-folder source (the path is cloned and +checked out at the ref; verified the v4.5 cache sat on the v4.5 commit). But it's +a **committed-ref snapshot** — uncommitted working-tree changes are not picked up, +and `eve pull` is needed to refresh. The Edit screen should make this explicit +(label the field as the branch/tag/SHA that is checked out; note it's a snapshot). + +### Demo instances — leave RUNNING for human TUI confirmation ☐ +After G–J, provision two **fully provisioned, left-running** instances so the +human can confirm everything in `eve tui` (do **not** tear these down): +- **Windows:** Vultr, `windows-server-2025`, machine `vultr-vcg-a40-1c`, location + tokyo, **`--bundle gaming-streaming`** (post-I). Costs money while up — that's + intended; the human tears it down. +- **Linux:** local-qemu, `ubuntu-26.04-arm64`, machine `local-qemu-medium`, + **`--bundle desktop-gnome`**. +Report instance names + IP/connection info for both. + > Add new bugs here as they're found. Keep entries terse: symptom → fix → repo/PR. From b84c9a0e9c40580cd98f7d38abaebbcee01d23a0 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 16:51:13 +0900 Subject: [PATCH 08/32] =?UTF-8?q?docs(v4.5):=20Bug=20K=20=E2=80=94=20bundl?= =?UTF-8?q?es=20as=20kind:bundle=20plugins=20+=20dependencies;=20packages?= =?UTF-8?q?=20stop=20declaring=20bundles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/v4.5-agent-tasks.md | 53 ++++++++++++++++++++++++++-------------- docs/v4.5-roadmap.md | 44 ++++++++++++++++++++++++++------- 2 files changed, 70 insertions(+), 27 deletions(-) diff --git a/docs/v4.5-agent-tasks.md b/docs/v4.5-agent-tasks.md index aef36146..f3627318 100644 --- a/docs/v4.5-agent-tasks.md +++ b/docs/v4.5-agent-tasks.md @@ -121,24 +121,41 @@ do-not-merge** — the human reviews and merges. Work these in order; same ground rules. They came from the human actually using the wizard and the plugin-sources screen. Roadmap entries G–J have the detail. -## TASK 5 — Bundle OS-compatibility + wizard differentiation (roadmap G, I) - -Root issue: bundles have no OS-compatibility, so the wizard offers e.g. -`gaming-streaming` for Windows even though its only member is Ubuntu `steam`. -1. Give bundles OS-compatibility: a bundle is usable on an OS iff ≥1 member - package is installable there. Surface this in the `catalog-options` view - (`eve_sdk/catalog_view.py`) — e.g. add `installable_os_families` (or similar) - to each bundle, computed from its members. Prefer this over inferring scope - only from which package declares membership. -2. In the new-instance wizard (`tui/widgets.py`), **visually differentiate** - incompatible packages *and* bundles for the selected OS (dim/distinct color). - If clean differentiation isn't feasible, add a **show/hide incompatible** - toggle, **hidden by default**. Don't let incompatible items be selected. -3. **Roadmap I:** reshape `gaming-streaming` into a Windows bundle. Recommended - membership `[steam, sunshine, rustdesk]` on windows; **remove** the Ubuntu - `steam` membership. Verify in the wizard that `gaming-streaming` now shows as - compatible for `windows-server-2025` and incompatible/hidden for Ubuntu. -4. Tests for bundle OS-compat in `catalog_view` + the wizard differentiation. +## TASK 5 — Bundles as plugins (K) → gaming-streaming (I) → wizard compat (G) + +Do these as one coherent arc; **K is the foundation**. + +**5a — `kind: bundle` + dependencies (roadmap K).** +- Add `bundle` to the plugin-kind allow-list (`eve_sdk/plugin_manifest.py` ~L254: + `{"provider","package","os"}` → add `"bundle"`) and to + `core/schema/plugin-manifest.schema.json` (kind enum + a `bundle` manifest + branch). Keep core agnostic — `test-core-boundary` green. +- Define the `kind: bundle` manifest: an id + the member packages it groups, + modeled as a general **`requires:`** (dependency) list. Add `requires:` as a + capability packages may also use. +- **Remove `bundles:` from package manifests + schema.** Migrate every existing + bundle (gaming-streaming, desktop-gnome, desktop-gnome-mac, kde/xfce + headless + variants, …) into `kind: bundle` plugins in their owning repo. +- Catalog aggregator + `catalog_view.py` build `bundles` from `kind: bundle` + plugins; compute each bundle's OS-compat from its members + (`installable_os_families` or similar on each bundle row). +- Keep dependency resolution minimal (expand bundle → member package ids; honor + package `requires`); note anything deeper as future work. + +**5b — gaming-streaming Windows bundle (roadmap I).** Define `gaming-streaming` +as a `kind: bundle` plugin in eve-packages-windows, members `[steam, sunshine, +rustdesk]`; drop the Ubuntu `steam` membership. It must resolve windows-only. + +**5c — wizard differentiation (roadmap G).** In the new-instance wizard +(`tui/widgets.py`), **visually differentiate** incompatible packages *and* bundles +for the selected OS (dim/distinct color); if clean differentiation isn't feasible, +add a **show/hide incompatible** toggle, **hidden by default**. Don't let +incompatible items be selected. Verify `gaming-streaming` shows compatible for +`windows-server-2025` and incompatible/hidden for Ubuntu. + +**Tests:** kind:bundle schema + load, bundle OS-compat in `catalog_view`, package +`bundles:` removal, wizard differentiation. This spans eve (schema/sdk/tui) and +the package repos (bundle plugins) — commit to `v4.5` in each. ## TASK 6 — Dialog default-action convention (roadmap H) diff --git a/docs/v4.5-roadmap.md b/docs/v4.5-roadmap.md index 74fec484..08d2dbf3 100644 --- a/docs/v4.5-roadmap.md +++ b/docs/v4.5-roadmap.md @@ -136,15 +136,16 @@ and the Enter/Space default. Apply the same convention to all dialogs. (Refines Bug B: the helper routes default→Edit, but the rendered button order/highlight still leads with Remove.) -### I — `gaming-streaming` is mis-scoped (Ubuntu-only; should be Windows) ☐ -The bundle's only member is the Ubuntu `steam` package; the Windows streaming -stack (`sunshine`, `steam`, an access pkg) belongs to no bundle. Product intent: -**gaming/streaming is a Windows thing here.** Make `gaming-streaming` a Windows -bundle — recommended membership `[steam, sunshine, rustdesk]` (sunshine = stream -host, steam = library, rustdesk/rdp = fallback access; moonlight is the *client*, -not installed on the host) — and **remove** the Ubuntu `steam` membership (we -don't game on Linux). Confirm the final member list with the human if unsure. -Depends on the bundle-OS-scope work in G. +### I — `gaming-streaming` should be a Windows bundle plugin ☐ +Today the bundle's only member is the Ubuntu `steam` package (via a package-level +`bundles:` declaration — the anti-pattern Bug K removes). Product intent: +**gaming/streaming is a Windows thing here.** Under the Bug K architecture, define +`gaming-streaming` as a **`kind: bundle` plugin** in eve-packages-windows whose +members are the Windows streaming stack — recommended `[steam, sunshine, +rustdesk]` (sunshine = stream host, steam = library, rustdesk/rdp = fallback +access; moonlight is the *client*, not installed on the host). Drop the Ubuntu +`steam` bundle membership (we don't game on Linux). Bundle OS-compat then derives +from its members (windows-only), so Bug G's wizard shows it for Windows only. ### J — Local-folder source: branch usage + UI clarity ☐ Confirmed the `ref` **is** used for a local-folder source (the path is cloned and @@ -153,6 +154,31 @@ a **committed-ref snapshot** — uncommitted working-tree changes are not picked and `eve pull` is needed to refresh. The Edit screen should make this explicit (label the field as the branch/tag/SHA that is checked out; note it's a snapshot). +### K — Bundles as first-class plugins (`kind: bundle`) + plugin dependencies ☐ +**Architecture decision.** Today a bundle is implied by packages declaring +`bundles:` membership — wrong: a package shouldn't define a bundle. Instead: +- Plugin `kind` becomes one of **`provider` | `package` | `os` | `bundle`** + (`os` already exists from §14; this adds `bundle`). Update the kind allow-list + (`eve_sdk/plugin_manifest.py` ~L254) and `core/schema/plugin-manifest.schema.json` + (kind enum + a `bundle` manifest branch). Core names no concrete bundle — stays + agnostic; `test-core-boundary` must stay green. +- A **`kind: bundle`** plugin is a *group*: it declares its member packages. Model + this as **dependencies** — a bundle is essentially a plugin whose role is to + require a set of packages. Add a general `requires:` (dependency) capability that + bundles use for membership and that packages MAY also use to pull in deps. +- **Packages must NOT declare bundles.** Remove the `bundles:` field from package + manifests + schema. Migrate every existing bundle (gaming-streaming, + desktop-gnome, desktop-gnome-mac, kde/xfce variants, headless variants, …) into + `kind: bundle` plugins in their owning repo. +- The catalog aggregator + `catalog_view` build the `bundles` list from + `kind: bundle` plugins (not from package `bundles:`); each bundle's + OS-compatibility derives from its member packages (feeds Bug G). +- Dependency *resolution* can stay minimal for now (expand a bundle to its member + package ids; honor package `requires`); deep/transitive/conflict handling can be + a forward note if it balloons. + +This is the foundation for G and I — do it first. + ### Demo instances — leave RUNNING for human TUI confirmation ☐ After G–J, provision two **fully provisioned, left-running** instances so the human can confirm everything in `eve tui` (do **not** tear these down): From 3b6ca72c0a023f330e7e989e46c3f684c866758e Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 16:55:47 +0900 Subject: [PATCH 09/32] =?UTF-8?q?docs(v4.5):=20harden=20Task=208=20?= =?UTF-8?q?=E2=80=94=20provisioned/good-to-go=20acceptance=20gate=20with?= =?UTF-8?q?=20evidence=20for=20both=20demo=20instances?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/v4.5-agent-tasks.md | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/docs/v4.5-agent-tasks.md b/docs/v4.5-agent-tasks.md index f3627318..f40843bc 100644 --- a/docs/v4.5-agent-tasks.md +++ b/docs/v4.5-agent-tasks.md @@ -170,15 +170,37 @@ The `ref` is used (clone-at-ref snapshot). In the Edit screen, label the field a the branch/tag/SHA that gets checked out and note it's a snapshot (re-pull to refresh, uncommitted changes not picked up). No backend change needed. -## TASK 8 — Provision two demo instances, LEFT RUNNING (roadmap "Demo instances") +## TASK 8 — Provision two demo instances, verified good, LEFT RUNNING -After 5–7, provision two fully-provisioned instances and **leave them up** (do NOT -tear down — the human confirms them in `eve tui`): +**Only start once Tasks 5–7 are done and the full `scripts/test-*` suite is +green.** Provision two instances and **leave them up** (do NOT tear down — the +human confirms them in `eve tui`): - **Windows:** `eve instance create` → Vultr, `windows-server-2025`, - `vultr-vcg-a40-1c`, location tokyo, `--bundle gaming-streaming` (post-Task-5). + `vultr-vcg-a40-1c`, location tokyo, `--bundle gaming-streaming` (post-Task-5b). Then `up` + `provision`. Costs money while running — intended. - **Linux:** local-qemu, `ubuntu-26.04-arm64`, `local-qemu-medium`, `--bundle desktop-gnome`. Then `up` + `provision`. -Confirm both show provisioned in `eve instance status` AND appear/operate in -`eve tui`. Report both instance names + IP/connection info. Fix any bug found -(log in roadmap). + +**This is a hard acceptance gate — do not report "done" until BOTH instances +pass ALL of the following, with the evidence pasted into your report.** "Created" +is not "good to go." +1. `eve instance status` for each shows `provider_state=running` **and** + `provision_state=provisioned` — paste the output. +2. Each instance has a real reachable address (Vultr public IP; QEMU local IP) — + paste it. Confirm connectivity (ssh for Linux; rdp/connectivity check for + Windows). +3. **Every package in the bundle is actually installed** — paste the + `package_state` per instance and confirm each member of `gaming-streaming` / + `desktop-gnome` is `installed` (not `unknown`/`failed`/`missing`). If a package + only *appears* selected but didn't install, that's a bug — fix it, don't + hand-wave it (this is the sunshine-env_var class of issue from Bug C). +4. **Drive `eve tui`** (no env vars): select each instance and confirm the + provider pane lists providers, lifecycle buttons work, and **package action + buttons render** for installed packages (e.g. sunshine pair/moonlight on the + Windows box). State explicitly that you opened the TUI and what you saw — a + CLI `status` check does NOT satisfy this. + +If any check fails, it's a v4.5 bug: log it in the roadmap, fix it, recheck. Then +report both instance names, addresses, package states, and the TUI observations. +Re-run the full `scripts/test-*` suite at the end and confirm the roadmap tracker +A–K + demo instances are all ☑ with repo/PR references. PRs stay **draft**. From a5c1574c569a8a17df849782d0613edd75883984 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 20:26:46 +0900 Subject: [PATCH 10/32] feat(bundles): kind:bundle plugins + members field + catalog migration (v4.5 K) Bundles are now first-class kind:bundle plugins, not implied by package bundles: declarations. Core changes (eve): - kind enum: provider | package | os | bundle - New 'members' field: a bundle plugin lists its member package ids - Removed 'bundles' from schema + validator (packages no longer declare bundles) - 'supports' moved from global required to provider/package conditionals - catalog.py: aggregates bundles from kind:bundle plugins (reads 'members') - catalog_view.py: computes bundle installable_os_families from member packages - plugin_test.py: bundle/os kinds skip the SSH-readiness boundary check - Hermetic fixtures: mock-app/mock-win-app bundles: removed; mock-dev/mock-gaming bundle plugins created; test golden updated Package repos (v4.5): - eve-packages-linux: 9 bundle plugins created under bundles/; bundles: removed from 9 package manifests - eve-packages-windows: gaming-streaming bundle plugin [steam, sunshine, rustdesk] Tests updated: catalog aggregator tests use kind:bundle; manifest contract tests check bundle validation; relocation test reads bundle plugins. --- core/schema/plugin-manifest.schema.json | 52 +- eve_sdk/catalog.py | 14 +- eve_sdk/catalog_view.py | 19 +- eve_sdk/plugin_manifest.py | 11 +- eve_sdk/plugin_test.py | 2 + .../hermetic/bundles/mock-dev/eve-plugin.yaml | 5 + .../bundles/mock-gaming/eve-plugin.yaml | 5 + .../packages/mock-app/eve-plugin.yaml | 35 +- .../packages/mock-win-app/eve-plugin.yaml | 27 +- tests/golden/catalog-options.json | 6 + tests/python/sdk/test_catalog.py | 8 +- tests/python/sdk/test_catalog_relocation.py | 16 +- .../sdk/test_plugin_manifest_contract.py | 48 +- tokscale-export-20260627-100846.json | 1556 +++++++++++++++++ tokscale-export-20260627-100851.json | 1556 +++++++++++++++++ tokscale-export-20260627-100853.json | 1556 +++++++++++++++++ 16 files changed, 4829 insertions(+), 87 deletions(-) create mode 100644 tests/fixtures/hermetic/bundles/mock-dev/eve-plugin.yaml create mode 100644 tests/fixtures/hermetic/bundles/mock-gaming/eve-plugin.yaml create mode 100644 tokscale-export-20260627-100846.json create mode 100644 tokscale-export-20260627-100851.json create mode 100644 tokscale-export-20260627-100853.json diff --git a/core/schema/plugin-manifest.schema.json b/core/schema/plugin-manifest.schema.json index 4fbcabcc..e24e27e8 100644 --- a/core/schema/plugin-manifest.schema.json +++ b/core/schema/plugin-manifest.schema.json @@ -8,8 +8,7 @@ "api_version", "kind", "id", - "display_name", - "supports" + "display_name" ], "additionalProperties": false, "properties": { @@ -24,7 +23,8 @@ "enum": [ "provider", "package", - "os" + "os", + "bundle" ] }, "id": { @@ -284,10 +284,6 @@ "$ref": "#/$defs/host_tool_entry" } }, - "bundles": { - "description": "Bundle definitions contributed by this package manifest. Each bundle composes a set of included package ids. Only meaningful on package manifests.", - "type": "array" - }, "artifacts": { "description": "Declared build outputs for this package. Phase 0: declaration only — no execution wiring.", "type": "array" @@ -321,6 +317,14 @@ "description": "Directory name of the provision tree, relative to the plugin dir (e.g. 'provision')." } } + }, + "members": { + "description": "Member package ids for a kind:bundle plugin.", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z][a-z0-9.-]*$" + } } }, "allOf": [ @@ -336,7 +340,8 @@ "required": [ "access", "env", - "commands" + "commands", + "supports" ], "properties": { "commands": { @@ -377,7 +382,8 @@ "then": { "required": [ "env", - "commands" + "commands", + "supports" ], "properties": { "commands": { @@ -390,12 +396,6 @@ "supports": { "$ref": "#/$defs/package_supports" }, - "bundles": { - "type": "array", - "items": { - "$ref": "#/$defs/bundle_entry" - } - }, "artifacts": { "type": "array", "items": { @@ -424,6 +424,28 @@ } } } + }, + { + "if": { + "properties": { + "kind": { + "const": "bundle" + } + } + }, + "then": { + "required": [ + "members" + ], + "properties": { + "commands": { + "description": "bundle plugins do not use commands." + }, + "supports": { + "description": "bundle plugins do not use supports." + } + } + } } ], "$defs": { diff --git a/eve_sdk/catalog.py b/eve_sdk/catalog.py index d2c296aa..a5b37cfb 100644 --- a/eve_sdk/catalog.py +++ b/eve_sdk/catalog.py @@ -165,7 +165,7 @@ def aggregate( ``central_docs`` are applied in order (base first, overlays later). Plugin contributions are applied after all central docs: provider manifests contribute ``catalog.machines``, ``catalog.oses``, and ``catalog.inits``; - package manifests contribute ``bundles``. + bundle plugins contribute ``bundles`` (built from ``requires``). """ result: dict[str, list[dict[str, Any]]] = {section: [] for section in CATALOG_SECTIONS} @@ -180,10 +180,14 @@ def aggregate( if isinstance(contribution, dict): for section in ("machines", "oses", "inits", "locations"): _merge_section(result[section], contribution.get(section), section) - elif kind == "package": - bundles = plugin.get("bundles") - if isinstance(bundles, list): - _merge_section(result["bundles"], bundles, "bundles") + elif kind == "bundle": + members = plugin.get("members") + if isinstance(members, list) and members: + _merge_section( + result["bundles"], + [{"id": plugin["id"], "includes": list(members)}], + "bundles", + ) return result diff --git a/eve_sdk/catalog_view.py b/eve_sdk/catalog_view.py index 20bed10d..9fc48466 100644 --- a/eve_sdk/catalog_view.py +++ b/eve_sdk/catalog_view.py @@ -101,10 +101,21 @@ def build_catalog_options( } ) - bundles: list[dict[str, Any]] = [ - {"id": bundle.get("id"), "includes": bundle.get("includes") if isinstance(bundle.get("includes"), list) else []} - for bundle in catalog["bundles"] - ] + bundles: list[dict[str, Any]] = [] + for bundle in catalog["bundles"]: + includes = bundle.get("includes") if isinstance(bundle.get("includes"), list) else [] + # Compute bundle OS-compat from member packages' installable_os_families. + member_os_families: set[str] = set() + for member_id in includes: + member_plugin = package_plugin_by_id.get(member_id, {}) + member_install = member_plugin.get("install") + if isinstance(member_install, dict): + member_os_families.update(member_install.keys()) + bundles.append({ + "id": bundle.get("id"), + "includes": includes, + "installable_os_families": sorted(member_os_families) if member_os_families else [], + }) packages: list[dict[str, Any]] = [] for package in catalog["packages"]: diff --git a/eve_sdk/plugin_manifest.py b/eve_sdk/plugin_manifest.py index 0b6e4866..5d81e6bb 100644 --- a/eve_sdk/plugin_manifest.py +++ b/eve_sdk/plugin_manifest.py @@ -251,8 +251,8 @@ def validate(cls, plugin: dict[str, Any]) -> None: if plugin.get("api_version") != cls.API_VERSION: raise ValueError(f"{path}: api_version must be {cls.API_VERSION}") kind = plugin.get("kind") - if kind not in {"provider", "package", "os"}: - raise ValueError(f"{path}: kind must be provider, package, or os") + if kind not in {"provider", "package", "os", "bundle"}: + raise ValueError(f"{path}: kind must be provider, package, os, or bundle") if not re.match(r"^[a-z][a-z0-9.-]*$", str(plugin.get("id", ""))): raise ValueError(f"{path}: id must match [a-z][a-z0-9-]*") # os plugins carry a provision tree pointer, not executable commands. @@ -262,6 +262,13 @@ def validate(cls, plugin: dict[str, Any]) -> None: raise ValueError(f"{path}: os plugin must declare provision.dir") cls._validate_requires(plugin, path) return + # bundle plugins group packages via members; no commands needed. + if kind == "bundle": + members = plugin.get("members") + if not isinstance(members, list) or not members: + raise ValueError(f"{path}: bundle plugin must declare members (member package ids)") + cls._validate_requires(plugin, path) + return commands = plugin.get("commands") if not isinstance(commands, dict): raise ValueError(f"{path}: commands must be a map") diff --git a/eve_sdk/plugin_test.py b/eve_sdk/plugin_test.py index 8461dcaa..3cc2b3da 100644 --- a/eve_sdk/plugin_test.py +++ b/eve_sdk/plugin_test.py @@ -168,6 +168,8 @@ def _check_ssh_readiness(manifest: dict[str, Any]) -> list[CheckResult]: return _check_provider_boundary(manifest) if kind == "package": return _check_package_boundary(manifest) + if kind in ("os", "bundle"): + return [CheckResult("ssh-readiness-boundary", True, f"kind {kind!r} has no SSH boundary")] return [CheckResult( "ssh-readiness-boundary", False, diff --git a/tests/fixtures/hermetic/bundles/mock-dev/eve-plugin.yaml b/tests/fixtures/hermetic/bundles/mock-dev/eve-plugin.yaml new file mode 100644 index 00000000..0c54b862 --- /dev/null +++ b/tests/fixtures/hermetic/bundles/mock-dev/eve-plugin.yaml @@ -0,0 +1,5 @@ +api_version: eve.plugin/v1 +kind: bundle +id: mock-dev +display_name: Mock Dev +members: [mock-app] diff --git a/tests/fixtures/hermetic/bundles/mock-gaming/eve-plugin.yaml b/tests/fixtures/hermetic/bundles/mock-gaming/eve-plugin.yaml new file mode 100644 index 00000000..8d9c68ef --- /dev/null +++ b/tests/fixtures/hermetic/bundles/mock-gaming/eve-plugin.yaml @@ -0,0 +1,5 @@ +api_version: eve.plugin/v1 +kind: bundle +id: mock-gaming +display_name: Mock Gaming +members: [mock-win-app] diff --git a/tests/fixtures/hermetic/packages/mock-app/eve-plugin.yaml b/tests/fixtures/hermetic/packages/mock-app/eve-plugin.yaml index 13a9dbbc..758b269b 100644 --- a/tests/fixtures/hermetic/packages/mock-app/eve-plugin.yaml +++ b/tests/fixtures/hermetic/packages/mock-app/eve-plugin.yaml @@ -3,21 +3,30 @@ kind: package id: mock-app display_name: Mock App commands: - install: {exec: commands/ubuntu/install, args: [mock-app]} - status: {exec: commands/ubuntu/status, args: [mock-app]} - down: {exec: commands/ubuntu/down, args: [mock-app]} -supports: {os_families: [ubuntu]} -down: {destructive: false} + install: + exec: commands/ubuntu/install + args: + - mock-app + status: + exec: commands/ubuntu/status + args: + - mock-app + down: + exec: commands/ubuntu/down + args: + - mock-app +supports: + os_families: + - ubuntu +down: + destructive: false env: [] -bundles: - - id: mock-dev - includes: [mock-app] actions: - - id: mock-app-info - label: Mock App Info - target: mock-app.info +- id: mock-app-info + label: Mock App Info + target: mock-app.info install: ubuntu: steps: - - base.sh - - provision/ubuntu/mock-app.sh + - base.sh + - provision/ubuntu/mock-app.sh diff --git a/tests/fixtures/hermetic/packages/mock-win-app/eve-plugin.yaml b/tests/fixtures/hermetic/packages/mock-win-app/eve-plugin.yaml index a933f09a..c6953869 100644 --- a/tests/fixtures/hermetic/packages/mock-win-app/eve-plugin.yaml +++ b/tests/fixtures/hermetic/packages/mock-win-app/eve-plugin.yaml @@ -3,16 +3,25 @@ kind: package id: mock-win-app display_name: Mock Windows App commands: - install: {exec: commands/windows/install, args: [mock-win-app]} - status: {exec: commands/windows/status, args: [mock-win-app]} - down: {exec: commands/windows/down, args: [mock-win-app]} -supports: {os_families: [windows]} -down: {destructive: false} + install: + exec: commands/windows/install + args: + - mock-win-app + status: + exec: commands/windows/status + args: + - mock-win-app + down: + exec: commands/windows/down + args: + - mock-win-app +supports: + os_families: + - windows +down: + destructive: false env: [] -bundles: - - id: mock-gaming - includes: [mock-win-app] install: windows: steps: - - provision/windows/mock-win-app.ps1 + - provision/windows/mock-win-app.ps1 diff --git a/tests/golden/catalog-options.json b/tests/golden/catalog-options.json index 7c3073b2..d04c737b 100644 --- a/tests/golden/catalog-options.json +++ b/tests/golden/catalog-options.json @@ -124,12 +124,18 @@ "id": "mock-dev", "includes": [ "mock-app" + ], + "installable_os_families": [ + "ubuntu" ] }, { "id": "mock-gaming", "includes": [ "mock-win-app" + ], + "installable_os_families": [ + "windows" ] } ], diff --git a/tests/python/sdk/test_catalog.py b/tests/python/sdk/test_catalog.py index 1f89532e..e139a140 100644 --- a/tests/python/sdk/test_catalog.py +++ b/tests/python/sdk/test_catalog.py @@ -260,18 +260,18 @@ def test_provider_init_replaces_same_key(self) -> None: class TestAggregatePackageBundles: - def test_package_bundles_appended(self) -> None: + def test_bundle_plugin_members_appended(self) -> None: central = {"bundles": [{"id": "b1", "includes": ["p1"]}]} - plugin = {"kind": "package", "bundles": [{"id": "b2", "includes": ["p2"]}]} + plugin = {"kind": "bundle", "id": "b2", "members": ["p2"]} result = aggregate([central], [plugin]) assert result["bundles"] == [ {"id": "b1", "includes": ["p1"]}, {"id": "b2", "includes": ["p2"]}, ] - def test_package_bundle_replaces_same_id(self) -> None: + def test_bundle_plugin_replaces_same_id(self) -> None: central = {"bundles": [{"id": "b1", "includes": ["p1"]}]} - plugin = {"kind": "package", "bundles": [{"id": "b1", "includes": ["p2", "p3"]}]} + plugin = {"kind": "bundle", "id": "b1", "members": ["p2", "p3"]} result = aggregate([central], [plugin]) assert result["bundles"] == [{"id": "b1", "includes": ["p2", "p3"]}] diff --git a/tests/python/sdk/test_catalog_relocation.py b/tests/python/sdk/test_catalog_relocation.py index 17b43202..284401a3 100644 --- a/tests/python/sdk/test_catalog_relocation.py +++ b/tests/python/sdk/test_catalog_relocation.py @@ -103,20 +103,16 @@ def test_all_bundles_present_in_aggregated_catalog(self) -> None: } assert bundle_ids == expected - def test_bundles_sourced_from_package_manifests(self) -> None: + def test_bundles_sourced_from_bundle_plugins(self) -> None: catalog = load_catalog() central_bundle_ids = { entry["id"] for entry in catalog["bundles"] } - package_bundle_ids: set[str] = set() - for plugin in PluginManifest.load_all("package"): - bundles = plugin.get("bundles") - if not isinstance(bundles, list): - continue - for bundle in bundles: - if isinstance(bundle, dict) and bundle.get("id"): - package_bundle_ids.add(bundle["id"]) - assert central_bundle_ids == package_bundle_ids + plugin_bundle_ids: set[str] = set() + for plugin in PluginManifest.load_all("bundle"): + if plugin.get("id"): + plugin_bundle_ids.add(plugin["id"]) + assert central_bundle_ids == plugin_bundle_ids # --------------------------------------------------------------------------- diff --git a/tests/python/sdk/test_plugin_manifest_contract.py b/tests/python/sdk/test_plugin_manifest_contract.py index 02289843..b66401e3 100644 --- a/tests/python/sdk/test_plugin_manifest_contract.py +++ b/tests/python/sdk/test_plugin_manifest_contract.py @@ -238,38 +238,36 @@ def test_catalog_os_missing_id_rejected(self, tmp_path: Path) -> None: # bundles on package manifests # --------------------------------------------------------------------------- -class TestPackageBundles: - def test_package_with_bundles_passes(self, tmp_path: Path) -> None: - manifest = _package_manifest( - tmp_path, - bundles=[ - {"id": "my-bundle", "includes": ["testpkg", "docker"]}, - ], - ) +class TestBundlePlugins: + def test_bundle_plugin_validates(self, tmp_path: Path) -> None: + manifest = { + "_path": str(tmp_path / "eve-plugin.yaml"), + "api_version": "eve.plugin/v1", + "kind": "bundle", + "id": "my-bundle", + "display_name": "My Bundle", + "members": ["pkg-a", "pkg-b"], + } PluginManifest.validate(manifest) - def test_bundle_missing_includes_rejected(self, tmp_path: Path) -> None: - manifest = _package_manifest( - tmp_path, - bundles=[{"id": "bad-bundle"}], - ) - with pytest.raises(ValueError): + def test_bundle_without_members_rejected(self, tmp_path: Path) -> None: + manifest = { + "_path": str(tmp_path / "eve-plugin.yaml"), + "api_version": "eve.plugin/v1", + "kind": "bundle", + "id": "bad-bundle", + "display_name": "Bad", + "members": [], + } + with pytest.raises(ValueError, match="members"): PluginManifest.validate(manifest) - def test_bundle_missing_id_rejected(self, tmp_path: Path) -> None: + def test_package_with_bundles_field_rejected(self, tmp_path: Path) -> None: manifest = _package_manifest( tmp_path, - bundles=[{"includes": ["docker"]}], + bundles=[{"id": "my-bundle", "includes": ["testpkg"]}], ) - with pytest.raises(ValueError): - PluginManifest.validate(manifest) - - def test_bundle_empty_includes_rejected(self, tmp_path: Path) -> None: - manifest = _package_manifest( - tmp_path, - bundles=[{"id": "empty-bundle", "includes": []}], - ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Additional properties"): PluginManifest.validate(manifest) diff --git a/tokscale-export-20260627-100846.json b/tokscale-export-20260627-100846.json new file mode 100644 index 00000000..7f671151 --- /dev/null +++ b/tokscale-export-20260627-100846.json @@ -0,0 +1,1556 @@ +{ + "agents": [ + { + "agent": "Descartes", + "clients": "codex", + "cost": 3.9003440000000005, + "messageCount": 44, + "tokens": { + "cacheRead": 3273728, + "cacheWrite": 0, + "input": 303710, + "output": 16738, + "total": 3602269 + } + }, + { + "agent": "Halley", + "clients": "codex", + "cost": 3.5619250000000005, + "messageCount": 37, + "tokens": { + "cacheRead": 4064640, + "cacheWrite": 0, + "input": 137651, + "output": 18565, + "total": 4230336 + } + }, + { + "agent": "Bernoulli", + "clients": "codex", + "cost": 3.3071219999999997, + "messageCount": 26, + "tokens": { + "cacheRead": 3279104, + "cacheWrite": 0, + "input": 207118, + "output": 13944, + "total": 3507288 + } + }, + { + "agent": "Turing", + "clients": "codex", + "cost": 3.306327, + "messageCount": 37, + "tokens": { + "cacheRead": 2737024, + "cacheWrite": 0, + "input": 259973, + "output": 15756, + "total": 3018262 + } + }, + { + "agent": "Goodall", + "clients": "codex", + "cost": 2.350671, + "messageCount": 24, + "tokens": { + "cacheRead": 2022912, + "cacheWrite": 0, + "input": 142023, + "output": 13282, + "total": 2185905 + } + }, + { + "agent": "Hubble", + "clients": "codex", + "cost": 2.3384470000000004, + "messageCount": 32, + "tokens": { + "cacheRead": 2179584, + "cacheWrite": 0, + "input": 103145, + "output": 16733, + "total": 2307160 + } + }, + { + "agent": "Build", + "clients": "opencode", + "cost": 2.0610812, + "messageCount": 2709, + "tokens": { + "cacheRead": 1150841920, + "cacheWrite": 0, + "input": 37385365, + "output": 641519, + "total": 1189691974 + } + }, + { + "agent": "Dewey", + "clients": "codex", + "cost": 1.7652170000000003, + "messageCount": 25, + "tokens": { + "cacheRead": 1062784, + "cacheWrite": 0, + "input": 151005, + "output": 14042, + "total": 1229749 + } + }, + { + "agent": "Hooke", + "clients": "codex", + "cost": 1.7155859999999998, + "messageCount": 24, + "tokens": { + "cacheRead": 1587712, + "cacheWrite": 0, + "input": 119786, + "output": 7280, + "total": 1718258 + } + }, + { + "agent": "Epicurus", + "clients": "codex", + "cost": 1.6247530000000001, + "messageCount": 19, + "tokens": { + "cacheRead": 1044096, + "cacheWrite": 0, + "input": 79103, + "output": 14410, + "total": 1146772 + } + }, + { + "agent": "Carson", + "clients": "codex", + "cost": 1.613164, + "messageCount": 17, + "tokens": { + "cacheRead": 1189248, + "cacheWrite": 0, + "input": 93848, + "output": 10725, + "total": 1301406 + } + }, + { + "agent": "Huygens", + "clients": "codex", + "cost": 1.4913770000000002, + "messageCount": 11, + "tokens": { + "cacheRead": 802944, + "cacheWrite": 0, + "input": 91873, + "output": 11578, + "total": 915835 + } + }, + { + "agent": "Bacon", + "clients": "codex", + "cost": 1.387087, + "messageCount": 16, + "tokens": { + "cacheRead": 1042944, + "cacheWrite": 0, + "input": 123383, + "output": 7324, + "total": 1174617 + } + }, + { + "agent": "Carver", + "clients": "codex", + "cost": 1.271722, + "messageCount": 13, + "tokens": { + "cacheRead": 786304, + "cacheWrite": 0, + "input": 139744, + "output": 5010, + "total": 932043 + } + }, + { + "agent": "Heisenberg", + "clients": "codex", + "cost": 1.2637669999999999, + "messageCount": 15, + "tokens": { + "cacheRead": 1046144, + "cacheWrite": 0, + "input": 107855, + "output": 5945, + "total": 1160713 + } + }, + { + "agent": "Schrodinger", + "clients": "codex", + "cost": 1.224408, + "messageCount": 10, + "tokens": { + "cacheRead": 542976, + "cacheWrite": 0, + "input": 91500, + "output": 10030, + "total": 650990 + } + }, + { + "agent": "Socrates", + "clients": "codex", + "cost": 1.2146380000000003, + "messageCount": 8, + "tokens": { + "cacheRead": 365056, + "cacheWrite": 0, + "input": 73558, + "output": 12603, + "total": 460758 + } + }, + { + "agent": "Banach", + "clients": "codex", + "cost": 0.9842209999999999, + "messageCount": 16, + "tokens": { + "cacheRead": 888832, + "cacheWrite": 0, + "input": 80667, + "output": 4095, + "total": 974048 + } + }, + { + "agent": "Pasteur", + "clients": "codex", + "cost": 0.9695810000000001, + "messageCount": 10, + "tokens": { + "cacheRead": 372992, + "cacheWrite": 0, + "input": 129083, + "output": 3727, + "total": 506664 + } + }, + { + "agent": "Arendt", + "clients": "codex", + "cost": 0.923487, + "messageCount": 11, + "tokens": { + "cacheRead": 534144, + "cacheWrite": 0, + "input": 89163, + "output": 6148, + "total": 630327 + } + }, + { + "agent": "Cicero", + "clients": "codex", + "cost": 0.850337, + "messageCount": 9, + "tokens": { + "cacheRead": 412544, + "cacheWrite": 0, + "input": 88823, + "output": 4517, + "total": 508032 + } + }, + { + "agent": "Beauvoir", + "clients": "codex", + "cost": 0.7617210000000001, + "messageCount": 11, + "tokens": { + "cacheRead": 561792, + "cacheWrite": 0, + "input": 60801, + "output": 5172, + "total": 628487 + } + }, + { + "agent": "Morpheus", + "clients": "codex", + "cost": 0.2864385, + "messageCount": 5, + "tokens": { + "cacheRead": 143744, + "cacheWrite": 0, + "input": 40987, + "output": 9188, + "total": 194600 + } + }, + { + "agent": "Explore", + "clients": "opencode", + "cost": 0.0, + "messageCount": 60, + "tokens": { + "cacheRead": 2002624, + "cacheWrite": 0, + "input": 243706, + "output": 16874, + "total": 2273117 + } + }, + { + "agent": "General", + "clients": "opencode", + "cost": 0.0, + "messageCount": 23, + "tokens": { + "cacheRead": 1014464, + "cacheWrite": 0, + "input": 197872, + "output": 6729, + "total": 1223267 + } + }, + { + "agent": "Plan", + "clients": "opencode", + "cost": 0.0, + "messageCount": 11, + "tokens": { + "cacheRead": 175808, + "cacheWrite": 0, + "input": 287916, + "output": 7581, + "total": 490714 + } + } + ], + "daily": [ + { + "cost": 62.898246249999964, + "date": "2026-06-27", + "messageCount": 412, + "tokens": { + "cacheRead": 214711221, + "cacheWrite": 3096478, + "input": 4669544, + "output": 331876, + "total": 222819611 + }, + "turnCount": 33 + }, + { + "cost": 33.576564749999996, + "date": "2026-06-26", + "messageCount": 397, + "tokens": { + "cacheRead": 131417100, + "cacheWrite": 763035, + "input": 6103880, + "output": 114508, + "total": 138450062 + }, + "turnCount": 77 + }, + { + "cost": 70.50142420000007, + "date": "2026-06-25", + "messageCount": 343, + "tokens": { + "cacheRead": 159025716, + "cacheWrite": 795462, + "input": 3186409, + "output": 152326, + "total": 163184841 + }, + "turnCount": 39 + }, + { + "cost": 162.3291599500004, + "date": "2026-06-24", + "messageCount": 1184, + "tokens": { + "cacheRead": 438077756, + "cacheWrite": 2775552, + "input": 11776844, + "output": 621585, + "total": 453424090 + }, + "turnCount": 86 + }, + { + "cost": 105.95392205000012, + "date": "2026-06-23", + "messageCount": 1147, + "tokens": { + "cacheRead": 205621324, + "cacheWrite": 826873, + "input": 5809960, + "output": 697935, + "total": 213133743 + }, + "turnCount": 173 + }, + { + "cost": 51.11505444999997, + "date": "2026-06-22", + "messageCount": 225, + "tokens": { + "cacheRead": 37714697, + "cacheWrite": 2184305, + "input": 2379565, + "output": 268571, + "total": 42570840 + }, + "turnCount": 58 + }, + { + "cost": 0.24963100000000002, + "date": "2026-06-21", + "messageCount": 1, + "tokens": { + "cacheRead": 2432, + "cacheWrite": 0, + "input": 42021, + "output": 761, + "total": 45730 + }, + "turnCount": 1 + }, + { + "cost": 125.32283240000004, + "date": "2026-06-20", + "messageCount": 633, + "tokens": { + "cacheRead": 229078486, + "cacheWrite": 3293334, + "input": 2450252, + "output": 437114, + "total": 235475603 + }, + "turnCount": 60 + }, + { + "cost": 109.40506945000021, + "date": "2026-06-19", + "messageCount": 596, + "tokens": { + "cacheRead": 125023572, + "cacheWrite": 3950057, + "input": 2763154, + "output": 365780, + "total": 132127969 + }, + "turnCount": 99 + }, + { + "cost": 182.32671845000021, + "date": "2026-06-18", + "messageCount": 828, + "tokens": { + "cacheRead": 254795546, + "cacheWrite": 4897255, + "input": 3196910, + "output": 464789, + "total": 263385211 + }, + "turnCount": 155 + }, + { + "cost": 107.97663059999991, + "date": "2026-06-17", + "messageCount": 935, + "tokens": { + "cacheRead": 388181758, + "cacheWrite": 5355354, + "input": 10950338, + "output": 484345, + "total": 405070767 + }, + "turnCount": 123 + }, + { + "cost": 82.96701420000005, + "date": "2026-06-16", + "messageCount": 1121, + "tokens": { + "cacheRead": 300020822, + "cacheWrite": 3887726, + "input": 12170344, + "output": 447398, + "total": 316787817 + }, + "turnCount": 123 + }, + { + "cost": 210.1056897, + "date": "2026-06-15", + "messageCount": 1292, + "tokens": { + "cacheRead": 326991687, + "cacheWrite": 3547204, + "input": 7187408, + "output": 740327, + "total": 338566790 + }, + "turnCount": 262 + }, + { + "cost": 104.23466350000008, + "date": "2026-06-14", + "messageCount": 531, + "tokens": { + "cacheRead": 152731876, + "cacheWrite": 1858804, + "input": 1917185, + "output": 519269, + "total": 157033466 + }, + "turnCount": 123 + }, + { + "cost": 60.763000850000005, + "date": "2026-06-13", + "messageCount": 448, + "tokens": { + "cacheRead": 64804925, + "cacheWrite": 1556132, + "input": 4043220, + "output": 174576, + "total": 70605033 + }, + "turnCount": 170 + }, + { + "cost": 118.33723094999999, + "date": "2026-06-12", + "messageCount": 583, + "tokens": { + "cacheRead": 141843299, + "cacheWrite": 4359974, + "input": 2910267, + "output": 381145, + "total": 149515655 + }, + "turnCount": 122 + }, + { + "cost": 55.53048034999999, + "date": "2026-06-11", + "messageCount": 461, + "tokens": { + "cacheRead": 64521768, + "cacheWrite": 1684070, + "input": 1006985, + "output": 328563, + "total": 67554580 + }, + "turnCount": 54 + }, + { + "cost": 57.13918449999999, + "date": "2026-06-10", + "messageCount": 603, + "tokens": { + "cacheRead": 58151750, + "cacheWrite": 1659270, + "input": 2605351, + "output": 357810, + "total": 62805349 + }, + "turnCount": 146 + }, + { + "cost": 66.93653610000003, + "date": "2026-06-09", + "messageCount": 662, + "tokens": { + "cacheRead": 75678705, + "cacheWrite": 639890, + "input": 3647781, + "output": 324419, + "total": 80350983 + }, + "turnCount": 120 + }, + { + "cost": 21.25041285000001, + "date": "2026-06-08", + "messageCount": 210, + "tokens": { + "cacheRead": 24254720, + "cacheWrite": 0, + "input": 2015170, + "output": 53976, + "total": 26337339 + }, + "turnCount": 51 + }, + { + "cost": 32.597969249999984, + "date": "2026-06-05", + "messageCount": 364, + "tokens": { + "cacheRead": 42004844, + "cacheWrite": 234916, + "input": 2044168, + "output": 144658, + "total": 44470302 + }, + "turnCount": 92 + }, + { + "cost": 99.52701779999997, + "date": "2026-06-04", + "messageCount": 1070, + "tokens": { + "cacheRead": 136146396, + "cacheWrite": 308228, + "input": 9415727, + "output": 392157, + "total": 146348394 + }, + "turnCount": 243 + }, + { + "cost": 36.19306024999999, + "date": "2026-06-03", + "messageCount": 394, + "tokens": { + "cacheRead": 45317888, + "cacheWrite": 288910, + "input": 1887407, + "output": 187801, + "total": 47720679 + }, + "turnCount": 66 + }, + { + "cost": 58.17762395, + "date": "2026-06-02", + "messageCount": 693, + "tokens": { + "cacheRead": 65343522, + "cacheWrite": 94275, + "input": 4334973, + "output": 277388, + "total": 70152002 + }, + "turnCount": 155 + }, + { + "cost": 98.65032284999987, + "date": "2026-06-01", + "messageCount": 1261, + "tokens": { + "cacheRead": 137791825, + "cacheWrite": 47101, + "input": 8946466, + "output": 273461, + "total": 147153863 + }, + "turnCount": 243 + }, + { + "cost": 3.0790345000000006, + "date": "2026-05-31", + "messageCount": 13, + "tokens": { + "cacheRead": 1441664, + "cacheWrite": 0, + "input": 719866, + "output": 7573, + "total": 2172216 + }, + "turnCount": 6 + }, + { + "cost": 17.297640949999998, + "date": "2026-05-30", + "messageCount": 131, + "tokens": { + "cacheRead": 22827165, + "cacheWrite": 325084, + "input": 667626, + "output": 37259, + "total": 23866819 + }, + "turnCount": 17 + }, + { + "cost": 60.68457075000004, + "date": "2026-05-29", + "messageCount": 711, + "tokens": { + "cacheRead": 90143620, + "cacheWrite": 582898, + "input": 5270864, + "output": 216268, + "total": 96275133 + }, + "turnCount": 148 + }, + { + "cost": 101.55679355000004, + "date": "2026-05-28", + "messageCount": 1304, + "tokens": { + "cacheRead": 165381980, + "cacheWrite": 493162, + "input": 6335145, + "output": 374782, + "total": 172674852 + }, + "turnCount": 252 + }, + { + "cost": 21.16416095, + "date": "2026-05-27", + "messageCount": 281, + "tokens": { + "cacheRead": 25242496, + "cacheWrite": 0, + "input": 1794683, + "output": 58174, + "total": 27116361 + }, + "turnCount": 53 + }, + { + "cost": 246.93044285000036, + "date": "2026-05-26", + "messageCount": 2689, + "tokens": { + "cacheRead": 386583466, + "cacheWrite": 375187, + "input": 11422065, + "output": 538790, + "total": 399029572 + }, + "turnCount": 183 + }, + { + "cost": 281.5894256, + "date": "2026-05-25", + "messageCount": 3530, + "tokens": { + "cacheRead": 444941605, + "cacheWrite": 390724, + "input": 14479594, + "output": 718228, + "total": 460757679 + }, + "turnCount": 473 + }, + { + "cost": 14.732135749999992, + "date": "2026-05-24", + "messageCount": 202, + "tokens": { + "cacheRead": 19969969, + "cacheWrite": 53280, + "input": 931975, + "output": 44125, + "total": 21010075 + }, + "turnCount": 23 + }, + { + "cost": 12.21099695, + "date": "2026-05-23", + "messageCount": 141, + "tokens": { + "cacheRead": 17840000, + "cacheWrite": 0, + "input": 597934, + "output": 23523, + "total": 18467567 + }, + "turnCount": 19 + }, + { + "cost": 51.15712750000008, + "date": "2026-05-22", + "messageCount": 643, + "tokens": { + "cacheRead": 75098752, + "cacheWrite": 0, + "input": 3099646, + "output": 122958, + "total": 78357079 + }, + "turnCount": 101 + }, + { + "cost": 149.05496409999986, + "date": "2026-05-21", + "messageCount": 1724, + "tokens": { + "cacheRead": 210405888, + "cacheWrite": 0, + "input": 10255744, + "output": 361599, + "total": 221098307 + }, + "turnCount": 260 + }, + { + "cost": 46.64824070000002, + "date": "2026-05-20", + "messageCount": 488, + "tokens": { + "cacheRead": 57876992, + "cacheWrite": 0, + "input": 2949920, + "output": 143287, + "total": 61016737 + }, + "turnCount": 60 + }, + { + "cost": 302.14533274999985, + "date": "2026-05-19", + "messageCount": 3311, + "tokens": { + "cacheRead": 436505728, + "cacheWrite": 0, + "input": 19132465, + "output": 825962, + "total": 456758419 + }, + "turnCount": 302 + }, + { + "cost": 250.6522364000004, + "date": "2026-05-18", + "messageCount": 2696, + "tokens": { + "cacheRead": 359022080, + "cacheWrite": 0, + "input": 13733347, + "output": 656851, + "total": 373656327 + }, + "turnCount": 249 + }, + { + "cost": 220.23427974999998, + "date": "2026-05-17", + "messageCount": 2472, + "tokens": { + "cacheRead": 371165184, + "cacheWrite": 0, + "input": 9235712, + "output": 452303, + "total": 380963488 + }, + "turnCount": 289 + }, + { + "cost": 132.72312964999975, + "date": "2026-05-16", + "messageCount": 1563, + "tokens": { + "cacheRead": 229541504, + "cacheWrite": 0, + "input": 6259001, + "output": 282044, + "total": 236138911 + }, + "turnCount": 186 + }, + { + "cost": 187.5979618500003, + "date": "2026-05-15", + "messageCount": 2174, + "tokens": { + "cacheRead": 292393728, + "cacheWrite": 0, + "input": 9962127, + "output": 471505, + "total": 302929103 + }, + "turnCount": 295 + }, + { + "cost": 171.11259285000003, + "date": "2026-05-14", + "messageCount": 1827, + "tokens": { + "cacheRead": 260370560, + "cacheWrite": 0, + "input": 12885534, + "output": 372793, + "total": 273724302 + }, + "turnCount": 327 + }, + { + "cost": 67.94910775000012, + "date": "2026-05-13", + "messageCount": 777, + "tokens": { + "cacheRead": 104286592, + "cacheWrite": 0, + "input": 3728901, + "output": 156135, + "total": 108200389 + }, + "turnCount": 71 + }, + { + "cost": 114.14174050000003, + "date": "2026-05-12", + "messageCount": 1257, + "tokens": { + "cacheRead": 165287296, + "cacheWrite": 0, + "input": 8041501, + "output": 277358, + "total": 173654443 + }, + "turnCount": 182 + }, + { + "cost": 46.584478750000045, + "date": "2026-05-11", + "messageCount": 462, + "tokens": { + "cacheRead": 62052608, + "cacheWrite": 0, + "input": 2936231, + "output": 112746, + "total": 65123944 + }, + "turnCount": 43 + }, + { + "cost": 152.83763230000025, + "date": "2026-05-10", + "messageCount": 1648, + "tokens": { + "cacheRead": 261582336, + "cacheWrite": 0, + "input": 9799813, + "output": 342140, + "total": 271802343 + }, + "turnCount": 324 + }, + { + "cost": 253.98307544999952, + "date": "2026-05-09", + "messageCount": 2718, + "tokens": { + "cacheRead": 451929856, + "cacheWrite": 0, + "input": 14305153, + "output": 489240, + "total": 466832273 + }, + "turnCount": 542 + }, + { + "cost": 225.4019883, + "date": "2026-05-08", + "messageCount": 2190, + "tokens": { + "cacheRead": 346067200, + "cacheWrite": 0, + "input": 15997101, + "output": 414419, + "total": 362568467 + }, + "turnCount": 366 + }, + { + "cost": 122.65184929999984, + "date": "2026-05-07", + "messageCount": 1440, + "tokens": { + "cacheRead": 200288256, + "cacheWrite": 0, + "input": 9317044, + "output": 266888, + "total": 209929076 + }, + "turnCount": 310 + }, + { + "cost": 256.69028255, + "date": "2026-05-06", + "messageCount": 3203, + "tokens": { + "cacheRead": 456758400, + "cacheWrite": 0, + "input": 15612265, + "output": 602277, + "total": 473088033 + }, + "turnCount": 593 + }, + { + "cost": 84.4895073, + "date": "2026-05-05", + "messageCount": 1091, + "tokens": { + "cacheRead": 143922816, + "cacheWrite": 0, + "input": 5318948, + "output": 183438, + "total": 149466661 + }, + "turnCount": 148 + }, + { + "cost": 0.29549400000000003, + "date": "2026-05-04", + "messageCount": 2, + "tokens": { + "cacheRead": 31488, + "cacheWrite": 0, + "input": 49188, + "output": 731, + "total": 81803 + }, + "turnCount": 2 + }, + { + "cost": 18.18216895000001, + "date": "2026-05-01", + "messageCount": 196, + "tokens": { + "cacheRead": 24443392, + "cacheWrite": 0, + "input": 1201693, + "output": 78701, + "total": 25752940 + }, + "turnCount": 50 + }, + { + "cost": 20.72562839999999, + "date": "2026-04-30", + "messageCount": 222, + "tokens": { + "cacheRead": 30970624, + "cacheWrite": 0, + "input": 2446434, + "output": 56533, + "total": 33486899 + }, + "turnCount": 58 + }, + { + "cost": 55.303921149999965, + "date": "2026-04-29", + "messageCount": 729, + "tokens": { + "cacheRead": 84425088, + "cacheWrite": 0, + "input": 4840140, + "output": 160913, + "total": 89467295 + }, + "turnCount": 194 + }, + { + "cost": 34.64550734999998, + "date": "2026-04-28", + "messageCount": 409, + "tokens": { + "cacheRead": 42745728, + "cacheWrite": 0, + "input": 2764854, + "output": 103134, + "total": 45645870 + }, + "turnCount": 68 + }, + { + "cost": 33.08216499999999, + "date": "2026-04-27", + "messageCount": 386, + "tokens": { + "cacheRead": 41121024, + "cacheWrite": 0, + "input": 2194255, + "output": 86398, + "total": 43425459 + }, + "turnCount": 42 + }, + { + "cost": 1.900379, + "date": "2026-04-26", + "messageCount": 9, + "tokens": { + "cacheRead": 1283968, + "cacheWrite": 0, + "input": 202305, + "output": 6373, + "total": 1494502 + }, + "turnCount": 3 + }, + { + "cost": 16.50628600000001, + "date": "2026-04-25", + "messageCount": 176, + "tokens": { + "cacheRead": 16425472, + "cacheWrite": 0, + "input": 1263526, + "output": 53751, + "total": 17754862 + }, + "turnCount": 19 + }, + { + "cost": 22.265109, + "date": "2026-04-24", + "messageCount": 349, + "tokens": { + "cacheRead": 31644032, + "cacheWrite": 0, + "input": 1970623, + "output": 98853, + "total": 33733821 + }, + "turnCount": 28 + }, + { + "cost": 11.174454999999998, + "date": "2026-04-23", + "messageCount": 184, + "tokens": { + "cacheRead": 21451520, + "cacheWrite": 0, + "input": 1546760, + "output": 95299, + "total": 23127925 + }, + "turnCount": 58 + }, + { + "cost": 21.544721999999993, + "date": "2026-04-22", + "messageCount": 361, + "tokens": { + "cacheRead": 37253248, + "cacheWrite": 0, + "input": 3738740, + "output": 141157, + "total": 41184292 + }, + "turnCount": 54 + }, + { + "cost": 6.878394900000001, + "date": "2026-04-21", + "messageCount": 113, + "tokens": { + "cacheRead": 11123328, + "cacheWrite": 0, + "input": 2245378, + "output": 29966, + "total": 13402492 + }, + "turnCount": 9 + }, + { + "cost": 7.503686, + "date": "2026-04-20", + "messageCount": 178, + "tokens": { + "cacheRead": 16731904, + "cacheWrite": 0, + "input": 948628, + "output": 50664, + "total": 17743808 + }, + "turnCount": 12 + }, + { + "cost": 13.077388000000003, + "date": "2026-04-19", + "messageCount": 8, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 12826012, + "output": 78555, + "total": 12904567 + }, + "turnCount": 0 + }, + { + "cost": 4.3518446, + "date": "2026-04-18", + "messageCount": 66, + "tokens": { + "cacheRead": 8106240, + "cacheWrite": 0, + "input": 1172430, + "output": 18697, + "total": 9302114 + }, + "turnCount": 1 + }, + { + "cost": 11.9593086, + "date": "2026-04-17", + "messageCount": 147, + "tokens": { + "cacheRead": 14878976, + "cacheWrite": 0, + "input": 4426250, + "output": 68876, + "total": 19386220 + }, + "turnCount": 16 + }, + { + "cost": 7.787204500000003, + "date": "2026-04-16", + "messageCount": 95, + "tokens": { + "cacheRead": 13781248, + "cacheWrite": 0, + "input": 1503039, + "output": 32230, + "total": 15323240 + }, + "turnCount": 18 + }, + { + "cost": 194.5845475, + "date": "2026-04-15", + "messageCount": 11, + "tokens": { + "cacheRead": 1336960, + "cacheWrite": 0, + "input": 38755320, + "output": 169570, + "total": 40262246 + }, + "turnCount": 3 + }, + { + "cost": 46.75895299999998, + "date": "2026-04-14", + "messageCount": 590, + "tokens": { + "cacheRead": 65797632, + "cacheWrite": 0, + "input": 10736846, + "output": 180419, + "total": 76765640 + }, + "turnCount": 111 + }, + { + "cost": 2.4843545, + "date": "2026-04-13", + "messageCount": 74, + "tokens": { + "cacheRead": 2289408, + "cacheWrite": 0, + "input": 483209, + "output": 36385, + "total": 2819549 + }, + "turnCount": 17 + }, + { + "cost": 7.691671500000001, + "date": "2026-04-10", + "messageCount": 126, + "tokens": { + "cacheRead": 15201536, + "cacheWrite": 0, + "input": 1345627, + "output": 28877, + "total": 16582311 + }, + "turnCount": 7 + }, + { + "cost": 29.540619000000024, + "date": "2026-04-09", + "messageCount": 506, + "tokens": { + "cacheRead": 46094976, + "cacheWrite": 0, + "input": 6170646, + "output": 139003, + "total": 52438306 + }, + "turnCount": 73 + }, + { + "cost": 9.918214, + "date": "2026-04-08", + "messageCount": 156, + "tokens": { + "cacheRead": 18131456, + "cacheWrite": 0, + "input": 1808996, + "output": 45397, + "total": 19997976 + }, + "turnCount": 22 + } + ], + "models": [ + { + "client": "codex", + "cost": 4476.965901000025, + "model": "gpt-5.5", + "performance": { + "msPer1KTokens": 4524.020180595458, + "sampleCount": 44616, + "timedTokens": 6328004754, + "tokenCoverage": 1.0, + "totalDurationMs": 28628021210 + }, + "provider": "openai", + "sessionCount": 121, + "tokens": { + "cacheRead": 6113071232, + "cacheWrite": 0, + "input": 201103015, + "output": 10839269, + "total": 6328004754 + } + }, + { + "client": "claude", + "cost": 1306.033157000002, + "model": "claude-opus-4-8", + "performance": { + "msPer1KTokens": 50.03828284427112, + "sampleCount": 4702, + "timedTokens": 1774442346, + "tokenCoverage": 0.9923299290410101, + "totalDurationMs": 88790048 + }, + "provider": "anthropic", + "sessionCount": 19, + "tokens": { + "cacheRead": 1733100519, + "cacheWrite": 48029010, + "input": 1820062, + "output": 5208051, + "total": 1788157642 + } + }, + { + "client": "opencode, codex, goose", + "cost": 400.8453585000001, + "model": "gpt-5.4", + "performance": { + "msPer1KTokens": 5007.590199461738, + "sampleCount": 3222, + "timedTokens": 379225212, + "tokenCoverage": 0.9081412800891782, + "totalDurationMs": 1899004455 + }, + "provider": "openai, chatgpt_codex", + "sessionCount": 37, + "tokens": { + "cacheRead": 336189824, + "cacheWrite": 0, + "input": 79914106, + "output": 1189409, + "total": 417583938 + } + }, + { + "client": "claude", + "cost": 43.60413899999995, + "model": "claude-opus-4-7", + "performance": { + "msPer1KTokens": 88.288596602198, + "sampleCount": 282, + "timedTokens": 44946586, + "tokenCoverage": 0.9708623063422688, + "totalDurationMs": 3968271 + }, + "provider": "anthropic", + "sessionCount": 4, + "tokens": { + "cacheRead": 43571353, + "cacheWrite": 2295530, + "input": 162240, + "output": 266408, + "total": 46295531 + } + }, + { + "client": "codex", + "cost": 42.4404918999999, + "model": "codex-auto-review", + "performance": { + "msPer1KTokens": 40.30321610779724, + "sampleCount": 6470, + "timedTokens": 743616959, + "tokenCoverage": 1.0, + "totalDurationMs": 29970155 + }, + "provider": "openai", + "sessionCount": 221, + "tokens": { + "cacheRead": 643499776, + "cacheWrite": 0, + "input": 99360782, + "output": 564770, + "total": 743616959 + } + }, + { + "client": "goose", + "cost": 17.5749466, + "model": "glm-5.1", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "custom_z.ai", + "sessionCount": 15, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 17204217, + "output": 115853, + "total": 17320070 + } + }, + { + "client": "goose", + "cost": 0.041356000000000004, + "model": "gpt-5.3-codex", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "chatgpt_codex", + "sessionCount": 1, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 17144, + "output": 811, + "total": 17955 + } + }, + { + "client": "opencode", + "cost": 0.0228622, + "model": "zai-org/glm-5-maas", + "performance": { + "msPer1KTokens": 202.61784735119838, + "sampleCount": 1, + "timedTokens": 22614, + "tokenCoverage": 1.0, + "totalDurationMs": 4582 + }, + "provider": "google_vertex", + "sessionCount": 1, + "tokens": { + "cacheRead": 64, + "cacheWrite": 0, + "input": 22411, + "output": 139, + "total": 22614 + } + }, + { + "client": "claude", + "cost": 0.0, + "model": "", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "unknown", + "sessionCount": 9, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 0, + "output": 0, + "total": 0 + } + }, + { + "client": "opencode", + "cost": 0.0, + "model": "gemini-3.5-flash", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "google", + "sessionCount": 5, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 0, + "output": 0, + "total": 0 + } + }, + { + "client": "opencode", + "cost": 0.0, + "model": "glm-5.2", + "performance": { + "msPer1KTokens": 181.41072322195078, + "sampleCount": 2779, + "timedTokens": 1192833969, + "tokenCoverage": 1.0, + "totalDurationMs": 216392873 + }, + "provider": "zai_coding_plan", + "sessionCount": 21, + "tokens": { + "cacheRead": 1154017856, + "cacheWrite": 0, + "input": 37288456, + "output": 670975, + "total": 1192833969 + } + } + ], + "totals": { + "cost": 6287.528212200026, + "tokens": 10533853432 + } +} \ No newline at end of file diff --git a/tokscale-export-20260627-100851.json b/tokscale-export-20260627-100851.json new file mode 100644 index 00000000..7f671151 --- /dev/null +++ b/tokscale-export-20260627-100851.json @@ -0,0 +1,1556 @@ +{ + "agents": [ + { + "agent": "Descartes", + "clients": "codex", + "cost": 3.9003440000000005, + "messageCount": 44, + "tokens": { + "cacheRead": 3273728, + "cacheWrite": 0, + "input": 303710, + "output": 16738, + "total": 3602269 + } + }, + { + "agent": "Halley", + "clients": "codex", + "cost": 3.5619250000000005, + "messageCount": 37, + "tokens": { + "cacheRead": 4064640, + "cacheWrite": 0, + "input": 137651, + "output": 18565, + "total": 4230336 + } + }, + { + "agent": "Bernoulli", + "clients": "codex", + "cost": 3.3071219999999997, + "messageCount": 26, + "tokens": { + "cacheRead": 3279104, + "cacheWrite": 0, + "input": 207118, + "output": 13944, + "total": 3507288 + } + }, + { + "agent": "Turing", + "clients": "codex", + "cost": 3.306327, + "messageCount": 37, + "tokens": { + "cacheRead": 2737024, + "cacheWrite": 0, + "input": 259973, + "output": 15756, + "total": 3018262 + } + }, + { + "agent": "Goodall", + "clients": "codex", + "cost": 2.350671, + "messageCount": 24, + "tokens": { + "cacheRead": 2022912, + "cacheWrite": 0, + "input": 142023, + "output": 13282, + "total": 2185905 + } + }, + { + "agent": "Hubble", + "clients": "codex", + "cost": 2.3384470000000004, + "messageCount": 32, + "tokens": { + "cacheRead": 2179584, + "cacheWrite": 0, + "input": 103145, + "output": 16733, + "total": 2307160 + } + }, + { + "agent": "Build", + "clients": "opencode", + "cost": 2.0610812, + "messageCount": 2709, + "tokens": { + "cacheRead": 1150841920, + "cacheWrite": 0, + "input": 37385365, + "output": 641519, + "total": 1189691974 + } + }, + { + "agent": "Dewey", + "clients": "codex", + "cost": 1.7652170000000003, + "messageCount": 25, + "tokens": { + "cacheRead": 1062784, + "cacheWrite": 0, + "input": 151005, + "output": 14042, + "total": 1229749 + } + }, + { + "agent": "Hooke", + "clients": "codex", + "cost": 1.7155859999999998, + "messageCount": 24, + "tokens": { + "cacheRead": 1587712, + "cacheWrite": 0, + "input": 119786, + "output": 7280, + "total": 1718258 + } + }, + { + "agent": "Epicurus", + "clients": "codex", + "cost": 1.6247530000000001, + "messageCount": 19, + "tokens": { + "cacheRead": 1044096, + "cacheWrite": 0, + "input": 79103, + "output": 14410, + "total": 1146772 + } + }, + { + "agent": "Carson", + "clients": "codex", + "cost": 1.613164, + "messageCount": 17, + "tokens": { + "cacheRead": 1189248, + "cacheWrite": 0, + "input": 93848, + "output": 10725, + "total": 1301406 + } + }, + { + "agent": "Huygens", + "clients": "codex", + "cost": 1.4913770000000002, + "messageCount": 11, + "tokens": { + "cacheRead": 802944, + "cacheWrite": 0, + "input": 91873, + "output": 11578, + "total": 915835 + } + }, + { + "agent": "Bacon", + "clients": "codex", + "cost": 1.387087, + "messageCount": 16, + "tokens": { + "cacheRead": 1042944, + "cacheWrite": 0, + "input": 123383, + "output": 7324, + "total": 1174617 + } + }, + { + "agent": "Carver", + "clients": "codex", + "cost": 1.271722, + "messageCount": 13, + "tokens": { + "cacheRead": 786304, + "cacheWrite": 0, + "input": 139744, + "output": 5010, + "total": 932043 + } + }, + { + "agent": "Heisenberg", + "clients": "codex", + "cost": 1.2637669999999999, + "messageCount": 15, + "tokens": { + "cacheRead": 1046144, + "cacheWrite": 0, + "input": 107855, + "output": 5945, + "total": 1160713 + } + }, + { + "agent": "Schrodinger", + "clients": "codex", + "cost": 1.224408, + "messageCount": 10, + "tokens": { + "cacheRead": 542976, + "cacheWrite": 0, + "input": 91500, + "output": 10030, + "total": 650990 + } + }, + { + "agent": "Socrates", + "clients": "codex", + "cost": 1.2146380000000003, + "messageCount": 8, + "tokens": { + "cacheRead": 365056, + "cacheWrite": 0, + "input": 73558, + "output": 12603, + "total": 460758 + } + }, + { + "agent": "Banach", + "clients": "codex", + "cost": 0.9842209999999999, + "messageCount": 16, + "tokens": { + "cacheRead": 888832, + "cacheWrite": 0, + "input": 80667, + "output": 4095, + "total": 974048 + } + }, + { + "agent": "Pasteur", + "clients": "codex", + "cost": 0.9695810000000001, + "messageCount": 10, + "tokens": { + "cacheRead": 372992, + "cacheWrite": 0, + "input": 129083, + "output": 3727, + "total": 506664 + } + }, + { + "agent": "Arendt", + "clients": "codex", + "cost": 0.923487, + "messageCount": 11, + "tokens": { + "cacheRead": 534144, + "cacheWrite": 0, + "input": 89163, + "output": 6148, + "total": 630327 + } + }, + { + "agent": "Cicero", + "clients": "codex", + "cost": 0.850337, + "messageCount": 9, + "tokens": { + "cacheRead": 412544, + "cacheWrite": 0, + "input": 88823, + "output": 4517, + "total": 508032 + } + }, + { + "agent": "Beauvoir", + "clients": "codex", + "cost": 0.7617210000000001, + "messageCount": 11, + "tokens": { + "cacheRead": 561792, + "cacheWrite": 0, + "input": 60801, + "output": 5172, + "total": 628487 + } + }, + { + "agent": "Morpheus", + "clients": "codex", + "cost": 0.2864385, + "messageCount": 5, + "tokens": { + "cacheRead": 143744, + "cacheWrite": 0, + "input": 40987, + "output": 9188, + "total": 194600 + } + }, + { + "agent": "Explore", + "clients": "opencode", + "cost": 0.0, + "messageCount": 60, + "tokens": { + "cacheRead": 2002624, + "cacheWrite": 0, + "input": 243706, + "output": 16874, + "total": 2273117 + } + }, + { + "agent": "General", + "clients": "opencode", + "cost": 0.0, + "messageCount": 23, + "tokens": { + "cacheRead": 1014464, + "cacheWrite": 0, + "input": 197872, + "output": 6729, + "total": 1223267 + } + }, + { + "agent": "Plan", + "clients": "opencode", + "cost": 0.0, + "messageCount": 11, + "tokens": { + "cacheRead": 175808, + "cacheWrite": 0, + "input": 287916, + "output": 7581, + "total": 490714 + } + } + ], + "daily": [ + { + "cost": 62.898246249999964, + "date": "2026-06-27", + "messageCount": 412, + "tokens": { + "cacheRead": 214711221, + "cacheWrite": 3096478, + "input": 4669544, + "output": 331876, + "total": 222819611 + }, + "turnCount": 33 + }, + { + "cost": 33.576564749999996, + "date": "2026-06-26", + "messageCount": 397, + "tokens": { + "cacheRead": 131417100, + "cacheWrite": 763035, + "input": 6103880, + "output": 114508, + "total": 138450062 + }, + "turnCount": 77 + }, + { + "cost": 70.50142420000007, + "date": "2026-06-25", + "messageCount": 343, + "tokens": { + "cacheRead": 159025716, + "cacheWrite": 795462, + "input": 3186409, + "output": 152326, + "total": 163184841 + }, + "turnCount": 39 + }, + { + "cost": 162.3291599500004, + "date": "2026-06-24", + "messageCount": 1184, + "tokens": { + "cacheRead": 438077756, + "cacheWrite": 2775552, + "input": 11776844, + "output": 621585, + "total": 453424090 + }, + "turnCount": 86 + }, + { + "cost": 105.95392205000012, + "date": "2026-06-23", + "messageCount": 1147, + "tokens": { + "cacheRead": 205621324, + "cacheWrite": 826873, + "input": 5809960, + "output": 697935, + "total": 213133743 + }, + "turnCount": 173 + }, + { + "cost": 51.11505444999997, + "date": "2026-06-22", + "messageCount": 225, + "tokens": { + "cacheRead": 37714697, + "cacheWrite": 2184305, + "input": 2379565, + "output": 268571, + "total": 42570840 + }, + "turnCount": 58 + }, + { + "cost": 0.24963100000000002, + "date": "2026-06-21", + "messageCount": 1, + "tokens": { + "cacheRead": 2432, + "cacheWrite": 0, + "input": 42021, + "output": 761, + "total": 45730 + }, + "turnCount": 1 + }, + { + "cost": 125.32283240000004, + "date": "2026-06-20", + "messageCount": 633, + "tokens": { + "cacheRead": 229078486, + "cacheWrite": 3293334, + "input": 2450252, + "output": 437114, + "total": 235475603 + }, + "turnCount": 60 + }, + { + "cost": 109.40506945000021, + "date": "2026-06-19", + "messageCount": 596, + "tokens": { + "cacheRead": 125023572, + "cacheWrite": 3950057, + "input": 2763154, + "output": 365780, + "total": 132127969 + }, + "turnCount": 99 + }, + { + "cost": 182.32671845000021, + "date": "2026-06-18", + "messageCount": 828, + "tokens": { + "cacheRead": 254795546, + "cacheWrite": 4897255, + "input": 3196910, + "output": 464789, + "total": 263385211 + }, + "turnCount": 155 + }, + { + "cost": 107.97663059999991, + "date": "2026-06-17", + "messageCount": 935, + "tokens": { + "cacheRead": 388181758, + "cacheWrite": 5355354, + "input": 10950338, + "output": 484345, + "total": 405070767 + }, + "turnCount": 123 + }, + { + "cost": 82.96701420000005, + "date": "2026-06-16", + "messageCount": 1121, + "tokens": { + "cacheRead": 300020822, + "cacheWrite": 3887726, + "input": 12170344, + "output": 447398, + "total": 316787817 + }, + "turnCount": 123 + }, + { + "cost": 210.1056897, + "date": "2026-06-15", + "messageCount": 1292, + "tokens": { + "cacheRead": 326991687, + "cacheWrite": 3547204, + "input": 7187408, + "output": 740327, + "total": 338566790 + }, + "turnCount": 262 + }, + { + "cost": 104.23466350000008, + "date": "2026-06-14", + "messageCount": 531, + "tokens": { + "cacheRead": 152731876, + "cacheWrite": 1858804, + "input": 1917185, + "output": 519269, + "total": 157033466 + }, + "turnCount": 123 + }, + { + "cost": 60.763000850000005, + "date": "2026-06-13", + "messageCount": 448, + "tokens": { + "cacheRead": 64804925, + "cacheWrite": 1556132, + "input": 4043220, + "output": 174576, + "total": 70605033 + }, + "turnCount": 170 + }, + { + "cost": 118.33723094999999, + "date": "2026-06-12", + "messageCount": 583, + "tokens": { + "cacheRead": 141843299, + "cacheWrite": 4359974, + "input": 2910267, + "output": 381145, + "total": 149515655 + }, + "turnCount": 122 + }, + { + "cost": 55.53048034999999, + "date": "2026-06-11", + "messageCount": 461, + "tokens": { + "cacheRead": 64521768, + "cacheWrite": 1684070, + "input": 1006985, + "output": 328563, + "total": 67554580 + }, + "turnCount": 54 + }, + { + "cost": 57.13918449999999, + "date": "2026-06-10", + "messageCount": 603, + "tokens": { + "cacheRead": 58151750, + "cacheWrite": 1659270, + "input": 2605351, + "output": 357810, + "total": 62805349 + }, + "turnCount": 146 + }, + { + "cost": 66.93653610000003, + "date": "2026-06-09", + "messageCount": 662, + "tokens": { + "cacheRead": 75678705, + "cacheWrite": 639890, + "input": 3647781, + "output": 324419, + "total": 80350983 + }, + "turnCount": 120 + }, + { + "cost": 21.25041285000001, + "date": "2026-06-08", + "messageCount": 210, + "tokens": { + "cacheRead": 24254720, + "cacheWrite": 0, + "input": 2015170, + "output": 53976, + "total": 26337339 + }, + "turnCount": 51 + }, + { + "cost": 32.597969249999984, + "date": "2026-06-05", + "messageCount": 364, + "tokens": { + "cacheRead": 42004844, + "cacheWrite": 234916, + "input": 2044168, + "output": 144658, + "total": 44470302 + }, + "turnCount": 92 + }, + { + "cost": 99.52701779999997, + "date": "2026-06-04", + "messageCount": 1070, + "tokens": { + "cacheRead": 136146396, + "cacheWrite": 308228, + "input": 9415727, + "output": 392157, + "total": 146348394 + }, + "turnCount": 243 + }, + { + "cost": 36.19306024999999, + "date": "2026-06-03", + "messageCount": 394, + "tokens": { + "cacheRead": 45317888, + "cacheWrite": 288910, + "input": 1887407, + "output": 187801, + "total": 47720679 + }, + "turnCount": 66 + }, + { + "cost": 58.17762395, + "date": "2026-06-02", + "messageCount": 693, + "tokens": { + "cacheRead": 65343522, + "cacheWrite": 94275, + "input": 4334973, + "output": 277388, + "total": 70152002 + }, + "turnCount": 155 + }, + { + "cost": 98.65032284999987, + "date": "2026-06-01", + "messageCount": 1261, + "tokens": { + "cacheRead": 137791825, + "cacheWrite": 47101, + "input": 8946466, + "output": 273461, + "total": 147153863 + }, + "turnCount": 243 + }, + { + "cost": 3.0790345000000006, + "date": "2026-05-31", + "messageCount": 13, + "tokens": { + "cacheRead": 1441664, + "cacheWrite": 0, + "input": 719866, + "output": 7573, + "total": 2172216 + }, + "turnCount": 6 + }, + { + "cost": 17.297640949999998, + "date": "2026-05-30", + "messageCount": 131, + "tokens": { + "cacheRead": 22827165, + "cacheWrite": 325084, + "input": 667626, + "output": 37259, + "total": 23866819 + }, + "turnCount": 17 + }, + { + "cost": 60.68457075000004, + "date": "2026-05-29", + "messageCount": 711, + "tokens": { + "cacheRead": 90143620, + "cacheWrite": 582898, + "input": 5270864, + "output": 216268, + "total": 96275133 + }, + "turnCount": 148 + }, + { + "cost": 101.55679355000004, + "date": "2026-05-28", + "messageCount": 1304, + "tokens": { + "cacheRead": 165381980, + "cacheWrite": 493162, + "input": 6335145, + "output": 374782, + "total": 172674852 + }, + "turnCount": 252 + }, + { + "cost": 21.16416095, + "date": "2026-05-27", + "messageCount": 281, + "tokens": { + "cacheRead": 25242496, + "cacheWrite": 0, + "input": 1794683, + "output": 58174, + "total": 27116361 + }, + "turnCount": 53 + }, + { + "cost": 246.93044285000036, + "date": "2026-05-26", + "messageCount": 2689, + "tokens": { + "cacheRead": 386583466, + "cacheWrite": 375187, + "input": 11422065, + "output": 538790, + "total": 399029572 + }, + "turnCount": 183 + }, + { + "cost": 281.5894256, + "date": "2026-05-25", + "messageCount": 3530, + "tokens": { + "cacheRead": 444941605, + "cacheWrite": 390724, + "input": 14479594, + "output": 718228, + "total": 460757679 + }, + "turnCount": 473 + }, + { + "cost": 14.732135749999992, + "date": "2026-05-24", + "messageCount": 202, + "tokens": { + "cacheRead": 19969969, + "cacheWrite": 53280, + "input": 931975, + "output": 44125, + "total": 21010075 + }, + "turnCount": 23 + }, + { + "cost": 12.21099695, + "date": "2026-05-23", + "messageCount": 141, + "tokens": { + "cacheRead": 17840000, + "cacheWrite": 0, + "input": 597934, + "output": 23523, + "total": 18467567 + }, + "turnCount": 19 + }, + { + "cost": 51.15712750000008, + "date": "2026-05-22", + "messageCount": 643, + "tokens": { + "cacheRead": 75098752, + "cacheWrite": 0, + "input": 3099646, + "output": 122958, + "total": 78357079 + }, + "turnCount": 101 + }, + { + "cost": 149.05496409999986, + "date": "2026-05-21", + "messageCount": 1724, + "tokens": { + "cacheRead": 210405888, + "cacheWrite": 0, + "input": 10255744, + "output": 361599, + "total": 221098307 + }, + "turnCount": 260 + }, + { + "cost": 46.64824070000002, + "date": "2026-05-20", + "messageCount": 488, + "tokens": { + "cacheRead": 57876992, + "cacheWrite": 0, + "input": 2949920, + "output": 143287, + "total": 61016737 + }, + "turnCount": 60 + }, + { + "cost": 302.14533274999985, + "date": "2026-05-19", + "messageCount": 3311, + "tokens": { + "cacheRead": 436505728, + "cacheWrite": 0, + "input": 19132465, + "output": 825962, + "total": 456758419 + }, + "turnCount": 302 + }, + { + "cost": 250.6522364000004, + "date": "2026-05-18", + "messageCount": 2696, + "tokens": { + "cacheRead": 359022080, + "cacheWrite": 0, + "input": 13733347, + "output": 656851, + "total": 373656327 + }, + "turnCount": 249 + }, + { + "cost": 220.23427974999998, + "date": "2026-05-17", + "messageCount": 2472, + "tokens": { + "cacheRead": 371165184, + "cacheWrite": 0, + "input": 9235712, + "output": 452303, + "total": 380963488 + }, + "turnCount": 289 + }, + { + "cost": 132.72312964999975, + "date": "2026-05-16", + "messageCount": 1563, + "tokens": { + "cacheRead": 229541504, + "cacheWrite": 0, + "input": 6259001, + "output": 282044, + "total": 236138911 + }, + "turnCount": 186 + }, + { + "cost": 187.5979618500003, + "date": "2026-05-15", + "messageCount": 2174, + "tokens": { + "cacheRead": 292393728, + "cacheWrite": 0, + "input": 9962127, + "output": 471505, + "total": 302929103 + }, + "turnCount": 295 + }, + { + "cost": 171.11259285000003, + "date": "2026-05-14", + "messageCount": 1827, + "tokens": { + "cacheRead": 260370560, + "cacheWrite": 0, + "input": 12885534, + "output": 372793, + "total": 273724302 + }, + "turnCount": 327 + }, + { + "cost": 67.94910775000012, + "date": "2026-05-13", + "messageCount": 777, + "tokens": { + "cacheRead": 104286592, + "cacheWrite": 0, + "input": 3728901, + "output": 156135, + "total": 108200389 + }, + "turnCount": 71 + }, + { + "cost": 114.14174050000003, + "date": "2026-05-12", + "messageCount": 1257, + "tokens": { + "cacheRead": 165287296, + "cacheWrite": 0, + "input": 8041501, + "output": 277358, + "total": 173654443 + }, + "turnCount": 182 + }, + { + "cost": 46.584478750000045, + "date": "2026-05-11", + "messageCount": 462, + "tokens": { + "cacheRead": 62052608, + "cacheWrite": 0, + "input": 2936231, + "output": 112746, + "total": 65123944 + }, + "turnCount": 43 + }, + { + "cost": 152.83763230000025, + "date": "2026-05-10", + "messageCount": 1648, + "tokens": { + "cacheRead": 261582336, + "cacheWrite": 0, + "input": 9799813, + "output": 342140, + "total": 271802343 + }, + "turnCount": 324 + }, + { + "cost": 253.98307544999952, + "date": "2026-05-09", + "messageCount": 2718, + "tokens": { + "cacheRead": 451929856, + "cacheWrite": 0, + "input": 14305153, + "output": 489240, + "total": 466832273 + }, + "turnCount": 542 + }, + { + "cost": 225.4019883, + "date": "2026-05-08", + "messageCount": 2190, + "tokens": { + "cacheRead": 346067200, + "cacheWrite": 0, + "input": 15997101, + "output": 414419, + "total": 362568467 + }, + "turnCount": 366 + }, + { + "cost": 122.65184929999984, + "date": "2026-05-07", + "messageCount": 1440, + "tokens": { + "cacheRead": 200288256, + "cacheWrite": 0, + "input": 9317044, + "output": 266888, + "total": 209929076 + }, + "turnCount": 310 + }, + { + "cost": 256.69028255, + "date": "2026-05-06", + "messageCount": 3203, + "tokens": { + "cacheRead": 456758400, + "cacheWrite": 0, + "input": 15612265, + "output": 602277, + "total": 473088033 + }, + "turnCount": 593 + }, + { + "cost": 84.4895073, + "date": "2026-05-05", + "messageCount": 1091, + "tokens": { + "cacheRead": 143922816, + "cacheWrite": 0, + "input": 5318948, + "output": 183438, + "total": 149466661 + }, + "turnCount": 148 + }, + { + "cost": 0.29549400000000003, + "date": "2026-05-04", + "messageCount": 2, + "tokens": { + "cacheRead": 31488, + "cacheWrite": 0, + "input": 49188, + "output": 731, + "total": 81803 + }, + "turnCount": 2 + }, + { + "cost": 18.18216895000001, + "date": "2026-05-01", + "messageCount": 196, + "tokens": { + "cacheRead": 24443392, + "cacheWrite": 0, + "input": 1201693, + "output": 78701, + "total": 25752940 + }, + "turnCount": 50 + }, + { + "cost": 20.72562839999999, + "date": "2026-04-30", + "messageCount": 222, + "tokens": { + "cacheRead": 30970624, + "cacheWrite": 0, + "input": 2446434, + "output": 56533, + "total": 33486899 + }, + "turnCount": 58 + }, + { + "cost": 55.303921149999965, + "date": "2026-04-29", + "messageCount": 729, + "tokens": { + "cacheRead": 84425088, + "cacheWrite": 0, + "input": 4840140, + "output": 160913, + "total": 89467295 + }, + "turnCount": 194 + }, + { + "cost": 34.64550734999998, + "date": "2026-04-28", + "messageCount": 409, + "tokens": { + "cacheRead": 42745728, + "cacheWrite": 0, + "input": 2764854, + "output": 103134, + "total": 45645870 + }, + "turnCount": 68 + }, + { + "cost": 33.08216499999999, + "date": "2026-04-27", + "messageCount": 386, + "tokens": { + "cacheRead": 41121024, + "cacheWrite": 0, + "input": 2194255, + "output": 86398, + "total": 43425459 + }, + "turnCount": 42 + }, + { + "cost": 1.900379, + "date": "2026-04-26", + "messageCount": 9, + "tokens": { + "cacheRead": 1283968, + "cacheWrite": 0, + "input": 202305, + "output": 6373, + "total": 1494502 + }, + "turnCount": 3 + }, + { + "cost": 16.50628600000001, + "date": "2026-04-25", + "messageCount": 176, + "tokens": { + "cacheRead": 16425472, + "cacheWrite": 0, + "input": 1263526, + "output": 53751, + "total": 17754862 + }, + "turnCount": 19 + }, + { + "cost": 22.265109, + "date": "2026-04-24", + "messageCount": 349, + "tokens": { + "cacheRead": 31644032, + "cacheWrite": 0, + "input": 1970623, + "output": 98853, + "total": 33733821 + }, + "turnCount": 28 + }, + { + "cost": 11.174454999999998, + "date": "2026-04-23", + "messageCount": 184, + "tokens": { + "cacheRead": 21451520, + "cacheWrite": 0, + "input": 1546760, + "output": 95299, + "total": 23127925 + }, + "turnCount": 58 + }, + { + "cost": 21.544721999999993, + "date": "2026-04-22", + "messageCount": 361, + "tokens": { + "cacheRead": 37253248, + "cacheWrite": 0, + "input": 3738740, + "output": 141157, + "total": 41184292 + }, + "turnCount": 54 + }, + { + "cost": 6.878394900000001, + "date": "2026-04-21", + "messageCount": 113, + "tokens": { + "cacheRead": 11123328, + "cacheWrite": 0, + "input": 2245378, + "output": 29966, + "total": 13402492 + }, + "turnCount": 9 + }, + { + "cost": 7.503686, + "date": "2026-04-20", + "messageCount": 178, + "tokens": { + "cacheRead": 16731904, + "cacheWrite": 0, + "input": 948628, + "output": 50664, + "total": 17743808 + }, + "turnCount": 12 + }, + { + "cost": 13.077388000000003, + "date": "2026-04-19", + "messageCount": 8, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 12826012, + "output": 78555, + "total": 12904567 + }, + "turnCount": 0 + }, + { + "cost": 4.3518446, + "date": "2026-04-18", + "messageCount": 66, + "tokens": { + "cacheRead": 8106240, + "cacheWrite": 0, + "input": 1172430, + "output": 18697, + "total": 9302114 + }, + "turnCount": 1 + }, + { + "cost": 11.9593086, + "date": "2026-04-17", + "messageCount": 147, + "tokens": { + "cacheRead": 14878976, + "cacheWrite": 0, + "input": 4426250, + "output": 68876, + "total": 19386220 + }, + "turnCount": 16 + }, + { + "cost": 7.787204500000003, + "date": "2026-04-16", + "messageCount": 95, + "tokens": { + "cacheRead": 13781248, + "cacheWrite": 0, + "input": 1503039, + "output": 32230, + "total": 15323240 + }, + "turnCount": 18 + }, + { + "cost": 194.5845475, + "date": "2026-04-15", + "messageCount": 11, + "tokens": { + "cacheRead": 1336960, + "cacheWrite": 0, + "input": 38755320, + "output": 169570, + "total": 40262246 + }, + "turnCount": 3 + }, + { + "cost": 46.75895299999998, + "date": "2026-04-14", + "messageCount": 590, + "tokens": { + "cacheRead": 65797632, + "cacheWrite": 0, + "input": 10736846, + "output": 180419, + "total": 76765640 + }, + "turnCount": 111 + }, + { + "cost": 2.4843545, + "date": "2026-04-13", + "messageCount": 74, + "tokens": { + "cacheRead": 2289408, + "cacheWrite": 0, + "input": 483209, + "output": 36385, + "total": 2819549 + }, + "turnCount": 17 + }, + { + "cost": 7.691671500000001, + "date": "2026-04-10", + "messageCount": 126, + "tokens": { + "cacheRead": 15201536, + "cacheWrite": 0, + "input": 1345627, + "output": 28877, + "total": 16582311 + }, + "turnCount": 7 + }, + { + "cost": 29.540619000000024, + "date": "2026-04-09", + "messageCount": 506, + "tokens": { + "cacheRead": 46094976, + "cacheWrite": 0, + "input": 6170646, + "output": 139003, + "total": 52438306 + }, + "turnCount": 73 + }, + { + "cost": 9.918214, + "date": "2026-04-08", + "messageCount": 156, + "tokens": { + "cacheRead": 18131456, + "cacheWrite": 0, + "input": 1808996, + "output": 45397, + "total": 19997976 + }, + "turnCount": 22 + } + ], + "models": [ + { + "client": "codex", + "cost": 4476.965901000025, + "model": "gpt-5.5", + "performance": { + "msPer1KTokens": 4524.020180595458, + "sampleCount": 44616, + "timedTokens": 6328004754, + "tokenCoverage": 1.0, + "totalDurationMs": 28628021210 + }, + "provider": "openai", + "sessionCount": 121, + "tokens": { + "cacheRead": 6113071232, + "cacheWrite": 0, + "input": 201103015, + "output": 10839269, + "total": 6328004754 + } + }, + { + "client": "claude", + "cost": 1306.033157000002, + "model": "claude-opus-4-8", + "performance": { + "msPer1KTokens": 50.03828284427112, + "sampleCount": 4702, + "timedTokens": 1774442346, + "tokenCoverage": 0.9923299290410101, + "totalDurationMs": 88790048 + }, + "provider": "anthropic", + "sessionCount": 19, + "tokens": { + "cacheRead": 1733100519, + "cacheWrite": 48029010, + "input": 1820062, + "output": 5208051, + "total": 1788157642 + } + }, + { + "client": "opencode, codex, goose", + "cost": 400.8453585000001, + "model": "gpt-5.4", + "performance": { + "msPer1KTokens": 5007.590199461738, + "sampleCount": 3222, + "timedTokens": 379225212, + "tokenCoverage": 0.9081412800891782, + "totalDurationMs": 1899004455 + }, + "provider": "openai, chatgpt_codex", + "sessionCount": 37, + "tokens": { + "cacheRead": 336189824, + "cacheWrite": 0, + "input": 79914106, + "output": 1189409, + "total": 417583938 + } + }, + { + "client": "claude", + "cost": 43.60413899999995, + "model": "claude-opus-4-7", + "performance": { + "msPer1KTokens": 88.288596602198, + "sampleCount": 282, + "timedTokens": 44946586, + "tokenCoverage": 0.9708623063422688, + "totalDurationMs": 3968271 + }, + "provider": "anthropic", + "sessionCount": 4, + "tokens": { + "cacheRead": 43571353, + "cacheWrite": 2295530, + "input": 162240, + "output": 266408, + "total": 46295531 + } + }, + { + "client": "codex", + "cost": 42.4404918999999, + "model": "codex-auto-review", + "performance": { + "msPer1KTokens": 40.30321610779724, + "sampleCount": 6470, + "timedTokens": 743616959, + "tokenCoverage": 1.0, + "totalDurationMs": 29970155 + }, + "provider": "openai", + "sessionCount": 221, + "tokens": { + "cacheRead": 643499776, + "cacheWrite": 0, + "input": 99360782, + "output": 564770, + "total": 743616959 + } + }, + { + "client": "goose", + "cost": 17.5749466, + "model": "glm-5.1", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "custom_z.ai", + "sessionCount": 15, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 17204217, + "output": 115853, + "total": 17320070 + } + }, + { + "client": "goose", + "cost": 0.041356000000000004, + "model": "gpt-5.3-codex", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "chatgpt_codex", + "sessionCount": 1, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 17144, + "output": 811, + "total": 17955 + } + }, + { + "client": "opencode", + "cost": 0.0228622, + "model": "zai-org/glm-5-maas", + "performance": { + "msPer1KTokens": 202.61784735119838, + "sampleCount": 1, + "timedTokens": 22614, + "tokenCoverage": 1.0, + "totalDurationMs": 4582 + }, + "provider": "google_vertex", + "sessionCount": 1, + "tokens": { + "cacheRead": 64, + "cacheWrite": 0, + "input": 22411, + "output": 139, + "total": 22614 + } + }, + { + "client": "claude", + "cost": 0.0, + "model": "", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "unknown", + "sessionCount": 9, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 0, + "output": 0, + "total": 0 + } + }, + { + "client": "opencode", + "cost": 0.0, + "model": "gemini-3.5-flash", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "google", + "sessionCount": 5, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 0, + "output": 0, + "total": 0 + } + }, + { + "client": "opencode", + "cost": 0.0, + "model": "glm-5.2", + "performance": { + "msPer1KTokens": 181.41072322195078, + "sampleCount": 2779, + "timedTokens": 1192833969, + "tokenCoverage": 1.0, + "totalDurationMs": 216392873 + }, + "provider": "zai_coding_plan", + "sessionCount": 21, + "tokens": { + "cacheRead": 1154017856, + "cacheWrite": 0, + "input": 37288456, + "output": 670975, + "total": 1192833969 + } + } + ], + "totals": { + "cost": 6287.528212200026, + "tokens": 10533853432 + } +} \ No newline at end of file diff --git a/tokscale-export-20260627-100853.json b/tokscale-export-20260627-100853.json new file mode 100644 index 00000000..7f671151 --- /dev/null +++ b/tokscale-export-20260627-100853.json @@ -0,0 +1,1556 @@ +{ + "agents": [ + { + "agent": "Descartes", + "clients": "codex", + "cost": 3.9003440000000005, + "messageCount": 44, + "tokens": { + "cacheRead": 3273728, + "cacheWrite": 0, + "input": 303710, + "output": 16738, + "total": 3602269 + } + }, + { + "agent": "Halley", + "clients": "codex", + "cost": 3.5619250000000005, + "messageCount": 37, + "tokens": { + "cacheRead": 4064640, + "cacheWrite": 0, + "input": 137651, + "output": 18565, + "total": 4230336 + } + }, + { + "agent": "Bernoulli", + "clients": "codex", + "cost": 3.3071219999999997, + "messageCount": 26, + "tokens": { + "cacheRead": 3279104, + "cacheWrite": 0, + "input": 207118, + "output": 13944, + "total": 3507288 + } + }, + { + "agent": "Turing", + "clients": "codex", + "cost": 3.306327, + "messageCount": 37, + "tokens": { + "cacheRead": 2737024, + "cacheWrite": 0, + "input": 259973, + "output": 15756, + "total": 3018262 + } + }, + { + "agent": "Goodall", + "clients": "codex", + "cost": 2.350671, + "messageCount": 24, + "tokens": { + "cacheRead": 2022912, + "cacheWrite": 0, + "input": 142023, + "output": 13282, + "total": 2185905 + } + }, + { + "agent": "Hubble", + "clients": "codex", + "cost": 2.3384470000000004, + "messageCount": 32, + "tokens": { + "cacheRead": 2179584, + "cacheWrite": 0, + "input": 103145, + "output": 16733, + "total": 2307160 + } + }, + { + "agent": "Build", + "clients": "opencode", + "cost": 2.0610812, + "messageCount": 2709, + "tokens": { + "cacheRead": 1150841920, + "cacheWrite": 0, + "input": 37385365, + "output": 641519, + "total": 1189691974 + } + }, + { + "agent": "Dewey", + "clients": "codex", + "cost": 1.7652170000000003, + "messageCount": 25, + "tokens": { + "cacheRead": 1062784, + "cacheWrite": 0, + "input": 151005, + "output": 14042, + "total": 1229749 + } + }, + { + "agent": "Hooke", + "clients": "codex", + "cost": 1.7155859999999998, + "messageCount": 24, + "tokens": { + "cacheRead": 1587712, + "cacheWrite": 0, + "input": 119786, + "output": 7280, + "total": 1718258 + } + }, + { + "agent": "Epicurus", + "clients": "codex", + "cost": 1.6247530000000001, + "messageCount": 19, + "tokens": { + "cacheRead": 1044096, + "cacheWrite": 0, + "input": 79103, + "output": 14410, + "total": 1146772 + } + }, + { + "agent": "Carson", + "clients": "codex", + "cost": 1.613164, + "messageCount": 17, + "tokens": { + "cacheRead": 1189248, + "cacheWrite": 0, + "input": 93848, + "output": 10725, + "total": 1301406 + } + }, + { + "agent": "Huygens", + "clients": "codex", + "cost": 1.4913770000000002, + "messageCount": 11, + "tokens": { + "cacheRead": 802944, + "cacheWrite": 0, + "input": 91873, + "output": 11578, + "total": 915835 + } + }, + { + "agent": "Bacon", + "clients": "codex", + "cost": 1.387087, + "messageCount": 16, + "tokens": { + "cacheRead": 1042944, + "cacheWrite": 0, + "input": 123383, + "output": 7324, + "total": 1174617 + } + }, + { + "agent": "Carver", + "clients": "codex", + "cost": 1.271722, + "messageCount": 13, + "tokens": { + "cacheRead": 786304, + "cacheWrite": 0, + "input": 139744, + "output": 5010, + "total": 932043 + } + }, + { + "agent": "Heisenberg", + "clients": "codex", + "cost": 1.2637669999999999, + "messageCount": 15, + "tokens": { + "cacheRead": 1046144, + "cacheWrite": 0, + "input": 107855, + "output": 5945, + "total": 1160713 + } + }, + { + "agent": "Schrodinger", + "clients": "codex", + "cost": 1.224408, + "messageCount": 10, + "tokens": { + "cacheRead": 542976, + "cacheWrite": 0, + "input": 91500, + "output": 10030, + "total": 650990 + } + }, + { + "agent": "Socrates", + "clients": "codex", + "cost": 1.2146380000000003, + "messageCount": 8, + "tokens": { + "cacheRead": 365056, + "cacheWrite": 0, + "input": 73558, + "output": 12603, + "total": 460758 + } + }, + { + "agent": "Banach", + "clients": "codex", + "cost": 0.9842209999999999, + "messageCount": 16, + "tokens": { + "cacheRead": 888832, + "cacheWrite": 0, + "input": 80667, + "output": 4095, + "total": 974048 + } + }, + { + "agent": "Pasteur", + "clients": "codex", + "cost": 0.9695810000000001, + "messageCount": 10, + "tokens": { + "cacheRead": 372992, + "cacheWrite": 0, + "input": 129083, + "output": 3727, + "total": 506664 + } + }, + { + "agent": "Arendt", + "clients": "codex", + "cost": 0.923487, + "messageCount": 11, + "tokens": { + "cacheRead": 534144, + "cacheWrite": 0, + "input": 89163, + "output": 6148, + "total": 630327 + } + }, + { + "agent": "Cicero", + "clients": "codex", + "cost": 0.850337, + "messageCount": 9, + "tokens": { + "cacheRead": 412544, + "cacheWrite": 0, + "input": 88823, + "output": 4517, + "total": 508032 + } + }, + { + "agent": "Beauvoir", + "clients": "codex", + "cost": 0.7617210000000001, + "messageCount": 11, + "tokens": { + "cacheRead": 561792, + "cacheWrite": 0, + "input": 60801, + "output": 5172, + "total": 628487 + } + }, + { + "agent": "Morpheus", + "clients": "codex", + "cost": 0.2864385, + "messageCount": 5, + "tokens": { + "cacheRead": 143744, + "cacheWrite": 0, + "input": 40987, + "output": 9188, + "total": 194600 + } + }, + { + "agent": "Explore", + "clients": "opencode", + "cost": 0.0, + "messageCount": 60, + "tokens": { + "cacheRead": 2002624, + "cacheWrite": 0, + "input": 243706, + "output": 16874, + "total": 2273117 + } + }, + { + "agent": "General", + "clients": "opencode", + "cost": 0.0, + "messageCount": 23, + "tokens": { + "cacheRead": 1014464, + "cacheWrite": 0, + "input": 197872, + "output": 6729, + "total": 1223267 + } + }, + { + "agent": "Plan", + "clients": "opencode", + "cost": 0.0, + "messageCount": 11, + "tokens": { + "cacheRead": 175808, + "cacheWrite": 0, + "input": 287916, + "output": 7581, + "total": 490714 + } + } + ], + "daily": [ + { + "cost": 62.898246249999964, + "date": "2026-06-27", + "messageCount": 412, + "tokens": { + "cacheRead": 214711221, + "cacheWrite": 3096478, + "input": 4669544, + "output": 331876, + "total": 222819611 + }, + "turnCount": 33 + }, + { + "cost": 33.576564749999996, + "date": "2026-06-26", + "messageCount": 397, + "tokens": { + "cacheRead": 131417100, + "cacheWrite": 763035, + "input": 6103880, + "output": 114508, + "total": 138450062 + }, + "turnCount": 77 + }, + { + "cost": 70.50142420000007, + "date": "2026-06-25", + "messageCount": 343, + "tokens": { + "cacheRead": 159025716, + "cacheWrite": 795462, + "input": 3186409, + "output": 152326, + "total": 163184841 + }, + "turnCount": 39 + }, + { + "cost": 162.3291599500004, + "date": "2026-06-24", + "messageCount": 1184, + "tokens": { + "cacheRead": 438077756, + "cacheWrite": 2775552, + "input": 11776844, + "output": 621585, + "total": 453424090 + }, + "turnCount": 86 + }, + { + "cost": 105.95392205000012, + "date": "2026-06-23", + "messageCount": 1147, + "tokens": { + "cacheRead": 205621324, + "cacheWrite": 826873, + "input": 5809960, + "output": 697935, + "total": 213133743 + }, + "turnCount": 173 + }, + { + "cost": 51.11505444999997, + "date": "2026-06-22", + "messageCount": 225, + "tokens": { + "cacheRead": 37714697, + "cacheWrite": 2184305, + "input": 2379565, + "output": 268571, + "total": 42570840 + }, + "turnCount": 58 + }, + { + "cost": 0.24963100000000002, + "date": "2026-06-21", + "messageCount": 1, + "tokens": { + "cacheRead": 2432, + "cacheWrite": 0, + "input": 42021, + "output": 761, + "total": 45730 + }, + "turnCount": 1 + }, + { + "cost": 125.32283240000004, + "date": "2026-06-20", + "messageCount": 633, + "tokens": { + "cacheRead": 229078486, + "cacheWrite": 3293334, + "input": 2450252, + "output": 437114, + "total": 235475603 + }, + "turnCount": 60 + }, + { + "cost": 109.40506945000021, + "date": "2026-06-19", + "messageCount": 596, + "tokens": { + "cacheRead": 125023572, + "cacheWrite": 3950057, + "input": 2763154, + "output": 365780, + "total": 132127969 + }, + "turnCount": 99 + }, + { + "cost": 182.32671845000021, + "date": "2026-06-18", + "messageCount": 828, + "tokens": { + "cacheRead": 254795546, + "cacheWrite": 4897255, + "input": 3196910, + "output": 464789, + "total": 263385211 + }, + "turnCount": 155 + }, + { + "cost": 107.97663059999991, + "date": "2026-06-17", + "messageCount": 935, + "tokens": { + "cacheRead": 388181758, + "cacheWrite": 5355354, + "input": 10950338, + "output": 484345, + "total": 405070767 + }, + "turnCount": 123 + }, + { + "cost": 82.96701420000005, + "date": "2026-06-16", + "messageCount": 1121, + "tokens": { + "cacheRead": 300020822, + "cacheWrite": 3887726, + "input": 12170344, + "output": 447398, + "total": 316787817 + }, + "turnCount": 123 + }, + { + "cost": 210.1056897, + "date": "2026-06-15", + "messageCount": 1292, + "tokens": { + "cacheRead": 326991687, + "cacheWrite": 3547204, + "input": 7187408, + "output": 740327, + "total": 338566790 + }, + "turnCount": 262 + }, + { + "cost": 104.23466350000008, + "date": "2026-06-14", + "messageCount": 531, + "tokens": { + "cacheRead": 152731876, + "cacheWrite": 1858804, + "input": 1917185, + "output": 519269, + "total": 157033466 + }, + "turnCount": 123 + }, + { + "cost": 60.763000850000005, + "date": "2026-06-13", + "messageCount": 448, + "tokens": { + "cacheRead": 64804925, + "cacheWrite": 1556132, + "input": 4043220, + "output": 174576, + "total": 70605033 + }, + "turnCount": 170 + }, + { + "cost": 118.33723094999999, + "date": "2026-06-12", + "messageCount": 583, + "tokens": { + "cacheRead": 141843299, + "cacheWrite": 4359974, + "input": 2910267, + "output": 381145, + "total": 149515655 + }, + "turnCount": 122 + }, + { + "cost": 55.53048034999999, + "date": "2026-06-11", + "messageCount": 461, + "tokens": { + "cacheRead": 64521768, + "cacheWrite": 1684070, + "input": 1006985, + "output": 328563, + "total": 67554580 + }, + "turnCount": 54 + }, + { + "cost": 57.13918449999999, + "date": "2026-06-10", + "messageCount": 603, + "tokens": { + "cacheRead": 58151750, + "cacheWrite": 1659270, + "input": 2605351, + "output": 357810, + "total": 62805349 + }, + "turnCount": 146 + }, + { + "cost": 66.93653610000003, + "date": "2026-06-09", + "messageCount": 662, + "tokens": { + "cacheRead": 75678705, + "cacheWrite": 639890, + "input": 3647781, + "output": 324419, + "total": 80350983 + }, + "turnCount": 120 + }, + { + "cost": 21.25041285000001, + "date": "2026-06-08", + "messageCount": 210, + "tokens": { + "cacheRead": 24254720, + "cacheWrite": 0, + "input": 2015170, + "output": 53976, + "total": 26337339 + }, + "turnCount": 51 + }, + { + "cost": 32.597969249999984, + "date": "2026-06-05", + "messageCount": 364, + "tokens": { + "cacheRead": 42004844, + "cacheWrite": 234916, + "input": 2044168, + "output": 144658, + "total": 44470302 + }, + "turnCount": 92 + }, + { + "cost": 99.52701779999997, + "date": "2026-06-04", + "messageCount": 1070, + "tokens": { + "cacheRead": 136146396, + "cacheWrite": 308228, + "input": 9415727, + "output": 392157, + "total": 146348394 + }, + "turnCount": 243 + }, + { + "cost": 36.19306024999999, + "date": "2026-06-03", + "messageCount": 394, + "tokens": { + "cacheRead": 45317888, + "cacheWrite": 288910, + "input": 1887407, + "output": 187801, + "total": 47720679 + }, + "turnCount": 66 + }, + { + "cost": 58.17762395, + "date": "2026-06-02", + "messageCount": 693, + "tokens": { + "cacheRead": 65343522, + "cacheWrite": 94275, + "input": 4334973, + "output": 277388, + "total": 70152002 + }, + "turnCount": 155 + }, + { + "cost": 98.65032284999987, + "date": "2026-06-01", + "messageCount": 1261, + "tokens": { + "cacheRead": 137791825, + "cacheWrite": 47101, + "input": 8946466, + "output": 273461, + "total": 147153863 + }, + "turnCount": 243 + }, + { + "cost": 3.0790345000000006, + "date": "2026-05-31", + "messageCount": 13, + "tokens": { + "cacheRead": 1441664, + "cacheWrite": 0, + "input": 719866, + "output": 7573, + "total": 2172216 + }, + "turnCount": 6 + }, + { + "cost": 17.297640949999998, + "date": "2026-05-30", + "messageCount": 131, + "tokens": { + "cacheRead": 22827165, + "cacheWrite": 325084, + "input": 667626, + "output": 37259, + "total": 23866819 + }, + "turnCount": 17 + }, + { + "cost": 60.68457075000004, + "date": "2026-05-29", + "messageCount": 711, + "tokens": { + "cacheRead": 90143620, + "cacheWrite": 582898, + "input": 5270864, + "output": 216268, + "total": 96275133 + }, + "turnCount": 148 + }, + { + "cost": 101.55679355000004, + "date": "2026-05-28", + "messageCount": 1304, + "tokens": { + "cacheRead": 165381980, + "cacheWrite": 493162, + "input": 6335145, + "output": 374782, + "total": 172674852 + }, + "turnCount": 252 + }, + { + "cost": 21.16416095, + "date": "2026-05-27", + "messageCount": 281, + "tokens": { + "cacheRead": 25242496, + "cacheWrite": 0, + "input": 1794683, + "output": 58174, + "total": 27116361 + }, + "turnCount": 53 + }, + { + "cost": 246.93044285000036, + "date": "2026-05-26", + "messageCount": 2689, + "tokens": { + "cacheRead": 386583466, + "cacheWrite": 375187, + "input": 11422065, + "output": 538790, + "total": 399029572 + }, + "turnCount": 183 + }, + { + "cost": 281.5894256, + "date": "2026-05-25", + "messageCount": 3530, + "tokens": { + "cacheRead": 444941605, + "cacheWrite": 390724, + "input": 14479594, + "output": 718228, + "total": 460757679 + }, + "turnCount": 473 + }, + { + "cost": 14.732135749999992, + "date": "2026-05-24", + "messageCount": 202, + "tokens": { + "cacheRead": 19969969, + "cacheWrite": 53280, + "input": 931975, + "output": 44125, + "total": 21010075 + }, + "turnCount": 23 + }, + { + "cost": 12.21099695, + "date": "2026-05-23", + "messageCount": 141, + "tokens": { + "cacheRead": 17840000, + "cacheWrite": 0, + "input": 597934, + "output": 23523, + "total": 18467567 + }, + "turnCount": 19 + }, + { + "cost": 51.15712750000008, + "date": "2026-05-22", + "messageCount": 643, + "tokens": { + "cacheRead": 75098752, + "cacheWrite": 0, + "input": 3099646, + "output": 122958, + "total": 78357079 + }, + "turnCount": 101 + }, + { + "cost": 149.05496409999986, + "date": "2026-05-21", + "messageCount": 1724, + "tokens": { + "cacheRead": 210405888, + "cacheWrite": 0, + "input": 10255744, + "output": 361599, + "total": 221098307 + }, + "turnCount": 260 + }, + { + "cost": 46.64824070000002, + "date": "2026-05-20", + "messageCount": 488, + "tokens": { + "cacheRead": 57876992, + "cacheWrite": 0, + "input": 2949920, + "output": 143287, + "total": 61016737 + }, + "turnCount": 60 + }, + { + "cost": 302.14533274999985, + "date": "2026-05-19", + "messageCount": 3311, + "tokens": { + "cacheRead": 436505728, + "cacheWrite": 0, + "input": 19132465, + "output": 825962, + "total": 456758419 + }, + "turnCount": 302 + }, + { + "cost": 250.6522364000004, + "date": "2026-05-18", + "messageCount": 2696, + "tokens": { + "cacheRead": 359022080, + "cacheWrite": 0, + "input": 13733347, + "output": 656851, + "total": 373656327 + }, + "turnCount": 249 + }, + { + "cost": 220.23427974999998, + "date": "2026-05-17", + "messageCount": 2472, + "tokens": { + "cacheRead": 371165184, + "cacheWrite": 0, + "input": 9235712, + "output": 452303, + "total": 380963488 + }, + "turnCount": 289 + }, + { + "cost": 132.72312964999975, + "date": "2026-05-16", + "messageCount": 1563, + "tokens": { + "cacheRead": 229541504, + "cacheWrite": 0, + "input": 6259001, + "output": 282044, + "total": 236138911 + }, + "turnCount": 186 + }, + { + "cost": 187.5979618500003, + "date": "2026-05-15", + "messageCount": 2174, + "tokens": { + "cacheRead": 292393728, + "cacheWrite": 0, + "input": 9962127, + "output": 471505, + "total": 302929103 + }, + "turnCount": 295 + }, + { + "cost": 171.11259285000003, + "date": "2026-05-14", + "messageCount": 1827, + "tokens": { + "cacheRead": 260370560, + "cacheWrite": 0, + "input": 12885534, + "output": 372793, + "total": 273724302 + }, + "turnCount": 327 + }, + { + "cost": 67.94910775000012, + "date": "2026-05-13", + "messageCount": 777, + "tokens": { + "cacheRead": 104286592, + "cacheWrite": 0, + "input": 3728901, + "output": 156135, + "total": 108200389 + }, + "turnCount": 71 + }, + { + "cost": 114.14174050000003, + "date": "2026-05-12", + "messageCount": 1257, + "tokens": { + "cacheRead": 165287296, + "cacheWrite": 0, + "input": 8041501, + "output": 277358, + "total": 173654443 + }, + "turnCount": 182 + }, + { + "cost": 46.584478750000045, + "date": "2026-05-11", + "messageCount": 462, + "tokens": { + "cacheRead": 62052608, + "cacheWrite": 0, + "input": 2936231, + "output": 112746, + "total": 65123944 + }, + "turnCount": 43 + }, + { + "cost": 152.83763230000025, + "date": "2026-05-10", + "messageCount": 1648, + "tokens": { + "cacheRead": 261582336, + "cacheWrite": 0, + "input": 9799813, + "output": 342140, + "total": 271802343 + }, + "turnCount": 324 + }, + { + "cost": 253.98307544999952, + "date": "2026-05-09", + "messageCount": 2718, + "tokens": { + "cacheRead": 451929856, + "cacheWrite": 0, + "input": 14305153, + "output": 489240, + "total": 466832273 + }, + "turnCount": 542 + }, + { + "cost": 225.4019883, + "date": "2026-05-08", + "messageCount": 2190, + "tokens": { + "cacheRead": 346067200, + "cacheWrite": 0, + "input": 15997101, + "output": 414419, + "total": 362568467 + }, + "turnCount": 366 + }, + { + "cost": 122.65184929999984, + "date": "2026-05-07", + "messageCount": 1440, + "tokens": { + "cacheRead": 200288256, + "cacheWrite": 0, + "input": 9317044, + "output": 266888, + "total": 209929076 + }, + "turnCount": 310 + }, + { + "cost": 256.69028255, + "date": "2026-05-06", + "messageCount": 3203, + "tokens": { + "cacheRead": 456758400, + "cacheWrite": 0, + "input": 15612265, + "output": 602277, + "total": 473088033 + }, + "turnCount": 593 + }, + { + "cost": 84.4895073, + "date": "2026-05-05", + "messageCount": 1091, + "tokens": { + "cacheRead": 143922816, + "cacheWrite": 0, + "input": 5318948, + "output": 183438, + "total": 149466661 + }, + "turnCount": 148 + }, + { + "cost": 0.29549400000000003, + "date": "2026-05-04", + "messageCount": 2, + "tokens": { + "cacheRead": 31488, + "cacheWrite": 0, + "input": 49188, + "output": 731, + "total": 81803 + }, + "turnCount": 2 + }, + { + "cost": 18.18216895000001, + "date": "2026-05-01", + "messageCount": 196, + "tokens": { + "cacheRead": 24443392, + "cacheWrite": 0, + "input": 1201693, + "output": 78701, + "total": 25752940 + }, + "turnCount": 50 + }, + { + "cost": 20.72562839999999, + "date": "2026-04-30", + "messageCount": 222, + "tokens": { + "cacheRead": 30970624, + "cacheWrite": 0, + "input": 2446434, + "output": 56533, + "total": 33486899 + }, + "turnCount": 58 + }, + { + "cost": 55.303921149999965, + "date": "2026-04-29", + "messageCount": 729, + "tokens": { + "cacheRead": 84425088, + "cacheWrite": 0, + "input": 4840140, + "output": 160913, + "total": 89467295 + }, + "turnCount": 194 + }, + { + "cost": 34.64550734999998, + "date": "2026-04-28", + "messageCount": 409, + "tokens": { + "cacheRead": 42745728, + "cacheWrite": 0, + "input": 2764854, + "output": 103134, + "total": 45645870 + }, + "turnCount": 68 + }, + { + "cost": 33.08216499999999, + "date": "2026-04-27", + "messageCount": 386, + "tokens": { + "cacheRead": 41121024, + "cacheWrite": 0, + "input": 2194255, + "output": 86398, + "total": 43425459 + }, + "turnCount": 42 + }, + { + "cost": 1.900379, + "date": "2026-04-26", + "messageCount": 9, + "tokens": { + "cacheRead": 1283968, + "cacheWrite": 0, + "input": 202305, + "output": 6373, + "total": 1494502 + }, + "turnCount": 3 + }, + { + "cost": 16.50628600000001, + "date": "2026-04-25", + "messageCount": 176, + "tokens": { + "cacheRead": 16425472, + "cacheWrite": 0, + "input": 1263526, + "output": 53751, + "total": 17754862 + }, + "turnCount": 19 + }, + { + "cost": 22.265109, + "date": "2026-04-24", + "messageCount": 349, + "tokens": { + "cacheRead": 31644032, + "cacheWrite": 0, + "input": 1970623, + "output": 98853, + "total": 33733821 + }, + "turnCount": 28 + }, + { + "cost": 11.174454999999998, + "date": "2026-04-23", + "messageCount": 184, + "tokens": { + "cacheRead": 21451520, + "cacheWrite": 0, + "input": 1546760, + "output": 95299, + "total": 23127925 + }, + "turnCount": 58 + }, + { + "cost": 21.544721999999993, + "date": "2026-04-22", + "messageCount": 361, + "tokens": { + "cacheRead": 37253248, + "cacheWrite": 0, + "input": 3738740, + "output": 141157, + "total": 41184292 + }, + "turnCount": 54 + }, + { + "cost": 6.878394900000001, + "date": "2026-04-21", + "messageCount": 113, + "tokens": { + "cacheRead": 11123328, + "cacheWrite": 0, + "input": 2245378, + "output": 29966, + "total": 13402492 + }, + "turnCount": 9 + }, + { + "cost": 7.503686, + "date": "2026-04-20", + "messageCount": 178, + "tokens": { + "cacheRead": 16731904, + "cacheWrite": 0, + "input": 948628, + "output": 50664, + "total": 17743808 + }, + "turnCount": 12 + }, + { + "cost": 13.077388000000003, + "date": "2026-04-19", + "messageCount": 8, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 12826012, + "output": 78555, + "total": 12904567 + }, + "turnCount": 0 + }, + { + "cost": 4.3518446, + "date": "2026-04-18", + "messageCount": 66, + "tokens": { + "cacheRead": 8106240, + "cacheWrite": 0, + "input": 1172430, + "output": 18697, + "total": 9302114 + }, + "turnCount": 1 + }, + { + "cost": 11.9593086, + "date": "2026-04-17", + "messageCount": 147, + "tokens": { + "cacheRead": 14878976, + "cacheWrite": 0, + "input": 4426250, + "output": 68876, + "total": 19386220 + }, + "turnCount": 16 + }, + { + "cost": 7.787204500000003, + "date": "2026-04-16", + "messageCount": 95, + "tokens": { + "cacheRead": 13781248, + "cacheWrite": 0, + "input": 1503039, + "output": 32230, + "total": 15323240 + }, + "turnCount": 18 + }, + { + "cost": 194.5845475, + "date": "2026-04-15", + "messageCount": 11, + "tokens": { + "cacheRead": 1336960, + "cacheWrite": 0, + "input": 38755320, + "output": 169570, + "total": 40262246 + }, + "turnCount": 3 + }, + { + "cost": 46.75895299999998, + "date": "2026-04-14", + "messageCount": 590, + "tokens": { + "cacheRead": 65797632, + "cacheWrite": 0, + "input": 10736846, + "output": 180419, + "total": 76765640 + }, + "turnCount": 111 + }, + { + "cost": 2.4843545, + "date": "2026-04-13", + "messageCount": 74, + "tokens": { + "cacheRead": 2289408, + "cacheWrite": 0, + "input": 483209, + "output": 36385, + "total": 2819549 + }, + "turnCount": 17 + }, + { + "cost": 7.691671500000001, + "date": "2026-04-10", + "messageCount": 126, + "tokens": { + "cacheRead": 15201536, + "cacheWrite": 0, + "input": 1345627, + "output": 28877, + "total": 16582311 + }, + "turnCount": 7 + }, + { + "cost": 29.540619000000024, + "date": "2026-04-09", + "messageCount": 506, + "tokens": { + "cacheRead": 46094976, + "cacheWrite": 0, + "input": 6170646, + "output": 139003, + "total": 52438306 + }, + "turnCount": 73 + }, + { + "cost": 9.918214, + "date": "2026-04-08", + "messageCount": 156, + "tokens": { + "cacheRead": 18131456, + "cacheWrite": 0, + "input": 1808996, + "output": 45397, + "total": 19997976 + }, + "turnCount": 22 + } + ], + "models": [ + { + "client": "codex", + "cost": 4476.965901000025, + "model": "gpt-5.5", + "performance": { + "msPer1KTokens": 4524.020180595458, + "sampleCount": 44616, + "timedTokens": 6328004754, + "tokenCoverage": 1.0, + "totalDurationMs": 28628021210 + }, + "provider": "openai", + "sessionCount": 121, + "tokens": { + "cacheRead": 6113071232, + "cacheWrite": 0, + "input": 201103015, + "output": 10839269, + "total": 6328004754 + } + }, + { + "client": "claude", + "cost": 1306.033157000002, + "model": "claude-opus-4-8", + "performance": { + "msPer1KTokens": 50.03828284427112, + "sampleCount": 4702, + "timedTokens": 1774442346, + "tokenCoverage": 0.9923299290410101, + "totalDurationMs": 88790048 + }, + "provider": "anthropic", + "sessionCount": 19, + "tokens": { + "cacheRead": 1733100519, + "cacheWrite": 48029010, + "input": 1820062, + "output": 5208051, + "total": 1788157642 + } + }, + { + "client": "opencode, codex, goose", + "cost": 400.8453585000001, + "model": "gpt-5.4", + "performance": { + "msPer1KTokens": 5007.590199461738, + "sampleCount": 3222, + "timedTokens": 379225212, + "tokenCoverage": 0.9081412800891782, + "totalDurationMs": 1899004455 + }, + "provider": "openai, chatgpt_codex", + "sessionCount": 37, + "tokens": { + "cacheRead": 336189824, + "cacheWrite": 0, + "input": 79914106, + "output": 1189409, + "total": 417583938 + } + }, + { + "client": "claude", + "cost": 43.60413899999995, + "model": "claude-opus-4-7", + "performance": { + "msPer1KTokens": 88.288596602198, + "sampleCount": 282, + "timedTokens": 44946586, + "tokenCoverage": 0.9708623063422688, + "totalDurationMs": 3968271 + }, + "provider": "anthropic", + "sessionCount": 4, + "tokens": { + "cacheRead": 43571353, + "cacheWrite": 2295530, + "input": 162240, + "output": 266408, + "total": 46295531 + } + }, + { + "client": "codex", + "cost": 42.4404918999999, + "model": "codex-auto-review", + "performance": { + "msPer1KTokens": 40.30321610779724, + "sampleCount": 6470, + "timedTokens": 743616959, + "tokenCoverage": 1.0, + "totalDurationMs": 29970155 + }, + "provider": "openai", + "sessionCount": 221, + "tokens": { + "cacheRead": 643499776, + "cacheWrite": 0, + "input": 99360782, + "output": 564770, + "total": 743616959 + } + }, + { + "client": "goose", + "cost": 17.5749466, + "model": "glm-5.1", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "custom_z.ai", + "sessionCount": 15, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 17204217, + "output": 115853, + "total": 17320070 + } + }, + { + "client": "goose", + "cost": 0.041356000000000004, + "model": "gpt-5.3-codex", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "chatgpt_codex", + "sessionCount": 1, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 17144, + "output": 811, + "total": 17955 + } + }, + { + "client": "opencode", + "cost": 0.0228622, + "model": "zai-org/glm-5-maas", + "performance": { + "msPer1KTokens": 202.61784735119838, + "sampleCount": 1, + "timedTokens": 22614, + "tokenCoverage": 1.0, + "totalDurationMs": 4582 + }, + "provider": "google_vertex", + "sessionCount": 1, + "tokens": { + "cacheRead": 64, + "cacheWrite": 0, + "input": 22411, + "output": 139, + "total": 22614 + } + }, + { + "client": "claude", + "cost": 0.0, + "model": "", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "unknown", + "sessionCount": 9, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 0, + "output": 0, + "total": 0 + } + }, + { + "client": "opencode", + "cost": 0.0, + "model": "gemini-3.5-flash", + "performance": { + "msPer1KTokens": null, + "sampleCount": 0, + "timedTokens": 0, + "tokenCoverage": 0.0, + "totalDurationMs": 0 + }, + "provider": "google", + "sessionCount": 5, + "tokens": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 0, + "output": 0, + "total": 0 + } + }, + { + "client": "opencode", + "cost": 0.0, + "model": "glm-5.2", + "performance": { + "msPer1KTokens": 181.41072322195078, + "sampleCount": 2779, + "timedTokens": 1192833969, + "tokenCoverage": 1.0, + "totalDurationMs": 216392873 + }, + "provider": "zai_coding_plan", + "sessionCount": 21, + "tokens": { + "cacheRead": 1154017856, + "cacheWrite": 0, + "input": 37288456, + "output": 670975, + "total": 1192833969 + } + } + ], + "totals": { + "cost": 6287.528212200026, + "tokens": 10533853432 + } +} \ No newline at end of file From b3543a6f6b50453ff9e4ecea50c0a950519af39f Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 20:32:55 +0900 Subject: [PATCH 11/32] feat(tui): wizard differentiation + dialog convention + edit label (v4.5 G, H, J) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wizard (Bug G): incompatible bundles and packages are now rendered dim in the new-instance wizard. The support_reason/conflict checks already prevented selection; now they're visually differentiated too. Dialog convention (Bug H): plugin-sources screen reordered — Edit is leftmost, primary-colored, and the default action (Enter/Space). The Remove toggle button is warning-colored and no longer flips. Edit label (Bug J): the EditSourceScreen ref field is now labeled 'Branch / ref (snapshot — re-pull to refresh)' making it explicit that uncommitted changes aren't picked up. --- tests/python/test_tui_smoke.py | 7 +++---- tui/widgets.py | 31 ++++++++++++++++++------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/python/test_tui_smoke.py b/tests/python/test_tui_smoke.py index ebd7c612..9c5f96ff 100644 --- a/tests/python/test_tui_smoke.py +++ b/tests/python/test_tui_smoke.py @@ -168,14 +168,13 @@ async def _run() -> None: await app.push_screen(screen) await pilot.pause() toggle = screen.query_one("#plugins-toggle", Button) - assert str(toggle.label) == "Add selected" + # Toggle button is always "Remove" (warning); Edit is the primary default. + assert str(toggle.label) == "Remove" assert screen._rows[0][0] == "rec" # empty override -> all recommended sid = screen._rows[0][1] - screen._activate_current() # space/enter/button on a recommended row -> add + screen._activate_current() # space/enter on a recommended row -> add await pilot.pause() assert any(r["id"] == sid for r in p.configured_rows()) - # the added source is now configured at row 0 -> button flips - assert str(toggle.label) == "Remove" asyncio.run(_run()) diff --git a/tui/widgets.py b/tui/widgets.py index 649f6701..62057d7d 100644 --- a/tui/widgets.py +++ b/tui/widgets.py @@ -1027,7 +1027,12 @@ def sync_bundle_options(self) -> None: if not bundle_id: continue bundle_ids.append(bundle_id) - bundle_list.add_row("✓" if bundle_id in selected else "", bundle_id, key=bundle_id) + reason = reasons.get(bundle_id) + if reason: + label = f"[dim]{bundle_id}[/]" + else: + label = bundle_id + bundle_list.add_row("✓" if bundle_id in selected else "", label, key=bundle_id) if bundle_ids: if self.highlighted_bundle_id not in bundle_ids: self.highlighted_bundle_id = bundle_ids[0] @@ -1065,7 +1070,12 @@ def sync_package_options(self) -> None: if package_id in bundled: continue package_values.append(package_id) - package_list.add_row("✓" if package_id in selected else "", package_id, key=package_id) + reason = reasons.get(package_id) + if reason: + label = f"[dim]{package_id}[/]" + else: + label = package_id + package_list.add_row("✓" if package_id in selected else "", label, key=package_id) if package_values: if self.highlighted_package_id not in package_values: self.highlighted_package_id = package_values[0] @@ -2359,7 +2369,7 @@ def compose(self) -> ComposeResult: yield Input(value=str(self._source.get("id", "")), id="editsrc-id") yield Label("URL / folder", classes="editsrc-field-label") yield Input(value=str(self._source.get("url", "")), id="editsrc-url") - yield Label("Branch / ref", classes="editsrc-field-label") + yield Label("Branch / ref (snapshot — re-pull to refresh)", classes="editsrc-field-label") ref = str(self._source.get("ref", "")) yield Input(value="" if ref == "(unpinned)" else ref, id="editsrc-ref") yield Label("Folder", classes="editsrc-field-label") @@ -2545,8 +2555,8 @@ def compose(self) -> ComposeResult: table.cursor_type = "row" yield table with Horizontal(id="plugins-controls"): - yield Button("Add selected", id="plugins-toggle", variant="primary") - yield Button("Edit", id="plugins-edit") + yield Button("Edit", id="plugins-edit", variant="primary") + yield Button("Remove", id="plugins-toggle", variant="warning") yield Button("Add URL…", id="plugins-add-url") yield Button("Pull", id="plugins-pull") yield Button("Close", id="close") @@ -2626,15 +2636,10 @@ def action_remove_row(self) -> None: self._reload() def _update_toggle(self) -> None: - """Toggle the main button between Add/Remove based on the highlighted row.""" - current = self._current() + """The toggle button is always 'Remove' (warning) — Edit is the primary default.""" button = self.query_one("#plugins-toggle", Button) - if current is not None and current[0] == "cfg": - button.label = "Remove" - button.variant = "warning" - else: - button.label = "Add selected" - button.variant = "primary" + button.label = "Remove" + button.variant = "warning" def _activate_current(self) -> None: """Default action on the highlighted row. From a63d0f74969f3ba55371702028e48a85a0f160f6 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sat, 27 Jun 2026 20:34:09 +0900 Subject: [PATCH 12/32] docs(v4.5): Bugs G, H, I, J, K done --- docs/v4.5-roadmap.md | 100 +++++++++++++++---------------------------- 1 file changed, 35 insertions(+), 65 deletions(-) diff --git a/docs/v4.5-roadmap.md b/docs/v4.5-roadmap.md index 08d2dbf3..5ac50033 100644 --- a/docs/v4.5-roadmap.md +++ b/docs/v4.5-roadmap.md @@ -113,71 +113,41 @@ required=True`, so the `--instance.json` files were created by old scripts that passed raw `argv` before argparse was adopted (pre-v3). Current code doesn't reproduce these. Remaining `.json`/`.lock` files are normal per-instance state. -### G — Wizard shows incompatible packages/bundles with no differentiation ☐ -The new-instance wizard lists **all** packages and **all** bundles regardless of -whether they work on the selected OS. Bundles carry no OS-compat in the catalog -view at all (`catalog_view` emits only `{id, includes}`), and the wizard renders -every bundle unconditionally. Net effect: e.g. `gaming-streaming` (Ubuntu-only) -appears selectable for a Windows OS but would install nothing — this is what made -it *look* like a Windows bundle. Fix: -- Compute bundle OS-compatibility (a bundle is usable on an OS iff ≥1 member - package is installable there; consider giving bundles explicit OS scope so it - isn't inferred only from which package happens to declare membership — see the - note under I). -- In the wizard, **visually differentiate** incompatible packages *and* bundles - (distinct color/dim). If that's not feasible cleanly, add a **show/hide - incompatible** toggle button, **hidden by default**. - -### H — Dialog default action / button convention ☐ -Every dialog's default action (Enter/Space) must be the **highlighted/primary** -button. The plugin-sources screen currently makes **Remove** the default for a -configured entry; it must be **Edit** — placed on the **left**, primary-colored, -and the Enter/Space default. Apply the same convention to all dialogs. (Refines -Bug B: the helper routes default→Edit, but the rendered button order/highlight -still leads with Remove.) - -### I — `gaming-streaming` should be a Windows bundle plugin ☐ -Today the bundle's only member is the Ubuntu `steam` package (via a package-level -`bundles:` declaration — the anti-pattern Bug K removes). Product intent: -**gaming/streaming is a Windows thing here.** Under the Bug K architecture, define -`gaming-streaming` as a **`kind: bundle` plugin** in eve-packages-windows whose -members are the Windows streaming stack — recommended `[steam, sunshine, -rustdesk]` (sunshine = stream host, steam = library, rustdesk/rdp = fallback -access; moonlight is the *client*, not installed on the host). Drop the Ubuntu -`steam` bundle membership (we don't game on Linux). Bundle OS-compat then derives -from its members (windows-only), so Bug G's wizard shows it for Windows only. - -### J — Local-folder source: branch usage + UI clarity ☐ -Confirmed the `ref` **is** used for a local-folder source (the path is cloned and -checked out at the ref; verified the v4.5 cache sat on the v4.5 commit). But it's -a **committed-ref snapshot** — uncommitted working-tree changes are not picked up, -and `eve pull` is needed to refresh. The Edit screen should make this explicit -(label the field as the branch/tag/SHA that is checked out; note it's a snapshot). - -### K — Bundles as first-class plugins (`kind: bundle`) + plugin dependencies ☐ -**Architecture decision.** Today a bundle is implied by packages declaring -`bundles:` membership — wrong: a package shouldn't define a bundle. Instead: -- Plugin `kind` becomes one of **`provider` | `package` | `os` | `bundle`** - (`os` already exists from §14; this adds `bundle`). Update the kind allow-list - (`eve_sdk/plugin_manifest.py` ~L254) and `core/schema/plugin-manifest.schema.json` - (kind enum + a `bundle` manifest branch). Core names no concrete bundle — stays - agnostic; `test-core-boundary` must stay green. -- A **`kind: bundle`** plugin is a *group*: it declares its member packages. Model - this as **dependencies** — a bundle is essentially a plugin whose role is to - require a set of packages. Add a general `requires:` (dependency) capability that - bundles use for membership and that packages MAY also use to pull in deps. -- **Packages must NOT declare bundles.** Remove the `bundles:` field from package - manifests + schema. Migrate every existing bundle (gaming-streaming, - desktop-gnome, desktop-gnome-mac, kde/xfce variants, headless variants, …) into - `kind: bundle` plugins in their owning repo. -- The catalog aggregator + `catalog_view` build the `bundles` list from - `kind: bundle` plugins (not from package `bundles:`); each bundle's - OS-compatibility derives from its member packages (feeds Bug G). -- Dependency *resolution* can stay minimal for now (expand a bundle to its member - package ids; honor package `requires`); deep/transitive/conflict handling can be - a forward note if it balloons. - -This is the foundation for G and I — do it first. +### G — Wizard shows incompatible packages/bundles with no differentiation ☑ +Incompatible bundles and packages are now rendered **dim** in the new-instance +wizard. The support_reason/conflict checks already prevented selection; now +they're visually differentiated too. `gaming-streaming` shows as compatible for +`windows-server-2025` and dim/incompatible for Ubuntu. +- eve: `feat(tui): wizard differentiation + dialog convention + edit label` (eve#55) + +### H — Dialog default action / button convention ☑ +Edit is now the leftmost, primary-colored, default action (Enter/Space) on the +plugin-sources screen. The Remove toggle button is warning-colored and no longer +flips between Add/Remove states. +- eve: same commit as G + +### I — `gaming-streaming` should be a Windows bundle plugin ☑ +`gaming-streaming` is now a `kind: bundle` plugin in eve-packages-windows with +members `[steam, sunshine, rustdesk]`. The Ubuntu `steam` package's `bundles:` +declaration is removed (all bundles are first-class plugins now). +- eve-packages-windows: `feat(bundles): gaming-streaming as kind:bundle plugin` + +### J — Local-folder source: branch usage + UI clarity ☑ +The EditSourceScreen ref field is now labeled "Branch / ref (snapshot — re-pull +to refresh)", making it explicit that uncommitted changes aren't picked up. +- eve: same commit as G + +### K — Bundles as first-class plugins (`kind: bundle`) + plugin dependencies ☑ +Plugin kind is now `provider | package | os | bundle`. A `kind: bundle` plugin +declares member packages via the `members:` field (avoids collision with the +existing `requires:` version-gate field). All 10 bundles migrated to kind:bundle +plugins in eve-packages-linux (9) and eve-packages-windows (1). The `bundles:` +field is removed from package manifests + schema. Catalog aggregator builds +bundles from kind:bundle plugins; `catalog_view` computes each bundle's +`installable_os_families` from its members. +- eve: `feat(bundles): kind:bundle plugins + members field + catalog migration` +- eve-packages-linux: `feat(bundles): migrate to kind:bundle plugins` +- eve-packages-windows: `feat(bundles): gaming-streaming as kind:bundle plugin` ### Demo instances — leave RUNNING for human TUI confirmation ☐ After G–J, provision two **fully provisioned, left-running** instances so the From 39e9944042e27ba4bd27a1a22960cf00aab8a67a Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sun, 28 Jun 2026 01:13:22 +0900 Subject: [PATCH 13/32] docs(v4.5): Bug L - VDD Code-43 init fix, VDD-only requirement, drop a40-1c machine --- docs/multi-purpose-vm-blueprint.md | 6 ------ docs/v4.5-agent-tasks.md | 2 +- docs/v4.5-roadmap.md | 24 +++++++++++++++++++++++- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/multi-purpose-vm-blueprint.md b/docs/multi-purpose-vm-blueprint.md index 4c98bf29..b89a1ddd 100644 --- a/docs/multi-purpose-vm-blueprint.md +++ b/docs/multi-purpose-vm-blueprint.md @@ -110,12 +110,6 @@ machines: instance_type: g5.2xlarge disk_gb: 200 - - name: vultr-vcg-a40-1c # 1 vCPU, 5 GB RAM, 90 GB NVMe, 3 TB/mo, $0.075/hr, 1/24 NVIDIA A40 (2 GB VRAM) - provider: vultr - kind: vm - defaults: - plan: vcg-a40-1c-5g-2vram - - name: vultr-vcg-a40-2c # 2 vCPUs, 10 GB RAM, 180 GB NVMe, 4 TB/mo, $0.144/hr, 1/12 NVIDIA A40 (4 GB VRAM) provider: vultr kind: vm diff --git a/docs/v4.5-agent-tasks.md b/docs/v4.5-agent-tasks.md index f40843bc..a8283c36 100644 --- a/docs/v4.5-agent-tasks.md +++ b/docs/v4.5-agent-tasks.md @@ -176,7 +176,7 @@ refresh, uncommitted changes not picked up). No backend change needed. green.** Provision two instances and **leave them up** (do NOT tear down — the human confirms them in `eve tui`): - **Windows:** `eve instance create` → Vultr, `windows-server-2025`, - `vultr-vcg-a40-1c`, location tokyo, `--bundle gaming-streaming` (post-Task-5b). + `vultr-vcg-a40-2c`, location tokyo, `--bundle gaming-streaming` (post-Task-5b). Then `up` + `provision`. Costs money while running — intended. - **Linux:** local-qemu, `ubuntu-26.04-arm64`, `local-qemu-medium`, `--bundle desktop-gnome`. Then `up` + `provision`. diff --git a/docs/v4.5-roadmap.md b/docs/v4.5-roadmap.md index 5ac50033..6cdbe1c2 100644 --- a/docs/v4.5-roadmap.md +++ b/docs/v4.5-roadmap.md @@ -152,11 +152,33 @@ bundles from kind:bundle plugins; `catalog_view` computes each bundle's ### Demo instances — leave RUNNING for human TUI confirmation ☐ After G–J, provision two **fully provisioned, left-running** instances so the human can confirm everything in `eve tui` (do **not** tear these down): -- **Windows:** Vultr, `windows-server-2025`, machine `vultr-vcg-a40-1c`, location +- **Windows:** Vultr, `windows-server-2025`, machine `vultr-vcg-a40-2c`, location tokyo, **`--bundle gaming-streaming`** (post-I). Costs money while up — that's intended; the human tears it down. - **Linux:** local-qemu, `ubuntu-26.04-arm64`, machine `local-qemu-medium`, **`--bundle desktop-gnome`**. Report instance names + IP/connection info for both. +### L — Windows VDD: Code-43 init + VDD-only required + drop 1-vCPU machine ☐ +Investigated the Vultr windows gaming box failing at sunshine step 22. Findings +(all box-confirmed via SSH on a live A40-2Q instance): +- **NOT an env/config regression** — `env.json` carried the correct + `sunshine_version` (pin 2025.924.154138, actually installed), `display_resolution`, + password; `vdd_settings.xml` rendered correctly; runner + scripts byte-identical + to June 11. The whole env-passing surface is clean. +- **Real bug:** the VirtualDrivers VDD comes up **Code 43** after install; an OS + reboot does *not* clear it — only a device **disable/enable** does (the "manual + fix" we half-remembered). Fixed: step 22 now cycles the VDD if not OK and waits + until it presents a mode before reconfiguring the desktop (eve-packages-windows). +- **VDD-only is required** (deactivate NVIDIA + Basic Display, else Sunshine + captures the wrong display). Reverted the agent's band-aid; step 22 fails hard + if a non-VDD display remains. +- **Single-vCPU vGPU is unsupported:** `vultr-vcg-a40-1c` can't render 2560×1440 + / drive the VDD reliably (the box where the NVIDIA primary wouldn't release was + this 1-vCPU profile). **Removed `vultr-vcg-a40-1c` from the catalog**; demo + + realistic usage use `vultr-vcg-a40-2c` (≥2 vCPU). Known-good target: 5120×1440 + ~60fps via VDD on a multi-vCPU A40. +- TODO: re-provision the demo on `vultr-vcg-a40-2c` and confirm VDD-only succeeds + (NVIDIA + Basic deactivated, VDD sole) there. + > Add new bugs here as they're found. Keep entries terse: symptom → fix → repo/PR. From 6422cd27613836927ca347d1a26f5290503117f3 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sun, 28 Jun 2026 13:54:53 +0900 Subject: [PATCH 14/32] fix(catalog_view): extract isinstance check to variable for mypy narrowing --- eve_sdk/catalog_view.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eve_sdk/catalog_view.py b/eve_sdk/catalog_view.py index 9fc48466..44abb52e 100644 --- a/eve_sdk/catalog_view.py +++ b/eve_sdk/catalog_view.py @@ -103,7 +103,8 @@ def build_catalog_options( bundles: list[dict[str, Any]] = [] for bundle in catalog["bundles"]: - includes = bundle.get("includes") if isinstance(bundle.get("includes"), list) else [] + includes_raw = bundle.get("includes") + includes = includes_raw if isinstance(includes_raw, list) else [] # Compute bundle OS-compat from member packages' installable_os_families. member_os_families: set[str] = set() for member_id in includes: From d8048a090d8feaf7cfc7f7c44d3072c9453ecf8f Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sun, 28 Jun 2026 17:35:54 +0900 Subject: [PATCH 15/32] fix(providers-status): resolve config-env + secrets like dispatch (v4.5 Bug M) The status probe ran each provider's connectivity command with a bare environment, so a provider whose 'configured' gate is a secret (Vultr's VULTR_API_KEY, which lives only in the secrets file) reported unconfigured even though dispatch could create instances with it; AWS meanwhile reported 'configured' off ambient ~/.aws, bypassing eve's config entirely. The probe now resolves the same ConfigEnv.environment() + _inject_secrets the dispatch path uses, so status and dispatch see identical config/secrets. Also records the v4.5 architecture workstream (Bugs M-R): one shared config/ secret resolution, a central FSM platform with settings as machine context, XDG config layout, TUI-over-CLI parity, and streaming<->non-streaming parity. --- docs/v4.5-roadmap.md | 55 ++++++++++++++++++++++++++++++++++++++++ scripts/providers-status | 15 ++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/docs/v4.5-roadmap.md b/docs/v4.5-roadmap.md index 6cdbe1c2..8f5e2607 100644 --- a/docs/v4.5-roadmap.md +++ b/docs/v4.5-roadmap.md @@ -181,4 +181,59 @@ Investigated the Vultr windows gaming box failing at sunshine step 22. Findings - TODO: re-provision the demo on `vultr-vcg-a40-2c` and confirm VDD-only succeeds (NVIDIA + Basic deactivated, VDD sole) there. +## Architecture workstream (the unified flow) + +v4.5 is "make eve usable + fix everything milestone 4 broke." The provider-status +bug below exposed that there is **no single flow** for config/state — each plugin +invents its own notion of "configured," and status vs dispatch resolve config +differently. The fix is structural: one config/secret flow and one state-machine +platform, used by everything. + +### M — Status ≡ dispatch: one config/secret resolution ☑(partial) +`scripts/providers-status` ran connectivity with a **bare environment** (no +secrets), so Vultr (gate = `VULTR_API_KEY`, secrets-only) showed ✗ while dispatch +could still create instances; AWS showed "configured" off **ambient `~/.aws`**, +bypassing eve entirely. Immediate fix landed: the probe now resolves the same +`ConfigEnv.environment()` + `_inject_secrets` that dispatch uses (vultr → yes/yes; +aws now honestly reflects eve config). **Systemic (open):** make that resolution +**one shared helper** that is the *only* way any code path gets a provider/ +package/instance's config+secrets; no plugin may read ambient external config +behind eve's back. Folds into N/O. +- eve: `fix(providers-status): resolve config-env + secrets like dispatch` + +### N — Central state-machine platform ☐ +Today state logic is scattered: `state.py` (enums + validation), `state_machine.py` +(derivation helpers), and `desired_state_for`/`provider_state_for`/`record_*` +across dispatch/provider_command/package_dispatch. Consolidate into **one core FSM +platform** behind a clean interface — states, transitions (guards/actions), and a +**context of state variables**; pluggable host-vs-client storage. **Providers, +instances, packages, OS, and bundles are all machines** managed by it; none +reimplement state. Interface kept clean enough to later swap the implementation +for a cns-based FSM (ref: github.com/fruwe/cns_statemachine). Provider +"configured/reachable" becomes an FSM-computed state (not a separate probe). + +### O — Settings are FSM context, declared per plugin ☐ +Each plugin declares its config params, defaults, and an explicit **unset** state +(empty string only where sufficient). Resolution order: plugin default → global +override (settings menu / CLI / yaml) → **per-instance override** (instance +settings / CLI / yaml), for ALL config (not just packages). The FSM platform +resolves and **provides settings as machine context** — so status, dispatch, and +connectivity always see identical config. This is what makes M structurally +impossible to regress. + +### P — Config → XDG, user-global ☐ +Move off project-local `.eve/`: `~/.config/eve/` (config.yaml, secrets/), +`~/.local/state/eve/` (instances/, state/), `~/.cache/eve/` (sources/ plugin +cache), `~/.local/share/eve/` (plugins/). Include a migration from an existing +`.eve/`. eve becomes a user-level tool, not per-checkout. + +### Q — TUI is a view over the CLI surface ☐ +Everything visible/doable in the TUI must be reproducible as a CLI command — one +shared command surface, with 1:1 parity enforced by tests. (Warm in-process engine +is fine for speed; the invariant is parity, not literally shelling out.) + +### R — CLI streaming ≡ non-streaming ☐ +No command or option may exist in only streaming or only non-streaming mode; +enforce parity with a test. + > Add new bugs here as they're found. Keep entries terse: symptom → fix → repo/PR. diff --git a/scripts/providers-status b/scripts/providers-status index c77e3eb9..397d1e0f 100755 --- a/scripts/providers-status +++ b/scripts/providers-status @@ -14,6 +14,7 @@ JSON object: {"configured","reachable","notes"}. from __future__ import annotations import json +import os import subprocess import sys from pathlib import Path @@ -22,8 +23,10 @@ from typing import Any ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) +from eve_sdk.config import ConfigEnv # noqa: E402 from eve_sdk.dispatch import command_vector # noqa: E402 from eve_sdk.plugin_manifest import PluginManifest # noqa: E402 +from eve_sdk.provider_command import _inject_secrets # noqa: E402 HEADER = "Provider Configured Reachable Notes" SEPARATOR = "----------------- ---------- --------- -------------------------------------------" @@ -45,7 +48,17 @@ def _run_connectivity(plugin: dict[str, Any]) -> tuple[str, str, str]: """Run a provider's connectivity command, parse JSON → (configured, reachable, notes).""" spec = plugin["commands"]["connectivity"] cmd = command_vector(plugin, spec) - result = subprocess.run(cmd, capture_output=True, text=True, check=False) + # Resolve the SAME env the real dispatch path gives a provider command: + # global config-env plus the provider's injected secrets. Without this the + # probe ran with a bare environment, so a provider whose "configured" gate is + # a secret (e.g. Vultr's VULTR_API_KEY, which lives only in the secrets file) + # was reported unconfigured even though dispatch could create instances with + # it. Status and dispatch must resolve config/secrets identically. (v4.5: this + # injection should become the one shared helper the FSM platform provides.) + env = dict(os.environ) + env.update(ConfigEnv.environment()) + env = _inject_secrets(str(plugin["id"]), plugin, env) + result = subprocess.run(cmd, env=env, capture_output=True, text=True, check=False) if result.returncode != 0: return "no", "no", f"connectivity check failed (exit {result.returncode})" try: From f8fefdf6a78df90c73c5ac9b116887e9c3f99a93 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sun, 28 Jun 2026 17:48:46 +0900 Subject: [PATCH 16/32] docs(v4.5): FSM platform + settings-as-context design (plugin-declared YAML machines) --- docs/v4.5-fsm-design.md | 155 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 docs/v4.5-fsm-design.md diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md new file mode 100644 index 00000000..1aae44e2 --- /dev/null +++ b/docs/v4.5-fsm-design.md @@ -0,0 +1,155 @@ +# v4.5 — FSM platform + settings-as-context (design) + +Spine doc for roadmap bugs **N** (central FSM platform) and **O** (settings are +FSM context). Engine lives in core; **every machine is declared in YAML by the +plugin that owns it**. The interface is kept thin so the engine can later be +replaced by a cns-based implementation (ref: github.com/fruwe/cns_statemachine). + +## Why + +Today state is scattered (`eve_sdk/state.py` enums, `state_machine.py` derivation, +`desired_state_for`/`provider_state_for`/`record_*` across dispatch) and **config +lives outside it entirely**. So `providers-status` could resolve config one way +and dispatch another — Vultr ✗ while creatable, AWS "configured" off ambient +`~/.aws`. The cure: one engine that owns both **state** and **context (settings)** +and hands them to every command identically. If config is always read from the +machine's context, status and dispatch *physically cannot* diverge. + +## Core model + +- **Machine definition** — declarative YAML a plugin ships. Describes one *kind* + of machine (a provider, a package, …): its states, transitions, and the shape + of its context (variables + settings). +- **Machine instance** — a live entity the engine tracks: `(definition, key, + current_state, context)`. Keys: a provider id; an instance name; an + `(instance, package)` pair; etc. +- **Context** — the machine's state variables, including **resolved settings**. + The single source commands read config from. +- **Event** — a verb that may cause a transition. Events are the existing CLI + verbs (`configure`, `up`, `down`, `provision`, `install`, `probe`, …). +- **Transition** — `from --on(event)--> to`, optionally gated by a **guard** over + context, effected by an **action** (a plugin command). Action succeeds → land + in `to`; fails → an error state. +- **Observe/reconcile** — a read-only command maps real-world status back into a + state (e.g. connectivity → `reachable`, terraform status → `running`). Keeps + the machine honest about drift. +- **Storage** — per machine kind: `host` (persisted state record) or `client` + (derived live, not persisted). The engine owns persistence; machines don't. + +## Plugin-declared YAML (the schema) + +A `state_machine:` block in `eve-plugin.yaml` (or a referenced `fsm.yaml`): + +```yaml +state_machine: + scope: provider # provider | instance | package | os | bundle + initial: unconfigured + storage: host # host | client + + # Context = state variables. `setting`-sourced vars are resolved by the engine + # (default -> global override -> per-instance override) and exposed to every + # command of this machine via env_var. `null` default == the explicit "unset". + context: + api_key: + source: secret # setting | secret | observed | derived + type: secret + default: null # unset + scope: global # global | instance | both + env_var: VULTR_API_KEY + region: + source: setting + type: string + default: "" + scope: both + env_var: VULTR_REGION + + states: + unconfigured: {} + configured: {} + reachable: {} + + transitions: + - { from: "*", on: resolve, to: configured, guard: "api_key != null" } + - { from: configured, on: probe, to: reachable, action: {command: connectivity}, guard_on_result: "reachable == 'yes'" } + - { from: "*", on: resolve, to: unconfigured, guard: "api_key == null" } + + observe: # read-only reconciliation + command: connectivity # emits {configured, reachable, notes} + maps: { reachable: reachable, configured: configured } +``` + +Notes: +- **Guards** are simple expressions over context (no arbitrary code); a small, + safe evaluator. Result-guards (`guard_on_result`) check an action's JSON output. +- **Actions/commands** are the plugin's existing `commands:` — the FSM references + them by name, it doesn't introduce a second command surface. +- A package machine looks the same with `scope: package`, states + `{missing, installing, installed, failed, removed}`, events + `{install, status, down}`, and context for its settings. + +## The platform interface (core engine) + +Thin, storage- and engine-agnostic — the seam for a future cns swap: + +```python +class Machine: # one definition + key + state: str + context: Mapping[str, str] # resolved settings + observed vars + +class FSM: + def load(kind, plugin) -> Definition # parse + validate YAML + def machine(scope, key, *, registry=None) -> Machine + def context(scope, key) -> Mapping[str, str] # resolved settings (THE helper M/O need) + def fire(scope, key, event, *, extra_args=()) -> Result # run transition (guard->action->persist) + def observe(scope, key) -> Machine # reconcile from the observe command + def states(scope, key) -> StateView # current state + available events (for TUI/CLI) +``` + +`context()` is the one shared resolver from bug M: `providers-status`, dispatch, +and connectivity all call it, so they cannot see different config/secrets. + +## Settings-as-context (the bug-killer) + +Resolution order, done once by the engine: **plugin default → global override +(`~/.config/eve/config.yaml` / `eve settings ...` / TUI) → per-instance override +(instance entry / `eve instance settings ...` / TUI)**. The resolved value is put +into context and exposed to every command of that machine via `env_var`. There is +no other path to config — no plugin reads `~/.aws` or ambient env behind eve's +back; if a provider needs ambient creds, it declares them as a `setting` eve +resolves and injects. + +## Mapping current eve → machines + +| Machine (scope) | States (today → YAML) | Owner plugin | +|---|---|---| +| provider | unconfigured/configured/reachable (new; was the secret-less probe) | provider | +| instance | absent/creating/running/stopped/error (`PROVIDER_STATES`) + desired | provider | +| provision | unknown/provisioning/provisioned/error (`PROVISION_STATES`) | provider (os bootstrap) | +| package | missing/installing/installed/failed/removed (`PACKAGE_STATES`) | package | +| os | (image/bootstrap readiness) | os | +| bundle | derived from member packages | bundle | + +`state.py`'s hardcoded enums move into the owning plugins' YAML; `state_machine.py` ++ the `*_state_for`/`record_*` functions collapse into `FSM`. `Engine`/`Workdir` +keep persistence, but behind `FSM`'s storage seam. + +## Storage (host vs client) + +- **host** — durable per-entity state record (instances, provision, packages). + Lives under `~/.local/state/eve/` (roadmap P). +- **client** — derived live, not persisted (provider configured/reachable is + recomputed via `observe`; cached briefly only). + +## Open decisions (need your call) + +1. **Instance lifecycle ownership** — does the *provider* plugin declare the + instance FSM (lifecycles differ: vultr vs docker vs local-qemu), or is there a + shared core instance FSM with provider-supplied transition actions? I lean + **provider-declared** for honesty; it's more YAML per provider. +2. **Guard expression language** — minimal safe DSL (`==`, `!=`, `&&`, presence) + vs. something richer. I lean **minimal**. +3. **Bundle/OS machines** — model now, or stub their states until B/§14 settle? + I lean **stub** (define scope, minimal states) to keep N/O focused. +4. **One YAML block vs separate `fsm.yaml`** — inline in `eve-plugin.yaml` keeps + it together; a separate file keeps manifests small. I lean **inline** with an + optional `$ref`. From d6a27e1eb31db7a20abdbe04b67865e9ca5c3c7a Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sun, 28 Jun 2026 17:50:37 +0900 Subject: [PATCH 17/32] docs(v4.5): FSM engine as a separate generic repo (engine vs eve-adapter split) --- docs/v4.5-fsm-design.md | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index 1aae44e2..0af6b687 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -87,9 +87,26 @@ Notes: `{missing, installing, installed, failed, removed}`, events `{install, status, down}`, and context for its settings. -## The platform interface (core engine) +## Repository split (generic engine vs eve adapters) -Thin, storage- and engine-agnostic — the seam for a future cns swap: +The engine is its **own repository** — a generic, eve-agnostic FSM library; eve +depends on it one-way (like every other agnostic-core boundary). This makes the +"swap for a cns-based engine later" goal enforceable: the repo is the swappable +unit with a published interface, not a module that can drift into eve internals. + +| Lives in the **engine repo** (generic) | Lives in **eve** (adapters) | +|---|---| +| Parse/validate a machine definition (YAML) | The `state_machine:` YAML in each plugin | +| Hold `(state, context)`; evaluate guards; fire transitions | Resolve settings → context (default→global→instance); inject secrets | +| Determine valid events; expose current state | Map eve commands ↔ FSM actions; entity→machine wiring | +| **Interfaces** for storage, action execution, context provider | **Implement** those interfaces (Workdir storage, run plugin command, ConfigEnv/Secrets) | + +The engine never imports eve and never names a provider/package/OS — it just runs +whatever machine YAML + callbacks it is handed. + +## The platform interface (the engine↔eve boundary) + +Thin, storage- and engine-agnostic — this *is* the repo boundary and the cns seam: ```python class Machine: # one definition + key @@ -152,4 +169,10 @@ keep persistence, but behind `FSM`'s storage seam. I lean **stub** (define scope, minimal states) to keep N/O focused. 4. **One YAML block vs separate `fsm.yaml`** — inline in `eve-plugin.yaml` keeps it together; a separate file keeps manifests small. I lean **inline** with an - optional `$ref`. + optional `$ref`. (Orthogonal to the engine repo split — plugins declare their + machine YAML in eve either way; the engine just parses it.) +5. **Engine repo home + name** — a generic FSM library, eve-agnostic. Under the + eve org (`fruwehq/…`) or your personal org alongside cns (`fruwe/…`)? Generic + name (`statemachine`, a modern cns) vs eve-scoped (`eve-fsm`)? I lean a + **generic name** since the whole point is reuse + a future cns-based swap, but + it's your call where it lives. From ee0b69d180cc125456fc2eb86acbe015da1421f3 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Thu, 2 Jul 2026 00:16:46 +0900 Subject: [PATCH 18/32] feat(v4.5 N/O): FSM platform (EveFsm) over harel + settings-as-context The eve<->harel seam: EveFsm wraps a harel.Host, loads plugin-declared harel machines, and seeds each machine's declared `external` esvs from one pluggable ContextResolver (the shared config/secret resolver M/O need). Settings changes arrive as harel's `env` event + `refresh`; side effects stay host-driven (eve runs the probe, delivers `probe_ok`/`probe_fail`). Proven on a synthetic provider machine: configured/reachable are FSM-computed from injected context and re-derive on a settings change (status and dispatch physically cannot diverge). Test importorskips harel so CI stays green until the dependency is pinned (gated on a harel-python release tag). Reconciles docs/v4.5-fsm-design.md to harel's real YAML format. --- docs/v4.5-fsm-design.md | 54 ++++++++++++++++ eve_sdk/fsm.py | 103 +++++++++++++++++++++++++++++++ tests/fsm_fixtures/provider.yaml | 35 +++++++++++ tests/test_fsm_platform.py | 70 +++++++++++++++++++++ 4 files changed, 262 insertions(+) create mode 100644 eve_sdk/fsm.py create mode 100644 tests/fsm_fixtures/provider.yaml create mode 100644 tests/test_fsm_platform.py diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index 0af6b687..d1505285 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -5,6 +5,13 @@ FSM context). Engine lives in core; **every machine is declared in YAML by the plugin that owns it**. The interface is kept thin so the engine can later be replaced by a cns-based implementation (ref: github.com/fruwe/cns_statemachine). +> **Status (v4.5):** the engine is now real — [**harel**](https://github.com/fruwehq/harel) +> (spec) + [harel-python](https://github.com/fruwehq/harel-python) (reference impl, +> passing the conformance suite). The original `state_machine:`/`transitions:` sketch +> below is **superseded by harel's actual YAML** (`top`/`esvs`/`events`/`on_events`/ +> `transition_to`) — see **"Reconciliation with harel"** at the end. The eve↔harel seam +> is `eve_sdk/fsm.py` (`EveFsm`); increment 1 (platform + a proving test) has landed. + ## Why Today state is scattered (`eve_sdk/state.py` enums, `state_machine.py` derivation, @@ -176,3 +183,50 @@ keep persistence, but behind `FSM`'s storage seam. name (`statemachine`, a modern cns) vs eve-scoped (`eve-fsm`)? I lean a **generic name** since the whole point is reuse + a future cns-based swap, but it's your call where it lives. + +## Reconciliation with harel (as built) + +The engine is real now, so this supersedes the speculative `state_machine:` schema above. + +- **Machines are harel YAML.** A plugin ships a harel machine (in its own repo). States + are `states:` under `top:`; transitions are `on_events: { : { transition_to, + guard, action } }`; the machine's config lives in `esvs:`. Guards are **CEL**; actions + are harel's structured set (`assign`/`publish`/`refresh`/`spawn`/`stop`). +- **Settings are `external` esvs.** eve resolves a machine's declared `external` esvs + (settings + secrets) via one shared resolver and seeds them at instance creation. A + settings change is delivered as harel's reserved **`env` event**; the machine's `env` + handler adopts it via **`refresh`**. One context ⇒ status and dispatch cannot diverge. +- **Side effects are host-driven.** harel actions are sandboxed (no host calls), so eve + runs the real command (connectivity probe, terraform, provision) and delivers the + outcome back as an event (`probe_ok`/`probe_fail`). `configured` is a guarded + transition on the secret; `reachable` is a substate of `configured` reached by the + probe result. +- **The platform** is `eve_sdk/fsm.py` — `EveFsm` wraps a `harel.Host` and takes a + pluggable `ContextResolver` (`(scope, key, external_names) -> {name: value}`), the seam + where the single ConfigEnv+secrets resolver plugs in. Surface: `load` / `instance` / + `fire` / `refresh` / `state` / `context` / `view`. + +### Landed / next +- ☑ **Increment 1** — `EveFsm` + a synthetic provider machine + (`tests/fsm_fixtures/provider.yaml`) + `tests/test_fsm_platform.py` proving + configured/reachable are FSM-computed from injected context and re-derive on a settings + change. harel is editable-installed for dev; the test `importorskip`s it. +- ☐ **Dependency** — pin harel-python in `pyproject` (git dep) once it has a release tag, + then `poetry lock`; the test then runs in CI. +- ☐ **Real resolver** — implement the `ContextResolver` over `ConfigEnv.environment()` + + `_inject_secrets` (the one path; no plugin reads ambient config), mapping declared esv + names ↔ eve config/env keys. +- ☐ **Provider machine** — ship the real provider machine in `eve-providers`; rewire + `scripts/providers-status` to read state via `EveFsm` (kills M/N for providers). +- ☐ **Instance/package machines** — same pattern; retire `state.py` enums + + `state_machine.py` into the platform. + +### Open decisions — resolved +1. Engine repo/name → **harel** (`fruwehq/harel` spec, `fruwehq/harel-python` impl). +2. Guard language → **CEL** (harel's). +3. Machine format → **harel YAML** (not the `state_machine:` sketch). + +Still open (unchanged leans): provider-declared vs shared instance FSM (lean +provider-declared); bundle/OS machines (lean stub); one YAML block vs a separate +`fsm.yaml` (now that machines are full harel documents, lean **a separate `fsm.yaml`** +per plugin). diff --git a/eve_sdk/fsm.py b/eve_sdk/fsm.py new file mode 100644 index 00000000..1b1a566b --- /dev/null +++ b/eve_sdk/fsm.py @@ -0,0 +1,103 @@ +"""eve FSM platform — the single seam between eve and the harel statechart engine. + +Machines are declared in **harel YAML** by the plugin that owns them (a provider, a +package, ...). eve resolves each machine's ``external`` esvs (settings + secrets) +through **one shared resolver** and seeds them as the machine's context, so status, +dispatch, and connectivity can never see different config (v4.5 bugs N/O). The harel +engine is the swappable unit; this module is the only place eve touches it. + +Side effects are host-driven: harel actions are sandboxed, so eve runs the real +command (probe, terraform, provision) and delivers the outcome back as an event +(e.g. ``probe_ok`` / ``probe_fail``). The machine only ever holds state + context. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any + +import harel + +# Resolve a machine's declared ``external`` esvs from eve settings/secrets: +# (scope, key, external_names) -> {name: value} +# Injected so resolution lives in exactly one place (the bug-M/O helper) and the +# platform stays testable without real config on disk. +ContextResolver = Callable[[str, str, list[str]], dict[str, Any]] + + +class EveFsm: + """Loads plugin-declared harel machines and tracks their live instances.""" + + def __init__(self, resolve_context: ContextResolver) -> None: + self._resolve = resolve_context + self._host = harel.Host() + self._raw: dict[str, dict[str, Any]] = {} + self._machine: dict[str, Any] = {} + + def load(self, scope: str, machine_yaml: str) -> None: + """Register the harel machine a plugin ships for ``scope`` (its first document).""" + defs = harel.load_definitions(machine_yaml) + for definition in defs: + harel.validate(definition.raw) + self._host.register_all(defs) + root = defs[0] + self._raw[scope] = root.raw + self._machine[scope] = self._host.machines[root.raw["id"]] + + @staticmethod + def _external_names(raw: Mapping[str, Any]) -> list[str]: + esvs = raw.get("top", {}).get("esvs", {}) or {} + return [name for name, spec in esvs.items() if isinstance(spec, Mapping) and spec.get("external")] + + def _iid(self, scope: str, key: str) -> str: + return f"{scope}:{key}" + + def instance(self, scope: str, key: str) -> Any: + """The live instance for ``(scope, key)``; created and seeded on first use.""" + iid = self._iid(scope, key) + inst = self._host.instances.get(iid) + if inst is None: + external = self._resolve(scope, key, self._external_names(self._raw[scope])) + inst = self._host.create_root(self._machine[scope], iid, external=external) + self._host.run_to_quiescence() + return inst + + def fire(self, scope: str, key: str, event: str, payload: Mapping[str, Any] | None = None) -> bool: + """Deliver an event and run to quiescence. Returns False if it was rejected.""" + self.instance(scope, key) + accepted = self._host.deliver(self._iid(scope, key), event, dict(payload or {})) + self._host.run_to_quiescence() + return accepted + + def refresh(self, scope: str, key: str) -> dict[str, Any]: + """Re-resolve settings/secrets; deliver an ``env`` event with what changed. + + The machine's ``env`` handler adopts the new values via ``refresh``. Returns + the changed subset (empty if nothing moved). + """ + inst = self.instance(scope, key) + current = dict(inst.resolved_esvs()) + latest = self._resolve(scope, key, self._external_names(self._raw[scope])) + changed = {name: value for name, value in latest.items() if current.get(name) != value} + if changed: + self._host.deliver(self._iid(scope, key), "env", {"changed": changed}) + self._host.run_to_quiescence() + return changed + + def state(self, scope: str, key: str) -> str: + """The active leaf state (first, for non-orthogonal machines).""" + leaves = self.instance(scope, key).active_leaf_names() + return leaves[0] if leaves else "" + + def context(self, scope: str, key: str) -> dict[str, Any]: + """The machine's resolved context (esvs) — the one place commands read config.""" + return dict(self.instance(scope, key).resolved_esvs()) + + def view(self, scope: str, key: str) -> dict[str, Any]: + """State + context + status, for status/TUI/CLI rendering.""" + inst = self.instance(scope, key) + return { + "state": list(inst.active_leaf_names()), + "context": dict(inst.resolved_esvs()), + "status": inst.status.name.lower(), + } diff --git a/tests/fsm_fixtures/provider.yaml b/tests/fsm_fixtures/provider.yaml new file mode 100644 index 00000000..a11bec44 --- /dev/null +++ b/tests/fsm_fixtures/provider.yaml @@ -0,0 +1,35 @@ +# Synthetic provider machine (harel format) for the FSM-platform test — not a real +# plugin. `configured` = a secret is present; `reachable` is a substate of configured +# reached by a live probe. eve seeds api_key/region as external esvs from its shared +# resolver, so status and dispatch read identical config. +id: provider +events: + resolve: {} + probe_ok: {} + probe_fail: {} +top: + esvs: + api_key: { type: string, external: true } # secret, eve-resolved + region: { type: string, external: true } # setting, eve-resolved + on_events: + env: + action: [ { refresh: {} } ] # adopt changed external values + initial: { transition_to: unconfigured } + states: + unconfigured: + on_events: + resolve: + - { guard: "api_key != null && api_key != ''", transition_to: configured } + configured: + type: composite + initial: { transition_to: unreachable } + on_events: + resolve: + - { guard: "api_key == null || api_key == ''", transition_to: unconfigured } + states: + unreachable: + on_events: + probe_ok: { transition_to: reachable } + reachable: + on_events: + probe_fail: { transition_to: unreachable } diff --git a/tests/test_fsm_platform.py b/tests/test_fsm_platform.py new file mode 100644 index 00000000..56bbb9e8 --- /dev/null +++ b/tests/test_fsm_platform.py @@ -0,0 +1,70 @@ +"""FSM platform (eve_sdk.fsm.EveFsm) over the harel engine. + +Proves the v4.5 N/O invariant on a synthetic provider machine: `configured` and +`reachable` are FSM-computed from context that comes from *one* resolver, and a +settings change re-derives state. No real secrets/config on disk — the resolver is +injected (as the real platform will inject the shared ConfigEnv+secrets resolver). +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +# The harel engine is a new dependency; until it is pinned in pyproject (gated on a +# harel-python release tag), skip cleanly where it isn't installed so CI stays green. +pytest.importorskip("harel") + +from eve_sdk.fsm import EveFsm # noqa: E402 (must follow importorskip) + +FIXTURE = Path(__file__).parent / "fsm_fixtures" / "provider.yaml" + + +def _make(settings: dict[str, str]) -> tuple[EveFsm, dict[str, dict[str, str]]]: + holder = {"cur": dict(settings)} + + def resolve(scope: str, key: str, names: list[str]) -> dict[str, Any]: + return {name: holder["cur"].get(name, "") for name in names} + + fsm = EveFsm(resolve) + fsm.load("provider", FIXTURE.read_text()) + return fsm, holder + + +def _leaves(fsm: EveFsm) -> list[str]: + return list(fsm.instance("provider", "vultr").active_leaf_names()) + + +def test_unconfigured_without_secret() -> None: + fsm, _ = _make({"api_key": "", "region": "ewr"}) + fsm.fire("provider", "vultr", "resolve") + assert _leaves(fsm) == ["unconfigured"] + + +def test_configured_then_reachable_via_probe() -> None: + fsm, _ = _make({"api_key": "KEY", "region": "ewr"}) + fsm.fire("provider", "vultr", "resolve") + assert _leaves(fsm) == ["unreachable"] # configured, not yet probed + assert fsm.context("provider", "vultr")["region"] == "ewr" + + fsm.fire("provider", "vultr", "probe_ok") + assert _leaves(fsm) == ["reachable"] + assert fsm.view("provider", "vultr")["status"] == "active" + + +def test_settings_change_reevaluates() -> None: + fsm, holder = _make({"api_key": "KEY", "region": "ewr"}) + fsm.fire("provider", "vultr", "resolve") + fsm.fire("provider", "vultr", "probe_ok") + assert _leaves(fsm) == ["reachable"] + + # secret removed at the source -> refresh delivers `env`, machine adopts it, + # re-resolve derives `unconfigured`. Status and dispatch cannot disagree because + # both read this same context. + holder["cur"]["api_key"] = "" + changed = fsm.refresh("provider", "vultr") + assert changed == {"api_key": ""} + fsm.fire("provider", "vultr", "resolve") + assert _leaves(fsm) == ["unconfigured"] From 484d704a06f06feb8aac16ade979d5c8cbebb86b Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sun, 5 Jul 2026 21:59:50 +0900 Subject: [PATCH 19/32] feat(v4.5 M/N): real provider context resolver + engine-rename resilience - provider_context_resolver(): resolves a provider machine's external esvs (named as their eve env vars, e.g. VULTR_API_KEY) from the one path dispatch uses (os.environ + ConfigEnv.environment() + _inject_secrets), so a provider's FSM context is byte-identical to what dispatch runs with (bug M). Hermetically tested. - Import the statechart engine behind one name (determa_state, falling back to the pre-rename harel) so the upcoming rename is a no-op for eve. --- docs/v4.5-fsm-design.md | 25 +++++++++++++++-------- eve_sdk/fsm.py | 41 ++++++++++++++++++++++++++++++++++---- tests/test_fsm_platform.py | 34 +++++++++++++++++++++++++++---- 3 files changed, 84 insertions(+), 16 deletions(-) diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index d1505285..66dc6dc2 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -206,18 +206,27 @@ The engine is real now, so this supersedes the speculative `state_machine:` sche where the single ConfigEnv+secrets resolver plugs in. Surface: `load` / `instance` / `fire` / `refresh` / `state` / `context` / `view`. +> **Engine rename:** harel is being renamed **determa-state**; the package isn't +> published under the new name yet. `eve_sdk/fsm.py` imports it behind one name +> (`import determa_state as engine`, falling back to `harel`), so the swap is a no-op +> for eve — nothing else in eve names the engine. + ### Landed / next - ☑ **Increment 1** — `EveFsm` + a synthetic provider machine (`tests/fsm_fixtures/provider.yaml`) + `tests/test_fsm_platform.py` proving configured/reachable are FSM-computed from injected context and re-derive on a settings - change. harel is editable-installed for dev; the test `importorskip`s it. -- ☐ **Dependency** — pin harel-python in `pyproject` (git dep) once it has a release tag, - then `poetry lock`; the test then runs in CI. -- ☐ **Real resolver** — implement the `ContextResolver` over `ConfigEnv.environment()` + - `_inject_secrets` (the one path; no plugin reads ambient config), mapping declared esv - names ↔ eve config/env keys. -- ☐ **Provider machine** — ship the real provider machine in `eve-providers`; rewire - `scripts/providers-status` to read state via `EveFsm` (kills M/N for providers). + change. The engine is editable-installed for dev; the test skips if neither + `determa_state` nor `harel` is installed. +- ☑ **Real resolver** — `provider_context_resolver()` in `eve_sdk/fsm.py`: resolves each + declared external esv (named as its eve env var) from the one path dispatch uses + (`os.environ` + `ConfigEnv.environment()` + `_inject_secrets`), hermetically tested. So + a provider's FSM context ≡ what dispatch runs with (bug M). Systemic follow-up: drop + ambient `os.environ` from both this and dispatch. +- ☐ **Dependency** — pin the engine in `pyproject` (git dep) once `determa-state` is + published + tagged, then `poetry lock`; the test then runs in CI. +- ☐ **Provider machine** — ship the real provider machine in `eve-providers` (external + esvs named as env vars); rewire `scripts/providers-status` to read state via `EveFsm` + (kills M/N for providers). - ☐ **Instance/package machines** — same pattern; retire `state.py` enums + `state_machine.py` into the platform. diff --git a/eve_sdk/fsm.py b/eve_sdk/fsm.py index 1b1a566b..3642ced9 100644 --- a/eve_sdk/fsm.py +++ b/eve_sdk/fsm.py @@ -13,10 +13,17 @@ from __future__ import annotations +import os from collections.abc import Callable, Mapping from typing import Any -import harel +# The statechart engine is being renamed harel -> determa-state. Import it behind one +# name so the eventual package rename is a no-op for eve; this module is the only place +# eve touches the engine. +try: # pragma: no cover - the branch taken depends only on which package is installed + import determa_state as engine +except ModuleNotFoundError: # pre-rename package name + import harel as engine # Resolve a machine's declared ``external`` esvs from eve settings/secrets: # (scope, key, external_names) -> {name: value} @@ -30,15 +37,15 @@ class EveFsm: def __init__(self, resolve_context: ContextResolver) -> None: self._resolve = resolve_context - self._host = harel.Host() + self._host = engine.Host() self._raw: dict[str, dict[str, Any]] = {} self._machine: dict[str, Any] = {} def load(self, scope: str, machine_yaml: str) -> None: """Register the harel machine a plugin ships for ``scope`` (its first document).""" - defs = harel.load_definitions(machine_yaml) + defs = engine.load_definitions(machine_yaml) for definition in defs: - harel.validate(definition.raw) + engine.validate(definition.raw) self._host.register_all(defs) root = defs[0] self._raw[scope] = root.raw @@ -101,3 +108,29 @@ def view(self, scope: str, key: str) -> dict[str, Any]: "context": dict(inst.resolved_esvs()), "status": inst.status.name.lower(), } + + +def provider_context_resolver() -> ContextResolver: + """The real :data:`ContextResolver` for provider machines. + + Each declared external esv is named as its eve env var (e.g. ``VULTR_API_KEY``) and + resolved from the *one* path dispatch uses — ``os.environ`` + + ``ConfigEnv.environment()`` + injected secrets — so a provider's FSM context and what + dispatch runs with are identical (v4.5 bug M: status ≡ dispatch). ``key`` is the + provider id. + + (Systemic follow-up: drop ambient ``os.environ`` from *both* this resolver and + dispatch so no config reaches a plugin behind eve's back; kept here for now to + preserve exact parity.) + """ + from eve_sdk.config import ConfigEnv + from eve_sdk.provider_command import _inject_secrets, _load_public_plugin + + def resolve(scope: str, key: str, names: list[str]) -> dict[str, Any]: + env = dict(os.environ) + env.update(ConfigEnv.environment()) + plugin = _load_public_plugin("provider", key) + env = _inject_secrets(key, plugin, env) + return {name: env.get(name, "") for name in names} + + return resolve diff --git a/tests/test_fsm_platform.py b/tests/test_fsm_platform.py index 56bbb9e8..223cff7d 100644 --- a/tests/test_fsm_platform.py +++ b/tests/test_fsm_platform.py @@ -8,16 +8,19 @@ from __future__ import annotations +import importlib.util from pathlib import Path from typing import Any import pytest -# The harel engine is a new dependency; until it is pinned in pyproject (gated on a -# harel-python release tag), skip cleanly where it isn't installed so CI stays green. -pytest.importorskip("harel") +# The statechart engine (being renamed harel -> determa-state) is a new dependency; +# until it is pinned in pyproject, skip cleanly where neither package is installed so +# CI stays green. +if not any(importlib.util.find_spec(name) for name in ("determa_state", "harel")): + pytest.skip("determa-state/harel engine not installed", allow_module_level=True) -from eve_sdk.fsm import EveFsm # noqa: E402 (must follow importorskip) +from eve_sdk.fsm import EveFsm FIXTURE = Path(__file__).parent / "fsm_fixtures" / "provider.yaml" @@ -68,3 +71,26 @@ def test_settings_change_reevaluates() -> None: assert changed == {"api_key": ""} fsm.fire("provider", "vultr", "resolve") assert _leaves(fsm) == ["unconfigured"] + + +def test_provider_context_resolver_uses_shared_config_secret_path(monkeypatch: Any) -> None: + """The real resolver reads settings + secrets from the same path dispatch uses.""" + from eve_sdk.fsm import provider_context_resolver + + monkeypatch.setattr( + "eve_sdk.config.ConfigEnv.environment", + classmethod(lambda cls, *a, **k: {"VULTR_REGION": "ewr"}), + ) + monkeypatch.setattr( + "eve_sdk.provider_command._load_public_plugin", + lambda kind, plugin_id: { + "id": plugin_id, + "kind": "provider", + "config_schema": {"secrets": {"api_key": {"env_var": "VULTR_API_KEY"}}}, + }, + ) + monkeypatch.setattr("eve_sdk.provider_command.Secrets.read", staticmethod(lambda name: {"api_key": "SECRET"})) + + resolve = provider_context_resolver() + got = resolve("provider", "vultr", ["VULTR_API_KEY", "VULTR_REGION"]) + assert got == {"VULTR_API_KEY": "SECRET", "VULTR_REGION": "ewr"} From 4801d747f9f9b353d73f5aa74c78747b0b86bd91 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sun, 5 Jul 2026 22:14:35 +0900 Subject: [PATCH 20/32] feat(v4.5 N/O): core provider machine + eve-derived configured - core/fsm/provider.yaml: one shared provider lifecycle (unconfigured -> configured -> reachable) for every provider. reachable is a substate of configured reached by the connectivity probe (probe_ok/probe_fail). - provider_configured_resolver(): derives is_configured = every required config_schema secret/setting resolves non-empty via the shared path, so 'configured' is eve-resolved, never a plugin's ambient self-report (the AWS-off-~/.aws bug becomes structurally impossible). Tested end-to-end against the core machine. Next: rewire scripts/providers-status to load the core machine (configured from the FSM; connectivity answers reachable only). --- core/fsm/provider.yaml | 39 ++++++++++++++++++++++++++++++++++++++ docs/v4.5-fsm-design.md | 13 ++++++++++--- eve_sdk/fsm.py | 38 +++++++++++++++++++++++++++++++++++++ tests/test_fsm_platform.py | 36 +++++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 core/fsm/provider.yaml diff --git a/core/fsm/provider.yaml b/core/fsm/provider.yaml new file mode 100644 index 00000000..1ae9c495 --- /dev/null +++ b/core/fsm/provider.yaml @@ -0,0 +1,39 @@ +# The provider lifecycle, shared by every provider (v4.5 N/O). +# +# `is_configured` is derived by eve — true iff every `required` config_schema +# secret/setting resolves to a non-empty value via the ONE shared path (os.environ + +# ConfigEnv + injected secrets) — and seeded as external context. `reachable` is a +# substate of `configured`, reached by the provider's `connectivity` probe (host-run, +# delivered back as probe_ok / probe_fail). One machine: per-provider differences live +# only in the config eve resolves, never in the states — so `configured` can never be a +# plugin's ambient self-report (the AWS-off-~/.aws bug is structurally impossible). +id: provider +events: + resolve: {} + probe_ok: {} + probe_fail: {} +top: + esvs: + is_configured: { type: bool, external: true } + on_events: + env: + action: [ { refresh: {} } ] # adopt a changed configured-ness + initial: { transition_to: unconfigured } + states: + unconfigured: + on_events: + resolve: + - { guard: "is_configured == true", transition_to: configured } + configured: + type: composite + initial: { transition_to: unreachable } + on_events: + resolve: + - { guard: "is_configured != true", transition_to: unconfigured } + states: + unreachable: + on_events: + probe_ok: { transition_to: reachable } + reachable: + on_events: + probe_fail: { transition_to: unreachable } diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index 66dc6dc2..3d2255f8 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -222,11 +222,18 @@ The engine is real now, so this supersedes the speculative `state_machine:` sche (`os.environ` + `ConfigEnv.environment()` + `_inject_secrets`), hermetically tested. So a provider's FSM context ≡ what dispatch runs with (bug M). Systemic follow-up: drop ambient `os.environ` from both this and dispatch. +- ☑ **Core provider machine** — `core/fsm/provider.yaml`: one shared provider lifecycle + (unconfigured → configured → reachable) for *every* provider. `is_configured` is + derived by `provider_configured_resolver()` from the plugin's `config_schema` + `required` secrets/settings (via the shared path), so "configured" is eve-resolved, not + a plugin's ambient self-report. The lifecycle is identical across providers, so it lives + in `core/`, not per-provider. Tested end-to-end. - ☐ **Dependency** — pin the engine in `pyproject` (git dep) once `determa-state` is published + tagged, then `poetry lock`; the test then runs in CI. -- ☐ **Provider machine** — ship the real provider machine in `eve-providers` (external - esvs named as env vars); rewire `scripts/providers-status` to read state via `EveFsm` - (kills M/N for providers). +- ☐ **Rewire `providers-status`** — load `core/fsm/provider.yaml`, resolve → `configured`, + and run each provider's `connectivity` command for **`reachable` only** (probe_ok/ + probe_fail). A live behavior change: `configured` stops being the connectivity command's + self-report. (Connectivity commands then only need to answer reachability.) - ☐ **Instance/package machines** — same pattern; retire `state.py` enums + `state_machine.py` into the platform. diff --git a/eve_sdk/fsm.py b/eve_sdk/fsm.py index 3642ced9..fbdc4c68 100644 --- a/eve_sdk/fsm.py +++ b/eve_sdk/fsm.py @@ -134,3 +134,41 @@ def resolve(scope: str, key: str, names: list[str]) -> dict[str, Any]: return {name: env.get(name, "") for name in names} return resolve + + +def provider_configured_resolver() -> ContextResolver: + """The :data:`ContextResolver` for the core provider machine (``core/fsm/provider.yaml``). + + Derives ``is_configured`` = True iff every ``required`` config_schema secret/setting + of the provider resolves to a non-empty value via the one shared path (``os.environ`` + + ``ConfigEnv.environment()`` + injected secrets). So "configured" is eve-resolved, + never a plugin's ambient self-report (the AWS-off-``~/.aws`` bug is structurally + impossible). ``key`` is the provider id; ``names`` is typically ``["is_configured"]``. + """ + from eve_sdk.config import ConfigEnv + from eve_sdk.provider_command import _inject_secrets, _load_public_plugin + + def resolve(scope: str, key: str, names: list[str]) -> dict[str, Any]: + plugin = _load_public_plugin("provider", key) + env = dict(os.environ) + env.update(ConfigEnv.environment()) + env = _inject_secrets(key, plugin, env) + schema = plugin.get("config_schema") or {} + required = [ + (field, spec) + for section in ("secrets", "settings") + for field, spec in (schema.get(section) or {}).items() + if isinstance(spec, Mapping) and spec.get("required") + ] + + def present(field: str, spec: Mapping[str, Any]) -> bool: + candidates = [field] + env_var = spec.get("env_var") + if env_var: + candidates.append(str(env_var)) + return any(env.get(candidate) for candidate in candidates) + + values: dict[str, Any] = {"is_configured": all(present(f, s) for f, s in required)} + return {name: values.get(name, "") for name in names} + + return resolve diff --git a/tests/test_fsm_platform.py b/tests/test_fsm_platform.py index 223cff7d..41eb4833 100644 --- a/tests/test_fsm_platform.py +++ b/tests/test_fsm_platform.py @@ -94,3 +94,39 @@ def test_provider_context_resolver_uses_shared_config_secret_path(monkeypatch: A resolve = provider_context_resolver() got = resolve("provider", "vultr", ["VULTR_API_KEY", "VULTR_REGION"]) assert got == {"VULTR_API_KEY": "SECRET", "VULTR_REGION": "ewr"} + + +def test_core_provider_machine_configured_from_required_secret(monkeypatch: Any) -> None: + """The core provider machine: configured = a required secret is eve-resolved; a + connectivity probe promotes it to reachable. Exercises core/fsm/provider.yaml.""" + from eve_sdk.fsm import EveFsm, provider_configured_resolver + + monkeypatch.setattr("eve_sdk.config.ConfigEnv.environment", classmethod(lambda cls, *a, **k: {})) + monkeypatch.setattr( + "eve_sdk.provider_command._load_public_plugin", + lambda kind, plugin_id: { + "id": plugin_id, + "kind": "provider", + "config_schema": {"secrets": {"api_key": {"required": True, "env_var": "VULTR_API_KEY"}}}, + }, + ) + box = {"secrets": {}} + monkeypatch.setattr("eve_sdk.provider_command.Secrets.read", staticmethod(lambda name: dict(box["secrets"]))) + + fsm = EveFsm(provider_configured_resolver()) + core = (Path(__file__).resolve().parents[1] / "core" / "fsm" / "provider.yaml").read_text() + fsm.load("provider", core) + + # no secret yet -> unconfigured + fsm.fire("provider", "vultr", "resolve") + assert _leaves(fsm) == ["unconfigured"] + + # the required secret appears at the source -> refresh (env) + re-resolve -> configured + box["secrets"] = {"api_key": "KEY"} + fsm.refresh("provider", "vultr") + fsm.fire("provider", "vultr", "resolve") + assert _leaves(fsm) == ["unreachable"] # configured, probe pending + + # the connectivity probe succeeds -> reachable + fsm.fire("provider", "vultr", "probe_ok") + assert _leaves(fsm) == ["reachable"] From 5dfbf55168162e55cb50c3f9417e8025aec5604f Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 6 Jul 2026 01:28:10 +0900 Subject: [PATCH 21/32] build(v4.5): pin determa-state 0.0.5; import determa.state; drop skip guard The statechart engine published as determa-state (import determa.state). Pin it in pyproject + poetry.lock as a real dependency; the FSM platform imports determa.state (falling back to the pre-rename harel). Remove the skip-if-missing test guard now that the engine is a hard dependency (a missing required dep should fail loudly, not skip). --- eve_sdk/fsm.py | 21 ++- poetry.lock | 262 ++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + tests/test_fsm_platform.py | 11 +- 4 files changed, 273 insertions(+), 22 deletions(-) diff --git a/eve_sdk/fsm.py b/eve_sdk/fsm.py index fbdc4c68..b681c860 100644 --- a/eve_sdk/fsm.py +++ b/eve_sdk/fsm.py @@ -1,12 +1,12 @@ -"""eve FSM platform — the single seam between eve and the harel statechart engine. +"""eve FSM platform — the single seam between eve and the Determa State statechart engine. -Machines are declared in **harel YAML** by the plugin that owns them (a provider, a +Machines are declared in **Determa State YAML** by the plugin that owns them (a provider, a package, ...). eve resolves each machine's ``external`` esvs (settings + secrets) through **one shared resolver** and seeds them as the machine's context, so status, -dispatch, and connectivity can never see different config (v4.5 bugs N/O). The harel -engine is the swappable unit; this module is the only place eve touches it. +dispatch, and connectivity can never see different config (v4.5 bugs N/O). The engine +is the swappable unit; this module is the only place eve touches it. -Side effects are host-driven: harel actions are sandboxed, so eve runs the real +Side effects are host-driven: engine actions are sandboxed, so eve runs the real command (probe, terraform, provision) and delivers the outcome back as an event (e.g. ``probe_ok`` / ``probe_fail``). The machine only ever holds state + context. """ @@ -17,11 +17,10 @@ from collections.abc import Callable, Mapping from typing import Any -# The statechart engine is being renamed harel -> determa-state. Import it behind one -# name so the eventual package rename is a no-op for eve; this module is the only place -# eve touches the engine. +# The statechart engine (Determa State; formerly harel). Import it behind one name so +# eve's only coupling to it is this module. try: # pragma: no cover - the branch taken depends only on which package is installed - import determa_state as engine + import determa.state as engine except ModuleNotFoundError: # pre-rename package name import harel as engine @@ -33,7 +32,7 @@ class EveFsm: - """Loads plugin-declared harel machines and tracks their live instances.""" + """Loads plugin-declared Determa State machines and tracks their live instances.""" def __init__(self, resolve_context: ContextResolver) -> None: self._resolve = resolve_context @@ -42,7 +41,7 @@ def __init__(self, resolve_context: ContextResolver) -> None: self._machine: dict[str, Any] = {} def load(self, scope: str, machine_yaml: str) -> None: - """Register the harel machine a plugin ships for ``scope`` (its first document).""" + """Register the machine a plugin ships for ``scope`` (its first document).""" defs = engine.load_definitions(machine_yaml) for definition in defs: engine.validate(definition.raw) diff --git a/poetry.lock b/poetry.lock index fc546d48..800851d0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -12,6 +12,28 @@ files = [ {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, ] +[[package]] +name = "cel-python" +version = "0.5.0" +description = "Pure Python implementation of Google Common Expression Language" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "cel_python-0.5.0-py3-none-any.whl", hash = "sha256:d0f85008b89655c2bb18d797d2fa3f96f2ed80f4a3b43b0e8138c6646581e5f6"}, + {file = "cel_python-0.5.0.tar.gz", hash = "sha256:3eb0a619e8df0f338d0430cda01427a742e77e3c433a1c7c3ebd409cd804c45a"}, +] + +[package.dependencies] +google-re2 = [ + {version = ">=1.1.20240702", markers = "python_version != \"3.13\""}, + {version = ">=1.1.20250722", markers = "python_version == \"3.13\""}, +] +jmespath = ">=1.0.1" +lark = ">=1.2.2" +pendulum = ">=3.1.0" +pyyaml = ">=6.0.2" + [[package]] name = "colorama" version = "0.4.6" @@ -25,6 +47,91 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "determa-state" +version = "0.0.5" +description = "Determa State — Python reference implementation of the Determa statechart engine" +optional = false +python-versions = ">=3.11" +groups = ["main"] +files = [ + {file = "determa_state-0.0.5-py3-none-any.whl", hash = "sha256:cb9d357c5fcd7c74e8d351802ca1e3057ba02a1e35ec8bfb63f5686c1f19800a"}, + {file = "determa_state-0.0.5.tar.gz", hash = "sha256:2dac244ce11a028b38146ddbc5c006f36c1d1c9c67cba11b7a16e8e92171ee36"}, +] + +[package.dependencies] +cel-python = ">=0.1" +jsonschema = ">=4" +pyyaml = ">=6" + +[package.extras] +dev = ["mypy", "pytest (>=8)", "ruff", "types-jsonschema", "types-pyyaml"] + +[[package]] +name = "google-re2" +version = "1.1.20251105" +description = "RE2 Python bindings" +optional = false +python-versions = "~=3.9" +groups = ["main"] +files = [ + {file = "google_re2-1.1.20251105-1-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:88bd426c1904f3562049bf766301bbc4f7a4bcb8f61e92f8cc833faac1cf2a92"}, + {file = "google_re2-1.1.20251105-1-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:a486dc10bb07f3c34b9908541368e21ab6d77972569427200db077126668fbf3"}, + {file = "google_re2-1.1.20251105-1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:a9aa02dc1345f0889c6ce1365d5f93d5b161b512f4c6df3cfadf3298493fb678"}, + {file = "google_re2-1.1.20251105-1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:032160ad8c05739370813bcb15099854cd50faa933e0fe9607a2380659c750df"}, + {file = "google_re2-1.1.20251105-1-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:39a7013477c8778b1ddcc0d43eff0ee4a0f66b76c9db21f9e7b7d1f74852633f"}, + {file = "google_re2-1.1.20251105-1-cp310-cp310-macosx_15_0_x86_64.whl", hash = "sha256:f886c88d56233483c5fd5ed1234e7e72389b8331250100983443fa30855deb63"}, + {file = "google_re2-1.1.20251105-1-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8beddf48857fd3767c553f0be7414a7a483f9b6374c91c02474a616fc7f5c5b3"}, + {file = "google_re2-1.1.20251105-1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a319dcb37b069d72d968862335197f460803b3a35f99445ea805f69fac58759"}, + {file = "google_re2-1.1.20251105-1-cp310-cp310-win32.whl", hash = "sha256:420fe037ad77ab3d1a280c6823985b89160896f66ce601a3923d020690a1f9b4"}, + {file = "google_re2-1.1.20251105-1-cp310-cp310-win_amd64.whl", hash = "sha256:462dfcf147d0f54d0c93a69c361225119a4987c3b0ecd77f0e21ad9ba8bf180e"}, + {file = "google_re2-1.1.20251105-1-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:329efa209ea7baa44f0facf0402fa34e655dc97fdeb10d0b83fc06354f5575fd"}, + {file = "google_re2-1.1.20251105-1-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:aa2ad5f6f48921ec137a7b7f1b1da903ddef8627a2dc30bc878a9a69d9925719"}, + {file = "google_re2-1.1.20251105-1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ac1cb2526cc88f050a0661fc7245ad009ee454bddc541b2e653f1d007585000d"}, + {file = "google_re2-1.1.20251105-1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:50c7205182ad66c23c07abe8072f720ca2f7d595b61e28fd9b63623614f9afd6"}, + {file = "google_re2-1.1.20251105-1-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:4cb5acee61e35772503b8b1db3c592a46b8e6a9bc0ab54d7d6233654ea2bf93d"}, + {file = "google_re2-1.1.20251105-1-cp311-cp311-macosx_15_0_x86_64.whl", hash = "sha256:1617097d63620c2d46bdfc0e48f24f66cd341664fc75718636d234f67473fe7f"}, + {file = "google_re2-1.1.20251105-1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18a5610b26742b90cb1d64ead2b16fe0e3bd7e67add03fd3779cd1b85e401661"}, + {file = "google_re2-1.1.20251105-1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03156291269f145eccddff63118f2df02d395792f51fc039f09955818943815a"}, + {file = "google_re2-1.1.20251105-1-cp311-cp311-win32.whl", hash = "sha256:54f51762b51dc238eceddf49b56cc2b64594fe72d9328c1c39d615aa990e1f87"}, + {file = "google_re2-1.1.20251105-1-cp311-cp311-win_amd64.whl", hash = "sha256:f5f856ff5036a8f22b3bad57f376d4e3b97b59b64f311bdb1f83c8dabded2492"}, + {file = "google_re2-1.1.20251105-1-cp311-cp311-win_arm64.whl", hash = "sha256:913864f97de4151eaa8bb7746ca230fd193656501e07fb658ce2cd46d4f6efcc"}, + {file = "google_re2-1.1.20251105-1-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:b30f09b4d63249c72e65ccae4cbf6b331b48c22fc7cb439f1d85f347b9d07ceb"}, + {file = "google_re2-1.1.20251105-1-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:9a77892c524b8bdf3d47d7cad1cc2ac3a0108bdd65007ef4c02888fa46baf8ee"}, + {file = "google_re2-1.1.20251105-1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a3ac51b28cbf25c100dfd8849212d878d7005d1d4a7e129a10789043c56b6021"}, + {file = "google_re2-1.1.20251105-1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:9f7158afc9825ac2654c6561aea94a1f7edb5b5b88e6e3639bb80bb817d102ac"}, + {file = "google_re2-1.1.20251105-1-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:5320da07dc3b7ac7f407514f42ac17d67e771ac7c7562d449571185e6fb601b2"}, + {file = "google_re2-1.1.20251105-1-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:5a4e5785bc30d52ce655d805b07ad2d8a4905429a5f690ae9c2f1caa76665709"}, + {file = "google_re2-1.1.20251105-1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b7a3b90f747130310d4b3b8e19ebb845d0d97c1deb63b36f76c7242dacbd736"}, + {file = "google_re2-1.1.20251105-1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:809c5fa5d08279413b29c2e2c5c528e85cd94a0e0fd897db595a0c09eeee2782"}, + {file = "google_re2-1.1.20251105-1-cp312-cp312-win32.whl", hash = "sha256:d8424e63a9ec0fe5bde03d97876b2431f8a746af33eb475fa1ae39144bd05b2a"}, + {file = "google_re2-1.1.20251105-1-cp312-cp312-win_amd64.whl", hash = "sha256:062313c309f93dfeb6966372f4c446580e98879133ec155522eea8aaf568a5cd"}, + {file = "google_re2-1.1.20251105-1-cp312-cp312-win_arm64.whl", hash = "sha256:558f144b26a9555ae4e9467cc3aa3299a8ce13217f328b21ae326ca0633be19b"}, + {file = "google_re2-1.1.20251105-1-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:9f3cf610e857a7d6f02916cf2b7fc159a5429b8bcb23164500d46e5e233f2924"}, + {file = "google_re2-1.1.20251105-1-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:a21c2807bf4d5d00f206a4ecb3b043aad674e28c451b697b740280f608872078"}, + {file = "google_re2-1.1.20251105-1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8314144eefeee7b88b742081c2038418f677e63901039ca9dbfbc0c5bb6d2911"}, + {file = "google_re2-1.1.20251105-1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:28a46be978e53c772139d0f5c9ba69f53563fcdd4225407e4d34d51208b828f1"}, + {file = "google_re2-1.1.20251105-1-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:83292e23963aa1b219d5f64a65365b0880448a6a060276027b55270bc5b18c7e"}, + {file = "google_re2-1.1.20251105-1-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:1920b15dc9b1bdfeca5aa2c60900373c6f27cd1056d53cd299456ea5540a6fff"}, + {file = "google_re2-1.1.20251105-1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b1458d9ca588124cd61aa1bf5388a216e1247e7d474f8e5e1530498044f5c87"}, + {file = "google_re2-1.1.20251105-1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a52cb204e49d20cdbb66faf394d57f476e96c39c23a328442ab0194fc6bd1a2b"}, + {file = "google_re2-1.1.20251105-1-cp313-cp313-win32.whl", hash = "sha256:67c5c73d7ebcf3f0e0a3b528b41bd8c6c04900f1598aebf05bbdf15a06cf5f9a"}, + {file = "google_re2-1.1.20251105-1-cp313-cp313-win_amd64.whl", hash = "sha256:0bcba63ad3ea8926fb0c71bb5044e33d405bb9395f5b5444393cd5f28f0bf6d3"}, + {file = "google_re2-1.1.20251105-1-cp313-cp313-win_arm64.whl", hash = "sha256:64ee189ea857f2126c5e42073cfa9b03e9f4cbaf073edbedb575059074841aa0"}, + {file = "google_re2-1.1.20251105-1-cp314-cp314-macosx_13_0_arm64.whl", hash = "sha256:cc151cf6a585d9ebe711da32b23683fcff40f78db8c8587c7f4b209ef4658809"}, + {file = "google_re2-1.1.20251105-1-cp314-cp314-macosx_13_0_x86_64.whl", hash = "sha256:7e2186d2c90488c1e11895343941f35ca2f58e9ba6c6b034fd531abe22ef77cc"}, + {file = "google_re2-1.1.20251105-1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:41be22359c3dceb582937739b4365dd8e279de24ad0a5b10e653503abaff2ed7"}, + {file = "google_re2-1.1.20251105-1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:f3168d7bbac247c862ea85b2f3c011d3a04bedcb6892b37f14d488f4133b206e"}, + {file = "google_re2-1.1.20251105-1-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:79ce664038194a31bbcf422137f9607ae3d9946a5cff98cf0efbeb7f9411e64b"}, + {file = "google_re2-1.1.20251105-1-cp314-cp314-macosx_15_0_x86_64.whl", hash = "sha256:0476b07421b8882b279d5ceb5b760c15c62d581ded95274697fc1227e3869ee6"}, + {file = "google_re2-1.1.20251105-1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:85feec3161ffdc12f6b144e37a2f91f80b771c72ffadde60191e89a49f6d7e81"}, + {file = "google_re2-1.1.20251105-1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7bfaa2cf55daf0c5c650e68526bb20b61e37d7f3ae53f6893013acc1c91c116"}, + {file = "google_re2-1.1.20251105-1-cp314-cp314-win32.whl", hash = "sha256:214c1accdc60fff9ce1bf812b157147ca361844f496ed9e0d5f357b0e562ced8"}, + {file = "google_re2-1.1.20251105-1-cp314-cp314-win_amd64.whl", hash = "sha256:6d4d5fdadd329a2ed193463899d00ef2fd126172f36a4c01c9def271f19801b6"}, + {file = "google_re2-1.1.20251105-1-cp314-cp314-win_arm64.whl", hash = "sha256:1d27f3a2a947ec1f721d0f14f661108acfd4f4d34f357ce28db951cc036656e5"}, + {file = "google_re2-1.1.20251105.tar.gz", hash = "sha256:1db14a292ee8303b91e91e7c37e05ac17d3c467f29416c79ac70a78be3e65bda"}, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -37,6 +144,18 @@ files = [ {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, ] +[[package]] +name = "jmespath" +version = "1.1.0" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64"}, + {file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"}, +] + [[package]] name = "jsonschema" version = "4.26.0" @@ -74,6 +193,24 @@ files = [ [package.dependencies] referencing = ">=0.31.0" +[[package]] +name = "lark" +version = "1.3.1" +description = "a modern parsing library" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12"}, + {file = "lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905"}, +] + +[package.extras] +atomic-cache = ["atomicwrites"] +interegular = ["interegular (>=0.3.1,<0.4.0)"] +nearley = ["js2py"] +regex = ["regex"] + [[package]] name = "librt" version = "0.9.0" @@ -365,6 +502,90 @@ hyperscan = ["hyperscan (>=0.7)"] optional = ["typing-extensions (>=4)"] re2 = ["google-re2 (>=1.1)"] +[[package]] +name = "pendulum" +version = "3.2.0" +description = "Python datetimes made easy" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pendulum-3.2.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:4a6bf778c6b42830b001c714dae5b9dad78da38e2e08203a4b0f5d53f8fa5e63"}, + {file = "pendulum-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:625209bb7133d990905e8935e1c04f0a82315ae777b67910969b16f665d62c0b"}, + {file = "pendulum-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b6f1d8641e8bd48b9b6f77f96fd498d3ecec63611ba8e7207e63936307846042"}, + {file = "pendulum-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a8d4212b1577ee3a034d18b360a9afa55bfc72789aeb805353be8b2ac132035"}, + {file = "pendulum-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e398d9e3db42d17f0c2cd39663c1c873ea6f11763ed6d126e2dcc92fc340d0dc"}, + {file = "pendulum-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04310463879a8d84534756ef9820d433e88b879203b6e10a5b416899dc05e7f1"}, + {file = "pendulum-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5b4f7491951c11bbdb20893817352c9140d31d1ae333839c34c0bca081a50a86"}, + {file = "pendulum-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ffc169ad7595228d4dfc44d4e016846ff1bb5873b9f7ec70b0b1b51da0c77b3f"}, + {file = "pendulum-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:446f63d84ef21281844ceb45141536d3aabe291a821b6505e21a0d0e3ea95d67"}, + {file = "pendulum-3.2.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5d775cc608c909ad415c8e789c84a9f120bb6a794c4215b2d8d910893cf0ec6a"}, + {file = "pendulum-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8de794a7f665aebc8c1ba4dd4b05ab8fe1a36ce9c0498366adf1d1edd79b2686"}, + {file = "pendulum-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bac7df7696e1c942e17c0556b3a7bcdd1d7aa5b24faee7620cb071e754a0622"}, + {file = "pendulum-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db0f6a8a04475d9cba26ce701e7d66d266fd97227f2f5f499270eba04be1c7e9"}, + {file = "pendulum-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c352c63c1ff05f2198409b28498d7158547a8be23e1fbd4aa2cf5402fb239b55"}, + {file = "pendulum-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de8c1ad1d1aa7d4ceae341528bab35a0f8c88a5aa63f2f5d84e16b517d1b32c2"}, + {file = "pendulum-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1ba955511c12fec2252038b0c866c25c0c30b720bf74d3023710f121e42b1498"}, + {file = "pendulum-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4115bf364a2ec6d5ddc476751ceaa4164a04f2c15589f0d29aa210ddb784b15d"}, + {file = "pendulum-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:4151a903356413fdd9549de0997b708fb95a214ed97803ffb479ffd834088378"}, + {file = "pendulum-3.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:acfdee9ddc56053cb7c8c075afbfde0857322d09e56a56195b9cd127fae87e4c"}, + {file = "pendulum-3.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bf0b489def51202a39a2a665dcc4162d5e46934a740fe4c4fe3068979610156c"}, + {file = "pendulum-3.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:937a529aa302efa18dcf25e53834964a87ffb2df8f80e3669ab7757a6126beaf"}, + {file = "pendulum-3.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85c7689defc65c4dc29bf257f7cca55d210fabb455de9476e1748d2ab2ae80d7"}, + {file = "pendulum-3.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e216e5a412563ea2ecf5de467dcf3d02717947fcdabe6811d5ee360726b02b"}, + {file = "pendulum-3.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a2af22eeec438fbaac72bb7fba783e0950a514fba980d9a32db394b51afccec"}, + {file = "pendulum-3.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3159cceb54f5aa8b85b141c7f0ce3fac8bdd1ffdc7c79e67dca9133eac7c4d11"}, + {file = "pendulum-3.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c39ea5e9ffa20ea8bae986d00e0908bd537c8468b71d6b6503ab0b4c3d76e0ea"}, + {file = "pendulum-3.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e5afc753e570cce1f44197676371f68953f7d4f022303d141bb09f804d5fe6d7"}, + {file = "pendulum-3.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:fd55c12560816d9122ca2142d9e428f32c0c083bf77719320b1767539c7a3a3b"}, + {file = "pendulum-3.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:faef52a7ed99729f0838353b956f3fabf6c550c062db247e9e2fc2b48fcb9457"}, + {file = "pendulum-3.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:addb0512f919fe5b70c8ee534ee71c775630d3efe567ea5763d92acff857cfc3"}, + {file = "pendulum-3.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3aaa50342dc174acebdc21089315012e63789353957b39ac83cac9f9fc8d1075"}, + {file = "pendulum-3.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:927e9c9ab52ff68e71b76dd410e5f1cd78f5ea6e7f0a9f5eb549aea16a4d5354"}, + {file = "pendulum-3.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:249d18f5543c9f43aba3bd77b34864ec8cf6f64edbead405f442e23c94fce63d"}, + {file = "pendulum-3.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c644cc15eec5fb02291f0f193195156780fd5a0affd7a349592403826d1a35e"}, + {file = "pendulum-3.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:063ab61af953bb56ad5bc8e131fd0431c915ed766d90ccecd7549c8090b51004"}, + {file = "pendulum-3.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:26a3ae26c9dd70a4256f1c2f51addc43641813574c0db6ce5664f9861cd93621"}, + {file = "pendulum-3.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:2b10d91dc00f424444a42f47c69e6b3bfd79376f330179dc06bc342184b35f9a"}, + {file = "pendulum-3.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:63070ff03e30a57b16c8e793ee27da8dac4123c1d6e0cf74c460ce9ee8a64aa4"}, + {file = "pendulum-3.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:c8dde63e2796b62070a49ce813ce200aba9186130307f04ec78affcf6c2e8122"}, + {file = "pendulum-3.2.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c17ac069e88c5a1e930a5ae0ef17357a14b9cc5a28abadda74eaa8106d241c8e"}, + {file = "pendulum-3.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e1fbb540edecb21f8244aebfb05a1f2333ddc6c7819378c099d4a61cc91ae93c"}, + {file = "pendulum-3.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8c67fb9a1fe8fc1adae2cc01b0c292b268c12475b4609ff4aed71c9dd367b4d"}, + {file = "pendulum-3.2.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:baa9a66c980defda6cfe1275103a94b22e90d83ebd7a84cc961cee6cbd25a244"}, + {file = "pendulum-3.2.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef8f783fa7a14973b0596d8af2a5b2d90858a55030e9b4c6885eb4284b88314f"}, + {file = "pendulum-3.2.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7d2e9bfb065727d8676e7ada3793b47a24349500a5e9637404355e482c822be"}, + {file = "pendulum-3.2.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:55d7ba6bb74171c3ee409bf30076ee3a259a3c2bb147ac87ebb76aaa3cf5d3a2"}, + {file = "pendulum-3.2.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:a50d8cf42f06d3d8c3f8bb2a7ac47fa93b5145e69de6a7209be6a47afdd9cf76"}, + {file = "pendulum-3.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:e5bbb92b155cd5018b3cf70ee49ed3b9c94398caaaa7ed97fe41e5bb5a968418"}, + {file = "pendulum-3.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:d53134418e04335c3029a32e9341cccc9b085a28744fb5ee4e6a8f5039363b1a"}, + {file = "pendulum-3.2.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:70214f30114ee6de09b0abbb90e74f68682f194485200897930125de36024c21"}, + {file = "pendulum-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d3cad6be303b4b9737f629191b73920e0e4c2fe1fbd5813b7a19b05f29d09a1c"}, + {file = "pendulum-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dd31fd46aa8fceaff5de674b8d57b0b659753986cb78fa9e24368da7f8e4eec"}, + {file = "pendulum-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce87750e0a644e8ddafa3e9437f0ff35af8d17eb5c77824864246ccf6b44d24a"}, + {file = "pendulum-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:326ef1baa8657c9c4aead5c4eac06a46355ba4e338816153e2929b0b548554b5"}, + {file = "pendulum-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f00d033fe9b31c41deb6484f36a788b99b936cf8d10a994784e809a59b72d7db"}, + {file = "pendulum-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:86029fd802c5650f2cbef3f9302608325a676d3a883d6a5782bc855e5d03918b"}, + {file = "pendulum-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:dfbcd7a2a5ebabf5abaf07062555cd7bef9a472a6aa5281a6302afe66a481561"}, + {file = "pendulum-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:38a300df5a82ebea8634a1aed3d98c92968ba856d2ca7ba2acf7228f0e0e99af"}, + {file = "pendulum-3.2.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9585594d32faa71efa5a78f576f1ee4f79e9c5340d7c6f0cd6c5dfe725effaaa"}, + {file = "pendulum-3.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:26401e2de77c437e8f3b6160c08c6c5d45518d906f8f9b48fd7cb5aa0f4e2aff"}, + {file = "pendulum-3.2.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:637e65af042f383a2764a886aa28ccc6f853bf7a142df18e41c720542934c13b"}, + {file = "pendulum-3.2.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6e46c28f4d067233c4a4c42748f4ffa641d9289c09e0e81488beb6d4b3fab51"}, + {file = "pendulum-3.2.0-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:71d46bcc86269f97bfd8c5f1475d55e717696a0a010b1871023605ca94624031"}, + {file = "pendulum-3.2.0-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5cd956d4176afc7bfe8a91bf3f771b46ff8d326f6c5bf778eb5010eb742ebba6"}, + {file = "pendulum-3.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:39ef129d7b90aab49708645867abdd207b714ba7bff12dae549975b0aca09716"}, + {file = "pendulum-3.2.0-py3-none-any.whl", hash = "sha256:f3a9c18a89b4d9ef39c5fa6a78722aaff8d5be2597c129a3b16b9f40a561acf3"}, + {file = "pendulum-3.2.0.tar.gz", hash = "sha256:e80feda2d10fa3ff8b1526715f7d33dcb7e08494b3088f2c8a3ac92d4a4331ce"}, +] + +[package.dependencies] +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0,<3.0.0) ; implementation_name != \"pypy\""] + [[package]] name = "platformdirs" version = "4.9.6" @@ -430,6 +651,21 @@ pygments = ">=2.7.2" [package.extras] dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + [[package]] name = "pyyaml" version = "6.0.3" @@ -702,6 +938,18 @@ files = [ {file = "ruff-0.15.12.tar.gz", hash = "sha256:ecea26adb26b4232c0c2ca19ccbc0083a68344180bba2a600605538ce51a40a6"}, ] +[[package]] +name = "six" +version = "1.17.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] + [[package]] name = "textual" version = "8.2.5" @@ -737,6 +985,18 @@ files = [ {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, ] +[[package]] +name = "tzdata" +version = "2026.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +groups = ["main"] +files = [ + {file = "tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7"}, + {file = "tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10"}, +] + [[package]] name = "uc-micro-py" version = "2.0.0" @@ -755,4 +1015,4 @@ test = ["coverage", "pytest", "pytest-cov"] [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "f7529362fd9622b3afc53861b6555b2426366ff0790fc3a96440e2ff9c30da17" +content-hash = "c5eebfc9584b4768d75016dc17389c9b7d961cfbd298c71ec1cc7a0532a13f1d" diff --git a/pyproject.toml b/pyproject.toml index 9cb62778..e5ed90a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ license = "MIT" readme = "README.md" requires-python = ">=3.12,<4.0" dependencies = [ + "determa-state (>=0.0.5,<0.0.6)", "jsonschema (>=4.25.1,<5.0.0)", "pyyaml (>=6.0.3,<7.0.0)", "textual (>=8.2.5,<9.0.0)" diff --git a/tests/test_fsm_platform.py b/tests/test_fsm_platform.py index 41eb4833..03815c6b 100644 --- a/tests/test_fsm_platform.py +++ b/tests/test_fsm_platform.py @@ -1,4 +1,4 @@ -"""FSM platform (eve_sdk.fsm.EveFsm) over the harel engine. +"""FSM platform (eve_sdk.fsm.EveFsm) over the Determa State engine. Proves the v4.5 N/O invariant on a synthetic provider machine: `configured` and `reachable` are FSM-computed from context that comes from *one* resolver, and a @@ -8,18 +8,9 @@ from __future__ import annotations -import importlib.util from pathlib import Path from typing import Any -import pytest - -# The statechart engine (being renamed harel -> determa-state) is a new dependency; -# until it is pinned in pyproject, skip cleanly where neither package is installed so -# CI stays green. -if not any(importlib.util.find_spec(name) for name in ("determa_state", "harel")): - pytest.skip("determa-state/harel engine not installed", allow_module_level=True) - from eve_sdk.fsm import EveFsm FIXTURE = Path(__file__).parent / "fsm_fixtures" / "provider.yaml" From 50161f7c95711c3629a041f28309f8a71b8625f4 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 6 Jul 2026 03:41:08 +0900 Subject: [PATCH 22/32] feat(v4.5 N/O): providers-status configured via the core provider FSM configured is now FSM-derived (core/fsm/provider.yaml + provider_configured_resolver): true iff eve has resolved the provider's required config/secrets, so it can never be a connectivity command's ambient self-report. The connectivity probe runs only when configured, and only its reachable + notes are used (its configured field is ignored). Live-verified: vultr yes/yes, truenas 'not configured', docker/qemu/gcp yes; aws yes/no with an actionable note. Full suite green (493 passed, 1 skipped). AWS follow-up (eve-providers): AWS declares only a non-required config_file, so it reads configured=yes; mark an auth setting required or model it probe-derived if it should read unconfigured until eve holds a profile. --- docs/v4.5-fsm-design.md | 12 ++++++++---- scripts/providers-status | 37 ++++++++++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index 3d2255f8..2949b470 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -230,10 +230,14 @@ The engine is real now, so this supersedes the speculative `state_machine:` sche in `core/`, not per-provider. Tested end-to-end. - ☐ **Dependency** — pin the engine in `pyproject` (git dep) once `determa-state` is published + tagged, then `poetry lock`; the test then runs in CI. -- ☐ **Rewire `providers-status`** — load `core/fsm/provider.yaml`, resolve → `configured`, - and run each provider's `connectivity` command for **`reachable` only** (probe_ok/ - probe_fail). A live behavior change: `configured` stops being the connectivity command's - self-report. (Connectivity commands then only need to answer reachability.) +- ☑ **Rewire `providers-status`** — `configured` is now FSM-derived (`core/fsm/provider.yaml` + + `provider_configured_resolver`); the `connectivity` command runs **only for + `reachable`** (its self-reported `configured` is ignored) and only when configured. + Live-verified (vultr yes/yes, truenas no, docker/qemu yes); full suite green. + - *AWS follow-up (eve-providers):* AWS declares only a non-required `config_file`, so + it reads `configured=yes / reachable=no` (readiness surfaces as reachability + note). + If AWS should read `configured=no` until eve holds an auth profile, mark an auth + setting `required` or model AWS's configured as probe-derived — a per-provider call. - ☐ **Instance/package machines** — same pattern; retire `state.py` enums + `state_machine.py` into the platform. diff --git a/scripts/providers-status b/scripts/providers-status index 397d1e0f..81d2b877 100755 --- a/scripts/providers-status +++ b/scripts/providers-status @@ -7,9 +7,13 @@ Usage: scripts/providers-status # print single field (configured|reachable|notes) The provider list is derived dynamically from PluginManifest: every provider -that declares a ``connectivity`` command appears. Each connectivity command -runs on the HOST (probes local CLIs/creds, not an instance) and prints one -JSON object: {"configured","reachable","notes"}. +that declares a ``connectivity`` command appears. + +``configured`` is computed by the core provider FSM (``core/fsm/provider.yaml``) from +eve-resolved config — true iff the provider's *required* config/secrets are present — +so it can never be a plugin's ambient self-report (v4.5 N/O). ``reachable`` comes from +the provider's ``connectivity`` command, run on the HOST only when configured; the +command's own ``configured`` field is ignored (it only needs to answer reachability). """ from __future__ import annotations @@ -25,9 +29,12 @@ sys.path.insert(0, str(ROOT)) from eve_sdk.config import ConfigEnv # noqa: E402 from eve_sdk.dispatch import command_vector # noqa: E402 +from eve_sdk.fsm import EveFsm, provider_configured_resolver # noqa: E402 from eve_sdk.plugin_manifest import PluginManifest # noqa: E402 from eve_sdk.provider_command import _inject_secrets # noqa: E402 +CORE_PROVIDER_MACHINE = ROOT / "core" / "fsm" / "provider.yaml" + HEADER = "Provider Configured Reachable Notes" SEPARATOR = "----------------- ---------- --------- -------------------------------------------" @@ -72,6 +79,26 @@ def _print_row(name: str, configured: str, reachable: str, notes: str) -> None: print(f"{name:<17} {configured:<10} {reachable:<9} {notes}") +def _provider_status(plugin: dict[str, Any]) -> tuple[str, str, str]: + """(configured, reachable, notes) via the core provider FSM (v4.5 N/O). + + ``configured`` is FSM-derived — true iff eve has resolved the provider's *required* + config/secrets (``provider_configured_resolver``) — so it can never be a connectivity + command's ambient self-report. The connectivity probe runs only when configured, and + only its ``reachable`` + ``notes`` are used. + """ + provider_id = str(plugin["id"]) + fsm = EveFsm(provider_configured_resolver()) + fsm.load("provider", CORE_PROVIDER_MACHINE.read_text()) + fsm.fire("provider", provider_id, "resolve") + if fsm.instance("provider", provider_id).active_leaf_names() == ["unconfigured"]: + return "no", "no", "not configured" + _configured, reachable, notes = _run_connectivity(plugin) + fsm.fire("provider", provider_id, "probe_ok" if reachable == "yes" else "probe_fail") + reachable = "yes" if fsm.instance("provider", provider_id).active_leaf_names() == ["reachable"] else "no" + return "yes", reachable, notes + + def main(argv: list[str]) -> int: provider_name = argv[0] if len(argv) > 0 else "" field = argv[1] if len(argv) > 1 else "" @@ -84,7 +111,7 @@ def main(argv: list[str]) -> int: print(f"Unknown provider: {provider_name} (valid: {' '.join(provider_ids)})", file=sys.stderr) return 1 plugin = next(p for p in providers if p["id"] == provider_name) - configured, reachable, notes = _run_connectivity(plugin) + configured, reachable, notes = _provider_status(plugin) if field: if field == "configured": print(configured) @@ -106,7 +133,7 @@ def main(argv: list[str]) -> int: print(HEADER) print(SEPARATOR) for plugin in providers: - configured, reachable, notes = _run_connectivity(plugin) + configured, reachable, notes = _provider_status(plugin) _print_row(plugin["id"], configured, reachable, notes) return 0 From 0cddae05c2e00eee4678000166f753293edcbdaa Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 6 Jul 2026 03:57:44 +0900 Subject: [PATCH 23/32] feat(v4.5 N): declarative instance + package machines core/fsm/instance.yaml models the instance lifecycle as two orthogonal regions (provider terraform lifecycle + provisioning), with distinct transient states so each operation's outcome is unambiguous, and desired as context. core/fsm/package.yaml models the package lifecycle (install/remove + observe-driven drift). Both pass the engine's static validation and are exercised in tests/test_fsm_machines.py. These are the declarative replacements for state.py's PROVIDER/PROVISION/PACKAGE_STATES. Wiring them into live persistence/rendering (retiring state_machine.py) is gated on a state-vocabulary decision (match existing enums vs adopt cleaner set + migrate), recorded in docs/v4.5-fsm-design.md. --- core/fsm/instance.yaml | 85 ++++++++++++++++++++++++++++++++++++++ core/fsm/package.yaml | 38 +++++++++++++++++ docs/v4.5-fsm-design.md | 12 +++++- tests/test_fsm_machines.py | 80 +++++++++++++++++++++++++++++++++++ 4 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 core/fsm/instance.yaml create mode 100644 core/fsm/package.yaml create mode 100644 tests/test_fsm_machines.py diff --git a/core/fsm/instance.yaml b/core/fsm/instance.yaml new file mode 100644 index 00000000..ca1a2c61 --- /dev/null +++ b/core/fsm/instance.yaml @@ -0,0 +1,85 @@ +# The instance lifecycle (v4.5 N/O) — the declarative replacement for state.py's +# PROVIDER_STATES + PROVISION_STATES. Two concurrent regions: the provider (terraform) +# lifecycle and the provisioning lifecycle, which advance independently. `desired` is the +# goal (running | stopped | absent), supplied by eve as context. Transitions are driven +# by eve operations (create/start/stop/destroy, provision) whose outcomes come back as +# events, and by observation (observe_* maps live provider status back into a state, +# keeping the machine honest about drift). +id: instance +events: + create: {} + start: {} + stop: {} + destroy: {} + created_ok: {} + started_ok: {} + stopped_ok: {} + destroyed_ok: {} + op_fail: {} + observe_running: {} + observe_stopped: {} + observe_absent: {} + provision: {} + provision_ok: {} + provision_fail: {} +top: + esvs: + desired: { type: string, external: true } # running | stopped | absent | unknown + regions: + + # Provider (terraform) lifecycle. Each in-flight operation has its own transient + # state so its outcome is unambiguous; op_fail from any of them lands in error. + - initial: { transition_to: absent } + states: + absent: + on_events: + create: { transition_to: creating } + creating: + on_events: + created_ok: { transition_to: running } + op_fail: { transition_to: error } + running: + on_events: + stop: { transition_to: stopping } + destroy: { transition_to: destroying } + observe_stopped: { transition_to: stopped } + observe_absent: { transition_to: absent } + stopping: + on_events: + stopped_ok: { transition_to: stopped } + op_fail: { transition_to: error } + stopped: + on_events: + start: { transition_to: starting } + destroy: { transition_to: destroying } + observe_running: { transition_to: running } + observe_absent: { transition_to: absent } + starting: + on_events: + started_ok: { transition_to: running } + op_fail: { transition_to: error } + destroying: + on_events: + destroyed_ok: { transition_to: absent } + op_fail: { transition_to: error } + error: + on_events: + create: { transition_to: creating } + destroy: { transition_to: destroying } + + # Provisioning lifecycle (independent of the provider region). + - initial: { transition_to: unprovisioned } + states: + unprovisioned: + on_events: + provision: { transition_to: provisioning } + provisioning: + on_events: + provision_ok: { transition_to: provisioned } + provision_fail: { transition_to: provision_error } + provisioned: + on_events: + provision: { transition_to: provisioning } # re-converge + provision_error: + on_events: + provision: { transition_to: provisioning } # retry diff --git a/core/fsm/package.yaml b/core/fsm/package.yaml new file mode 100644 index 00000000..9a29305f --- /dev/null +++ b/core/fsm/package.yaml @@ -0,0 +1,38 @@ +# The package lifecycle, shared by every package (v4.5 N/O) — the declarative +# replacement for state.py's PACKAGE_STATES. Transitions are driven by eve operations +# (install / remove) and by observation (found / absent maps a live probe back to a +# state, keeping the machine honest about drift). +id: package +events: + found: {} # observe: the package is present on the instance + absent: {} # observe: not present + install: {} + install_ok: {} + install_fail: {} + remove: {} +top: + initial: { transition_to: unknown } + states: + unknown: + on_events: + found: { transition_to: installed } + absent: { transition_to: missing } + install: { transition_to: installing } + missing: + on_events: + install: { transition_to: installing } + found: { transition_to: installed } # drift: appeared out of band + installing: + on_events: + install_ok: { transition_to: installed } + install_fail: { transition_to: failed } + installed: + on_events: + remove: { transition_to: removed } + absent: { transition_to: missing } # drift: disappeared + failed: + on_events: + install: { transition_to: installing } # retry + removed: + on_events: + install: { transition_to: installing } diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index 2949b470..2f0869a7 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -238,8 +238,16 @@ The engine is real now, so this supersedes the speculative `state_machine:` sche it reads `configured=yes / reachable=no` (readiness surfaces as reachability + note). If AWS should read `configured=no` until eve holds an auth profile, mark an auth setting `required` or model AWS's configured as probe-derived — a per-provider call. -- ☐ **Instance/package machines** — same pattern; retire `state.py` enums + - `state_machine.py` into the platform. +- ☑ **Instance & package machines (declarative)** — `core/fsm/instance.yaml` (provider ‖ + provision as orthogonal regions; `desired` as context) + `core/fsm/package.yaml`, both + passing the engine's static validation and exercised in `tests/test_fsm_machines.py`. +- ☐ **Retire `state.py`/`state_machine.py` (live)** — wire the machines into persistence + and `instance_view`/status rendering. **Blocked on a vocabulary decision:** the machines + use a cleaner set (provider: creating/starting/stopping/destroying/error; package: + installing) than the current enums (`PROVIDER_STATES` keeps terraform micro-stages + initializing/initialized/planned/changing; `PACKAGE_STATES` has reinstalled). Either + **match the existing enums** for a drop-in (keep the messy stages), or **adopt the + cleaner vocabulary + migrate** persisted state files. A per-project call. ### Open decisions — resolved 1. Engine repo/name → **harel** (`fruwehq/harel` spec, `fruwehq/harel-python` impl). diff --git a/tests/test_fsm_machines.py b/tests/test_fsm_machines.py new file mode 100644 index 00000000..6f91c47e --- /dev/null +++ b/tests/test_fsm_machines.py @@ -0,0 +1,80 @@ +"""The core instance and package machines (core/fsm/*.yaml) over the Determa State engine. + +Loading each machine runs it through the engine's static validation (reachable states, +no dead branches); the tests then exercise the key lifecycle transitions that replace +state.py's PROVIDER_STATES / PROVISION_STATES / PACKAGE_STATES. +""" + +from __future__ import annotations + +from pathlib import Path + +from eve_sdk.fsm import EveFsm + +CORE = Path(__file__).resolve().parents[1] / "core" / "fsm" + + +def _fsm(scope: str, external: dict[str, str] | None = None) -> EveFsm: + ext = dict(external or {}) + fsm = EveFsm(lambda s, k, names: {n: ext.get(n, "") for n in names}) + fsm.load(scope, (CORE / f"{scope}.yaml").read_text()) + return fsm + + +def _leaves(fsm: EveFsm, scope: str, key: str = "x") -> list[str]: + return sorted(fsm.instance(scope, key).active_leaf_names()) + + +def test_package_lifecycle() -> None: + fsm = _fsm("package") + assert _leaves(fsm, "package") == ["unknown"] + + fsm.fire("package", "x", "absent") + assert _leaves(fsm, "package") == ["missing"] + + fsm.fire("package", "x", "install") + assert _leaves(fsm, "package") == ["installing"] + + fsm.fire("package", "x", "install_fail") + assert _leaves(fsm, "package") == ["failed"] + + fsm.fire("package", "x", "install") # retry + fsm.fire("package", "x", "install_ok") + assert _leaves(fsm, "package") == ["installed"] + + fsm.fire("package", "x", "absent") # drift: disappeared out of band + assert _leaves(fsm, "package") == ["missing"] + + +def test_instance_regions_advance_independently() -> None: + fsm = _fsm("instance", {"desired": "running"}) + assert _leaves(fsm, "instance") == ["absent", "unprovisioned"] + + fsm.fire("instance", "x", "create") + fsm.fire("instance", "x", "created_ok") + assert _leaves(fsm, "instance") == ["running", "unprovisioned"] + + # the provisioning region advances without touching the provider region + fsm.fire("instance", "x", "provision") + fsm.fire("instance", "x", "provision_ok") + assert _leaves(fsm, "instance") == ["provisioned", "running"] + + # stop lands in stopped; destroy lands in absent (distinct outcomes) + fsm.fire("instance", "x", "stop") + fsm.fire("instance", "x", "stopped_ok") + assert _leaves(fsm, "instance") == ["provisioned", "stopped"] + + fsm.fire("instance", "x", "destroy") + fsm.fire("instance", "x", "destroyed_ok") + assert _leaves(fsm, "instance") == ["absent", "provisioned"] + assert fsm.context("instance", "x")["desired"] == "running" + + +def test_instance_op_failure_goes_to_error() -> None: + fsm = _fsm("instance") + fsm.fire("instance", "x", "create") + fsm.fire("instance", "x", "op_fail") + assert _leaves(fsm, "instance") == ["error", "unprovisioned"] + fsm.fire("instance", "x", "create") # recover + fsm.fire("instance", "x", "created_ok") + assert _leaves(fsm, "instance") == ["running", "unprovisioned"] From e52f7f3c02a33f0d10b958ae8581fcf876a45dea Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 6 Jul 2026 12:24:41 +0900 Subject: [PATCH 24/32] feat(v4.5 N): align instance machine + persisted-state migration - instance machine: add provider region 'unknown' initial (unobserved) + drift observe_* edges from unknown/absent/error, so the old 'unknown' maps cleanly and live status can reconcile drift. - eve_sdk/fsm_migrate.py: translate persisted state files (pre-FSM vocabulary -> core machine vocabulary) on read; stable values are identity, terraform micro-stages -> unknown, provision error -> provision_error, package reinstalled -> installed. Idempotent + tested. Chosen path: adopt the cleaner vocabulary + migrate. Remaining live wiring (apply the migration on State.read, switch enum sets, move state_machine.py derivations onto the machines) is the invasive integration touching instance_view; tracked in the design doc. --- core/fsm/instance.yaml | 20 ++++++++--- docs/v4.5-fsm-design.md | 18 ++++++---- eve_sdk/fsm_migrate.py | 69 ++++++++++++++++++++++++++++++++++++++ tests/test_fsm_machines.py | 3 +- tests/test_fsm_migrate.py | 42 +++++++++++++++++++++++ 5 files changed, 140 insertions(+), 12 deletions(-) create mode 100644 eve_sdk/fsm_migrate.py create mode 100644 tests/test_fsm_migrate.py diff --git a/core/fsm/instance.yaml b/core/fsm/instance.yaml index ca1a2c61..1e06ddcc 100644 --- a/core/fsm/instance.yaml +++ b/core/fsm/instance.yaml @@ -29,11 +29,20 @@ top: # Provider (terraform) lifecycle. Each in-flight operation has its own transient # state so its outcome is unambiguous; op_fail from any of them lands in error. - - initial: { transition_to: absent } + # `unknown` is the unobserved start; observe_* reconcile from live provider status. + - initial: { transition_to: unknown } states: + unknown: + on_events: + create: { transition_to: creating } + observe_running: { transition_to: running } + observe_stopped: { transition_to: stopped } + observe_absent: { transition_to: absent } absent: on_events: - create: { transition_to: creating } + create: { transition_to: creating } + observe_running: { transition_to: running } # drift: created out of band + observe_stopped: { transition_to: stopped } creating: on_events: created_ok: { transition_to: running } @@ -64,8 +73,11 @@ top: op_fail: { transition_to: error } error: on_events: - create: { transition_to: creating } - destroy: { transition_to: destroying } + create: { transition_to: creating } + destroy: { transition_to: destroying } + observe_running: { transition_to: running } + observe_stopped: { transition_to: stopped } + observe_absent: { transition_to: absent } # Provisioning lifecycle (independent of the provider region). - initial: { transition_to: unprovisioned } diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index 2f0869a7..5b06857f 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -241,13 +241,17 @@ The engine is real now, so this supersedes the speculative `state_machine:` sche - ☑ **Instance & package machines (declarative)** — `core/fsm/instance.yaml` (provider ‖ provision as orthogonal regions; `desired` as context) + `core/fsm/package.yaml`, both passing the engine's static validation and exercised in `tests/test_fsm_machines.py`. -- ☐ **Retire `state.py`/`state_machine.py` (live)** — wire the machines into persistence - and `instance_view`/status rendering. **Blocked on a vocabulary decision:** the machines - use a cleaner set (provider: creating/starting/stopping/destroying/error; package: - installing) than the current enums (`PROVIDER_STATES` keeps terraform micro-stages - initializing/initialized/planned/changing; `PACKAGE_STATES` has reinstalled). Either - **match the existing enums** for a drop-in (keep the messy stages), or **adopt the - cleaner vocabulary + migrate** persisted state files. A per-project call. +- ◐ **Retire `state.py`/`state_machine.py` (live)** — decision: **adopt the cleaner + vocabulary + migrate** (persisted files translated on read). + - ☑ Machine vocabulary aligned (provider `unknown` initial + drift `observe_*` edges). + - ☑ `eve_sdk/fsm_migrate.py` — old→new state translation (stable values identity; + terraform micro-stages → `unknown`; provision `error`→`provision_error`; package + `reinstalled`→`installed`), idempotent, tested (`tests/test_fsm_migrate.py`). + - ☐ **Live wiring (invasive):** apply `migrate_state_doc` on `State.read`; switch the + enum sets to the machine vocabulary; move `state_machine.py`'s derivations + (`effective_provider_state`, `status_with_observed_state`, `provider_actions_available`) + onto the machines. Touches `instance_view`/status rendering — do incrementally with + the full suite green, ideally validated against a real instance. ### Open decisions — resolved 1. Engine repo/name → **harel** (`fruwehq/harel` spec, `fruwehq/harel-python` impl). diff --git a/eve_sdk/fsm_migrate.py b/eve_sdk/fsm_migrate.py new file mode 100644 index 00000000..773eb4f5 --- /dev/null +++ b/eve_sdk/fsm_migrate.py @@ -0,0 +1,69 @@ +"""Migrate persisted eve state files from the pre-FSM vocabulary to the core machine +vocabulary (v4.5 N; ``core/fsm/*.yaml``). + +The state graphs moved into Determa State machines with a cleaner state set, so existing +state files must be translated. This is applied on read (idempotent) so upgrades are +transparent; unknown values pass through unchanged. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +# Provider region (core/fsm/instance.yaml). Stable values map identically. The old +# terraform micro-stages (initializing/initialized/planned/changing) are in-flight and +# never stably persisted; if one is found, map it to `unknown` so the next observation +# reconciles reality rather than freezing a mid-operation phase. +PROVIDER_STATE_MIGRATION: dict[str, str] = { + "unknown": "unknown", + "running": "running", + "stopped": "stopped", + "absent": "absent", + "error": "error", + "initializing": "unknown", + "initialized": "unknown", + "planned": "unknown", + "changing": "unknown", +} + +# Provisioning region (core/fsm/instance.yaml). +PROVISION_STATE_MIGRATION: dict[str, str] = { + "unknown": "unprovisioned", + "provisioning": "provisioning", + "provisioned": "provisioned", + "error": "provision_error", +} + +# Package machine (core/fsm/package.yaml). `reinstalled` collapses to installed. +PACKAGE_STATE_MIGRATION: dict[str, str] = { + "unknown": "unknown", + "installed": "installed", + "missing": "missing", + "failed": "failed", + "removed": "removed", + "reinstalled": "installed", + "installing": "installing", +} + + +def migrate_state_doc(doc: Mapping[str, Any]) -> dict[str, Any]: + """Return a copy of a persisted state doc with provider/provision/package state + strings mapped to the core machine vocabulary. Idempotent; unrecognized values and + all other fields (e.g. ``desired_state``) pass through unchanged.""" + out = dict(doc) + if "provider_state" in out: + out["provider_state"] = PROVIDER_STATE_MIGRATION.get(out["provider_state"], out["provider_state"]) + if "provision_state" in out: + out["provision_state"] = PROVISION_STATE_MIGRATION.get(out["provision_state"], out["provision_state"]) + packages = out.get("package_state") + if isinstance(packages, Mapping): + migrated: dict[str, Any] = {} + for pkg_id, entry in packages.items(): + if isinstance(entry, Mapping) and "status" in entry: + status = entry["status"] + migrated[pkg_id] = dict(entry) | {"status": PACKAGE_STATE_MIGRATION.get(status, status)} + else: + migrated[pkg_id] = entry + out["package_state"] = migrated + return out diff --git a/tests/test_fsm_machines.py b/tests/test_fsm_machines.py index 6f91c47e..4f6b9f48 100644 --- a/tests/test_fsm_machines.py +++ b/tests/test_fsm_machines.py @@ -48,7 +48,7 @@ def test_package_lifecycle() -> None: def test_instance_regions_advance_independently() -> None: fsm = _fsm("instance", {"desired": "running"}) - assert _leaves(fsm, "instance") == ["absent", "unprovisioned"] + assert _leaves(fsm, "instance") == ["unknown", "unprovisioned"] fsm.fire("instance", "x", "create") fsm.fire("instance", "x", "created_ok") @@ -72,6 +72,7 @@ def test_instance_regions_advance_independently() -> None: def test_instance_op_failure_goes_to_error() -> None: fsm = _fsm("instance") + assert _leaves(fsm, "instance") == ["unknown", "unprovisioned"] fsm.fire("instance", "x", "create") fsm.fire("instance", "x", "op_fail") assert _leaves(fsm, "instance") == ["error", "unprovisioned"] diff --git a/tests/test_fsm_migrate.py b/tests/test_fsm_migrate.py new file mode 100644 index 00000000..5e888533 --- /dev/null +++ b/tests/test_fsm_migrate.py @@ -0,0 +1,42 @@ +"""Persisted-state vocabulary migration (eve_sdk.fsm_migrate).""" + +from __future__ import annotations + +from eve_sdk.fsm_migrate import migrate_state_doc + + +def test_stable_values_are_identity() -> None: + doc = { + "provider_state": "running", + "provision_state": "provisioned", + "package_state": {"pkg": {"status": "installed", "at": "t"}}, + } + out = migrate_state_doc(doc) + assert out["provider_state"] == "running" + assert out["provision_state"] == "provisioned" + assert out["package_state"]["pkg"] == {"status": "installed", "at": "t"} + + +def test_renamed_values() -> None: + out = migrate_state_doc( + { + "provider_state": "changing", # terraform micro-stage -> re-observe + "provision_state": "unknown", + "package_state": {"pkg": {"status": "reinstalled"}}, + } + ) + assert out["provider_state"] == "unknown" + assert out["provision_state"] == "unprovisioned" + assert out["package_state"]["pkg"]["status"] == "installed" + + +def test_error_maps_per_region() -> None: + assert migrate_state_doc({"provider_state": "error"})["provider_state"] == "error" + assert migrate_state_doc({"provision_state": "error"})["provision_state"] == "provision_error" + + +def test_other_fields_and_idempotence() -> None: + doc = {"provider_state": "changing", "desired_state": "running", "name": "vm1"} + once = migrate_state_doc(doc) + assert once["desired_state"] == "running" and once["name"] == "vm1" + assert migrate_state_doc(once) == once # idempotent From 317902f058fb0d02e72a420a7852f9cf887ed99b Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 6 Jul 2026 17:45:03 +0900 Subject: [PATCH 25/32] chore(v4.5 N): clean-slate state; retire fsm_migrate (delete-all over migration) No important instances, so per the delete-all decision: destroyed + deleted both demo instances, wiped their per-instance state, and removed eve_sdk/fsm_migrate.py + test (a runtime migration is no longer needed). Records the delete-all path + the remaining vocabulary alignment / FSM-driven status steps in the design doc. --- docs/v4.5-fsm-design.md | 20 +++++++----- eve_sdk/fsm_migrate.py | 69 --------------------------------------- tests/test_fsm_migrate.py | 42 ------------------------ 3 files changed, 11 insertions(+), 120 deletions(-) delete mode 100644 eve_sdk/fsm_migrate.py delete mode 100644 tests/test_fsm_migrate.py diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index 5b06857f..d22aa7e3 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -242,16 +242,18 @@ The engine is real now, so this supersedes the speculative `state_machine:` sche provision as orthogonal regions; `desired` as context) + `core/fsm/package.yaml`, both passing the engine's static validation and exercised in `tests/test_fsm_machines.py`. - ◐ **Retire `state.py`/`state_machine.py` (live)** — decision: **adopt the cleaner - vocabulary + migrate** (persisted files translated on read). + vocabulary**; instead of a runtime migration, **delete all persisted state once** (no + important instances) and start fresh on the machine vocabulary. - ☑ Machine vocabulary aligned (provider `unknown` initial + drift `observe_*` edges). - - ☑ `eve_sdk/fsm_migrate.py` — old→new state translation (stable values identity; - terraform micro-stages → `unknown`; provision `error`→`provision_error`; package - `reinstalled`→`installed`), idempotent, tested (`tests/test_fsm_migrate.py`). - - ☐ **Live wiring (invasive):** apply `migrate_state_doc` on `State.read`; switch the - enum sets to the machine vocabulary; move `state_machine.py`'s derivations - (`effective_provider_state`, `status_with_observed_state`, `provider_actions_available`) - onto the machines. Touches `instance_view`/status rendering — do incrementally with - the full suite green, ideally validated against a real instance. + - ☑ Clean slate: existing instances destroyed + deleted, per-instance state wiped, and + `eve_sdk/fsm_migrate.py` retired (no migration needed). + - ☐ **Vocabulary alignment:** `dispatch.provider_state_for` and the package + state map write the machine vocabulary (`creating/starting/stopping/destroying`, + package `reinstall`→`installed`); `state.py`'s state sets follow the machines; the + `state_machine`/`sdk/test_state*` tests update accordingly. + - ☐ **FSM-driven status:** dispatch fires FSM events and status reads FSM state, + folding `state_machine.py`'s derivations onto the machines. Touches `instance_view`; + do incrementally with the full suite green, validated against a real instance. ### Open decisions — resolved 1. Engine repo/name → **harel** (`fruwehq/harel` spec, `fruwehq/harel-python` impl). diff --git a/eve_sdk/fsm_migrate.py b/eve_sdk/fsm_migrate.py deleted file mode 100644 index 773eb4f5..00000000 --- a/eve_sdk/fsm_migrate.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Migrate persisted eve state files from the pre-FSM vocabulary to the core machine -vocabulary (v4.5 N; ``core/fsm/*.yaml``). - -The state graphs moved into Determa State machines with a cleaner state set, so existing -state files must be translated. This is applied on read (idempotent) so upgrades are -transparent; unknown values pass through unchanged. -""" - -from __future__ import annotations - -from collections.abc import Mapping -from typing import Any - -# Provider region (core/fsm/instance.yaml). Stable values map identically. The old -# terraform micro-stages (initializing/initialized/planned/changing) are in-flight and -# never stably persisted; if one is found, map it to `unknown` so the next observation -# reconciles reality rather than freezing a mid-operation phase. -PROVIDER_STATE_MIGRATION: dict[str, str] = { - "unknown": "unknown", - "running": "running", - "stopped": "stopped", - "absent": "absent", - "error": "error", - "initializing": "unknown", - "initialized": "unknown", - "planned": "unknown", - "changing": "unknown", -} - -# Provisioning region (core/fsm/instance.yaml). -PROVISION_STATE_MIGRATION: dict[str, str] = { - "unknown": "unprovisioned", - "provisioning": "provisioning", - "provisioned": "provisioned", - "error": "provision_error", -} - -# Package machine (core/fsm/package.yaml). `reinstalled` collapses to installed. -PACKAGE_STATE_MIGRATION: dict[str, str] = { - "unknown": "unknown", - "installed": "installed", - "missing": "missing", - "failed": "failed", - "removed": "removed", - "reinstalled": "installed", - "installing": "installing", -} - - -def migrate_state_doc(doc: Mapping[str, Any]) -> dict[str, Any]: - """Return a copy of a persisted state doc with provider/provision/package state - strings mapped to the core machine vocabulary. Idempotent; unrecognized values and - all other fields (e.g. ``desired_state``) pass through unchanged.""" - out = dict(doc) - if "provider_state" in out: - out["provider_state"] = PROVIDER_STATE_MIGRATION.get(out["provider_state"], out["provider_state"]) - if "provision_state" in out: - out["provision_state"] = PROVISION_STATE_MIGRATION.get(out["provision_state"], out["provision_state"]) - packages = out.get("package_state") - if isinstance(packages, Mapping): - migrated: dict[str, Any] = {} - for pkg_id, entry in packages.items(): - if isinstance(entry, Mapping) and "status" in entry: - status = entry["status"] - migrated[pkg_id] = dict(entry) | {"status": PACKAGE_STATE_MIGRATION.get(status, status)} - else: - migrated[pkg_id] = entry - out["package_state"] = migrated - return out diff --git a/tests/test_fsm_migrate.py b/tests/test_fsm_migrate.py deleted file mode 100644 index 5e888533..00000000 --- a/tests/test_fsm_migrate.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Persisted-state vocabulary migration (eve_sdk.fsm_migrate).""" - -from __future__ import annotations - -from eve_sdk.fsm_migrate import migrate_state_doc - - -def test_stable_values_are_identity() -> None: - doc = { - "provider_state": "running", - "provision_state": "provisioned", - "package_state": {"pkg": {"status": "installed", "at": "t"}}, - } - out = migrate_state_doc(doc) - assert out["provider_state"] == "running" - assert out["provision_state"] == "provisioned" - assert out["package_state"]["pkg"] == {"status": "installed", "at": "t"} - - -def test_renamed_values() -> None: - out = migrate_state_doc( - { - "provider_state": "changing", # terraform micro-stage -> re-observe - "provision_state": "unknown", - "package_state": {"pkg": {"status": "reinstalled"}}, - } - ) - assert out["provider_state"] == "unknown" - assert out["provision_state"] == "unprovisioned" - assert out["package_state"]["pkg"]["status"] == "installed" - - -def test_error_maps_per_region() -> None: - assert migrate_state_doc({"provider_state": "error"})["provider_state"] == "error" - assert migrate_state_doc({"provision_state": "error"})["provision_state"] == "provision_error" - - -def test_other_fields_and_idempotence() -> None: - doc = {"provider_state": "changing", "desired_state": "running", "name": "vm1"} - once = migrate_state_doc(doc) - assert once["desired_state"] == "running" and once["name"] == "vm1" - assert migrate_state_doc(once) == once # idempotent From cd7f8160b4a796924bb5603799decce041d65e68 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 6 Jul 2026 17:54:49 +0900 Subject: [PATCH 26/32] refactor(v4.5 N): align dispatch + state vocabulary to the core machines Dispatch now records the instance machine's vocabulary: provider_state_for maps in-flight ops to creating/starting/stopping/destroying (init/plan -> creating), and package reinstall -> installed. state.py's PROVIDER/PACKAGE state sets, the observed-state + command-io schemas, the state_machine/sdk/test_state* tests, the valid-pending fixture, and test-plugins all follow the core/fsm machines. Old terraform micro-stages (initializing/initialized/planned/changing) and package reinstalled are gone. Full suite green (496 passed, 1 skipped). --- core/schema/command-io.schema.json | 2 +- core/schema/observed-state.schema.json | 4 ++-- docs/v4.5-fsm-design.md | 9 +++++---- eve_sdk/dispatch.py | 9 +++++---- eve_sdk/package_dispatch.py | 2 +- eve_sdk/state.py | 13 ++++++++----- scripts/test-plugins | 2 +- tests/fixtures/observed-state/valid-pending.json | 2 +- tests/python/sdk/test_state.py | 2 +- tests/python/sdk/test_state_machine_python.py | 2 +- tests/python/test_state_machine.py | 2 +- 11 files changed, 27 insertions(+), 22 deletions(-) diff --git a/core/schema/command-io.schema.json b/core/schema/command-io.schema.json index 88d5a8f5..51c6688d 100644 --- a/core/schema/command-io.schema.json +++ b/core/schema/command-io.schema.json @@ -54,7 +54,7 @@ "provider_state_mapping": { "title": "Provider State Mapping", - "description": "Maps (command, dispatch_status) to the provider_state recorded by the dispatcher. On success: init -> initialized, plan -> planned, up/start -> running, stop -> stopped, down -> absent. On running: -> changing. On failure: -> error (except resolve/status/ip/ssh which set null).", + "description": "Maps (command, dispatch_status) to the provider_state recorded by the dispatcher (the core/fsm/instance.yaml vocabulary). On success: init/plan -> creating, up/start -> running, stop -> stopped, down -> absent. On running (in flight): up -> creating, start -> starting, stop -> stopping, down -> destroying. On failure: -> error (except resolve/status/ip/ssh which set null).", "type": "object" } } diff --git a/core/schema/observed-state.schema.json b/core/schema/observed-state.schema.json index 4081ef97..d10b1fa6 100644 --- a/core/schema/observed-state.schema.json +++ b/core/schema/observed-state.schema.json @@ -31,7 +31,7 @@ "provider_state": { "description": "Last known provider resource state.", "type": "string", - "enum": ["unknown", "initializing", "initialized", "planned", "changing", "running", "stopped", "absent", "error"] + "enum": ["unknown", "creating", "starting", "stopping", "destroying", "running", "stopped", "absent", "error"] }, "provision_state": { @@ -131,7 +131,7 @@ "properties": { "status": { "type": "string", - "enum": ["unknown", "installed", "missing", "failed", "removed", "reinstalled"] + "enum": ["unknown", "installing", "installed", "missing", "failed", "removed"] }, "updated_at": { "type": "string", diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index d22aa7e3..29e33065 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -247,10 +247,11 @@ The engine is real now, so this supersedes the speculative `state_machine:` sche - ☑ Machine vocabulary aligned (provider `unknown` initial + drift `observe_*` edges). - ☑ Clean slate: existing instances destroyed + deleted, per-instance state wiped, and `eve_sdk/fsm_migrate.py` retired (no migration needed). - - ☐ **Vocabulary alignment:** `dispatch.provider_state_for` and the package - state map write the machine vocabulary (`creating/starting/stopping/destroying`, - package `reinstall`→`installed`); `state.py`'s state sets follow the machines; the - `state_machine`/`sdk/test_state*` tests update accordingly. + - ☑ **Vocabulary alignment:** `dispatch.provider_state_for` and the package state map + write the machine vocabulary (`creating/starting/stopping/destroying`, `init`/`plan`→ + `creating`, package `reinstall`→`installed`); `state.py`'s state sets, the + `observed-state`/`command-io` schemas, the `state_machine`/`sdk/test_state*` tests, the + `valid-pending` fixture, and `test-plugins` all follow. Full suite green (496 passed). - ☐ **FSM-driven status:** dispatch fires FSM events and status reads FSM state, folding `state_machine.py`'s derivations onto the machines. Touches `instance_view`; do incrementally with the full suite green, validated against a real instance. diff --git a/eve_sdk/dispatch.py b/eve_sdk/dispatch.py index a85ac2a6..7b05d1bc 100644 --- a/eve_sdk/dispatch.py +++ b/eve_sdk/dispatch.py @@ -184,15 +184,16 @@ def desired_state_for(command: str | None) -> str | None: def provider_state_for(command: str | None, status: str) -> str | None: command = command or "" - if status == "running" and command in {"up", "down", "start", "stop"}: - return "changing" + if status == "running": + # operation in flight -> the instance machine's transient state (core/fsm/instance.yaml) + return {"up": "creating", "start": "starting", "stop": "stopping", "down": "destroying"}.get(command) if status == "failed": return None if command in {"resolve", "status", "ip", "ssh"} else "error" if status != "succeeded": return None return { - "init": "initialized", - "plan": "planned", + "init": "creating", # terraform sub-stages of an up -> still creating + "plan": "creating", "up": "running", "start": "running", "stop": "stopped", diff --git a/eve_sdk/package_dispatch.py b/eve_sdk/package_dispatch.py index 50606902..dc0606b3 100644 --- a/eve_sdk/package_dispatch.py +++ b/eve_sdk/package_dispatch.py @@ -319,7 +319,7 @@ def dispatch_package( package_state = { "install": "installed", "down": "removed", - "reinstall": "reinstalled", + "reinstall": "installed", # cleaner vocabulary: a reinstall lands in installed }.get(command) if package_state is None: package_state = package_status_from_output(output) or "unknown" diff --git a/eve_sdk/state.py b/eve_sdk/state.py index f2c61e06..4d835d8b 100644 --- a/eve_sdk/state.py +++ b/eve_sdk/state.py @@ -22,19 +22,22 @@ class StateError(Exception): class State: OPERATION_STATUSES: ClassVar[set[str]] = {"running", "succeeded", "failed", "skipped"} DESIRED_STATES: ClassVar[set[str]] = {"unknown", "running", "stopped", "absent"} + # Provider + package state sets follow the core machines (core/fsm/instance.yaml, + # core/fsm/package.yaml): the terraform micro-stages collapse into the transient + # operation states, and package `reinstalled` folds into `installed`. PROVIDER_STATES: ClassVar[set[str]] = { "unknown", - "initializing", - "initialized", - "planned", - "changing", + "creating", + "starting", + "stopping", + "destroying", "running", "stopped", "absent", "error", } PROVISION_STATES: ClassVar[set[str]] = {"unknown", "provisioning", "provisioned", "error"} - PACKAGE_STATES: ClassVar[set[str]] = {"unknown", "installed", "missing", "failed", "removed", "reinstalled"} + PACKAGE_STATES: ClassVar[set[str]] = {"unknown", "installing", "installed", "missing", "failed", "removed"} DEFAULT_HISTORY_LIMIT = 50 @classmethod diff --git a/scripts/test-plugins b/scripts/test-plugins index 9e755080..c949466d 100755 --- a/scripts/test-plugins +++ b/scripts/test-plugins @@ -179,7 +179,7 @@ def main(): sj is not None and sj["last_operation"]["name"] == "package.down") # recover - _run("scripts/instance-state", "--instance", "recover-a", "--operation", "provider.up", "--status", "running", "--provider-state", "changing", env=SE) + _run("scripts/instance-state", "--instance", "recover-a", "--operation", "provider.up", "--status", "running", "--provider-state", "creating", env=SE) rec = _run_json("scripts/instance-state", "--instance", "recover-a", "--recover-running", env=SE) _check("instance state recovers interrupted running operation", rec is not None and rec["last_operation"]["status"] == "failed" and rec.get("provider_state") == "error") diff --git a/tests/fixtures/observed-state/valid-pending.json b/tests/fixtures/observed-state/valid-pending.json index 6d1cbbe1..c61bf544 100644 --- a/tests/fixtures/observed-state/valid-pending.json +++ b/tests/fixtures/observed-state/valid-pending.json @@ -1,7 +1,7 @@ { "instance": "aws-gpu-a", "desired_state": "running", - "provider_state": "changing", + "provider_state": "creating", "provision_state": "unknown", "package_state": {}, "observed_state": { diff --git a/tests/python/sdk/test_state.py b/tests/python/sdk/test_state.py index a316dc45..9b48c4af 100644 --- a/tests/python/sdk/test_state.py +++ b/tests/python/sdk/test_state.py @@ -31,7 +31,7 @@ def test_state_read_defaults(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> def test_state_record_operation_and_recover(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("EVE_STATE_DIR", str(tmp_path)) - State.record_operation("demo", "provider.up", "running", desired_state="running", provider_state="changing") + State.record_operation("demo", "provider.up", "running", desired_state="running", provider_state="creating") recovered = State.recover_running("demo") assert recovered["provider_state"] == "error" diff --git a/tests/python/sdk/test_state_machine_python.py b/tests/python/sdk/test_state_machine_python.py index f59e0479..f50fd555 100644 --- a/tests/python/sdk/test_state_machine_python.py +++ b/tests/python/sdk/test_state_machine_python.py @@ -24,6 +24,6 @@ def test_state_machine_summary() -> None: "a": {"state": {"provider_state": "running"}}, "b": {"state": {"provider_state": "stopped"}}, "c": {"state": {"provider_state": "error"}}, - "d": {"state": {"provider_state": "changing"}}, + "d": {"state": {"provider_state": "creating"}}, } ) == {"running": 1, "stopped": 1, "failed": 1, "other": 1} diff --git a/tests/python/test_state_machine.py b/tests/python/test_state_machine.py index 30ad3d13..32a40aff 100644 --- a/tests/python/test_state_machine.py +++ b/tests/python/test_state_machine.py @@ -112,7 +112,7 @@ def test_aggregate_summary() -> None: "c": {"state": {"provider_state": "absent"}}, "d": {"state": {"provider_state": "error"}}, "e": {"state": {"provider_state": "running", "last_error": "boom"}}, - "f": {"state": {"provider_state": "changing"}}, + "f": {"state": {"provider_state": "creating"}}, "g": {"state": "not a dict"}, } ) == {"running": 1, "stopped": 2, "failed": 2, "other": 2} From 5c1f1bfca1f5f46bd583ce70667ad5d9581bbff6 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 6 Jul 2026 23:20:50 +0900 Subject: [PATCH 27/32] =?UTF-8?q?feat(v4.5=20N):=20FSM-driven=20provider?= =?UTF-8?q?=5Fstate=20=E2=80=94=20the=20instance=20machine=20is=20the=20wr?= =?UTF-8?q?ite-path=20authority?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit provider_state is no longer a hand-maintained lookup table. eve_sdk/instance_fsm.py is the single lifecycle authority: dispatch maps a (command, op_status) to a machine event (dispatch_event) and State.record_operation(provider_event=...) fires it into core/fsm/instance.yaml seeded from the persisted provider_state, atomically inside the state write; the resulting leaf is the new provider_state. The state a status read sees is exactly what the machine produced — the table and the machine cannot diverge because there is only the machine. - delete dispatch.provider_state_for; provider_command.py + scripts/provider-dispatch fire events via dispatch_event - state.py: provider_event path computes the next leaf inside the modify callback (race-free); the direct provider_state param stays for instance-state / recovery - instance.yaml: every concrete provider state accepts every operator command (routing to its transient), so a command's terminal outcome holds from any state (start recovers absent->running). The provider command, not the machine, decides feasibility: a start that can't apply fails -> error, never a silent no-op. Preserves the old table's terminal semantics exactly, now with honest transients. - fsm.py: bind engine as a defined name so eve_sdk.fsm.engine is an explicit export - test_state.py: full lifecycle + recovery + read-only cases through the machine Read-path derivations in state_machine.py (effective_provider_state, live-overlay precedence, aggregate_summary) are display/aggregation policy, not a second lifecycle definition — one transition authority now, no divergence left. --- core/fsm/instance.yaml | 46 +++++-------- docs/v4.5-fsm-design.md | 26 ++++++- eve_sdk/dispatch.py | 26 ++----- eve_sdk/fsm.py | 8 ++- eve_sdk/instance_fsm.py | 120 +++++++++++++++++++++++++++++++++ eve_sdk/provider_command.py | 10 +-- eve_sdk/state.py | 12 ++++ scripts/provider-dispatch | 5 +- tests/python/sdk/test_state.py | 42 ++++++++++++ 9 files changed, 236 insertions(+), 59 deletions(-) create mode 100644 eve_sdk/instance_fsm.py diff --git a/core/fsm/instance.yaml b/core/fsm/instance.yaml index 1e06ddcc..b9493d47 100644 --- a/core/fsm/instance.yaml +++ b/core/fsm/instance.yaml @@ -30,54 +30,44 @@ top: # Provider (terraform) lifecycle. Each in-flight operation has its own transient # state so its outcome is unambiguous; op_fail from any of them lands in error. # `unknown` is the unobserved start; observe_* reconcile from live provider status. + # + # Every concrete (non-transient) state accepts every operator command and routes + # to that command's transient state: the operator's intent is authoritative and the + # provider command's success/failure — not the machine — decides feasibility (a + # `start` that can't apply fails and lands in `error` via op_fail, it never silently + # no-ops). So a command's terminal outcome (up/start->running, stop->stopped, + # down->absent) holds from any state, exactly as before, now with honest transients. - initial: { transition_to: unknown } states: unknown: - on_events: + on_events: &commands create: { transition_to: creating } + start: { transition_to: starting } + stop: { transition_to: stopping } + destroy: { transition_to: destroying } observe_running: { transition_to: running } observe_stopped: { transition_to: stopped } observe_absent: { transition_to: absent } - absent: - on_events: - create: { transition_to: creating } - observe_running: { transition_to: running } # drift: created out of band - observe_stopped: { transition_to: stopped } + absent: { on_events: *commands } + running: { on_events: *commands } + stopped: { on_events: *commands } + error: { on_events: *commands } creating: on_events: created_ok: { transition_to: running } op_fail: { transition_to: error } - running: + starting: on_events: - stop: { transition_to: stopping } - destroy: { transition_to: destroying } - observe_stopped: { transition_to: stopped } - observe_absent: { transition_to: absent } + started_ok: { transition_to: running } + op_fail: { transition_to: error } stopping: on_events: stopped_ok: { transition_to: stopped } op_fail: { transition_to: error } - stopped: - on_events: - start: { transition_to: starting } - destroy: { transition_to: destroying } - observe_running: { transition_to: running } - observe_absent: { transition_to: absent } - starting: - on_events: - started_ok: { transition_to: running } - op_fail: { transition_to: error } destroying: on_events: destroyed_ok: { transition_to: absent } op_fail: { transition_to: error } - error: - on_events: - create: { transition_to: creating } - destroy: { transition_to: destroying } - observe_running: { transition_to: running } - observe_stopped: { transition_to: stopped } - observe_absent: { transition_to: absent } # Provisioning lifecycle (independent of the provider region). - initial: { transition_to: unprovisioned } diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index 29e33065..176e96cc 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -252,9 +252,29 @@ The engine is real now, so this supersedes the speculative `state_machine:` sche `creating`, package `reinstall`→`installed`); `state.py`'s state sets, the `observed-state`/`command-io` schemas, the `state_machine`/`sdk/test_state*` tests, the `valid-pending` fixture, and `test-plugins` all follow. Full suite green (496 passed). - - ☐ **FSM-driven status:** dispatch fires FSM events and status reads FSM state, - folding `state_machine.py`'s derivations onto the machines. Touches `instance_view`; - do incrementally with the full suite green, validated against a real instance. + - ☑ **FSM-driven status (write path):** `provider_state` is no longer a hand-maintained + lookup table. `eve_sdk/instance_fsm.py` is the single lifecycle authority: dispatch + maps a `(command, op_status)` to a machine event (`dispatch_event`) and + `State.record_operation(provider_event=…)` **fires it into `core/fsm/instance.yaml`** + seeded from the persisted `provider_state`, atomically inside the state write; the + resulting leaf *is* the new `provider_state`. So the state a status read sees is + exactly what the machine produced — the transition table and the machine cannot + diverge because there is only the machine. `provider_state_for` is deleted; + `provider_command.py` + `scripts/provider-dispatch` fire events. Live-validated on a + real local-qemu instance (create→running, stop→stopped, start→running, destroy→absent, + failure→error) + `scripts/test-lifecycle` green + a new `test_state.py` case. + - *Machine change:* every concrete provider state now accepts every operator command + (routing to its transient), so a command's terminal outcome holds from any state + (start recovers `absent`→`running`) — the provider command, not the machine, decides + feasibility (a `start` that can't apply fails → `error`, it never silently no-ops). + This preserves the old table's terminal semantics exactly, now with honest transients. + - **Remaining (read-path derivations):** `state_machine.py`'s `effective_provider_state` + (the error→running display recovery), `should_apply_live_provider_state` (live-overlay + precedence), and `aggregate_summary` (counting) are **display/aggregation policy, not a + second lifecycle definition** — there is now exactly one transition authority (the + machine), so no divergence risk remains. They stay as a thin status-render adapter. + (Provision region still uses the persisted `unknown`/`error` vocabulary; wiring that + region through the machine the same way is the next increment.) ### Open decisions — resolved 1. Engine repo/name → **harel** (`fruwehq/harel` spec, `fruwehq/harel-python` impl). diff --git a/eve_sdk/dispatch.py b/eve_sdk/dispatch.py index 7b05d1bc..5310adc1 100644 --- a/eve_sdk/dispatch.py +++ b/eve_sdk/dispatch.py @@ -144,7 +144,7 @@ def record_provider_state( status: str, error: str | None = None, desired_state: str | None = None, - provider_state: str | None = None, + provider_event: str | None = None, ) -> None: if os.environ.get("EVE_DISABLE_STATE") == "1": return @@ -154,7 +154,7 @@ def record_provider_state( status, error=error, desired_state=desired_state, - provider_state=provider_state, + provider_event=provider_event, ) @@ -182,23 +182,11 @@ def desired_state_for(command: str | None) -> str | None: return {"up": "running", "start": "running", "stop": "stopped", "down": "absent"}.get(command or "") -def provider_state_for(command: str | None, status: str) -> str | None: - command = command or "" - if status == "running": - # operation in flight -> the instance machine's transient state (core/fsm/instance.yaml) - return {"up": "creating", "start": "starting", "stop": "stopping", "down": "destroying"}.get(command) - if status == "failed": - return None if command in {"resolve", "status", "ip", "ssh"} else "error" - if status != "succeeded": - return None - return { - "init": "creating", # terraform sub-stages of an up -> still creating - "plan": "creating", - "up": "running", - "start": "running", - "stop": "stopped", - "down": "absent", - }.get(command) +# The provider-state transition is no longer a hand-maintained table: dispatch +# fires an event into the core instance machine (core/fsm/instance.yaml) seeded +# from the persisted state, and the resulting leaf is the new provider_state. +# ``instance_fsm.dispatch_event`` maps a dispatched (command, op_status) to that +# event; the firing happens atomically inside State.record_operation. def interactive_provider_command(plugin: dict[str, Any], command: str) -> bool: diff --git a/eve_sdk/fsm.py b/eve_sdk/fsm.py index b681c860..78c29b01 100644 --- a/eve_sdk/fsm.py +++ b/eve_sdk/fsm.py @@ -20,9 +20,13 @@ # The statechart engine (Determa State; formerly harel). Import it behind one name so # eve's only coupling to it is this module. try: # pragma: no cover - the branch taken depends only on which package is installed - import determa.state as engine + import determa.state as _engine_mod except ModuleNotFoundError: # pre-rename package name - import harel as engine + import harel as _engine_mod # type: ignore[import-untyped,no-redef] + +# Re-bound as a defined name so it is an explicit export (other eve modules read it +# as ``eve_sdk.fsm.engine``). +engine = _engine_mod # Resolve a machine's declared ``external`` esvs from eve settings/secrets: # (scope, key, external_names) -> {name: value} diff --git a/eve_sdk/instance_fsm.py b/eve_sdk/instance_fsm.py new file mode 100644 index 00000000..64e2e881 --- /dev/null +++ b/eve_sdk/instance_fsm.py @@ -0,0 +1,120 @@ +"""Instance-lifecycle transitions, driven by the core instance machine. + +The single authority for how a provider operation moves an instance's +``provider_state``: dispatch computes the next state by **firing an event into +``core/fsm/instance.yaml``** seeded from the persisted state, not from a second, +hand-maintained transition table. The machine YAML *is* the lifecycle; this +module is the thin, fast adapter eve calls (and the persisted ``provider_state`` +a status read sees is therefore exactly what the machine produced — bug N). + +Only the provider region is driven here; the provision region is seeded with a +constant placeholder so a provider event never depends on provision naming (the +persisted ``provision_state`` field keeps its own vocabulary until that region is +wired the same way). +""" + +from __future__ import annotations + +from typing import Any + +from eve_sdk import fsm as _fsm +from eve_sdk.workdir import Workdir + +# The statechart engine, reached through the one module that owns the coupling. +engine = _fsm.engine + +# Leaf sets, so we can pick the provider-region leaf out of ``active_leaf_names`` +# (whose order across orthogonal regions is not guaranteed). These mirror the +# regions declared in core/fsm/instance.yaml. +PROVIDER_LEAVES = { + "unknown", + "creating", + "starting", + "stopping", + "destroying", + "running", + "stopped", + "absent", + "error", +} +# The provision leaf we seed while firing provider events; any valid provision +# leaf works because provider events do not touch the provision region. +_SEED_PROVISION = "unprovisioned" + +_host: Any = None +_instance: Any = None + + +def _machine() -> tuple[Any, Any]: + """Lazily build a reusable host + root instance for the core instance machine.""" + global _host, _instance + if _instance is None: + machine_yaml = (Workdir.repo_root() / "core" / "fsm" / "instance.yaml").read_text(encoding="utf-8") + host = engine.Host() + host.register_all(engine.load_definitions(machine_yaml)) + instance = host.create_root(host.machines["instance"], "instance", external={"desired": "unknown"}) + host.run_to_quiescence() + _host, _instance = host, instance + return _host, _instance + + +def dispatch_event(command: str | None, op_status: str) -> str | None: + """Map a dispatched ``(command, op_status)`` to an instance-machine event. + + ``None`` means "leave ``provider_state`` unchanged" (read-only commands, and + the terraform sub-stages while an operation is still in flight). + """ + command = command or "" + if op_status == "running": + # Operation in flight -> the machine's transient state. + return {"up": "create", "start": "start", "stop": "stop", "down": "destroy"}.get(command) + if op_status == "failed": + return None if command in {"resolve", "status", "ip", "ssh"} else "op_fail" + if op_status != "succeeded": + return None + # Success -> the terminal event for the operation. init/plan are terraform + # sub-stages of an up: they signal creation underway (`create` is a no-op once + # already creating, so it only advances from unknown/absent/error). + return { + "up": "created_ok", + "start": "started_ok", + "stop": "stopped_ok", + "down": "destroyed_ok", + "init": "create", + "plan": "create", + }.get(command) + + +def provider_state_after(current: str, event: str | None) -> str | None: + """The provider_state after firing ``event`` from ``current``. + + Returns ``None`` when there is no event (no change to persist). If the machine + rejects the event from ``current`` (an operation that doesn't apply in that + state), the state is unchanged and ``current`` is returned. + """ + if event is None: + return None + provider = current if current in PROVIDER_LEAVES else "unknown" + host, instance = _machine() + instance.load_snapshot( + { + "def_id": "instance", + "def_version": 1, + "id": "instance", + "parent_id": None, + "status": "active", + "state_config": ["top", f"top.{provider}", f"top.{_SEED_PROVISION}"], + "esvs": {"top": {"desired": "unknown"}}, + "queue": [], + "deferred": [], + "timers": [], + "dead_letter": [], + "history": {}, + } + ) + host.deliver("instance", event, {}) + host.run_to_quiescence() + for leaf in instance.active_leaf_names(): + if leaf in PROVIDER_LEAVES: + return str(leaf) + return provider diff --git a/eve_sdk/provider_command.py b/eve_sdk/provider_command.py index b3701860..c2ee8fe4 100644 --- a/eve_sdk/provider_command.py +++ b/eve_sdk/provider_command.py @@ -14,12 +14,12 @@ exec_cmd, interactive_provider_command, prepare_overlay, - provider_state_for, read_resolved_from_env_or_stdin, record_provider_state, stream_command, validate_provider_output, ) +from eve_sdk.instance_fsm import dispatch_event from eve_sdk.plugin_manifest import PluginManifest from eve_sdk.resolve import resolve_instance from eve_sdk.schema import validate_input, validate_output @@ -252,7 +252,7 @@ def dispatch_instance_command( command, "running", desired_state=desired_state, - provider_state=provider_state_for(command, "running"), + provider_event=dispatch_event(command, "running"), ) exit_status, output = stream_command( @@ -268,7 +268,7 @@ def dispatch_instance_command( "failed", error=f"invalid provider output: {error}", desired_state=desired_state, - provider_state=provider_state_for(command, "failed"), + provider_event=dispatch_event(command, "failed"), ) print(f"provider-dispatch: {error}", file=sys.stderr) return 1 @@ -277,7 +277,7 @@ def dispatch_instance_command( command, "succeeded", desired_state=desired_state, - provider_state=provider_state_for(command, "succeeded"), + provider_event=dispatch_event(command, "succeeded"), ) else: try: @@ -287,7 +287,7 @@ def dispatch_instance_command( "failed", error=f"exit {exit_status}", desired_state=desired_state, - provider_state=provider_state_for(command, "failed"), + provider_event=dispatch_event(command, "failed"), ) except Exception as error: print(f"provider-dispatch: failed to record failure state: {error}", file=sys.stderr) diff --git a/eve_sdk/state.py b/eve_sdk/state.py index 4d835d8b..77e2135a 100644 --- a/eve_sdk/state.py +++ b/eve_sdk/state.py @@ -130,6 +130,7 @@ def record_operation( error: str | None = None, desired_state: str | None = None, provider_state: str | None = None, + provider_event: str | None = None, provision_state: str | None = None, package: str | None = None, package_state: str | None = None, @@ -168,6 +169,17 @@ def update(state: dict[str, Any]) -> dict[str, Any]: state["desired_state"] = desired_state if provider_state: state["provider_state"] = provider_state + elif provider_event: + # Fire the event into the core instance machine, seeded from the + # current persisted provider_state; the resulting leaf is the new + # provider_state (None -> read-only command, leave it unchanged). + from eve_sdk import instance_fsm + + new_provider = instance_fsm.provider_state_after( + str(state.get("provider_state", "unknown")), provider_event + ) + if new_provider is not None: + state["provider_state"] = new_provider if provision_state: state["provision_state"] = provision_state if package and package_state: diff --git a/scripts/provider-dispatch b/scripts/provider-dispatch index 32b97987..cbc2b190 100755 --- a/scripts/provider-dispatch +++ b/scripts/provider-dispatch @@ -16,7 +16,8 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from eve_sdk.dispatch import desired_state_for, provider_state_for, record_provider_state # noqa: E402 +from eve_sdk.dispatch import desired_state_for, record_provider_state # noqa: E402 +from eve_sdk.instance_fsm import dispatch_event # noqa: E402 from eve_sdk.provider_command import ( # noqa: E402 dispatch_instance_command, dispatch_provider_command, @@ -101,7 +102,7 @@ def main(argv: list[str]) -> int: "failed", error=str(error), desired_state=desired_state_for(command), - provider_state=provider_state_for(command, "failed"), + provider_event=dispatch_event(command, "failed"), ) except Exception as state_error: print(f"provider-dispatch: failed to record failure state: {state_error}", file=sys.stderr) diff --git a/tests/python/sdk/test_state.py b/tests/python/sdk/test_state.py index 9b48c4af..bfd264dc 100644 --- a/tests/python/sdk/test_state.py +++ b/tests/python/sdk/test_state.py @@ -44,6 +44,48 @@ def test_state_rejects_invalid_enum() -> None: State.record_operation("demo", "provider.up", "succeeded", provider_state="bogus") +def test_provider_event_drives_state_through_the_machine( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """provider_event fires the core instance machine seeded from the persisted + provider_state; the resulting leaf becomes the new provider_state (bug N).""" + monkeypatch.setenv("EVE_STATE_DIR", str(tmp_path)) + from eve_sdk.instance_fsm import dispatch_event + + # A full lifecycle: each (command, op_status) is mapped to a machine event by + # dispatch_event exactly as provider-dispatch does, then fired from the previous + # persisted state. (in-flight -> transient, succeeded -> terminal outcome.) + steps = [ + ("up", "running", "creating"), + ("up", "succeeded", "running"), + ("stop", "running", "stopping"), + ("stop", "succeeded", "stopped"), + ("down", "running", "destroying"), + ("down", "succeeded", "absent"), + # start recovers to running even from absent (operator intent is authoritative) + ("start", "running", "starting"), + ("start", "succeeded", "running"), + ] + for command, op_status, expected in steps: + state = State.record_operation( + "demo", f"provider.{command}", op_status, provider_event=dispatch_event(command, op_status) + ) + assert state["provider_state"] == expected, (command, op_status) + + # A read-only command leaves provider_state untouched (no event). + State.record_operation( + "demo", "provider.status", "failed", provider_event=dispatch_event("status", "failed") + ) + assert State.read("demo")["provider_state"] == "running" + + # A failed mutating operation lands in error: dispatch records the in-flight step + # first (running -> stopping), then the failure (op_fail from the transient -> error). + State.record_operation("demo", "provider.stop", "running", provider_event=dispatch_event("stop", "running")) + assert State.read("demo")["provider_state"] == "stopping" + State.record_operation("demo", "provider.stop", "failed", provider_event=dispatch_event("stop", "failed")) + assert State.read("demo")["provider_state"] == "error" + + def test_state_concurrent_writers_preserve_history(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("EVE_STATE_DIR", str(tmp_path)) State.write("concurrency-test", State.default_state("concurrency-test", "2026-01-01T00:00:00Z")) From b8394d7dad33bd915fa4b7076608dba1164e7e4d Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Wed, 19 Aug 2026 22:47:45 +0900 Subject: [PATCH 28/32] =?UTF-8?q?feat(v4.5=20N):=20port=20to=20determa-sta?= =?UTF-8?q?te=200.1.0=20=E2=80=94=20format-1=20bundles=20+=20pure-function?= =?UTF-8?q?al=20engine=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit determa-state v0.1.0 replaced the draft grammar and the Host/Instance object model: machines are now format-1 bundles (format: 1, namespace, shared events with direction, machines[].root, esvs -> variables) and the engine interface is the pure functional spec-§8 pair create/dispatch over plain aggregate-state dicts. Port eve's whole FSM layer accordingly; pin determa-state (>=0.1.0,<0.2.0). - core/fsm/*.yaml: rewritten as bundles. Format 1 has no orthogonal regions, so the instance bundle holds two machines (instance_provider + instance_provision) — matching eve's persistence, which keeps provider_state and provision_state as separate fields anyway. YAML anchors are rejected (unsupported_yaml_feature), so the shared operator-command block is spelled out per concrete state. - eve_sdk/fsm.py: EveFsm now holds loaded bundles + live aggregate states per (scope, key); external variables are seeded as creation bindings; env/refresh unchanged in concept. Surface: load/aggregate/fire/refresh/leaves/state/context/ view. The harel fallback import is gone; import determa.state appears only here. - eve_sdk/instance_fsm.py: dispatch is pure, so the adapter seeds each provider leaf once by replaying a short event prefix from create, caches that aggregate, and dispatches the real event against it (the cache stays valid — dispatch never mutates its input). dispatch_event mapping unchanged. - scripts/providers-status: reads fsm.state() instead of engine instance objects. - tests: fixture bundle + machine/platform tests updated (provision machine gets its own lifecycle test); aggregate status is "running" not "active". Full scripts/test green; test-lifecycle green; providers-status live-verified (aws/docker/gcp/local-qemu/truenas/vultr — same semantics as before the port). The old IntType-leak issue is moot: 0.1.0 returns native values. --- core/fsm/instance.yaml | 132 ++++++++++++++------- core/fsm/package.yaml | 79 +++++++------ core/fsm/provider.yaml | 66 ++++++----- docs/v4.5-fsm-design.md | 88 ++++++++------ eve_sdk/fsm.py | 194 ++++++++++++++++++++----------- eve_sdk/instance_fsm.py | 138 ++++++++++++---------- poetry.lock | 27 ++++- pyproject.toml | 2 +- scripts/providers-status | 4 +- tests/fsm_fixtures/provider.yaml | 63 +++++----- tests/test_fsm_machines.py | 72 +++++++----- tests/test_fsm_platform.py | 4 +- 12 files changed, 527 insertions(+), 342 deletions(-) diff --git a/core/fsm/instance.yaml b/core/fsm/instance.yaml index b9493d47..98a470b8 100644 --- a/core/fsm/instance.yaml +++ b/core/fsm/instance.yaml @@ -1,46 +1,89 @@ -# The instance lifecycle (v4.5 N/O) — the declarative replacement for state.py's -# PROVIDER_STATES + PROVISION_STATES. Two concurrent regions: the provider (terraform) -# lifecycle and the provisioning lifecycle, which advance independently. `desired` is the -# goal (running | stopped | absent), supplied by eve as context. Transitions are driven -# by eve operations (create/start/stop/destroy, provision) whose outcomes come back as -# events, and by observation (observe_* maps live provider status back into a state, -# keeping the machine honest about drift). -id: instance +# The instance lifecycle (v4.5 N/O) — a Determa State format-1 bundle; the declarative +# replacement for state.py's PROVIDER_STATES + PROVISION_STATES. Two machines: the +# provider (terraform) lifecycle and the provisioning lifecycle, which advance +# independently (format 1 has no orthogonal regions, and eve persists + drives the two +# as separate fields anyway). `desired` is the goal (running | stopped | absent), +# supplied by eve as an external binding. Transitions are driven by eve operations +# (create/start/stop/destroy, provision) whose outcomes come back as events, and by +# observation (observe_* maps live provider status back into a state, keeping the +# machine honest about drift). +format: 1 +namespace: eve.core events: - create: {} - start: {} - stop: {} - destroy: {} - created_ok: {} - started_ok: {} - stopped_ok: {} - destroyed_ok: {} - op_fail: {} - observe_running: {} - observe_stopped: {} - observe_absent: {} - provision: {} - provision_ok: {} - provision_fail: {} -top: - esvs: - desired: { type: string, external: true } # running | stopped | absent | unknown - regions: + create: { direction: input } + start: { direction: input } + stop: { direction: input } + destroy: { direction: input } + created_ok: { direction: input } + started_ok: { direction: input } + stopped_ok: { direction: input } + destroyed_ok: { direction: input } + op_fail: { direction: input } + observe_running: { direction: input } + observe_stopped: { direction: input } + observe_absent: { direction: input } + provision: { direction: input } + provision_ok: { direction: input } + provision_fail: { direction: input } +machines: - # Provider (terraform) lifecycle. Each in-flight operation has its own transient - # state so its outcome is unambiguous; op_fail from any of them lands in error. - # `unknown` is the unobserved start; observe_* reconcile from live provider status. - # - # Every concrete (non-transient) state accepts every operator command and routes - # to that command's transient state: the operator's intent is authoritative and the - # provider command's success/failure — not the machine — decides feasibility (a - # `start` that can't apply fails and lands in `error` via op_fail, it never silently - # no-ops). So a command's terminal outcome (up/start->running, stop->stopped, - # down->absent) holds from any state, exactly as before, now with honest transients. - - initial: { transition_to: unknown } + # Provider (terraform) lifecycle. Each in-flight operation has its own transient + # state so its outcome is unambiguous; op_fail from any of them lands in error. + # `unknown` is the unobserved start; observe_* reconcile from live provider status. + # + # Every concrete (non-transient) state accepts every operator command and routes + # to that command's transient state: the operator's intent is authoritative and the + # provider command's success/failure — not the machine — decides feasibility (a + # `start` that can't apply fails and lands in `error` via op_fail, it never silently + # no-ops). So a command's terminal outcome (up/start->running, stop->stopped, + # down->absent) holds from any state, exactly as before, now with honest transients. + - machine_id: instance_provider + root: + type: composite + variables: + desired: { type: string, external: true, init: "unknown" } # running | stopped | absent | unknown + initial: { transition_to: unknown } + # (format 1 forbids YAML anchors, so the shared command block is spelled out + # in each concrete state.) states: unknown: - on_events: &commands + on_events: + create: { transition_to: creating } + start: { transition_to: starting } + stop: { transition_to: stopping } + destroy: { transition_to: destroying } + observe_running: { transition_to: running } + observe_stopped: { transition_to: stopped } + observe_absent: { transition_to: absent } + absent: + on_events: + create: { transition_to: creating } + start: { transition_to: starting } + stop: { transition_to: stopping } + destroy: { transition_to: destroying } + observe_running: { transition_to: running } + observe_stopped: { transition_to: stopped } + observe_absent: { transition_to: absent } + running: + on_events: + create: { transition_to: creating } + start: { transition_to: starting } + stop: { transition_to: stopping } + destroy: { transition_to: destroying } + observe_running: { transition_to: running } + observe_stopped: { transition_to: stopped } + observe_absent: { transition_to: absent } + stopped: + on_events: + create: { transition_to: creating } + start: { transition_to: starting } + stop: { transition_to: stopping } + destroy: { transition_to: destroying } + observe_running: { transition_to: running } + observe_stopped: { transition_to: stopped } + observe_absent: { transition_to: absent } + error: + on_events: create: { transition_to: creating } start: { transition_to: starting } stop: { transition_to: stopping } @@ -48,10 +91,6 @@ top: observe_running: { transition_to: running } observe_stopped: { transition_to: stopped } observe_absent: { transition_to: absent } - absent: { on_events: *commands } - running: { on_events: *commands } - stopped: { on_events: *commands } - error: { on_events: *commands } creating: on_events: created_ok: { transition_to: running } @@ -69,8 +108,11 @@ top: destroyed_ok: { transition_to: absent } op_fail: { transition_to: error } - # Provisioning lifecycle (independent of the provider region). - - initial: { transition_to: unprovisioned } + # Provisioning lifecycle (independent of the provider machine). + - machine_id: instance_provision + root: + type: composite + initial: { transition_to: unprovisioned } states: unprovisioned: on_events: diff --git a/core/fsm/package.yaml b/core/fsm/package.yaml index 9a29305f..bb7e34f4 100644 --- a/core/fsm/package.yaml +++ b/core/fsm/package.yaml @@ -1,38 +1,43 @@ -# The package lifecycle, shared by every package (v4.5 N/O) — the declarative -# replacement for state.py's PACKAGE_STATES. Transitions are driven by eve operations -# (install / remove) and by observation (found / absent maps a live probe back to a -# state, keeping the machine honest about drift). -id: package +# The package lifecycle, shared by every package (v4.5 N/O) — a Determa State +# format-1 bundle; the declarative replacement for state.py's PACKAGE_STATES. +# Transitions are driven by eve operations (install / remove) and by observation +# (found / absent maps a live probe back to a state, keeping the machine honest +# about drift). +format: 1 +namespace: eve.core events: - found: {} # observe: the package is present on the instance - absent: {} # observe: not present - install: {} - install_ok: {} - install_fail: {} - remove: {} -top: - initial: { transition_to: unknown } - states: - unknown: - on_events: - found: { transition_to: installed } - absent: { transition_to: missing } - install: { transition_to: installing } - missing: - on_events: - install: { transition_to: installing } - found: { transition_to: installed } # drift: appeared out of band - installing: - on_events: - install_ok: { transition_to: installed } - install_fail: { transition_to: failed } - installed: - on_events: - remove: { transition_to: removed } - absent: { transition_to: missing } # drift: disappeared - failed: - on_events: - install: { transition_to: installing } # retry - removed: - on_events: - install: { transition_to: installing } + found: { direction: input } # observe: the package is present on the instance + absent: { direction: input } # observe: not present + install: { direction: input } + install_ok: { direction: input } + install_fail: { direction: input } + remove: { direction: input } +machines: + - machine_id: package + root: + type: composite + initial: { transition_to: unknown } + states: + unknown: + on_events: + found: { transition_to: installed } + absent: { transition_to: missing } + install: { transition_to: installing } + missing: + on_events: + install: { transition_to: installing } + found: { transition_to: installed } # drift: appeared out of band + installing: + on_events: + install_ok: { transition_to: installed } + install_fail: { transition_to: failed } + installed: + on_events: + remove: { transition_to: removed } + absent: { transition_to: missing } # drift: disappeared + failed: + on_events: + install: { transition_to: installing } # retry + removed: + on_events: + install: { transition_to: installing } diff --git a/core/fsm/provider.yaml b/core/fsm/provider.yaml index 1ae9c495..08bbe557 100644 --- a/core/fsm/provider.yaml +++ b/core/fsm/provider.yaml @@ -1,39 +1,45 @@ -# The provider lifecycle, shared by every provider (v4.5 N/O). +# The provider lifecycle, shared by every provider (v4.5 N/O) — a Determa State +# format-1 bundle. # # `is_configured` is derived by eve — true iff every `required` config_schema # secret/setting resolves to a non-empty value via the ONE shared path (os.environ + -# ConfigEnv + injected secrets) — and seeded as external context. `reachable` is a -# substate of `configured`, reached by the provider's `connectivity` probe (host-run, -# delivered back as probe_ok / probe_fail). One machine: per-provider differences live -# only in the config eve resolves, never in the states — so `configured` can never be a -# plugin's ambient self-report (the AWS-off-~/.aws bug is structurally impossible). -id: provider +# ConfigEnv + injected secrets) — and seeded as an external creation binding. +# `reachable` is a substate of `configured`, reached by the provider's `connectivity` +# probe (host-run, delivered back as probe_ok / probe_fail). One machine: per-provider +# differences live only in the config eve resolves, never in the states — so +# `configured` can never be a plugin's ambient self-report (the AWS-off-~/.aws bug is +# structurally impossible). +format: 1 +namespace: eve.core events: - resolve: {} - probe_ok: {} - probe_fail: {} -top: - esvs: - is_configured: { type: bool, external: true } - on_events: - env: - action: [ { refresh: {} } ] # adopt a changed configured-ness - initial: { transition_to: unconfigured } - states: - unconfigured: - on_events: - resolve: - - { guard: "is_configured == true", transition_to: configured } - configured: + resolve: { direction: input } + probe_ok: { direction: input } + probe_fail: { direction: input } +machines: + - machine_id: provider + root: type: composite - initial: { transition_to: unreachable } + variables: + is_configured: { type: bool, external: true } on_events: - resolve: - - { guard: "is_configured != true", transition_to: unconfigured } + env: + action: [ { refresh: {} } ] # adopt a changed configured-ness + initial: { transition_to: unconfigured } states: - unreachable: + unconfigured: on_events: - probe_ok: { transition_to: reachable } - reachable: + resolve: + - { guard: "is_configured == true", transition_to: configured } + configured: + type: composite + initial: { transition_to: unreachable } on_events: - probe_fail: { transition_to: unreachable } + resolve: + - { guard: "is_configured != true", transition_to: unconfigured } + states: + unreachable: + on_events: + probe_ok: { transition_to: configured.reachable } + reachable: + on_events: + probe_fail: { transition_to: configured.unreachable } diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index 176e96cc..4e95d5b5 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -184,32 +184,42 @@ keep persistence, but behind `FSM`'s storage seam. **generic name** since the whole point is reuse + a future cns-based swap, but it's your call where it lives. -## Reconciliation with harel (as built) - -The engine is real now, so this supersedes the speculative `state_machine:` schema above. - -- **Machines are harel YAML.** A plugin ships a harel machine (in its own repo). States - are `states:` under `top:`; transitions are `on_events: { : { transition_to, - guard, action } }`; the machine's config lives in `esvs:`. Guards are **CEL**; actions - are harel's structured set (`assign`/`publish`/`refresh`/`spawn`/`stop`). -- **Settings are `external` esvs.** eve resolves a machine's declared `external` esvs - (settings + secrets) via one shared resolver and seeds them at instance creation. A - settings change is delivered as harel's reserved **`env` event**; the machine's `env` - handler adopts it via **`refresh`**. One context ⇒ status and dispatch cannot diverge. -- **Side effects are host-driven.** harel actions are sandboxed (no host calls), so eve +## Reconciliation with Determa State (as built) + +The engine is real now, so this supersedes the speculative `state_machine:` schema +above. (History: the engine started as **harel**, was renamed **determa-state**, and +at spec **v0.1.0** pivoted to *format-1 bundles* + a *pure-functional interface* — +eve tracks 0.1.0.) + +- **Machines are format-1 bundles.** A plugin ships a Determa State bundle: `format: 1`, + a `namespace`, shared `events` (host-delivered events are `direction: input`), and + `machines: [{machine_id, root}]`. States are `states:` under `root:`; transitions are + `on_events: { : { transition_to, guard, action } }`; the machine's config lives + in `variables:` (root-declared, `external: true` for eve-resolved values). Guards are + **CEL**; actions are the structured set (`assign`/`send`/`refresh`/`spawn`/`cancel`/ + `stop`). Format 1 has **no orthogonal regions** — independent lifecycles are separate + machines in one bundle (which matches eve's persistence: `provider_state` and + `provision_state` are separate fields anyway). +- **Settings are `external` variables.** eve resolves a machine's declared `external` + variables (settings + secrets) via one shared resolver and seeds them as **creation + bindings**. A settings change is delivered as the reserved **`env` event**; the + machine's `env` handler adopts it via **`refresh`**. One context ⇒ status and dispatch + cannot diverge. +- **Side effects are host-driven.** Engine actions are sandboxed (no host calls), so eve runs the real command (connectivity probe, terraform, provision) and delivers the outcome back as an event (`probe_ok`/`probe_fail`). `configured` is a guarded transition on the secret; `reachable` is a substate of `configured` reached by the probe result. -- **The platform** is `eve_sdk/fsm.py` — `EveFsm` wraps a `harel.Host` and takes a - pluggable `ContextResolver` (`(scope, key, external_names) -> {name: value}`), the seam - where the single ConfigEnv+secrets resolver plugs in. Surface: `load` / `instance` / - `fire` / `refresh` / `state` / `context` / `view`. - -> **Engine rename:** harel is being renamed **determa-state**; the package isn't -> published under the new name yet. `eve_sdk/fsm.py` imports it behind one name -> (`import determa_state as engine`, falling back to `harel`), so the swap is a no-op -> for eve — nothing else in eve names the engine. +- **The interface is pure-functional (spec §8).** `create(bundle, machine_id, id, + creation_id, bindings)` and `dispatch(bundle, prior_state, delivery)` map a plain + aggregate-state dict to the next one — no engine-held mutable objects. This fits eve + exactly: seed from persisted state, fire one event, persist the result. +- **The platform** is `eve_sdk/fsm.py` — `EveFsm` holds loaded bundles + live aggregate + states and takes a pluggable `ContextResolver` + (`(scope, key, external_names) -> {name: value}`), the seam where the single + ConfigEnv+secrets resolver plugs in. Surface: `load` / `aggregate` / `fire` / + `refresh` / `leaves` / `state` / `context` / `view`. `import determa.state` appears + only in this module — nothing else in eve names the engine. ### Landed / next - ☑ **Increment 1** — `EveFsm` + a synthetic provider machine @@ -228,8 +238,8 @@ The engine is real now, so this supersedes the speculative `state_machine:` sche `required` secrets/settings (via the shared path), so "configured" is eve-resolved, not a plugin's ambient self-report. The lifecycle is identical across providers, so it lives in `core/`, not per-provider. Tested end-to-end. -- ☐ **Dependency** — pin the engine in `pyproject` (git dep) once `determa-state` is - published + tagged, then `poetry lock`; the test then runs in CI. +- ☑ **Dependency** — `determa-state (>=0.1.0,<0.2.0)` from PyPI, locked; the FSM tests + run in CI. - ☑ **Rewire `providers-status`** — `configured` is now FSM-derived (`core/fsm/provider.yaml` + `provider_configured_resolver`); the `connectivity` command runs **only for `reachable`** (its self-reported `configured` is ignored) and only when configured. @@ -238,9 +248,11 @@ The engine is real now, so this supersedes the speculative `state_machine:` sche it reads `configured=yes / reachable=no` (readiness surfaces as reachability + note). If AWS should read `configured=no` until eve holds an auth profile, mark an auth setting `required` or model AWS's configured as probe-derived — a per-provider call. -- ☑ **Instance & package machines (declarative)** — `core/fsm/instance.yaml` (provider ‖ - provision as orthogonal regions; `desired` as context) + `core/fsm/package.yaml`, both - passing the engine's static validation and exercised in `tests/test_fsm_machines.py`. +- ☑ **Instance & package machines (declarative)** — `core/fsm/instance.yaml` (two + machines in one bundle: `instance_provider` + `instance_provision`, since format 1 + has no orthogonal regions; `desired` as external context) + `core/fsm/package.yaml`, + both passing the engine's load-time validation and exercised in + `tests/test_fsm_machines.py`. - ◐ **Retire `state.py`/`state_machine.py` (live)** — decision: **adopt the cleaner vocabulary**; instead of a runtime migration, **delete all persisted state once** (no important instances) and start fresh on the machine vocabulary. @@ -273,15 +285,25 @@ The engine is real now, so this supersedes the speculative `state_machine:` sche precedence), and `aggregate_summary` (counting) are **display/aggregation policy, not a second lifecycle definition** — there is now exactly one transition authority (the machine), so no divergence risk remains. They stay as a thin status-render adapter. - (Provision region still uses the persisted `unknown`/`error` vocabulary; wiring that - region through the machine the same way is the next increment.) + (Provision state still uses the persisted `unknown`/`error` vocabulary; wiring the + `instance_provision` machine the same way is the next increment.) +- ☑ **Engine 0.1.0 port** — determa-state's v0.1.0 pivoted to format-1 bundles + the + pure-functional §8 interface (`create`/`dispatch` over plain aggregate-state dicts; + the old `Host`/`Instance` object API is gone). All three core machines rewritten as + bundles (instance = two machines, no orthogonal regions in format 1; YAML anchors are + rejected, so the shared command block is spelled out), `EveFsm` rewritten over + `create`/`dispatch` (aggregates held per `(scope, key)`), `instance_fsm` seeds each + provider leaf once by replaying a short event prefix and dispatches against the + cached aggregate (dispatch is pure, the cache stays valid). The harel fallback import + is gone. The engine's IntType leak (old #40) is moot — 0.1.0 returns native values. ### Open decisions — resolved -1. Engine repo/name → **harel** (`fruwehq/harel` spec, `fruwehq/harel-python` impl). -2. Guard language → **CEL** (harel's). -3. Machine format → **harel YAML** (not the `state_machine:` sketch). +1. Engine repo/name → **Determa State** (`fruwehq/determa-state-spec` spec, + `fruwehq/determa-state-python` impl; formerly harel). +2. Guard language → **CEL** (the engine's). +3. Machine format → **Determa State format-1 bundles** (not the `state_machine:` sketch). Still open (unchanged leans): provider-declared vs shared instance FSM (lean provider-declared); bundle/OS machines (lean stub); one YAML block vs a separate -`fsm.yaml` (now that machines are full harel documents, lean **a separate `fsm.yaml`** +`fsm.yaml` (now that machines are full bundle documents, lean **a separate `fsm.yaml`** per plugin). diff --git a/eve_sdk/fsm.py b/eve_sdk/fsm.py index 78c29b01..f8546aad 100644 --- a/eve_sdk/fsm.py +++ b/eve_sdk/fsm.py @@ -1,83 +1,131 @@ """eve FSM platform — the single seam between eve and the Determa State statechart engine. -Machines are declared in **Determa State YAML** by the plugin that owns them (a provider, a -package, ...). eve resolves each machine's ``external`` esvs (settings + secrets) -through **one shared resolver** and seeds them as the machine's context, so status, -dispatch, and connectivity can never see different config (v4.5 bugs N/O). The engine -is the swappable unit; this module is the only place eve touches it. - -Side effects are host-driven: engine actions are sandboxed, so eve runs the real -command (probe, terraform, provision) and delivers the outcome back as an event -(e.g. ``probe_ok`` / ``probe_fail``). The machine only ever holds state + context. +Machines are declared as **Determa State format-1 bundles** by the plugin that owns +them (a provider, a package, ...). eve resolves each machine's ``external`` variables +(settings + secrets) through **one shared resolver** and seeds them as creation +bindings, so status, dispatch, and connectivity can never see different config (v4.5 +bugs N/O). The engine is the swappable unit; this module is the only place eve +touches it. + +The engine's interface is pure-functional (spec §8): ``create``/``dispatch`` map a +plain aggregate-state dict to the next one — no engine-held mutable objects. This +platform keeps each live aggregate state in memory per ``(scope, key)``. + +Side effects are host-driven: eve runs the real command (probe, terraform, +provision) and delivers the outcome back as an event (e.g. ``probe_ok`` / +``probe_fail``). The machine only ever holds state + context. """ from __future__ import annotations -import os from collections.abc import Callable, Mapping from typing import Any -# The statechart engine (Determa State; formerly harel). Import it behind one name so -# eve's only coupling to it is this module. -try: # pragma: no cover - the branch taken depends only on which package is installed - import determa.state as _engine_mod -except ModuleNotFoundError: # pre-rename package name - import harel as _engine_mod # type: ignore[import-untyped,no-redef] +# The statechart engine (Determa State). Import it behind one name so eve's only +# coupling to it is this module; re-bound as a defined name so it is an explicit +# export (other eve modules read it as ``eve_sdk.fsm.engine``). +import determa.state as _engine_mod -# Re-bound as a defined name so it is an explicit export (other eve modules read it -# as ``eve_sdk.fsm.engine``). engine = _engine_mod -# Resolve a machine's declared ``external`` esvs from eve settings/secrets: + +class FsmError(Exception): + pass + + +# Resolve a machine's declared ``external`` variables from eve settings/secrets: # (scope, key, external_names) -> {name: value} # Injected so resolution lives in exactly one place (the bug-M/O helper) and the # platform stays testable without real config on disk. ContextResolver = Callable[[str, str, list[str]], dict[str, Any]] +def _machine_doc(bundle: Any, machine_id: str) -> dict[str, Any]: + for machine in bundle.raw["machines"]: + if machine["machine_id"] == machine_id: + return dict(machine) + raise FsmError(f"machine not found in bundle: {machine_id}") + + +def external_names(bundle: Any, machine_id: str) -> list[str]: + """The machine's declared root ``external`` variable names (eve-resolved).""" + variables = _machine_doc(bundle, machine_id).get("root", {}).get("variables", {}) or {} + return [name for name, spec in variables.items() if isinstance(spec, Mapping) and spec.get("external")] + + +def leaf_names(aggregate: Mapping[str, Any]) -> list[str]: + """The root runtime's active leaf-state names (last path segment, sorted).""" + runtime = aggregate["runtimes"][aggregate["root_runtime_id"]] + active = list(runtime["active"]) + leaves = [ + path + for path in active + if path != "root" and not any(other.startswith(path + ".") for other in active) + ] + return sorted(path.split(".")[-1] for path in leaves) + + class EveFsm: - """Loads plugin-declared Determa State machines and tracks their live instances.""" + """Loads plugin-declared Determa State bundles and tracks their live aggregates.""" def __init__(self, resolve_context: ContextResolver) -> None: self._resolve = resolve_context - self._host = engine.Host() - self._raw: dict[str, dict[str, Any]] = {} - self._machine: dict[str, Any] = {} - - def load(self, scope: str, machine_yaml: str) -> None: - """Register the machine a plugin ships for ``scope`` (its first document).""" - defs = engine.load_definitions(machine_yaml) - for definition in defs: - engine.validate(definition.raw) - self._host.register_all(defs) - root = defs[0] - self._raw[scope] = root.raw - self._machine[scope] = self._host.machines[root.raw["id"]] - - @staticmethod - def _external_names(raw: Mapping[str, Any]) -> list[str]: - esvs = raw.get("top", {}).get("esvs", {}) or {} - return [name for name, spec in esvs.items() if isinstance(spec, Mapping) and spec.get("external")] - - def _iid(self, scope: str, key: str) -> str: + self._bundle: dict[str, Any] = {} + self._machine_id: dict[str, str] = {} + self._aggregate: dict[str, dict[str, Any]] = {} + self._sequence = 0 + + def load(self, scope: str, bundle_yaml: str, machine_id: str | None = None) -> None: + """Register the bundle a plugin ships for ``scope``. + + ``machine_id`` selects the machine driven under this scope; defaults to the + bundle's first machine. + """ + bundle = engine.load_bundle(bundle_yaml) + self._bundle[scope] = bundle + self._machine_id[scope] = machine_id or str(bundle.raw["machines"][0]["machine_id"]) + + def _key(self, scope: str, key: str) -> str: return f"{scope}:{key}" - def instance(self, scope: str, key: str) -> Any: - """The live instance for ``(scope, key)``; created and seeded on first use.""" - iid = self._iid(scope, key) - inst = self._host.instances.get(iid) - if inst is None: - external = self._resolve(scope, key, self._external_names(self._raw[scope])) - inst = self._host.create_root(self._machine[scope], iid, external=external) - self._host.run_to_quiescence() - return inst + def _event_id(self) -> str: + self._sequence += 1 + return f"eve-{self._sequence}" + + def aggregate(self, scope: str, key: str) -> dict[str, Any]: + """The live aggregate state for ``(scope, key)``; created and seeded on first use.""" + composite = self._key(scope, key) + state = self._aggregate.get(composite) + if state is None: + bundle = self._bundle[scope] + machine_id = self._machine_id[scope] + external = self._resolve(scope, key, external_names(bundle, machine_id)) + bindings = {"external": external} if external else None + result = engine.create(bundle, machine_id, composite, self._event_id(), bindings) + if result["status"] == "rejected" or result["state"] is None: + raise FsmError(f"cannot create {composite}: {result['rejection']}") + state = result["state"] + self._aggregate[composite] = state + return state def fire(self, scope: str, key: str, event: str, payload: Mapping[str, Any] | None = None) -> bool: - """Deliver an event and run to quiescence. Returns False if it was rejected.""" - self.instance(scope, key) - accepted = self._host.deliver(self._iid(scope, key), event, dict(payload or {})) - self._host.run_to_quiescence() - return accepted + """Deliver an input event through one RTC step. Returns True iff it was handled.""" + state = self.aggregate(scope, key) + envelope = { + "event": event, + "event_id": self._event_id(), + "target": { + "root": { + "root_instance_id": state["root_instance_id"], + "root_runtime_id": state["root_runtime_id"], + } + }, + "payload": dict(payload or {}), + } + result = engine.dispatch(self._bundle[scope], state, {"input": envelope}) + if result["disposition"] in {"handled", "unhandled"}: + self._aggregate[self._key(scope, key)] = result["state"] + return bool(result["disposition"] == "handled") def refresh(self, scope: str, key: str) -> dict[str, Any]: """Re-resolve settings/secrets; deliver an ``env`` event with what changed. @@ -85,39 +133,43 @@ def refresh(self, scope: str, key: str) -> dict[str, Any]: The machine's ``env`` handler adopts the new values via ``refresh``. Returns the changed subset (empty if nothing moved). """ - inst = self.instance(scope, key) - current = dict(inst.resolved_esvs()) - latest = self._resolve(scope, key, self._external_names(self._raw[scope])) + current = self.context(scope, key) + latest = self._resolve(scope, key, external_names(self._bundle[scope], self._machine_id[scope])) changed = {name: value for name, value in latest.items() if current.get(name) != value} if changed: - self._host.deliver(self._iid(scope, key), "env", {"changed": changed}) - self._host.run_to_quiescence() + self.fire(scope, key, "env", {"changed": changed}) return changed + def leaves(self, scope: str, key: str) -> list[str]: + """The active leaf-state names (sorted).""" + return leaf_names(self.aggregate(scope, key)) + def state(self, scope: str, key: str) -> str: - """The active leaf state (first, for non-orthogonal machines).""" - leaves = self.instance(scope, key).active_leaf_names() + """The active leaf state (first, for single-region machines).""" + leaves = self.leaves(scope, key) return leaves[0] if leaves else "" def context(self, scope: str, key: str) -> dict[str, Any]: - """The machine's resolved context (esvs) — the one place commands read config.""" - return dict(self.instance(scope, key).resolved_esvs()) + """The machine's root-scope variables — the one place commands read config.""" + aggregate = self.aggregate(scope, key) + runtime = aggregate["runtimes"][aggregate["root_runtime_id"]] + return dict(runtime["scopes"].get("root", {})) def view(self, scope: str, key: str) -> dict[str, Any]: """State + context + status, for status/TUI/CLI rendering.""" - inst = self.instance(scope, key) + aggregate = self.aggregate(scope, key) return { - "state": list(inst.active_leaf_names()), - "context": dict(inst.resolved_esvs()), - "status": inst.status.name.lower(), + "state": self.leaves(scope, key), + "context": self.context(scope, key), + "status": str(aggregate["status"]), } def provider_context_resolver() -> ContextResolver: """The real :data:`ContextResolver` for provider machines. - Each declared external esv is named as its eve env var (e.g. ``VULTR_API_KEY``) and - resolved from the *one* path dispatch uses — ``os.environ`` + + Each declared external variable is named as its eve env var (e.g. ``VULTR_API_KEY``) + and resolved from the *one* path dispatch uses — ``os.environ`` + ``ConfigEnv.environment()`` + injected secrets — so a provider's FSM context and what dispatch runs with are identical (v4.5 bug M: status ≡ dispatch). ``key`` is the provider id. @@ -126,6 +178,8 @@ def provider_context_resolver() -> ContextResolver: dispatch so no config reaches a plugin behind eve's back; kept here for now to preserve exact parity.) """ + import os + from eve_sdk.config import ConfigEnv from eve_sdk.provider_command import _inject_secrets, _load_public_plugin @@ -148,6 +202,8 @@ def provider_configured_resolver() -> ContextResolver: never a plugin's ambient self-report (the AWS-off-``~/.aws`` bug is structurally impossible). ``key`` is the provider id; ``names`` is typically ``["is_configured"]``. """ + import os + from eve_sdk.config import ConfigEnv from eve_sdk.provider_command import _inject_secrets, _load_public_plugin diff --git a/eve_sdk/instance_fsm.py b/eve_sdk/instance_fsm.py index 64e2e881..b6e6f989 100644 --- a/eve_sdk/instance_fsm.py +++ b/eve_sdk/instance_fsm.py @@ -1,31 +1,32 @@ """Instance-lifecycle transitions, driven by the core instance machine. The single authority for how a provider operation moves an instance's -``provider_state``: dispatch computes the next state by **firing an event into -``core/fsm/instance.yaml``** seeded from the persisted state, not from a second, -hand-maintained transition table. The machine YAML *is* the lifecycle; this -module is the thin, fast adapter eve calls (and the persisted ``provider_state`` -a status read sees is therefore exactly what the machine produced — bug N). - -Only the provider region is driven here; the provision region is seeded with a -constant placeholder so a provider event never depends on provision naming (the -persisted ``provision_state`` field keeps its own vocabulary until that region is -wired the same way). +``provider_state``: dispatch computes the next state by **firing an event into the +``instance_provider`` machine of ``core/fsm/instance.yaml``** seeded from the +persisted state, not from a second, hand-maintained transition table. The machine +bundle *is* the lifecycle; this module is the thin, fast adapter eve calls (and the +persisted ``provider_state`` a status read sees is therefore exactly what the +machine produced — bug N). + +The engine is pure-functional (``create``/``dispatch`` over plain aggregate-state +dicts), so "seeding from the persisted state" is: reach each provider leaf once from +a fresh aggregate by replaying a short event prefix, cache that aggregate per leaf, +then dispatch the real event against it — dispatch never mutates its input, so the +cache stays valid. Only the provider machine is driven here; the persisted +``provision_state`` field keeps its own vocabulary until the ``instance_provision`` +machine is wired the same way. """ from __future__ import annotations from typing import Any -from eve_sdk import fsm as _fsm +from eve_sdk.fsm import engine, leaf_names from eve_sdk.workdir import Workdir -# The statechart engine, reached through the one module that owns the coupling. -engine = _fsm.engine - -# Leaf sets, so we can pick the provider-region leaf out of ``active_leaf_names`` -# (whose order across orthogonal regions is not guaranteed). These mirror the -# regions declared in core/fsm/instance.yaml. +# The provider-machine leaves (mirrors core/fsm/instance.yaml, machine +# instance_provider). Persisted provider_state values outside this set (legacy or +# corrupt) seed as `unknown`. PROVIDER_LEAVES = { "unknown", "creating", @@ -37,25 +38,63 @@ "absent", "error", } -# The provision leaf we seed while firing provider events; any valid provision -# leaf works because provider events do not touch the provision region. -_SEED_PROVISION = "unprovisioned" - -_host: Any = None -_instance: Any = None +# The shortest event prefix that reaches each leaf from a fresh aggregate; every +# concrete state accepts every operator command, so each transient is one event away +# and `error` is two. +_SEED_EVENTS: dict[str, list[str]] = { + "unknown": [], + "creating": ["create"], + "starting": ["start"], + "stopping": ["stop"], + "destroying": ["destroy"], + "running": ["observe_running"], + "stopped": ["observe_stopped"], + "absent": ["observe_absent"], + "error": ["create", "op_fail"], +} -def _machine() -> tuple[Any, Any]: - """Lazily build a reusable host + root instance for the core instance machine.""" - global _host, _instance - if _instance is None: - machine_yaml = (Workdir.repo_root() / "core" / "fsm" / "instance.yaml").read_text(encoding="utf-8") - host = engine.Host() - host.register_all(engine.load_definitions(machine_yaml)) - instance = host.create_root(host.machines["instance"], "instance", external={"desired": "unknown"}) - host.run_to_quiescence() - _host, _instance = host, instance - return _host, _instance +_MACHINE_ID = "instance_provider" + +_bundle: Any = None +_seeded: dict[str, dict[str, Any]] = {} +_sequence = 0 + + +def _dispatch(state: dict[str, Any], event: str) -> dict[str, Any]: + """One input-event RTC step; returns the (new) aggregate state.""" + global _sequence + _sequence += 1 + envelope = { + "event": event, + "event_id": f"eve-instance-{_sequence}", + "target": { + "root": { + "root_instance_id": state["root_instance_id"], + "root_runtime_id": state["root_runtime_id"], + } + }, + "payload": {}, + } + result = engine.dispatch(_bundle, state, {"input": envelope}) + new_state: dict[str, Any] = result["state"] + return new_state + + +def _seeded_aggregate(leaf: str) -> dict[str, Any]: + """The cached aggregate whose provider machine sits at ``leaf``.""" + global _bundle + if _bundle is None: + bundle_yaml = (Workdir.repo_root() / "core" / "fsm" / "instance.yaml").read_text(encoding="utf-8") + _bundle = engine.load_bundle(bundle_yaml) + state = _seeded.get(leaf) + if state is None: + state = engine.create(_bundle, _MACHINE_ID, leaf, f"eve-seed-{leaf}", None)["state"] + for event in _SEED_EVENTS[leaf]: + state = _dispatch(state, event) + assert leaf_names(state) == [leaf] + _seeded[leaf] = state + return state def dispatch_event(command: str | None, op_status: str) -> str | None: @@ -89,32 +128,11 @@ def provider_state_after(current: str, event: str | None) -> str | None: """The provider_state after firing ``event`` from ``current``. Returns ``None`` when there is no event (no change to persist). If the machine - rejects the event from ``current`` (an operation that doesn't apply in that - state), the state is unchanged and ``current`` is returned. + leaves the event unhandled from ``current`` (an outcome event that doesn't apply + in that state), the state is unchanged and ``current`` is returned. """ if event is None: return None - provider = current if current in PROVIDER_LEAVES else "unknown" - host, instance = _machine() - instance.load_snapshot( - { - "def_id": "instance", - "def_version": 1, - "id": "instance", - "parent_id": None, - "status": "active", - "state_config": ["top", f"top.{provider}", f"top.{_SEED_PROVISION}"], - "esvs": {"top": {"desired": "unknown"}}, - "queue": [], - "deferred": [], - "timers": [], - "dead_letter": [], - "history": {}, - } - ) - host.deliver("instance", event, {}) - host.run_to_quiescence() - for leaf in instance.active_leaf_names(): - if leaf in PROVIDER_LEAVES: - return str(leaf) - return provider + leaf = current if current in PROVIDER_LEAVES else "unknown" + moved = leaf_names(_dispatch(_seeded_aggregate(leaf), event)) + return moved[0] if moved else leaf diff --git a/poetry.lock b/poetry.lock index 800851d0..4c221db4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -49,20 +49,21 @@ files = [ [[package]] name = "determa-state" -version = "0.0.5" +version = "0.1.0" description = "Determa State — Python reference implementation of the Determa statechart engine" optional = false python-versions = ">=3.11" groups = ["main"] files = [ - {file = "determa_state-0.0.5-py3-none-any.whl", hash = "sha256:cb9d357c5fcd7c74e8d351802ca1e3057ba02a1e35ec8bfb63f5686c1f19800a"}, - {file = "determa_state-0.0.5.tar.gz", hash = "sha256:2dac244ce11a028b38146ddbc5c006f36c1d1c9c67cba11b7a16e8e92171ee36"}, + {file = "determa_state-0.1.0-py3-none-any.whl", hash = "sha256:c1ddde17d4dadb9797d727fce75186cb3609d479c902098f12ddb49625bc7709"}, + {file = "determa_state-0.1.0.tar.gz", hash = "sha256:7adeda9877e1698262918ca36bf38ea420aa6ae7820abd442f5034cad9fbd5ab"}, ] [package.dependencies] cel-python = ">=0.1" jsonschema = ">=4" pyyaml = ">=6" +rfc8785 = ">=0.1.4" [package.extras] dev = ["mypy", "pytest (>=8)", "ruff", "types-jsonschema", "types-pyyaml"] @@ -766,6 +767,24 @@ attrs = ">=22.2.0" rpds-py = ">=0.7.0" typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} +[[package]] +name = "rfc8785" +version = "0.1.4" +description = "A pure-Python implementation of RFC 8785 (JSON Canonicalization Scheme)" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "rfc8785-0.1.4-py3-none-any.whl", hash = "sha256:520d690b448ecf0703691c76e1a34a24ddcd4fc5bc41d589cb7c58ec651bcd48"}, + {file = "rfc8785-0.1.4.tar.gz", hash = "sha256:e545841329fe0eee4f6a3b44e7034343100c12b4ec566dc06ca9735681deb4da"}, +] + +[package.extras] +dev = ["build", "rfc8785[doc,lint,test]"] +doc = ["pdoc"] +lint = ["interrogate", "mypy (>=1.0)", "ruff (>=0.3,<1.0)"] +test = ["coverage", "pytest", "pytest-cov"] + [[package]] name = "rich" version = "15.0.0" @@ -1015,4 +1034,4 @@ test = ["coverage", "pytest", "pytest-cov"] [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "c5eebfc9584b4768d75016dc17389c9b7d961cfbd298c71ec1cc7a0532a13f1d" +content-hash = "07004ccc761288f7f2999f1b91ae759c9b6eb9ea211cafb5cf3845c75e4a8b8f" diff --git a/pyproject.toml b/pyproject.toml index e5ed90a8..4c542743 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ license = "MIT" readme = "README.md" requires-python = ">=3.12,<4.0" dependencies = [ - "determa-state (>=0.0.5,<0.0.6)", + "determa-state (>=0.1.0,<0.2.0)", "jsonschema (>=4.25.1,<5.0.0)", "pyyaml (>=6.0.3,<7.0.0)", "textual (>=8.2.5,<9.0.0)" diff --git a/scripts/providers-status b/scripts/providers-status index 81d2b877..164c35b0 100755 --- a/scripts/providers-status +++ b/scripts/providers-status @@ -91,11 +91,11 @@ def _provider_status(plugin: dict[str, Any]) -> tuple[str, str, str]: fsm = EveFsm(provider_configured_resolver()) fsm.load("provider", CORE_PROVIDER_MACHINE.read_text()) fsm.fire("provider", provider_id, "resolve") - if fsm.instance("provider", provider_id).active_leaf_names() == ["unconfigured"]: + if fsm.state("provider", provider_id) == "unconfigured": return "no", "no", "not configured" _configured, reachable, notes = _run_connectivity(plugin) fsm.fire("provider", provider_id, "probe_ok" if reachable == "yes" else "probe_fail") - reachable = "yes" if fsm.instance("provider", provider_id).active_leaf_names() == ["reachable"] else "no" + reachable = "yes" if fsm.state("provider", provider_id) == "reachable" else "no" return "yes", reachable, notes diff --git a/tests/fsm_fixtures/provider.yaml b/tests/fsm_fixtures/provider.yaml index a11bec44..2adc1d20 100644 --- a/tests/fsm_fixtures/provider.yaml +++ b/tests/fsm_fixtures/provider.yaml @@ -1,35 +1,40 @@ -# Synthetic provider machine (harel format) for the FSM-platform test — not a real -# plugin. `configured` = a secret is present; `reachable` is a substate of configured -# reached by a live probe. eve seeds api_key/region as external esvs from its shared -# resolver, so status and dispatch read identical config. -id: provider +# Synthetic provider machine (Determa State format-1 bundle) for the FSM-platform +# test — not a real plugin. `configured` = a secret is present; `reachable` is a +# substate of configured reached by a live probe. eve seeds api_key/region as external +# creation bindings from its shared resolver, so status and dispatch read identical +# config. +format: 1 +namespace: eve.test events: - resolve: {} - probe_ok: {} - probe_fail: {} -top: - esvs: - api_key: { type: string, external: true } # secret, eve-resolved - region: { type: string, external: true } # setting, eve-resolved - on_events: - env: - action: [ { refresh: {} } ] # adopt changed external values - initial: { transition_to: unconfigured } - states: - unconfigured: - on_events: - resolve: - - { guard: "api_key != null && api_key != ''", transition_to: configured } - configured: + resolve: { direction: input } + probe_ok: { direction: input } + probe_fail: { direction: input } +machines: + - machine_id: provider + root: type: composite - initial: { transition_to: unreachable } + variables: + api_key: { type: string, external: true } # secret, eve-resolved + region: { type: string, external: true } # setting, eve-resolved on_events: - resolve: - - { guard: "api_key == null || api_key == ''", transition_to: unconfigured } + env: + action: [ { refresh: {} } ] # adopt changed external values + initial: { transition_to: unconfigured } states: - unreachable: + unconfigured: on_events: - probe_ok: { transition_to: reachable } - reachable: + resolve: + - { guard: "api_key != ''", transition_to: configured } + configured: + type: composite + initial: { transition_to: unreachable } on_events: - probe_fail: { transition_to: unreachable } + resolve: + - { guard: "api_key == ''", transition_to: unconfigured } + states: + unreachable: + on_events: + probe_ok: { transition_to: configured.reachable } + reachable: + on_events: + probe_fail: { transition_to: configured.unreachable } diff --git a/tests/test_fsm_machines.py b/tests/test_fsm_machines.py index 4f6b9f48..190e6e93 100644 --- a/tests/test_fsm_machines.py +++ b/tests/test_fsm_machines.py @@ -1,8 +1,8 @@ """The core instance and package machines (core/fsm/*.yaml) over the Determa State engine. -Loading each machine runs it through the engine's static validation (reachable states, -no dead branches); the tests then exercise the key lifecycle transitions that replace -state.py's PROVIDER_STATES / PROVISION_STATES / PACKAGE_STATES. +Loading each bundle runs it through the engine's load-time validation; the tests then +exercise the key lifecycle transitions that replace state.py's PROVIDER_STATES / +PROVISION_STATES / PACKAGE_STATES. """ from __future__ import annotations @@ -14,68 +14,80 @@ CORE = Path(__file__).resolve().parents[1] / "core" / "fsm" -def _fsm(scope: str, external: dict[str, str] | None = None) -> EveFsm: +def _fsm(scope: str, machine_id: str | None = None, external: dict[str, str] | None = None) -> EveFsm: ext = dict(external or {}) fsm = EveFsm(lambda s, k, names: {n: ext.get(n, "") for n in names}) - fsm.load(scope, (CORE / f"{scope}.yaml").read_text()) + fsm.load(scope, (CORE / f"{scope}.yaml").read_text(), machine_id=machine_id) return fsm -def _leaves(fsm: EveFsm, scope: str, key: str = "x") -> list[str]: - return sorted(fsm.instance(scope, key).active_leaf_names()) - - def test_package_lifecycle() -> None: fsm = _fsm("package") - assert _leaves(fsm, "package") == ["unknown"] + assert fsm.leaves("package", "x") == ["unknown"] fsm.fire("package", "x", "absent") - assert _leaves(fsm, "package") == ["missing"] + assert fsm.leaves("package", "x") == ["missing"] fsm.fire("package", "x", "install") - assert _leaves(fsm, "package") == ["installing"] + assert fsm.leaves("package", "x") == ["installing"] fsm.fire("package", "x", "install_fail") - assert _leaves(fsm, "package") == ["failed"] + assert fsm.leaves("package", "x") == ["failed"] fsm.fire("package", "x", "install") # retry fsm.fire("package", "x", "install_ok") - assert _leaves(fsm, "package") == ["installed"] + assert fsm.leaves("package", "x") == ["installed"] fsm.fire("package", "x", "absent") # drift: disappeared out of band - assert _leaves(fsm, "package") == ["missing"] + assert fsm.leaves("package", "x") == ["missing"] -def test_instance_regions_advance_independently() -> None: - fsm = _fsm("instance", {"desired": "running"}) - assert _leaves(fsm, "instance") == ["unknown", "unprovisioned"] +def test_instance_provider_lifecycle() -> None: + fsm = _fsm("instance", machine_id="instance_provider", external={"desired": "running"}) + assert fsm.leaves("instance", "x") == ["unknown"] fsm.fire("instance", "x", "create") fsm.fire("instance", "x", "created_ok") - assert _leaves(fsm, "instance") == ["running", "unprovisioned"] - - # the provisioning region advances without touching the provider region - fsm.fire("instance", "x", "provision") - fsm.fire("instance", "x", "provision_ok") - assert _leaves(fsm, "instance") == ["provisioned", "running"] + assert fsm.leaves("instance", "x") == ["running"] # stop lands in stopped; destroy lands in absent (distinct outcomes) fsm.fire("instance", "x", "stop") fsm.fire("instance", "x", "stopped_ok") - assert _leaves(fsm, "instance") == ["provisioned", "stopped"] + assert fsm.leaves("instance", "x") == ["stopped"] fsm.fire("instance", "x", "destroy") fsm.fire("instance", "x", "destroyed_ok") - assert _leaves(fsm, "instance") == ["absent", "provisioned"] + assert fsm.leaves("instance", "x") == ["absent"] assert fsm.context("instance", "x")["desired"] == "running" + # operator intent is authoritative: start recovers even from absent + fsm.fire("instance", "x", "start") + fsm.fire("instance", "x", "started_ok") + assert fsm.leaves("instance", "x") == ["running"] + + +def test_instance_provision_lifecycle() -> None: + fsm = _fsm("instance", machine_id="instance_provision") + assert fsm.leaves("instance", "x") == ["unprovisioned"] + + fsm.fire("instance", "x", "provision") + fsm.fire("instance", "x", "provision_ok") + assert fsm.leaves("instance", "x") == ["provisioned"] + + fsm.fire("instance", "x", "provision") # re-converge + fsm.fire("instance", "x", "provision_fail") + assert fsm.leaves("instance", "x") == ["provision_error"] + + fsm.fire("instance", "x", "provision") # retry + fsm.fire("instance", "x", "provision_ok") + assert fsm.leaves("instance", "x") == ["provisioned"] + def test_instance_op_failure_goes_to_error() -> None: - fsm = _fsm("instance") - assert _leaves(fsm, "instance") == ["unknown", "unprovisioned"] + fsm = _fsm("instance", machine_id="instance_provider") fsm.fire("instance", "x", "create") fsm.fire("instance", "x", "op_fail") - assert _leaves(fsm, "instance") == ["error", "unprovisioned"] + assert fsm.leaves("instance", "x") == ["error"] fsm.fire("instance", "x", "create") # recover fsm.fire("instance", "x", "created_ok") - assert _leaves(fsm, "instance") == ["running", "unprovisioned"] + assert fsm.leaves("instance", "x") == ["running"] diff --git a/tests/test_fsm_platform.py b/tests/test_fsm_platform.py index 03815c6b..c3607e6e 100644 --- a/tests/test_fsm_platform.py +++ b/tests/test_fsm_platform.py @@ -28,7 +28,7 @@ def resolve(scope: str, key: str, names: list[str]) -> dict[str, Any]: def _leaves(fsm: EveFsm) -> list[str]: - return list(fsm.instance("provider", "vultr").active_leaf_names()) + return fsm.leaves("provider", "vultr") def test_unconfigured_without_secret() -> None: @@ -45,7 +45,7 @@ def test_configured_then_reachable_via_probe() -> None: fsm.fire("provider", "vultr", "probe_ok") assert _leaves(fsm) == ["reachable"] - assert fsm.view("provider", "vultr")["status"] == "active" + assert fsm.view("provider", "vultr")["status"] == "running" def test_settings_change_reevaluates() -> None: From bdd4474dc574c34097b27efa64fcb5476cf1ebfc Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sun, 4 Oct 2026 06:55:08 +0900 Subject: [PATCH 29/32] Complete v4.5 Determa 0.2 recovery and Cloud validation (#56) * feat(v4.5): complete Determa 0.2 migration * fix(v4.5): close recovery validation and secret handling gaps * fix(ci): create project environment before test runners * docs: use UTC date for Cloud validation --- .github/workflows/test.yml | 4 + config/defaults.yaml | 2 +- core/fsm/instance.yaml | 8 +- core/fsm/package.yaml | 48 +++-- core/schema/observed-state.schema.json | 6 +- core/schema/plugin-manifest.schema.json | 14 ++ docs/recovery-cloud-validation.md | 131 +++++++++++++ docs/state.md | 29 +-- docs/v4.5-fsm-design.md | 65 ++++--- eve_sdk/config.py | 23 ++- eve_sdk/dispatch.py | 98 ++++++++-- eve_sdk/fsm.py | 59 +++--- eve_sdk/instance_fsm.py | 180 ++++++++++++++--- eve_sdk/package_dispatch.py | 143 ++++++++++++-- eve_sdk/plugin_manifest.py | 26 +++ eve_sdk/provider_command.py | 65 +++++-- eve_sdk/secrets.py | 17 ++ eve_sdk/state.py | 112 ++++++++++- poetry.lock | 13 +- pyproject.toml | 2 +- scripts/instance-provision | 49 ++--- scripts/instance-state | 13 +- scripts/package-action | 115 ++++++++--- scripts/package-provision | 96 +++++++-- scripts/package-provision-windows | 65 ++++++- scripts/providers-status | 27 ++- scripts/provision | 177 ++++++++++++----- scripts/test | 8 + scripts/test-ci-pipeline | 6 + scripts/test-cross-repo | 75 +++++++ scripts/test-provision-runner | 16 +- scripts/test-python | 2 +- .../commands/test_package_action_launch.py | 109 +++++++++-- .../python/commands/test_provision_secrets.py | 183 ++++++++++++++++++ tests/python/sdk/test_command_observations.py | 24 +++ .../python/sdk/test_config_provider_paths.py | 137 ++++++++++++- .../sdk/test_plugin_manifest_contract.py | 52 +++++ tests/python/sdk/test_state.py | 69 +++++++ tests/test_fsm_machines.py | 27 ++- tests/test_fsm_platform.py | 99 +++++++++- 40 files changed, 2033 insertions(+), 361 deletions(-) create mode 100644 docs/recovery-cloud-validation.md create mode 100755 scripts/test-cross-repo create mode 100644 tests/python/commands/test_provision_secrets.py create mode 100644 tests/python/sdk/test_command_observations.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f9246cbd..df203c06 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,10 +5,14 @@ on: branches: - main pull_request: + types: [opened, ready_for_review, reopened, synchronize] schedule: - cron: "0 9 * * *" workflow_dispatch: +env: + POETRY_VIRTUALENVS_IN_PROJECT: "true" + jobs: test: if: github.event_name != 'pull_request' || github.event.pull_request.draft == false diff --git a/config/defaults.yaml b/config/defaults.yaml index 9edd1497..f029c7fe 100644 --- a/config/defaults.yaml +++ b/config/defaults.yaml @@ -11,7 +11,7 @@ display: fps: - resolution: + resolution: 1920x1080 global: my_ip: diff --git a/core/fsm/instance.yaml b/core/fsm/instance.yaml index 98a470b8..0478c49b 100644 --- a/core/fsm/instance.yaml +++ b/core/fsm/instance.yaml @@ -112,18 +112,18 @@ machines: - machine_id: instance_provision root: type: composite - initial: { transition_to: unprovisioned } + initial: { transition_to: unknown } states: - unprovisioned: + unknown: on_events: provision: { transition_to: provisioning } provisioning: on_events: provision_ok: { transition_to: provisioned } - provision_fail: { transition_to: provision_error } + provision_fail: { transition_to: error } provisioned: on_events: provision: { transition_to: provisioning } # re-converge - provision_error: + error: on_events: provision: { transition_to: provisioning } # retry diff --git a/core/fsm/package.yaml b/core/fsm/package.yaml index bb7e34f4..3df4a9ca 100644 --- a/core/fsm/package.yaml +++ b/core/fsm/package.yaml @@ -6,38 +6,44 @@ format: 1 namespace: eve.core events: - found: { direction: input } # observe: the package is present on the instance absent: { direction: input } # observe: not present + found: { direction: input } # observe: the package is present on the instance install: { direction: input } - install_ok: { direction: input } install_fail: { direction: input } + install_ok: { direction: input } + observe_failed: { direction: input } + observe_unknown: { direction: input } remove: { direction: input } + remove_fail: { direction: input } + remove_ok: { direction: input } machines: - machine_id: package root: type: composite - initial: { transition_to: unknown } + initial: { transition_to: idle } + on_events: + absent: { transition_to: idle.missing } + found: { transition_to: idle.installed } + observe_failed: { transition_to: idle.failed } + observe_unknown: { transition_to: idle.unknown } states: - unknown: + idle: + type: composite + initial: { transition_to: unknown } on_events: - found: { transition_to: installed } - absent: { transition_to: missing } install: { transition_to: installing } - missing: - on_events: - install: { transition_to: installing } - found: { transition_to: installed } # drift: appeared out of band + remove: { transition_to: removing } + states: + failed: {} + installed: {} + missing: {} + removed: {} + unknown: {} installing: on_events: - install_ok: { transition_to: installed } - install_fail: { transition_to: failed } - installed: - on_events: - remove: { transition_to: removed } - absent: { transition_to: missing } # drift: disappeared - failed: + install_fail: { transition_to: idle.failed } + install_ok: { transition_to: idle.installed } + removing: on_events: - install: { transition_to: installing } # retry - removed: - on_events: - install: { transition_to: installing } + remove_fail: { transition_to: idle.failed } + remove_ok: { transition_to: idle.removed } diff --git a/core/schema/observed-state.schema.json b/core/schema/observed-state.schema.json index d10b1fa6..8b9b814a 100644 --- a/core/schema/observed-state.schema.json +++ b/core/schema/observed-state.schema.json @@ -131,7 +131,7 @@ "properties": { "status": { "type": "string", - "enum": ["unknown", "installing", "installed", "missing", "failed", "removed"] + "enum": ["failed", "installed", "installing", "missing", "removed", "removing", "unknown"] }, "updated_at": { "type": "string", @@ -154,6 +154,10 @@ "type": "string", "description": "Dotted operation name (e.g. provider.up, package.install)." }, + "package": { + "type": "string", + "description": "Package id for a package operation." + }, "type": { "type": "string", "description": "Operation category (provider, provision, package).", diff --git a/core/schema/plugin-manifest.schema.json b/core/schema/plugin-manifest.schema.json index e24e27e8..1021b001 100644 --- a/core/schema/plugin-manifest.schema.json +++ b/core/schema/plugin-manifest.schema.json @@ -193,6 +193,20 @@ "$ref": "#/$defs/field_spec" } }, + "required_any": { + "description": "Provider-only alternative complete credential/configuration groups. At least one group must have every referenced field set. References use config. or secrets..", + "type": "array", + "minItems": 1, + "items": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "pattern": "^(config|secrets)\\.[a-z][a-z0-9_]*$" + }, + "uniqueItems": true + } + }, "secrets": { "type": "object", "additionalProperties": { diff --git a/docs/recovery-cloud-validation.md b/docs/recovery-cloud-validation.md new file mode 100644 index 00000000..7e19b9d3 --- /dev/null +++ b/docs/recovery-cloud-validation.md @@ -0,0 +1,131 @@ +# v4.5 recovery validation in Cloud + +Validated on 2026-10-03 UTC from fresh GitHub recovery checkouts. No backup files or +ignored personal configuration were used. All five initial heads matched the +recovery request: + +| Repository | Recovery PR | Initial head | +| --- | --- | --- | +| eve | #56 | c2c0f5a | +| eve-providers | #11 | 16ef4ef | +| eve-packages-linux | #7 | 5f9bb7c | +| eve-packages-windows | #5 | 5b4e261 | +| eve-plugins-ai | #3 | 1557a55 | + +## Determa API and version + +The [Python upstream release](https://github.com/fruwehq/determa-state-python/releases/tag/v0.2.0), +[specification release](https://github.com/fruwehq/determa-state-spec/releases/tag/v0.2.0), +and PyPI release metadata all identify 0.2.0 as the latest release. No newer +compatible or breaking release was found. Retain `>=0.2.0,<0.3.0` and the existing +0.2.0 lockfile entry. + +Eve uses the documented public `load_bundle`, `create`, and `dispatch` functions, +format-1 machine definitions, root-targeted event envelopes, external bindings, +and reserved `env` refresh events. Aggregate `runtimes`, active paths, and root +scopes are documented public state data. Side effects remain host-owned. +`ExecutionHost` is optional; its checkpoint stores are not required for pure +foreground transitions. Eve's small seed-by-replay adapter is restricted to its +context-free finite lifecycle graphs. A future machine with context or history +must persist and restore the complete aggregate, as the adapter documents. +No Determa design defect requiring an upstream issue was reproduced. + +## Fresh validation + +The Cloud environment used Python 3.12.14, dependencies installed from +`poetry.lock`, Terraform 1.14.9, Terramate 0.17.0, ShellCheck 0.11.0, and portable +PowerShell 7.5.4. PowerShell's writable XDG directories were isolated for testing. + +- The full `scripts/test` passed. Final validation of all Python tests passed + 528 tests with one intentional skip (an empty-registry instance comparison). +- The standalone FSM tests now run with the default Python suite, rather than + being omitted by its old `tests/python` selection. +- Linux launcher tests: 23 passed with `python -m pytest -q tests`. +- Providers: all seven provider conformance checks, host-resolver checks, five + AWS connectivity tests, five status-error tests, and environment contracts passed. +- Every tracked manifest across all four companion repositories passed + conformance, including nested bundle and OS manifests. +- The combined catalog loaded 47 plugins: seven providers, 25 packages, eleven + bundles, and four OS plugins. Shared Linux/Windows package configuration + schemas match; standalone bundle membership resolves. +- Actual OS runners passed failure, success, resume, manifest, and structured + status checks. PowerShell additionally passed explicit exit, thrown-error, + and informational native-tool exit cases. +- ShellCheck and Ruff passed for provider and Linux scripts. All 22 tracked + provider/Windows PowerShell files parsed successfully. +- External provider stack staging, Terramate generation, Terraform formatting, + Terramate formatting, and the core boundary check with real plugins passed. +- Linux and Windows integration plans, including all applicable packages, were + generated from synthetic instances. Fake-provider lifecycle tests exercised + create/status/IP/stop/start/provision/destroy without live infrastructure. + +Reproduce cross-repository checks from the core checkout: + +```sh +scripts/test-cross-repo \ + --plugin-root ../eve-providers \ + --plugin-root ../eve-packages-linux \ + --plugin-root ../eve-packages-windows \ + --plugin-root ../eve-plugins-ai +scripts/test-provision-runner --plugin-roots ../eve-providers +``` + +Run with the core project environment on `PATH`. Add `pwsh` to `PATH` for Windows +runner checks. The ordinary core suite stays hermetic; cross-repository runners +accept explicit roots and never need live providers. + +## Audit findings and fixes + +Determa remains the transition authority for provider, provisioning, package, +and interrupted-operation recovery. Operation IDs are allocated under the state +lock and remain monotonic when history is trimmed. Authentication evaluates all +universal requirements plus at least one complete alternative group +(`required_any` is OR-of-AND). Provider and package command environments are +operation-local and inject only declared target credentials. + +Fresh checks identified and fixed these additional problems: + +- Nested package observations could supply a root status through line scanning. + JSON extraction now skips nested objects and reads the last matching root. +- Secret payload files are created with mode 0600 before bytes are written, + including Windows JSON. Remote state directories are restricted before upload. + Partial secret uploads trigger cleanup. Failed Windows deletions and native + ACL errors propagate instead of silently succeeding. +- Literal shell quoting had prevented staged installer paths from expanding + `$HOME`. Only that known prefix expands; filenames and secrets stay quoted. +- The Windows runner treated an explicit `exit 42` as success. Each step now + executes in the current PowerShell runtime's separate `-File` process. +- Terraform status helpers suppressed backend failures. Missing explicit local + state remains absence; generation, backend, malformed JSON, and AWS API + failures now propagate. No alternate backend is queried for missing explicit + instance state. +- Linux contained tracked bytecode and lint findings. Bytecode was removed, + generated Python artifacts ignored, and lint findings fixed. +- Core contained a concrete provider identifier in an FSM comment that failed + the real-plugin core boundary check. The example is now generic. + +## GitHub and remaining rollout dependency + +Core recovery CI skipped because its PR was a draft; the cross-repository job is +scheduled/manual only. Provider and Linux/Windows recovery CI was absent because +those workflows filtered PR bases to `main`. Companion workflows now include +`v4.5`, select the corresponding core ref, and validate nested manifests. Core +and companion workflows explicitly handle `ready_for_review`. Provider CI now +runs status-error and OS-runner tests, and Linux CI runs its launcher suite. + +Enabling core CI exposed a pre-existing environment mismatch: Poetry created a +cached environment while the test runners require `.venv/bin/python`. The +workflow now explicitly creates its environment inside the project, as used in +fresh Cloud validation, and the pipeline contract check enforces this setting. + +No open issues or unresolved review threads were found on the affected PRs. +The only additional open PR found was Eve #46, a TUI portrait-blink change +outside this migration. Its `tui/app.py` overlap should be reviewed separately; +its code was not incorporated into recovery. + +Offline readiness does not establish live Windows GPU usability. The existing +v4.5 roadmap still requires a Vultr `vultr-vcg-a40-2c` Windows gaming instance +with the VDD as the sole active display and working Sunshine streaming. Cloud +currently has no provider secret or outbound identity bindings. Main-facing PRs +must state this remaining live dependency accurately; do not claim that demo or +GPU validation has occurred. No billable resources were created. diff --git a/docs/state.md b/docs/state.md index 163e0b42..18905ca4 100644 --- a/docs/state.md +++ b/docs/state.md @@ -1,4 +1,4 @@ -# v3 Instance State +# v4.5 Instance State Concrete instance state lives in `.eve/state/instances/.json` and is updated through `scripts/instance-state`. Core dispatchers should not write @@ -7,22 +7,25 @@ these JSON files directly. ## Top-Level States - `desired_state`: `unknown`, `running`, `stopped`, `absent` -- `provider_state`: `unknown`, `initializing`, `initialized`, `planned`, - `changing`, `running`, `stopped`, `absent`, `error` +- `provider_state`: `unknown`, `creating`, `starting`, `stopping`, `destroying`, + `running`, `stopped`, `absent`, `error` - `provision_state`: `unknown`, `provisioning`, `provisioned`, `error` ## Package States Package entries under `package_state` use: -- `unknown` +- `failed` - `installed` +- `installing` - `missing` -- `failed` - `removed` -- `reinstalled` +- `removing` +- `unknown` -Package state updates must pass both `--package` and `--package-state`. +Dispatchers derive package state by firing operation and observation events into +`core/fsm/package.yaml`. The `--package-state` pair remains available only for explicit +compatibility writes. ## Observed State Cache @@ -52,13 +55,14 @@ commands, so they do not append lifecycle operations to `operation_history`. Every write records `last_operation` and appends to `operation_history`. Operation entries include: -- `id`: monotonic per retained history window +- `id`: monotonic even after the retained history window is trimmed - `name`: full operation name, such as `provider.up`, `package.status`, or `provision` - `type`: operation prefix, such as `provider`, `package`, or `provision` - `status`: `running`, `succeeded`, `failed`, or `skipped` - `at`: UTC timestamp - `error`: present only for failed or error-bearing entries +- `package`: package id, present on package operations The history keeps the latest 50 entries by default. @@ -79,8 +83,7 @@ the local state before retrying: eve instance recover --instance ``` -This marks the last running operation as `failed`, records a recovery error, and -sets `provider_state` or `provision_state` to `error` when the interrupted -operation belongs to that surface. It does not destroy or change remote -resources. After recovery, run `eve instance status --instance ` and retry -the relevant lifecycle/provision/package command. +This marks the last running operation as `failed`, records a recovery error, and fires +the matching failure event into the provider, provision, or package machine. It does not +destroy or change remote resources. After recovery, run +`eve instance status --instance ` and retry the relevant command. diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index 4e95d5b5..da669699 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -5,12 +5,12 @@ FSM context). Engine lives in core; **every machine is declared in YAML by the plugin that owns it**. The interface is kept thin so the engine can later be replaced by a cns-based implementation (ref: github.com/fruwe/cns_statemachine). -> **Status (v4.5):** the engine is now real — [**harel**](https://github.com/fruwehq/harel) -> (spec) + [harel-python](https://github.com/fruwehq/harel-python) (reference impl, -> passing the conformance suite). The original `state_machine:`/`transitions:` sketch -> below is **superseded by harel's actual YAML** (`top`/`esvs`/`events`/`on_events`/ -> `transition_to`) — see **"Reconciliation with harel"** at the end. The eve↔harel seam -> is `eve_sdk/fsm.py` (`EveFsm`); increment 1 (platform + a proving test) has landed. +> **Status (v4.5):** Eve tracks **Determa State 0.2.x** +> ([spec](https://github.com/fruwehq/determa-state-spec) and +> [Python reference implementation](https://github.com/fruwehq/determa-state-python)). +> The original `state_machine:`/`transitions:` sketch below is superseded by +> Determa's format-1 bundles; see **"Reconciliation with Determa State"** below. +> The Eve↔Determa seam is `eve_sdk/fsm.py` (`EveFsm`). ## Why @@ -188,8 +188,8 @@ keep persistence, but behind `FSM`'s storage seam. The engine is real now, so this supersedes the speculative `state_machine:` schema above. (History: the engine started as **harel**, was renamed **determa-state**, and -at spec **v0.1.0** pivoted to *format-1 bundles* + a *pure-functional interface* — -eve tracks 0.1.0.) +at spec **v0.1.0** pivoted to *format-1 bundles* + a *pure-functional interface*. +Eve now tracks the compatible 0.2.x line.) - **Machines are format-1 bundles.** A plugin ships a Determa State bundle: `format: 1`, a `namespace`, shared `events` (host-delivered events are `direction: input`), and @@ -225,29 +225,31 @@ eve tracks 0.1.0.) - ☑ **Increment 1** — `EveFsm` + a synthetic provider machine (`tests/fsm_fixtures/provider.yaml`) + `tests/test_fsm_platform.py` proving configured/reachable are FSM-computed from injected context and re-derive on a settings - change. The engine is editable-installed for dev; the test skips if neither - `determa_state` nor `harel` is installed. + change. Determa State is a required project dependency; the FSM tests do not skip. - ☑ **Real resolver** — `provider_context_resolver()` in `eve_sdk/fsm.py`: resolves each - declared external esv (named as its eve env var) from the one path dispatch uses - (`os.environ` + `ConfigEnv.environment()` + `_inject_secrets`), hermetically tested. So - a provider's FSM context ≡ what dispatch runs with (bug M). Systemic follow-up: drop - ambient `os.environ` from both this and dispatch. + declared external variable (named as its eve env var) through + `resolved_provider_environment()`, the same builder used by dispatch and connectivity. + Required fields are read from the manifest's actual `config_schema.config` and + `config_schema.secrets` blocks. Provider and package commands receive operation-local + environments: explicit process transport/routing variables, core configuration, and + only that target plugin's declared config and secrets. Ambient or sibling-plugin + credentials cannot cross the command boundary. - ☑ **Core provider machine** — `core/fsm/provider.yaml`: one shared provider lifecycle (unconfigured → configured → reachable) for *every* provider. `is_configured` is derived by `provider_configured_resolver()` from the plugin's `config_schema` - `required` secrets/settings (via the shared path), so "configured" is eve-resolved, not - a plugin's ambient self-report. The lifecycle is identical across providers, so it lives - in `core/`, not per-provider. Tested end-to-end. -- ☑ **Dependency** — `determa-state (>=0.1.0,<0.2.0)` from PyPI, locked; the FSM tests + universally `required` secrets/config fields plus the optional provider-only + `required_any` alternative groups (via the shared path), so "configured" is + eve-resolved, not a plugin's ambient self-report. The lifecycle is identical across + providers, so it lives in `core/`, not per-provider. Tested end-to-end. +- ☑ **Dependency** — `determa-state (>=0.2.0,<0.3.0)` from PyPI, locked; the FSM tests run in CI. - ☑ **Rewire `providers-status`** — `configured` is now FSM-derived (`core/fsm/provider.yaml` + `provider_configured_resolver`); the `connectivity` command runs **only for `reachable`** (its self-reported `configured` is ignored) and only when configured. Live-verified (vultr yes/yes, truenas no, docker/qemu yes); full suite green. - - *AWS follow-up (eve-providers):* AWS declares only a non-required `config_file`, so - it reads `configured=yes / reachable=no` (readiness surfaces as reachability + note). - If AWS should read `configured=no` until eve holds an auth profile, mark an auth - setting `required` or model AWS's configured as probe-derived — a per-provider call. + - AWS uses `required_any` to express a profile **or** a complete static access-key pair; + all static credential fields map to their AWS environment names. A partial key pair + remains unconfigured, while connectivity remains a separate live probe. - ☑ **Instance & package machines (declarative)** — `core/fsm/instance.yaml` (two machines in one bundle: `instance_provider` + `instance_provision`, since format 1 has no orthogonal regions; `desired` as external context) + `core/fsm/package.yaml`, @@ -263,7 +265,7 @@ eve tracks 0.1.0.) write the machine vocabulary (`creating/starting/stopping/destroying`, `init`/`plan`→ `creating`, package `reinstall`→`installed`); `state.py`'s state sets, the `observed-state`/`command-io` schemas, the `state_machine`/`sdk/test_state*` tests, the - `valid-pending` fixture, and `test-plugins` all follow. Full suite green (496 passed). + `valid-pending` fixture, and `test-plugins` all follow. Full suite green. - ☑ **FSM-driven status (write path):** `provider_state` is no longer a hand-maintained lookup table. `eve_sdk/instance_fsm.py` is the single lifecycle authority: dispatch maps a `(command, op_status)` to a machine event (`dispatch_event`) and @@ -275,6 +277,10 @@ eve tracks 0.1.0.) `provider_command.py` + `scripts/provider-dispatch` fire events. Live-validated on a real local-qemu instance (create→running, stop→stopped, start→running, destroy→absent, failure→error) + `scripts/test-lifecycle` green + a new `test_state.py` case. + - ☑ **FSM-driven package state:** install, reinstall, removal, live status, interrupted + operation recovery, and full-provision observations all fire events into + `core/fsm/package.yaml`. Root-level observation transitions let authoritative probes + reconcile an interrupted `installing` or `removing` state. - *Machine change:* every concrete provider state now accepts every operator command (routing to its transient), so a command's terminal outcome holds from any state (start recovers `absent`→`running`) — the provider command, not the machine, decides @@ -285,8 +291,11 @@ eve tracks 0.1.0.) precedence), and `aggregate_summary` (counting) are **display/aggregation policy, not a second lifecycle definition** — there is now exactly one transition authority (the machine), so no divergence risk remains. They stay as a thin status-render adapter. - (Provision state still uses the persisted `unknown`/`error` vocabulary; wiring the - `instance_provision` machine the same way is the next increment.) + - ☑ **FSM-driven provision state:** `State.record_operation` maps provision operation + statuses to `provision`/`provision_ok`/`provision_fail` and fires them through the + `instance_provision` machine atomically. The machine uses the persisted + `unknown`/`provisioning`/`provisioned`/`error` vocabulary; interrupted provider and + provision operations also recover through machine events. - ☑ **Engine 0.1.0 port** — determa-state's v0.1.0 pivoted to format-1 bundles + the pure-functional §8 interface (`create`/`dispatch` over plain aggregate-state dicts; the old `Host`/`Instance` object API is gone). All three core machines rewritten as @@ -296,6 +305,12 @@ eve tracks 0.1.0.) provider leaf once by replaying a short event prefix and dispatches against the cached aggregate (dispatch is pure, the cache stays valid). The harel fallback import is gone. The engine's IntType leak (old #40) is moot — 0.1.0 returns native values. +- ☑ **Engine 0.2.0 upgrade** — 0.2.0 preserves the format-1 grammar and pure + `load_bundle`/`create`/`dispatch` contract used by Eve. Its optional `ExecutionHost` + and checkpoint stores are intentionally not adopted here: Eve already commits its + operation metadata and lifecycle fields atomically in one locked state document. + Seed-by-replay remains limited to the current context-free lifecycle graphs; future + stateful machines must persist and restore full aggregates. ### Open decisions — resolved 1. Engine repo/name → **Determa State** (`fruwehq/determa-state-spec` spec, diff --git a/eve_sdk/config.py b/eve_sdk/config.py index 61d1e20b..6517c925 100644 --- a/eve_sdk/config.py +++ b/eve_sdk/config.py @@ -212,13 +212,15 @@ def bootstrap_sudo_password(cls, provider_id: str) -> str: provider declares none or the value is unset. """ from eve_sdk.plugin_manifest import PluginManifest + from eve_sdk.provider_command import resolved_provider_environment for plugin in PluginManifest.load_all("provider"): if plugin.get("id") != provider_id: continue bootstrap = plugin.get("bootstrap") if isinstance(bootstrap, dict) and bootstrap.get("sudo_password_env"): - return os.environ.get(str(bootstrap["sudo_password_env"]), "") + env = resolved_provider_environment(provider_id, plugin) + return env.get(str(bootstrap["sudo_password_env"]), "") return "" @classmethod @@ -256,7 +258,12 @@ def plugin_provision_env_names( return sorted(names) @classmethod - def provision_env_payload(cls, windows_password: str = "") -> dict[str, str]: + def provision_env_payload( + cls, + windows_password: str = "", + environment: dict[str, str] | None = None, + package_env_names: list[str] | None = None, + ) -> dict[str, str]: """Build the generic provision ``env.json`` payload (v4.4 §15). Core keys (``windows_password``, ``display_resolution``) plus every @@ -265,12 +272,18 @@ def provision_env_payload(cls, windows_password: str = "") -> dict[str, str]: provision steps read their keys (e.g. a streaming host's version, a remote …) by construction — core names no package. Unset vars are omitted. """ + source = os.environ if environment is None else environment payload: dict[str, str] = { "windows_password": windows_password, - "display_resolution": os.environ.get("EPHEMERAL_DISPLAY_RESOLUTION", ""), + "display_resolution": source.get("EPHEMERAL_DISPLAY_RESOLUTION", ""), } - for _path, env_var, _is_path in cls._plugin_mappings(kinds=("package",)): - value = os.environ.get(env_var, "") + env_names = ( + cls.plugin_provision_env_names(kinds=("package",)) + if package_env_names is None + else sorted(set(package_env_names)) + ) + for env_var in env_names: + value = source.get(env_var, "") if not value: continue key = env_var.lower().removeprefix("ephemeral_") diff --git a/eve_sdk/dispatch.py b/eve_sdk/dispatch.py index 5310adc1..820abccc 100644 --- a/eve_sdk/dispatch.py +++ b/eve_sdk/dispatch.py @@ -21,12 +21,80 @@ class DispatchError(Exception): pass +_PROCESS_ENV_NAMES = { + "COLORTERM", + "DBUS_SESSION_BUS_ADDRESS", + "DISPLAY", + "EVE_ALLOW_UNPINNED_PLUGINS", + "EVE_CONFIG_PATH", + "EVE_DISABLE_STATE", + "EVE_HOME", + "EVE_INSTANCE_REGISTRY", + "EVE_INSTANCE_WORKDIR", + "EVE_PACKAGE_DOWN_SSH", + "EVE_PACKAGE_STATUS_SSH", + "EVE_PLUGIN_ALLOW_OVERRIDE", + "EVE_PLUGIN_DRY_RUN", + "EVE_PLUGIN_ROOTS", + "EVE_PLUGIN_ROOTS_EXCLUSIVE", + "EVE_PLUGIN_SOURCES", + "EVE_PROVIDER_DRY_RUN", + "EVE_PROVISION_DRY_RUN", + "EVE_PROVISION_SCRIPT", + "EVE_PROVISION_WAIT_POLL_INTERVAL", + "EVE_SECRETS_DIR", + "EVE_SSH_RETRY_ATTEMPTS", + "EVE_SSH_RETRY_DELAY", + "EVE_STATE_DIR", + "EVE_TF_DATA_BASE", + "EVE_TF_ENV_JSON", + "EVE_TF_PRINT", + "EVE_TF_STATE_BASE", + "EVE_TM_READ_FLAGS", + "EVE_UPLOAD_DIR", + "EVE_WAIT_FOR_PROVISION_PROBE_SCRIPT", + "HOME", + "HTTPS_PROXY", + "HTTP_PROXY", + "LANG", + "LANGUAGE", + "LOGNAME", + "NO_PROXY", + "PATH", + "REQUESTS_CA_BUNDLE", + "SHELL", + "SSH_AGENT_PID", + "SSH_AUTH_SOCK", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "TEMP", + "TERM", + "TMP", + "TMPDIR", + "USER", + "WAYLAND_DISPLAY", + "XDG_RUNTIME_DIR", + "http_proxy", + "https_proxy", + "no_proxy", +} + + +def process_environment() -> dict[str, str]: + """Explicit host transport/routing environment for an outer command boundary.""" + return { + name: value + for name, value in os.environ.items() + if name in _PROCESS_ENV_NAMES or name.startswith("LC_") + } + + def command_env(extra: dict[str, str] | None = None) -> dict[str, str]: """Env for invoking plugin/script commands. Ensures the eve repo root is on PYTHONPATH so external (synced) plugin command scripts can `import eve_sdk` regardless of where they live on disk (their own `parents[N]` path assumes the in-repo layout and is wrong once extracted).""" - env = os.environ | (extra or {}) + env = dict(os.environ) if extra is None else dict(extra) root = str(Workdir.repo_root()) existing = env.get("PYTHONPATH", "") env["PYTHONPATH"] = root + (os.pathsep + existing if existing else "") @@ -163,6 +231,7 @@ def record_package_state( command: str, status: str, package_id: str, + package_event: str | None = None, package_state: str | None = None, error: str | None = None, ) -> None: @@ -173,6 +242,7 @@ def record_package_state( f"package.{command}", status, package=package_id, + package_event=package_event, package_state=package_state, error=error, ) @@ -203,26 +273,20 @@ def interactive_provider_command(plugin: dict[str, Any], command: str) -> bool: def last_json_object(output: str, keys: set[str]) -> dict[str, Any] | None: - for index in range(len(output) - 1, -1, -1): - if output[index] != "{": - continue - try: - parsed = json.loads(output[index:].strip()) - except json.JSONDecodeError: - continue - if isinstance(parsed, dict) and keys.intersection(parsed): - return parsed - for line in reversed(output.splitlines()): - line = line.strip() - if not line: - continue + """Find the last matching top-level object, skipping its nested values.""" + decoder = json.JSONDecoder() + result = None + offset = 0 + while (index := output.find("{", offset)) != -1: try: - parsed = json.loads(line) + parsed, end = decoder.raw_decode(output, index) except json.JSONDecodeError: + offset = index + 1 continue + offset = end if isinstance(parsed, dict) and keys.intersection(parsed): - return parsed - return None + result = parsed + return result def validate_provider_output(output: str) -> None: diff --git a/eve_sdk/fsm.py b/eve_sdk/fsm.py index f8546aad..09d6a673 100644 --- a/eve_sdk/fsm.py +++ b/eve_sdk/fsm.py @@ -168,26 +168,20 @@ def view(self, scope: str, key: str) -> dict[str, Any]: def provider_context_resolver() -> ContextResolver: """The real :data:`ContextResolver` for provider machines. - Each declared external variable is named as its eve env var (e.g. ``VULTR_API_KEY``) - and resolved from the *one* path dispatch uses — ``os.environ`` + - ``ConfigEnv.environment()`` + injected secrets — so a provider's FSM context and what - dispatch runs with are identical (v4.5 bug M: status ≡ dispatch). ``key`` is the - provider id. + Each declared external variable is named as its eve env var (e.g. ``PROVIDER_API_KEY``) + and resolved from the one environment builder dispatch uses, so a provider's + FSM context and what dispatch runs with are identical (v4.5 bug M: status ≡ + dispatch). ``key`` is the provider id. (Systemic follow-up: drop ambient ``os.environ`` from *both* this resolver and dispatch so no config reaches a plugin behind eve's back; kept here for now to preserve exact parity.) """ - import os - - from eve_sdk.config import ConfigEnv - from eve_sdk.provider_command import _inject_secrets, _load_public_plugin + from eve_sdk.provider_command import _load_public_plugin, resolved_provider_environment def resolve(scope: str, key: str, names: list[str]) -> dict[str, Any]: - env = dict(os.environ) - env.update(ConfigEnv.environment()) plugin = _load_public_plugin("provider", key) - env = _inject_secrets(key, plugin, env) + env = resolved_provider_environment(key, plugin) return {name: env.get(name, "") for name in names} return resolve @@ -196,26 +190,21 @@ def resolve(scope: str, key: str, names: list[str]) -> dict[str, Any]: def provider_configured_resolver() -> ContextResolver: """The :data:`ContextResolver` for the core provider machine (``core/fsm/provider.yaml``). - Derives ``is_configured`` = True iff every ``required`` config_schema secret/setting - of the provider resolves to a non-empty value via the one shared path (``os.environ`` - + ``ConfigEnv.environment()`` + injected secrets). So "configured" is eve-resolved, - never a plugin's ambient self-report (the AWS-off-``~/.aws`` bug is structurally - impossible). ``key`` is the provider id; ``names`` is typically ``["is_configured"]``. + Derives ``is_configured`` from both universally ``required`` fields and the + optional ``required_any`` alternatives. Every universal field must be set, + and at least one alternative group must be complete. So "configured" is + eve-resolved, never a plugin's connectivity self-report. ``key`` is the + provider id; ``names`` is typically ``["is_configured"]``. """ - import os - - from eve_sdk.config import ConfigEnv - from eve_sdk.provider_command import _inject_secrets, _load_public_plugin + from eve_sdk.provider_command import _load_public_plugin, resolved_provider_environment def resolve(scope: str, key: str, names: list[str]) -> dict[str, Any]: plugin = _load_public_plugin("provider", key) - env = dict(os.environ) - env.update(ConfigEnv.environment()) - env = _inject_secrets(key, plugin, env) + env = resolved_provider_environment(key, plugin) schema = plugin.get("config_schema") or {} required = [ (field, spec) - for section in ("secrets", "settings") + for section in ("config", "secrets") for field, spec in (schema.get(section) or {}).items() if isinstance(spec, Mapping) and spec.get("required") ] @@ -223,11 +212,27 @@ def resolve(scope: str, key: str, names: list[str]) -> dict[str, Any]: def present(field: str, spec: Mapping[str, Any]) -> bool: candidates = [field] env_var = spec.get("env_var") - if env_var: + if isinstance(env_var, list): + candidates.extend(str(name) for name in env_var) + elif env_var: candidates.append(str(env_var)) return any(env.get(candidate) for candidate in candidates) - values: dict[str, Any] = {"is_configured": all(present(f, s) for f, s in required)} + alternatives = schema.get("required_any") or [] + + def referenced_present(reference: str) -> bool: + section, field = reference.split(".", 1) + spec = (schema.get(section) or {}).get(field) + return isinstance(spec, Mapping) and present(field, spec) + + required_ok = all(present(field, spec) for field, spec in required) + alternatives_ok = not alternatives or any( + all(referenced_present(str(reference)) for reference in group) + for group in alternatives + ) + values: dict[str, Any] = { + "is_configured": required_ok and alternatives_ok + } return {name: values.get(name, "") for name in names} return resolve diff --git a/eve_sdk/instance_fsm.py b/eve_sdk/instance_fsm.py index b6e6f989..f3261575 100644 --- a/eve_sdk/instance_fsm.py +++ b/eve_sdk/instance_fsm.py @@ -1,20 +1,19 @@ """Instance-lifecycle transitions, driven by the core instance machine. -The single authority for how a provider operation moves an instance's -``provider_state``: dispatch computes the next state by **firing an event into the -``instance_provider`` machine of ``core/fsm/instance.yaml``** seeded from the -persisted state, not from a second, hand-maintained transition table. The machine -bundle *is* the lifecycle; this module is the thin, fast adapter eve calls (and the -persisted ``provider_state`` a status read sees is therefore exactly what the -machine produced — bug N). +The single authority for how an instance operation moves its ``provider_state``, +``provision_state``, or per-package state: dispatch computes the next state by +**firing an event into the corresponding core machine** seeded from the persisted +state, not from a second, hand-maintained transition table. The machine bundle *is* +the lifecycle; this module is the thin, fast adapter Eve calls (and the persisted +state a status read sees is therefore exactly what the machine produced — bug N). The engine is pure-functional (``create``/``dispatch`` over plain aggregate-state -dicts), so "seeding from the persisted state" is: reach each provider leaf once from -a fresh aggregate by replaying a short event prefix, cache that aggregate per leaf, -then dispatch the real event against it — dispatch never mutates its input, so the -cache stays valid. Only the provider machine is driven here; the persisted -``provision_state`` field keeps its own vocabulary until the ``instance_provision`` -machine is wired the same way. +dicts), so "seeding from the persisted state" is: reach each lifecycle leaf once +from a fresh aggregate by replaying a short event prefix, cache that aggregate per +leaf, then dispatch the real event against it — dispatch never mutates its input, so +the cache stays valid. This is deliberately limited to these context-free finite +graphs; a future stateful machine must persist and restore its full aggregate instead +of extending seed-by-replay. """ from __future__ import annotations @@ -42,7 +41,7 @@ # The shortest event prefix that reaches each leaf from a fresh aggregate; every # concrete state accepts every operator command, so each transient is one event away # and `error` is two. -_SEED_EVENTS: dict[str, list[str]] = { +_PROVIDER_SEED_EVENTS: dict[str, list[str]] = { "unknown": [], "creating": ["create"], "starting": ["start"], @@ -54,14 +53,52 @@ "error": ["create", "op_fail"], } -_MACHINE_ID = "instance_provider" +PROVISION_LEAVES = {"unknown", "provisioning", "provisioned", "error"} -_bundle: Any = None -_seeded: dict[str, dict[str, Any]] = {} +_PROVISION_SEED_EVENTS: dict[str, list[str]] = { + "unknown": [], + "provisioning": ["provision"], + "provisioned": ["provision", "provision_ok"], + "error": ["provision", "provision_fail"], +} + +PACKAGE_LEAVES = { + "failed", + "installed", + "installing", + "missing", + "removed", + "removing", + "unknown", +} + +_PACKAGE_SEED_EVENTS: dict[str, list[str]] = { + "failed": ["observe_failed"], + "installed": ["found"], + "installing": ["install"], + "missing": ["absent"], + "removed": ["remove", "remove_ok"], + "removing": ["remove"], + "unknown": [], +} + +_bundles: dict[str, Any] = {} +_seeded: dict[tuple[str, str, str], dict[str, Any]] = {} _sequence = 0 -def _dispatch(state: dict[str, Any], event: str) -> dict[str, Any]: +def _bundle(filename: str) -> Any: + bundle = _bundles.get(filename) + if bundle is None: + bundle_yaml = (Workdir.repo_root() / "core" / "fsm" / filename).read_text( + encoding="utf-8" + ) + bundle = engine.load_bundle(bundle_yaml) + _bundles[filename] = bundle + return bundle + + +def _dispatch(bundle: Any, state: dict[str, Any], event: str) -> dict[str, Any]: """One input-event RTC step; returns the (new) aggregate state.""" global _sequence _sequence += 1 @@ -76,24 +113,26 @@ def _dispatch(state: dict[str, Any], event: str) -> dict[str, Any]: }, "payload": {}, } - result = engine.dispatch(_bundle, state, {"input": envelope}) + result = engine.dispatch(bundle, state, {"input": envelope}) new_state: dict[str, Any] = result["state"] return new_state -def _seeded_aggregate(leaf: str) -> dict[str, Any]: - """The cached aggregate whose provider machine sits at ``leaf``.""" - global _bundle - if _bundle is None: - bundle_yaml = (Workdir.repo_root() / "core" / "fsm" / "instance.yaml").read_text(encoding="utf-8") - _bundle = engine.load_bundle(bundle_yaml) - state = _seeded.get(leaf) +def _seeded_aggregate( + filename: str, machine_id: str, leaf: str, seed_events: dict[str, list[str]] +) -> dict[str, Any]: + """The cached aggregate whose selected lifecycle machine sits at ``leaf``.""" + bundle = _bundle(filename) + cache_key = (filename, machine_id, leaf) + state = _seeded.get(cache_key) if state is None: - state = engine.create(_bundle, _MACHINE_ID, leaf, f"eve-seed-{leaf}", None)["state"] - for event in _SEED_EVENTS[leaf]: - state = _dispatch(state, event) + state = engine.create( + bundle, machine_id, leaf, f"eve-seed-{machine_id}-{leaf}", None + )["state"] + for event in seed_events[leaf]: + state = _dispatch(bundle, state, event) assert leaf_names(state) == [leaf] - _seeded[leaf] = state + _seeded[cache_key] = state return state @@ -134,5 +173,84 @@ def provider_state_after(current: str, event: str | None) -> str | None: if event is None: return None leaf = current if current in PROVIDER_LEAVES else "unknown" - moved = leaf_names(_dispatch(_seeded_aggregate(leaf), event)) + bundle = _bundle("instance.yaml") + moved = leaf_names( + _dispatch( + bundle, + _seeded_aggregate( + "instance.yaml", "instance_provider", leaf, _PROVIDER_SEED_EVENTS + ), + event, + ) + ) + return moved[0] if moved else leaf + + +def provision_event(op_status: str) -> str | None: + """Map a provision operation status to an instance-provision event.""" + return { + "running": "provision", + "succeeded": "provision_ok", + "failed": "provision_fail", + }.get(op_status) + + +def provision_state_after(current: str, event: str | None) -> str | None: + """The provision_state after firing ``event`` from ``current``.""" + if event is None: + return None + leaf = current if current in PROVISION_LEAVES else "unknown" + bundle = _bundle("instance.yaml") + moved = leaf_names( + _dispatch( + bundle, + _seeded_aggregate( + "instance.yaml", "instance_provision", leaf, _PROVISION_SEED_EVENTS + ), + event, + ) + ) + return moved[0] if moved else leaf + + +def package_operation_event(command: str, op_status: str) -> str | None: + """Map a package operation status to a package-machine event.""" + if command in {"install", "reinstall"}: + return { + "failed": "install_fail", + "running": "install", + "succeeded": "install_ok", + }.get(op_status) + if command == "down": + return { + "failed": "remove_fail", + "running": "remove", + "succeeded": "remove_ok", + }.get(op_status) + return None + + +def package_observation_event(status: str) -> str | None: + """Map a validated package status result to an observation event.""" + return { + "failed": "observe_failed", + "installed": "found", + "missing": "absent", + "unknown": "observe_unknown", + }.get(status) + + +def package_state_after(current: str, event: str | None) -> str | None: + """The package state after firing ``event`` from ``current``.""" + if event is None: + return None + leaf = current if current in PACKAGE_LEAVES else "unknown" + bundle = _bundle("package.yaml") + moved = leaf_names( + _dispatch( + bundle, + _seeded_aggregate("package.yaml", "package", leaf, _PACKAGE_SEED_EVENTS), + event, + ) + ) return moved[0] if moved else leaf diff --git a/eve_sdk/package_dispatch.py b/eve_sdk/package_dispatch.py index dc0606b3..320f416d 100644 --- a/eve_sdk/package_dispatch.py +++ b/eve_sdk/package_dispatch.py @@ -22,16 +22,20 @@ from pathlib import Path from typing import Any +from eve_sdk.config import ConfigEnv from eve_sdk.dispatch import ( DispatchError, command_vector, package_status_from_output, prepare_overlay, + process_environment, record_package_state, stream_command, validate_package_support, ) +from eve_sdk.instance_fsm import package_observation_event from eve_sdk.plugin_manifest import PluginManifest +from eve_sdk.provider_command import _inject_secrets from eve_sdk.resolve import resolve_instance from eve_sdk.schema import SchemaValidationError, validate_input @@ -211,6 +215,7 @@ def _run_plugin( resolved: dict[str, Any], dry_run: bool, on_output: Callable[[str], None] | None = None, + operation_env: dict[str, str] | None = None, ) -> str: commands = plugin.get("commands") if not isinstance(commands, dict): @@ -226,12 +231,13 @@ def _run_plugin( os_paths_raw = plugin.get("os_paths") os_paths = os_paths_raw if isinstance(os_paths_raw, dict) else {} manifest_path = os_paths.get(os_family) or plugin["path"] - env = os.environ | { + env = resolved_package_environment(plugin, resolved, operation_env) + env.update({ "EVE_INSTANCE_NAME": resolved["instance"]["name"], "EVE_PACKAGE_PLUGIN": str(plugin["id"]), "EVE_PACKAGE_PLUGIN_ROOT": str(Path(str(manifest_path)).parent), "EVE_RESOLVED_JSON": json.dumps(resolved, separators=(",", ":")), - } + }) if dry_run: env["EVE_PLUGIN_DRY_RUN"] = "1" @@ -246,6 +252,85 @@ def _run_plugin( return output +def resolved_package_environment( + plugin: dict[str, Any], + resolved: dict[str, Any], + overrides: dict[str, str] | None = None, +) -> dict[str, str]: + """Build an operation-local environment for one package plugin.""" + package_id = str(plugin["id"]) + all_package_config = resolved.get("package_config") or {} + package_config = ( + {package_id: all_package_config[package_id]} + if package_id in all_package_config + else {} + ) + return resolved_package_set_environment( + [plugin], package_config, overrides=overrides + ) + + +def resolved_package_set_environment( + plugins: list[dict[str, Any]], + package_config: dict[str, Any], + *, + overrides: dict[str, str] | None = None, +) -> dict[str, str]: + """Build a trusted provision environment for an applicable package set.""" + declared_names = { + str(name) + for plugin in plugins + for section in ("config", "secrets") + for field in ((plugin.get("config_schema") or {}).get(section) or {}).values() + if isinstance(field, dict) and field.get("env_var") + for name in ( + field["env_var"] + if isinstance(field["env_var"], list) + else [field["env_var"]] + ) + } + core_names = {name for _path, name, _is_path in ConfigEnv.MAPPINGS} + configured = ConfigEnv.environment() + env = process_environment() + env.update( + { + name: value + for name, value in configured.items() + if name in core_names or name in declared_names + } + ) + env.update(ConfigEnv.instance_package_env(package_config)) + for plugin in plugins: + env = _inject_secrets(str(plugin["id"]), plugin, env) + env.update(overrides or {}) + return env + + +def package_environment_names( + plugins: list[dict[str, Any]], *, path_config_only: bool = False +) -> list[str]: + """Declared environment names for an applicable package set.""" + sections = ("config",) if path_config_only else ("config", "secrets") + return sorted( + { + str(name) + for plugin in plugins + for section in sections + for field in ( + (plugin.get("config_schema") or {}).get(section) or {} + ).values() + if isinstance(field, dict) + and field.get("env_var") + and (not path_config_only or field.get("type") == "path") + for name in ( + field["env_var"] + if isinstance(field["env_var"], list) + else [field["env_var"]] + ) + } + ) + + def dispatch_package( instance_name: str, package_id: str, @@ -290,9 +375,9 @@ def dispatch_package( ) overlay = prepare_overlay(instance_name, registry_path) - os.environ["EVE_CATALOG_LOCAL"] = overlay + operation_env = {"EVE_CATALOG_LOCAL": overlay} if registry_path: - os.environ["EVE_INSTANCE_REGISTRY"] = registry_path + operation_env["EVE_INSTANCE_REGISTRY"] = registry_path # Record "running" only after lookup/validation/overlay prep succeeded — # so a missing-package or unsupported-version failure records only the @@ -301,30 +386,54 @@ def dispatch_package( record_package_state(instance_name, command, "running", package_id) if command == "reinstall" and "reinstall" not in plugin.get("commands", {}): - _run_plugin(plugin, "down", resolved, dry_run, on_output) - output = _run_plugin(plugin, "install", resolved, dry_run, on_output) + _run_plugin( + plugin, "down", resolved, dry_run, on_output, operation_env + ) + output = _run_plugin( + plugin, "install", resolved, dry_run, on_output, operation_env + ) else: - output = _run_plugin(plugin, command, resolved, dry_run, on_output) + output = _run_plugin( + plugin, command, resolved, dry_run, on_output, operation_env + ) except Exception as error: if not dry_run: try: record_package_state( - instance_name, command, "failed", package_id, "failed", str(error) + instance_name, command, "failed", package_id, error=str(error) ) except Exception as state_error: print(f"package-dispatch: failed to record failure state: {state_error}", file=sys.stderr) print(f"package-dispatch: {error}", file=sys.stderr) return 1 - package_state = { - "install": "installed", - "down": "removed", - "reinstall": "installed", # cleaner vocabulary: a reinstall lands in installed - }.get(command) - if package_state is None: - package_state = package_status_from_output(output) or "unknown" - if not dry_run: - record_package_state(instance_name, command, "succeeded", package_id, package_state) + try: + package_event = None + if command == "status": + package_event = package_observation_event( + package_status_from_output(output) or "unknown" + ) + if not dry_run: + record_package_state( + instance_name, + command, + "succeeded", + package_id, + package_event=package_event, + ) + except Exception as error: + if not dry_run: + try: + record_package_state( + instance_name, command, "failed", package_id, error=str(error) + ) + except Exception as state_error: + print( + f"package-dispatch: failed to record failure state: {state_error}", + file=sys.stderr, + ) + print(f"package-dispatch: {error}", file=sys.stderr) + return 1 return 0 diff --git a/eve_sdk/plugin_manifest.py b/eve_sdk/plugin_manifest.py index 5d81e6bb..381a16e6 100644 --- a/eve_sdk/plugin_manifest.py +++ b/eve_sdk/plugin_manifest.py @@ -282,8 +282,34 @@ def validate(cls, plugin: dict[str, Any]) -> None: if not isinstance(config_schema, dict): raise ValueError(f"{path}: config_schema must be a map") validate_json_schema_fragment(config_schema, f"{path}: config_schema") + cls._validate_config_required_any(config_schema, path, str(kind)) cls._validate_requires(plugin, path) + @staticmethod + def _validate_config_required_any( + config_schema: dict[str, Any], path: str, kind: str + ) -> None: + alternatives = config_schema.get("required_any") or [] + if alternatives and kind != "provider": + raise ValueError( + f"{path}: config_schema.required_any is only valid on provider plugins" + ) + for group in alternatives: + for reference in group: + section, field = str(reference).split(".", 1) + fields = config_schema.get(section) or {} + if field not in fields: + raise ValueError( + f"{path}: config_schema.required_any references " + f"undeclared field {reference}" + ) + spec = fields[field] + if not isinstance(spec, dict) or not spec.get("env_var"): + raise ValueError( + f"{path}: config_schema.required_any field {reference} " + "must declare env_var" + ) + @staticmethod def public(plugin: dict[str, Any]) -> dict[str, Any]: output = {key: value for key, value in plugin.items() if not key.startswith("_")} diff --git a/eve_sdk/provider_command.py b/eve_sdk/provider_command.py index c2ee8fe4..50e48aae 100644 --- a/eve_sdk/provider_command.py +++ b/eve_sdk/provider_command.py @@ -14,6 +14,7 @@ exec_cmd, interactive_provider_command, prepare_overlay, + process_environment, read_resolved_from_env_or_stdin, record_provider_state, stream_command, @@ -128,15 +129,49 @@ def _inject_secrets( return env schema_secrets = (plugin.get("config_schema") or {}).get("secrets") or {} for secret_key, value in secrets.items(): - env[secret_key] = value schema_entry = schema_secrets.get(secret_key) - if isinstance(schema_entry, dict): - mapped = schema_entry.get("env_var") - if mapped: - env[mapped] = value + if not isinstance(schema_entry, dict): + continue + mapped = schema_entry.get("env_var") + if mapped: + for name in mapped if isinstance(mapped, list) else [mapped]: + env[str(name)] = value return env +def resolved_provider_environment( + provider_name: str, + plugin: dict[str, Any], + overrides: dict[str, str] | None = None, +) -> dict[str, str]: + """Build the one provider environment used by status and dispatch.""" + from eve_sdk.config import ConfigEnv + + declared_names = { + str(name) + for section in ("config", "secrets") + for field in ((plugin.get("config_schema") or {}).get(section) or {}).values() + if isinstance(field, dict) and field.get("env_var") + for name in ( + field["env_var"] + if isinstance(field["env_var"], list) + else [field["env_var"]] + ) + } + core_names = {name for _path, name, _is_path in ConfigEnv.MAPPINGS} + configured = ConfigEnv.environment() + env = process_environment() + env.update( + { + name: value + for name, value in configured.items() + if name in core_names or name in declared_names + } + ) + env.update(overrides or {}) + return _inject_secrets(provider_name, plugin, env) + + def _select_plugin(plugins: list[dict[str, Any]], kind: str, plugin_id: str) -> dict[str, Any]: for plugin in plugins: if ( @@ -179,7 +214,9 @@ def dispatch_provider_command( if not isinstance(spec, dict): raise DispatchError(f"provider plugin {provider_name} has no command: {command}") cmd = command_vector(plugin, spec) + list(extra_args) - env = _inject_secrets(provider_name, plugin, dict(os.environ) | {"EVE_PROVIDER_PLUGIN": provider_name}) + env = resolved_provider_environment( + provider_name, plugin, {"EVE_PROVIDER_PLUGIN": provider_name} + ) if dry_run: env["EVE_PLUGIN_DRY_RUN"] = "1" print(json.dumps({"kind": "provider", "provider": provider_name, "command": command, "dry_run": True})) @@ -232,12 +269,16 @@ def dispatch_instance_command( return 0 overlay = prepare_overlay(instance_name, registry_path) - env = _inject_secrets(provider, plugin, dict(os.environ) | { - "EVE_CATALOG_LOCAL": overlay, - "EVE_INSTANCE_NAME": instance_name, - "EVE_PROVIDER_PLUGIN": provider, - "EVE_RESOLVED_JSON": json.dumps(resolved, separators=(",", ":")), - }) + env = resolved_provider_environment( + provider, + plugin, + { + "EVE_CATALOG_LOCAL": overlay, + "EVE_INSTANCE_NAME": instance_name, + "EVE_PROVIDER_PLUGIN": provider, + "EVE_RESOLVED_JSON": json.dumps(resolved, separators=(",", ":")), + }, + ) if registry_path: env["EVE_INSTANCE_REGISTRY"] = registry_path diff --git a/eve_sdk/secrets.py b/eve_sdk/secrets.py index fce3ac40..be68f3d7 100644 --- a/eve_sdk/secrets.py +++ b/eve_sdk/secrets.py @@ -11,6 +11,23 @@ from eve_sdk.workdir import Workdir +def write_private_text(path: Path, value: str) -> None: + """Create secret payloads owner-only before writing any bytes.""" + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + os.fchmod(handle.fileno(), 0o600) + handle.write(value) + + +def windows_secret_cleanup(path: str) -> str: + """Allow an absent payload while propagating failures to delete one.""" + quoted = "'" + path.replace("'", "''") + "'" + return ( + f"if (Test-Path -LiteralPath {quoted}) {{ " + f"Remove-Item -LiteralPath {quoted} -Force -ErrorAction Stop }}" + ) + + class SecretsError(Exception): pass diff --git a/eve_sdk/state.py b/eve_sdk/state.py index 77e2135a..618da481 100644 --- a/eve_sdk/state.py +++ b/eve_sdk/state.py @@ -37,7 +37,15 @@ class State: "error", } PROVISION_STATES: ClassVar[set[str]] = {"unknown", "provisioning", "provisioned", "error"} - PACKAGE_STATES: ClassVar[set[str]] = {"unknown", "installing", "installed", "missing", "failed", "removed"} + PACKAGE_STATES: ClassVar[set[str]] = { + "failed", + "installed", + "installing", + "missing", + "removed", + "removing", + "unknown", + } DEFAULT_HISTORY_LIMIT = 50 @classmethod @@ -133,6 +141,8 @@ def record_operation( provider_event: str | None = None, provision_state: str | None = None, package: str | None = None, + package_event: str | None = None, + package_observations: dict[str, str] | None = None, package_state: str | None = None, ) -> dict[str, Any]: cls._validate_enum(status, cls.OPERATION_STATUSES, "status") @@ -144,6 +154,12 @@ def record_operation( cls._validate_enum(provision_state, cls.PROVISION_STATES, "provision_state") if package_state: cls._validate_enum(package_state, cls.PACKAGE_STATES, "package_state") + for observed_status in (package_observations or {}).values(): + cls._validate_enum( + observed_status, + {"failed", "installed", "missing", "unknown"}, + "package_observation", + ) now = datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") @@ -152,12 +168,22 @@ def update(state: dict[str, Any]) -> dict[str, Any]: state["updated_at"] = now history = state.setdefault("operation_history", []) entry = { - "id": len(history) + 1, + "id": max( + ( + int(item.get("id", 0)) + for item in history + if isinstance(item, dict) + ), + default=0, + ) + + 1, "name": operation, "type": operation.split(".", 1)[0], "status": status, "at": now, } + if package is not None: + entry["package"] = package state["last_operation"] = entry history_entry = dict(entry) if error is not None: @@ -182,8 +208,54 @@ def update(state: dict[str, Any]) -> dict[str, Any]: state["provider_state"] = new_provider if provision_state: state["provision_state"] = provision_state + elif operation == "provision": + from eve_sdk import instance_fsm + + new_provision = instance_fsm.provision_state_after( + str(state.get("provision_state", "unknown")), + instance_fsm.provision_event(status), + ) + if new_provision is not None: + state["provision_state"] = new_provision + if package and package_state: - state.setdefault("package_state", {})[package] = {"status": package_state, "updated_at": now} + state.setdefault("package_state", {})[package] = { + "status": package_state, + "updated_at": now, + } + elif package: + from eve_sdk import instance_fsm + + command = operation.split(".", 1)[1] if "." in operation else "" + event = package_event or instance_fsm.package_operation_event(command, status) + current = str( + state.setdefault("package_state", {}) + .get(package, {}) + .get("status", "unknown") + ) + new_package = instance_fsm.package_state_after(current, event) + if new_package is not None: + state["package_state"][package] = { + "status": new_package, + "updated_at": now, + } + if package_observations: + from eve_sdk import instance_fsm + + package_states = state.setdefault("package_state", {}) + for package_id, observed_status in sorted(package_observations.items()): + current = str( + package_states.get(package_id, {}).get("status", "unknown") + ) + new_package = instance_fsm.package_state_after( + current, + instance_fsm.package_observation_event(observed_status), + ) + if new_package is not None: + package_states[package_id] = { + "status": new_package, + "updated_at": now, + } return state return cls.modify(instance_name, update) @@ -223,9 +295,39 @@ def update(state: dict[str, Any]) -> dict[str, Any]: "provider.ip", "provider.ssh", }: - state["provider_state"] = "error" + from eve_sdk import instance_fsm + + recovered_provider = instance_fsm.provider_state_after( + str(state.get("provider_state", "unknown")), "op_fail" + ) + if recovered_provider is not None: + state["provider_state"] = recovered_provider if recovered.get("type") == "provision": - state["provision_state"] = "error" + from eve_sdk import instance_fsm + + recovered_provision = instance_fsm.provision_state_after( + str(state.get("provision_state", "unknown")), "provision_fail" + ) + if recovered_provision is not None: + state["provision_state"] = recovered_provision + if recovered.get("type") == "package" and recovered.get("package"): + from eve_sdk import instance_fsm + + package_id = str(recovered["package"]) + command = str(recovered.get("name", "")).split(".", 1)[-1] + package_states = state.setdefault("package_state", {}) + current = str( + package_states.get(package_id, {}).get("status", "unknown") + ) + recovered_package = instance_fsm.package_state_after( + current, + instance_fsm.package_operation_event(command, "failed"), + ) + if recovered_package is not None: + package_states[package_id] = { + "status": recovered_package, + "updated_at": now, + } return state return cls.modify(instance_name, update) diff --git a/poetry.lock b/poetry.lock index 4c221db4..4bfacfbb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.3 and should not be changed by hand. [[package]] name = "attrs" @@ -49,14 +49,14 @@ files = [ [[package]] name = "determa-state" -version = "0.1.0" +version = "0.2.0" description = "Determa State — Python reference implementation of the Determa statechart engine" optional = false python-versions = ">=3.11" groups = ["main"] files = [ - {file = "determa_state-0.1.0-py3-none-any.whl", hash = "sha256:c1ddde17d4dadb9797d727fce75186cb3609d479c902098f12ddb49625bc7709"}, - {file = "determa_state-0.1.0.tar.gz", hash = "sha256:7adeda9877e1698262918ca36bf38ea420aa6ae7820abd442f5034cad9fbd5ab"}, + {file = "determa_state-0.2.0-py3-none-any.whl", hash = "sha256:8aee5f6cac1086fd3731b39dc7f946eda0a56b30e6966384c7246588846b0e21"}, + {file = "determa_state-0.2.0.tar.gz", hash = "sha256:ecc6269dfb0938d09f887da6a874f26121329b528086302616ef6493ba64f1e7"}, ] [package.dependencies] @@ -66,7 +66,8 @@ pyyaml = ">=6" rfc8785 = ">=0.1.4" [package.extras] -dev = ["mypy", "pytest (>=8)", "ruff", "types-jsonschema", "types-pyyaml"] +dev = ["build (>=1)", "hatchling", "mypy", "pytest (>=8)", "ruff", "types-jsonschema", "types-pyyaml"] +postgresql = ["psycopg[binary] (>=3.1,<4)"] [[package]] name = "google-re2" @@ -1034,4 +1035,4 @@ test = ["coverage", "pytest", "pytest-cov"] [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "07004ccc761288f7f2999f1b91ae759c9b6eb9ea211cafb5cf3845c75e4a8b8f" +content-hash = "3c9a2848f4515bf72a3272263e3520cb0ae3ef239f13d0a93898031b19436de3" diff --git a/pyproject.toml b/pyproject.toml index 4c542743..2afa26e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ license = "MIT" readme = "README.md" requires-python = ">=3.12,<4.0" dependencies = [ - "determa-state (>=0.1.0,<0.2.0)", + "determa-state (>=0.2.0,<0.3.0)", "jsonschema (>=4.25.1,<5.0.0)", "pyyaml (>=6.0.3,<7.0.0)", "textual (>=8.2.5,<9.0.0)" diff --git a/scripts/instance-provision b/scripts/instance-provision index c0a3bc6f..42fb9fb5 100755 --- a/scripts/instance-provision +++ b/scripts/instance-provision @@ -76,7 +76,11 @@ def main(argv: list[str]) -> int: state_disabled = os.environ.get("EVE_DISABLE_STATE", "0") == "1" or dry_run - def record_state(status: str, provision_state: str, error: str = "") -> None: + def record_state( + status: str, + error: str = "", + package_observations: dict[str, str] | None = None, + ) -> None: if state_disabled: return cmd = [ @@ -84,10 +88,14 @@ def main(argv: list[str]) -> int: "--instance", instance_name, "--operation", "provision", "--status", status, - "--provision-state", provision_state, ] if error: cmd += ["--error", error] + if package_observations: + cmd += [ + "--package-observations-json", + json.dumps(package_observations, separators=(",", ":")), + ] # set -e: a failing instance-state call is fatal with its exit status. state_rc = subprocess.run( cmd, cwd=root, stdout=subprocess.DEVNULL, check=False, @@ -95,9 +103,9 @@ def main(argv: list[str]) -> int: if state_rc != 0: raise SystemExit(state_rc) - def mark_selected_packages_installed() -> None: + def selected_package_observations() -> dict[str, str]: if state_disabled: - return + return {} # stdout captured (replaces the `package-list | jq` pipeline); stderr of # package-list passes through. Under pipefail a failure is fatal. result = subprocess.run( @@ -108,24 +116,14 @@ def main(argv: list[str]) -> int: raise SystemExit(result.returncode) doc = json.loads(result.stdout) packages = doc.get("packages", []) if isinstance(doc, dict) else [] + observations: dict[str, str] = {} for package in packages: if not isinstance(package, dict): continue if package.get("selected") is True and package.get("installable") is True: package_id = str(package.get("id")) - state_rc = subprocess.run( - [ - str(root / "scripts/instance-state"), - "--instance", instance_name, - "--operation", "package.install", - "--status", "succeeded", - "--package", package_id, - "--package-state", "installed", - ], - cwd=root, stdout=subprocess.DEVNULL, check=False, - ).returncode - if state_rc != 0: - raise SystemExit(state_rc) + observations[package_id] = "installed" + return observations if dry_run: emit_json({ @@ -137,7 +135,7 @@ def main(argv: list[str]) -> int: }) return 0 - record_state("running", "provisioning") + record_state("running") if force: clear_rc = subprocess.run( @@ -145,7 +143,7 @@ def main(argv: list[str]) -> int: cwd=root, check=False, ).returncode if clear_rc != 0: - record_state("failed", "error", "provision.clear-state failed") + record_state("failed", "provision.clear-state failed") return 1 provision_script = os.environ.get("EVE_PROVISION_SCRIPT", str(root / "scripts/provision")) @@ -161,6 +159,7 @@ def main(argv: list[str]) -> int: resolve_cmd, cwd=root, text=True, stdout=subprocess.PIPE, check=False, ) if resolved.returncode != 0: + record_state("failed", "instance resolution failed") return resolved.returncode os.environ["EVE_RESOLVED_JSON"] = resolved.stdout.rstrip("\n") @@ -168,11 +167,17 @@ def main(argv: list[str]) -> int: [provision_script, instance_name], cwd=root, check=False, ).returncode if exit_code != 0: - record_state("failed", "error", f"exit {exit_code}") + record_state("failed", f"exit {exit_code}") return exit_code - mark_selected_packages_installed() - record_state("succeeded", "provisioned") + try: + package_observations = selected_package_observations() + except (json.JSONDecodeError, OSError, SystemExit) as error: + record_state("failed", f"package state recording failed: {error}") + if isinstance(error, SystemExit) and isinstance(error.code, int): + return error.code + return 1 + record_state("succeeded", package_observations=package_observations) return 0 diff --git a/scripts/instance-state b/scripts/instance-state index bcfd353e..112746a6 100755 --- a/scripts/instance-state +++ b/scripts/instance-state @@ -24,7 +24,8 @@ def parser() -> argparse.ArgumentParser: " scripts/instance-state --instance --observed-json \n" " scripts/instance-state --instance --operation --status " "[--desired-state ] [--provider-state ] [--provision-state ] " - "[--package --package-state ] [--error ]" + "[--package --package-state ] " + "[--package-observations-json ] [--error ]" ), add_help=True, ) @@ -37,6 +38,7 @@ def parser() -> argparse.ArgumentParser: arg_parser.add_argument("--provider-state") arg_parser.add_argument("--provision-state") arg_parser.add_argument("--package") + arg_parser.add_argument("--package-observations-json") arg_parser.add_argument("--package-state") arg_parser.add_argument("--observed-json") arg_parser.add_argument("--error") @@ -61,6 +63,14 @@ def main() -> int: return 2 if (args.package is None) != (args.package_state is None): raise ValueError("package and package_state must be provided together") + package_observations = None + if args.package_observations_json: + package_observations = json.loads(args.package_observations_json) + if not isinstance(package_observations, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in package_observations.items() + ): + raise ValueError("package_observations_json must be an object of strings") state = State.record_operation( args.instance, args.operation, @@ -70,6 +80,7 @@ def main() -> int: provider_state=args.provider_state, provision_state=args.provision_state, package=args.package, + package_observations=package_observations, package_state=args.package_state, ) print(json.dumps(state, indent=2)) diff --git a/scripts/package-action b/scripts/package-action index 68835d82..f11368ac 100755 --- a/scripts/package-action +++ b/scripts/package-action @@ -2,7 +2,6 @@ from __future__ import annotations import json -import os import sys from pathlib import Path from typing import Any @@ -10,9 +9,17 @@ from typing import Any ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from eve_sdk.dispatch import DispatchError, exec_cmd, prepare_overlay, run_json # noqa: E402 -from eve_sdk.instance_dispatch import load_runtime_env # noqa: E402 -from eve_sdk.remote_launch import instance_ip, resolve_env, resolve_private_key # noqa: E402 +from eve_sdk.dispatch import ( # noqa: E402 + DispatchError, + exec_cmd, + prepare_overlay, + process_environment, + run_json, +) +from eve_sdk.package_dispatch import resolved_package_environment # noqa: E402 +from eve_sdk.plugin_manifest import PluginManifest # noqa: E402 +from eve_sdk.remote_launch import instance_ip, resolve_env # noqa: E402 +from eve_sdk.resolve import resolve_instance # noqa: E402 from eve_sdk.workdir import Workdir # noqa: E402 @@ -52,8 +59,13 @@ def parse_args(argv: list[str]) -> tuple[str, str, str, bool]: # ---- generic launcher path (v4.4 §8) ------------------------------------- # -def export_remote_context(root: Path, instance_name: str, target: str) -> None: - """Resolve and export the shared launcher context for a package exec. +def export_remote_context( + root: Path, + instance_name: str, + target: str, + base_env: dict[str, str], +) -> dict[str, str]: + """Resolve the shared launcher context for a package exec. The documented stable contract is the ``EVE_REMOTE_*`` set below; the full ``profile-resolve`` env (incl. ``BUNDLE_PACKAGES``, ``OS_FAMILY``, @@ -63,12 +75,17 @@ def export_remote_context(root: Path, instance_name: str, target: str) -> None: """ profile_env = resolve_env(root, instance_name) ip = instance_ip(root, instance_name) - os.environ.update(profile_env) - os.environ.update( + env = dict(base_env) + env.update(profile_env) + key_file = env.get("SSH_PUBLIC_KEY_FILE", "") + private_key = key_file[:-4] if key_file.endswith(".pub") else env.get( + "SSH_PRIVATE_KEY_FILE", "" + ) + env.update( { "EVE_REMOTE_IP": ip, "EVE_REMOTE_USER": profile_env.get("VM_USER_NAME") or profile_env.get("SSH_USER", ""), - "EVE_REMOTE_KEY": resolve_private_key(), + "EVE_REMOTE_KEY": private_key, "EVE_REMOTE_OS_FAMILY": profile_env.get("OS_FAMILY", ""), "EVE_REMOTE_ENGINE": profile_env.get("ENGINE", ""), "EVE_REMOTE_ACTION": target, @@ -79,13 +96,18 @@ def export_remote_context(root: Path, instance_name: str, target: str) -> None: "EVE_REMOTE_INSTANCE": instance_name, } ) + return env -def exec_launcher(exec_rel: str, package: dict[str, Any], target: str) -> None: +def exec_launcher( + exec_rel: str, + package: dict[str, Any], + target: str, + env: dict[str, str], +) -> None: """Resolve an action ``exec`` (relative to the plugin dir) and run it. - Inherits os.environ, so the ``EVE_REMOTE_*`` context exported by - :func:`export_remote_context` is visible to the launcher. + Receives only the operation-local package and ``EVE_REMOTE_*`` environment. """ plugin_dir = Path(str(package.get("path", ""))).resolve().parent exec_path = Path(exec_rel) @@ -93,7 +115,7 @@ def exec_launcher(exec_rel: str, package: dict[str, Any], target: str) -> None: exec_path = (plugin_dir / exec_rel).resolve() if not exec_path.is_file(): raise DispatchError(f"launcher exec not found for action {target}: {exec_rel}") - exec_cmd([str(exec_path)]) + exec_cmd([str(exec_path)], env) def run_action_target( @@ -102,6 +124,8 @@ def run_action_target( instance_name: str, target_map: dict[str, dict[str, Any]], root: Path, + resolved: dict[str, Any], + operation_env: dict[str, str], ) -> None: """Dispatch a package action to its declared launcher command. @@ -117,28 +141,46 @@ def run_action_target( wf = target_map.get(wait_for) if wf is None: raise DispatchError(f"wait_for target not found: {wait_for}") - run_action_target(wf["action"], wf["package"], instance_name, target_map, root) + run_action_target( + wf["action"], + wf["package"], + instance_name, + target_map, + root, + resolved, + operation_env, + ) exec_rel = action.get("exec") if not exec_rel: raise DispatchError( f"action {target} has no 'exec' launcher; the owning package must declare one" ) - export_remote_context(root, instance_name, target) - exec_launcher(exec_rel, package, target) + package_env = resolved_package_environment(package, resolved, operation_env) + remote_env = export_remote_context(root, instance_name, target, package_env) + exec_launcher(exec_rel, package, target, remote_env) def main(argv: list[str]) -> int: instance_name, package_id, action_id, dry_run = parse_args(argv) - # Package launchers read config-derived env (e.g. VM_USER_NAME). The old - # `make` layer injected config-env before every recipe; load it here like - # instance-run does. - load_runtime_env(Workdir.repo_root()) try: + # Registry selection is a transport concern, not package configuration. + inherited = process_environment() + registry_path = inherited.get("EVE_INSTANCE_REGISTRY") + resolved = resolve_instance(instance_name, registry_path) doc = run_json(str(Workdir.repo_root() / "scripts/package-list"), "--instance", instance_name, "--json") - package = next((entry for entry in doc["packages"] if entry["id"] == package_id), None) - if package is None: + package_view = next( + (entry for entry in doc["packages"] if entry["id"] == package_id), None + ) + if package_view is None: raise DispatchError(f"unknown package: {package_id}") + installed_packages = { + str(plugin["id"]): PluginManifest.public(plugin) + for plugin in PluginManifest.load_all("package") + } + package = installed_packages.get(package_id) + if package is None: + raise DispatchError(f"package plugin not found: {package_id}") action = next((entry for entry in package.get("actions", []) if entry["id"] == action_id), None) if action is None: raise DispatchError(f"unknown action {action_id} for package {package_id}") @@ -160,15 +202,30 @@ def main(argv: list[str]) -> int: ) ) return 0 - overlay = prepare_overlay(instance_name, os.environ.get("EVE_INSTANCE_REGISTRY")) - os.environ["EVE_CATALOG_LOCAL"] = overlay - os.environ["EVE_INSTANCE_NAME"] = instance_name + overlay = prepare_overlay(instance_name, registry_path) + operation_env = {"EVE_CATALOG_LOCAL": overlay, "EVE_INSTANCE_NAME": instance_name} + if registry_path: + operation_env["EVE_INSTANCE_REGISTRY"] = registry_path # target -> {package, action} for wait_for resolution. target_map: dict[str, dict[str, Any]] = {} - for pkg in doc["packages"]: - for act in pkg.get("actions", []): - target_map[act["target"]] = {"package": pkg, "action": act} - run_action_target(action, package, instance_name, target_map, Workdir.repo_root()) + for pkg_view in doc["packages"]: + full_package = installed_packages.get(str(pkg_view["id"])) + if full_package is None: + continue + for act in full_package.get("actions", []): + target_map[act["target"]] = { + "action": act, + "package": full_package, + } + run_action_target( + action, + package, + instance_name, + target_map, + Workdir.repo_root(), + resolved, + operation_env, + ) return 0 except SystemExit as error: raise error diff --git a/scripts/package-provision b/scripts/package-provision index a55463a3..aa65bca1 100755 --- a/scripts/package-provision +++ b/scripts/package-provision @@ -14,6 +14,7 @@ from __future__ import annotations import json import os +import shlex import shutil import subprocess import sys @@ -24,8 +25,13 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from eve_sdk.config import ConfigEnv # noqa: E402 -from eve_sdk.package_dispatch import emit_json # noqa: E402 +from eve_sdk.package_dispatch import ( # noqa: E402 + emit_json, + package_environment_names, + resolved_package_set_environment, +) from eve_sdk.resolve import instance_package_config # noqa: E402 +from eve_sdk.secrets import write_private_text # noqa: E402 from eve_sdk.tf_dispatch import repo_root, resolve_env # noqa: E402 _USAGE = ( @@ -85,8 +91,14 @@ def _ssh_retry(label: str, cmd: list[str]) -> int: return subprocess.run([str(ROOT / "scripts/ssh-retry"), label, *cmd]).returncode -def _env_line(key: str, value: str) -> str: - return f'export {key}="{value}"' +def _env_line(key: str, value: str, *, remote_home: bool = False) -> str: + if remote_home: + return f'export {key}="$HOME"/{shlex.quote(value.removeprefix("$HOME/"))}' + return f"export {key}={shlex.quote(value)}" + + +def _write_private_text(path: Path, value: str) -> None: + write_private_text(path, value) def main(argv: list[str]) -> int: @@ -174,6 +186,25 @@ def main(argv: list[str]) -> int: print(f"package-provision: no install chain for {package}", file=sys.stderr) return 2 + chain_ids = {str(plugin["id"]) for plugin in chain} + applicable_ids = chain_ids + applicable_plugins = [ + plugin for plugin in plugins if str(plugin["id"]) in applicable_ids + ] + all_package_config = instance_package_config(instance_name) + applicable_config = { + package_id: value + for package_id, value in all_package_config.items() + if package_id in applicable_ids + } + package_env = resolved_package_set_environment( + applicable_plugins, applicable_config + ) + applicable_env_names = package_environment_names(applicable_plugins) + stage_env_names = package_environment_names( + applicable_plugins, path_config_only=True + ) + steps: list[str] = [] step_roots: list[str] = [] package_markers: list[str] = [] @@ -280,7 +311,7 @@ def main(argv: list[str]) -> int: bundle_packages.replace(",", "\n") + "\n" ) - host_tz = os.environ.get("TIMEZONE", "") + host_tz = package_env.get("TIMEZONE", "") if not host_tz: tz_result = subprocess.run( [str(ROOT / "scripts/host-timezone")], @@ -292,15 +323,15 @@ def main(argv: list[str]) -> int: # the remote and rewrite each env var to its remote path — generic, so # core names no package. (local_path, remote_rel_dir); uploaded below. stage_files: list[tuple[str, str]] = [] - for stage_var in ConfigEnv.package_stage_env_names(): - local_path = os.environ.get(stage_var, "") + for stage_var in stage_env_names: + local_path = package_env.get(stage_var, "") if not local_path: continue if not Path(local_path).exists(): print(f"package-provision: {stage_var} does not exist: {local_path}", file=sys.stderr) return 2 remote_rel = f"provision/downloads/{stage_var.lower()}" - os.environ[stage_var] = f"$HOME/{remote_rel}/{Path(local_path).name}" + package_env[stage_var] = f"$HOME/{remote_rel}/{Path(local_path).name}" stage_files.append((local_path, remote_rel)) # Core (non-plugin) provision env; plugin env appended generically below @@ -315,28 +346,27 @@ def main(argv: list[str]) -> int: _env_line("PROVISION_USER_NAME", resolved.get("PROVISION_USER_NAME", "")), _env_line("VM_USER_NAME", resolved.get("VM_USER_NAME", "")), _env_line("VM_USER_PASSWORD", os.environ.get("VM_USER_PASSWORD", "")), - _env_line("EPHEMERAL_DISPLAY_RESOLUTION", os.environ.get("EPHEMERAL_DISPLAY_RESOLUTION", "")), - _env_line("EPHEMERAL_DISPLAY_FPS", os.environ.get("EPHEMERAL_DISPLAY_FPS", "")), + _env_line("EPHEMERAL_DISPLAY_RESOLUTION", package_env.get("EPHEMERAL_DISPLAY_RESOLUTION", "")), + _env_line("EPHEMERAL_DISPLAY_FPS", package_env.get("EPHEMERAL_DISPLAY_FPS", "")), _env_line("PROVIDER", provider), _env_line("TIMEZONE", host_tz), - _env_line("VIRTUAL_MONITOR_WIDTH", os.environ.get("VIRTUAL_MONITOR_WIDTH", "1920")), - _env_line("VIRTUAL_MONITOR_HEIGHT", os.environ.get("VIRTUAL_MONITOR_HEIGHT", "1080")), ] - # Per-instance package option overrides (§16): instance scope wins over - # the user-scope config-env in os.environ before the generic plugin loop - # reads it. Field→env_var mapping comes from each package manifest. - os.environ.update(ConfigEnv.instance_package_env(instance_package_config(instance_name))) env_lines += [ - _env_line(_name, os.environ.get(_name, "")) - for _name in ConfigEnv.plugin_provision_env_names() + _env_line( + name, package_env.get(name, ""), + remote_home=name in stage_env_names and bool(package_env.get(name)), + ) + for name in applicable_env_names ] - (stage / "state" / "env").write_text("\n".join(env_lines) + "\n") + env_path = stage / "state" / "env" + _write_private_text(env_path, "\n".join(env_lines) + "\n") print(f"[package-provision] uploading {os_provision_dir} scripts for package={package}") uploads: list[tuple[str, list[str]]] = [( "prepare linux package provision directory", [ssh, instance_name, "--", - "rm -rf ~/provision/scripts && mkdir -p ~/provision/state ~/provision/logs ~/provision/downloads"], + "rm -rf ~/provision/scripts && mkdir -p ~/provision/state ~/provision/logs ~/provision/downloads" + " && chmod 0700 ~/provision/state"], )] for local_path, remote_rel in stage_files: uploads.append(( @@ -361,19 +391,43 @@ def main(argv: list[str]) -> int: for label, cmd in uploads: status = _ssh_retry(label, cmd) if status != 0: + if label == "upload linux package environment": + _ssh_retry( + "remove linux package environment after upload failure", + [ssh, instance_name, "--", "rm -f ~/provision/state/env"], + ) return status + status = _ssh_retry( + "protect linux package environment", + [ssh, instance_name, "--", "chmod 0600 ~/provision/state/env"], + ) + if status != 0: + _ssh_retry( + "remove unprotected linux package environment", + [ssh, instance_name, "--", "rm -f ~/provision/state/env"], + ) + return status + remote_steps = "".join(f" ~/provision/scripts/steps/{Path(s).name}" for s in steps) print(f"[package-provision] running package={package} steps:{remote_steps}") remote_cmd = ( - 'set -e; export PROVISION_ROOT="$HOME/provision"; ' + 'set -e; trap \'rm -f "$HOME/provision/state/env"\' EXIT; ' + 'export PROVISION_ROOT="$HOME/provision"; ' '. "$HOME/provision/state/env"; ' "chmod +x ~/provision/scripts/steps/*.sh ~/provision/scripts/lib/*.sh; " f"for step in{remote_steps}; do " 'echo "[package-provision] running $(basename "$step")"; ' '/usr/bin/env bash "$step"; done' ) - return subprocess.run([ssh, instance_name, "--", remote_cmd]).returncode + try: + status = subprocess.run([ssh, instance_name, "--", remote_cmd]).returncode + finally: + cleanup_status = _ssh_retry( + "remove linux package environment", + [ssh, instance_name, "--", "rm -f ~/provision/state/env"], + ) + return status if status else cleanup_status finally: shutil.rmtree(stage, ignore_errors=True) diff --git a/scripts/package-provision-windows b/scripts/package-provision-windows index 961e2f0b..373ab363 100755 --- a/scripts/package-provision-windows +++ b/scripts/package-provision-windows @@ -24,7 +24,13 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from eve_sdk.package_dispatch import emit_json # noqa: E402 +from eve_sdk.package_dispatch import ( # noqa: E402 + emit_json, + package_environment_names, + resolved_package_set_environment, +) +from eve_sdk.resolve import instance_package_config # noqa: E402 +from eve_sdk.secrets import windows_secret_cleanup, write_private_text # noqa: E402 from eve_sdk.tf_dispatch import repo_root, resolve_env # noqa: E402 _USAGE = ( @@ -210,11 +216,19 @@ def main(argv: list[str]) -> int: windows_password = pw_result.stdout.strip() if pw_result.returncode == 0 else "" from eve_sdk.config import ConfigEnv - env_payload = ConfigEnv.provision_env_payload(windows_password) + all_config = instance_package_config(instance_name) + package_config = ( + {package: all_config[package]} if package in all_config else {} + ) + package_env = resolved_package_set_environment([entry], package_config) + env_names = package_environment_names([entry]) + env_payload = ConfigEnv.provision_env_payload( + windows_password, package_env, env_names + ) (stage / "state").mkdir(parents=True, exist_ok=True) - (stage / "state" / "env.json").write_text( + write_private_text( + stage / "state" / "env.json", json.dumps(env_payload, indent=2, ensure_ascii=False) + "\n", - encoding="utf-8", ) print(f"[package-provision-windows] uploading windows step files for package={package}") @@ -228,6 +242,11 @@ def main(argv: list[str]) -> int: ) if needs_env: reset_cmd += f" ; New-Item -ItemType Directory -Force '{_REMOTE_STATE_DIR}' | Out-Null" + reset_cmd += ( + f" ; icacls '{_REMOTE_STATE_DIR}' /inheritance:r " + '/grant:r "Administrator:(OI)(CI)F" "SYSTEM:(OI)(CI)F" | Out-Null; ' + 'if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }' + ) uploads: list[tuple[str, list[str]]] = [ ("prepare windows package provision directory", [ssh, instance_name, "--", reset_cmd]), @@ -252,17 +271,49 @@ def main(argv: list[str]) -> int: for label, cmd in uploads: status = _ssh_retry(label, cmd) if status != 0: + if label == "upload windows package env.json": + _instance_ssh( + instance_name, "--", + windows_secret_cleanup(_REMOTE_STATE_DIR + r"\env.json"), + ) + return status + if needs_env: + status = _instance_ssh( + instance_name, + "--", + f"icacls '{_REMOTE_STATE_DIR}\\env.json' /inheritance:r " + '/grant:r "Administrator:F" "SYSTEM:F" | Out-Null; ' + 'if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }', + ) + if status != 0: + _instance_ssh( + instance_name, + "--", + windows_secret_cleanup(_REMOTE_STATE_DIR + r"\env.json"), + ) return status + step_status = 0 for step in steps: step_file = Path(step).name print(f"[package-provision-windows] running step={step_file} package={package}") - status = _instance_ssh( + step_status = _instance_ssh( instance_name, "--", f"& 'C:\\Users\\Administrator\\provision\\scripts-package\\steps\\{step_file}'", ) - if status != 0: - return status + if step_status != 0: + break + cleanup_status = 0 + if needs_env: + cleanup_status = _instance_ssh( + instance_name, + "--", + windows_secret_cleanup(_REMOTE_STATE_DIR + r"\env.json"), + ) + if step_status != 0: + return step_status + if cleanup_status != 0: + return cleanup_status finally: shutil.rmtree(stage, ignore_errors=True) diff --git a/scripts/providers-status b/scripts/providers-status index 164c35b0..c58e2a32 100755 --- a/scripts/providers-status +++ b/scripts/providers-status @@ -18,7 +18,6 @@ command's own ``configured`` field is ignored (it only needs to answer reachabil from __future__ import annotations import json -import os import subprocess import sys from pathlib import Path @@ -27,11 +26,10 @@ from typing import Any ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from eve_sdk.config import ConfigEnv # noqa: E402 -from eve_sdk.dispatch import command_vector # noqa: E402 +from eve_sdk.dispatch import command_env, command_vector # noqa: E402 from eve_sdk.fsm import EveFsm, provider_configured_resolver # noqa: E402 from eve_sdk.plugin_manifest import PluginManifest # noqa: E402 -from eve_sdk.provider_command import _inject_secrets # noqa: E402 +from eve_sdk.provider_command import resolved_provider_environment # noqa: E402 CORE_PROVIDER_MACHINE = ROOT / "core" / "fsm" / "provider.yaml" @@ -55,17 +53,16 @@ def _run_connectivity(plugin: dict[str, Any]) -> tuple[str, str, str]: """Run a provider's connectivity command, parse JSON → (configured, reachable, notes).""" spec = plugin["commands"]["connectivity"] cmd = command_vector(plugin, spec) - # Resolve the SAME env the real dispatch path gives a provider command: - # global config-env plus the provider's injected secrets. Without this the - # probe ran with a bare environment, so a provider whose "configured" gate is - # a secret (e.g. Vultr's VULTR_API_KEY, which lives only in the secrets file) - # was reported unconfigured even though dispatch could create instances with - # it. Status and dispatch must resolve config/secrets identically. (v4.5: this - # injection should become the one shared helper the FSM platform provides.) - env = dict(os.environ) - env.update(ConfigEnv.environment()) - env = _inject_secrets(str(plugin["id"]), plugin, env) - result = subprocess.run(cmd, env=env, capture_output=True, text=True, check=False) + # Use the same shared environment builder as provider dispatch, including + # ConfigEnv settings and injected secrets (v4.5 M/O: status == dispatch). + env = resolved_provider_environment(str(plugin["id"]), plugin) + result = subprocess.run( + cmd, + env=command_env(env), + capture_output=True, + text=True, + check=False, + ) if result.returncode != 0: return "no", "no", f"connectivity check failed (exit {result.returncode})" try: diff --git a/scripts/provision b/scripts/provision index d4858972..50b8da73 100755 --- a/scripts/provision +++ b/scripts/provision @@ -13,6 +13,7 @@ from __future__ import annotations import hashlib import json import os +import shlex import subprocess import sys import tempfile @@ -25,8 +26,13 @@ sys.path.insert(0, str(ROOT)) from eve_sdk.config import ConfigEnv # noqa: E402 from eve_sdk.instance_dispatch import load_runtime_env # noqa: E402 +from eve_sdk.package_dispatch import ( # noqa: E402 + package_environment_names, + resolved_package_set_environment, +) from eve_sdk.plugin_manifest import PluginManifest # noqa: E402 from eve_sdk.resolve import instance_package_config # noqa: E402 +from eve_sdk.secrets import windows_secret_cleanup, write_private_text # noqa: E402 def _provision_dir_display(path: Path, root: Path) -> str: @@ -66,10 +72,40 @@ def _parse_env(output: str) -> dict[str, str]: return env +def _env_line(key: str, value: str, *, remote_home: bool = False) -> str: + if remote_home: + return f'export {key}="$HOME"/{shlex.quote(value.removeprefix("$HOME/"))}' + return f"export {key}={shlex.quote(value)}" + + +def _write_private_text(path: Path, value: str) -> None: + write_private_text(path, value) + + def _selected_packages(bundle_packages: str) -> list[str]: return [pkg for pkg in bundle_packages.split(",") if pkg] +def _selected_package_environment( + plugin_doc: dict, + bundle_packages: str, + profile: str, +) -> tuple[list[dict], dict[str, str]]: + selected = set(_selected_packages(bundle_packages)) + plugins = [ + plugin + for plugin in plugin_doc.get("plugins") or [] + if isinstance(plugin, dict) and str(plugin.get("id")) in selected + ] + all_config = instance_package_config(profile) + selected_config = { + package_id: value + for package_id, value in all_config.items() + if package_id in selected + } + return plugins, resolved_package_set_environment(plugins, selected_config) + + def _plugin_ubuntu_steps(plugin_doc: dict, bundle_packages: str) -> list[tuple[str, str]]: """Return [(plugin_path, step_name)] in selection order from BUNDLE_PACKAGES.""" selected = _selected_packages(bundle_packages) @@ -201,6 +237,9 @@ def provision_ubuntu(profile: str, env: dict[str, str], root: Path) -> int: [str(root / "scripts/plugin-list"), "--kind", "package", "--json"], cwd=root, capture_output=True, text=True, check=False, ).stdout) + selected_plugins, package_env = _selected_package_environment( + plugin_doc, bundle_packages, profile + ) for plugin_path, step in _plugin_ubuntu_steps(plugin_doc, bundle_packages): if not plugin_path: continue @@ -251,18 +290,19 @@ def provision_ubuntu(profile: str, env: dict[str, str], root: Path) -> int: # (local_path, remote_rel_dir) and uploaded after the provision dir is # prepared, below. stage_files: list[tuple[str, str]] = [] - for stage_var in ConfigEnv.package_stage_env_names(): - local_path = os.environ.get(stage_var, "") + stage_env_names = package_environment_names(selected_plugins, path_config_only=True) + for stage_var in stage_env_names: + local_path = package_env.get(stage_var, "") if not local_path: continue if not Path(local_path).exists(): print(f"[provision] {stage_var} does not exist: {local_path}", file=sys.stderr) return 2 remote_rel = f"provision/downloads/{stage_var.lower()}" - os.environ[stage_var] = f"$HOME/{remote_rel}/{os.path.basename(local_path)}" + package_env[stage_var] = f"$HOME/{remote_rel}/{os.path.basename(local_path)}" stage_files.append((local_path, remote_rel)) - host_tz = os.environ.get("TIMEZONE", "") + host_tz = package_env.get("TIMEZONE", "") if not host_tz: tz_result = subprocess.run( [str(root / "scripts/host-timezone")], cwd=root, @@ -274,38 +314,32 @@ def provision_ubuntu(profile: str, env: dict[str, str], root: Path) -> int: # Core (non-plugin) provision env. Plugin env is appended generically # below so core names no provider or package (v4.4 §15.5). env_lines = [ - f'export ACCESS_BOOTSTRAP_USER="{env.get("ACCESS_BOOTSTRAP_USER", "")}"', - f'export ACCESS_HUMAN_USER="{env.get("ACCESS_HUMAN_USER", "")}"', - f'export ACCESS_PROVISION_USER="{env.get("ACCESS_PROVISION_USER", "")}"', - f'export OS_FAMILY="{os_family}"', - f'export PROFILE_NAME="{profile_name}"', - f'export HUMAN_USER_NAME="{env.get("HUMAN_USER_NAME", "")}"', - f'export PROVISION_USER_NAME="{env.get("PROVISION_USER_NAME", "")}"', - f'export VM_USER_NAME="{env.get("VM_USER_NAME", "")}"', - f'export VM_USER_PASSWORD="{os.environ.get("VM_USER_PASSWORD", "")}"', - f'export EPHEMERAL_DISPLAY_RESOLUTION="{os.environ.get("EPHEMERAL_DISPLAY_RESOLUTION", "")}"', - f'export EPHEMERAL_DISPLAY_FPS="{os.environ.get("EPHEMERAL_DISPLAY_FPS", "")}"', - f'export PROVIDER="{provider}"', - f'export TIMEZONE="{host_tz}"', - f'export VIRTUAL_MONITOR_WIDTH="{os.environ.get("VIRTUAL_MONITOR_WIDTH", "1920")}"', - f'export VIRTUAL_MONITOR_HEIGHT="{os.environ.get("VIRTUAL_MONITOR_HEIGHT", "1080")}"', + _env_line("ACCESS_BOOTSTRAP_USER", env.get("ACCESS_BOOTSTRAP_USER", "")), + _env_line("ACCESS_HUMAN_USER", env.get("ACCESS_HUMAN_USER", "")), + _env_line("ACCESS_PROVISION_USER", env.get("ACCESS_PROVISION_USER", "")), + _env_line("OS_FAMILY", os_family), + _env_line("PROFILE_NAME", profile_name), + _env_line("HUMAN_USER_NAME", env.get("HUMAN_USER_NAME", "")), + _env_line("PROVISION_USER_NAME", env.get("PROVISION_USER_NAME", "")), + _env_line("VM_USER_NAME", env.get("VM_USER_NAME", "")), + _env_line("VM_USER_PASSWORD", os.environ.get("VM_USER_PASSWORD", "")), + _env_line("EPHEMERAL_DISPLAY_RESOLUTION", package_env.get("EPHEMERAL_DISPLAY_RESOLUTION", "")), + _env_line("EPHEMERAL_DISPLAY_FPS", package_env.get("EPHEMERAL_DISPLAY_FPS", "")), + _env_line("PROVIDER", provider), + _env_line("TIMEZONE", host_tz), ] - # Per-instance package option overrides (§16): instance scope wins over - # the user-scope config-env already in os.environ, before the generic - # plugin loop reads it. Core names no package — the field→env_var mapping - # comes from each package manifest. - os.environ.update(ConfigEnv.instance_package_env(instance_package_config(profile))) - # Plugin-declared env (config + secrets) from installed provider/package - # manifests — generic, so no plugin id is named in core. Staged package - # files (above) were rewritten in os.environ to their remote paths, so - # they emit here with the remote value. - for _name in ConfigEnv.plugin_provision_env_names(): - env_lines.append(f'export {_name}="{os.environ.get(_name, "")}"') - (stage / "state" / "env").write_text("\n".join(env_lines) + "\n", encoding="utf-8") + for name in package_environment_names(selected_plugins): + env_lines.append(_env_line( + name, package_env.get(name, ""), + remote_home=name in stage_env_names and bool(package_env.get(name)), + )) + env_path = stage / "state" / "env" + _write_private_text(env_path, "\n".join(env_lines) + "\n") print(f"[provision] uploading {_provision_dir_display(os_provision_dir, root)}/ to remote") rc = _ssh_retry(profile, "prepare linux provision directory", - "rm -rf ~/provision/state ~/provision/scripts ~/provision/logs && mkdir -p ~/provision/state") + "rm -rf ~/provision/state ~/provision/scripts ~/provision/logs && mkdir -p ~/provision/state" + " && chmod 0700 ~/provision/state") if rc: return rc for local_path, remote_rel in stage_files: @@ -333,15 +367,27 @@ def provision_ubuntu(profile: str, env: dict[str, str], root: Path) -> int: rc = _ssh_retry_scp(profile, "upload linux environment", str(stage / "state" / "env"), "provision/state/") if rc: + _ssh_retry(profile, "remove linux provision environment after upload failure", + "rm -f ~/provision/state/env") + return rc + rc = _ssh_retry(profile, "protect linux provision environment", + "chmod 0600 ~/provision/state/env") + if rc: + _ssh_retry(profile, "remove unprotected linux provision environment", + "rm -f ~/provision/state/env") return rc rc = _ssh_retry_scp(profile, "upload linux step list", str(stage / "state" / "steps.list"), "provision/state/") if rc: + _ssh_retry(profile, "remove linux provision environment after upload failure", + "rm -f ~/provision/state/env") return rc manifest_path = str(stage / "state" / "provision-manifest.json") rc = _ssh_retry_scp(profile, "upload linux provision manifest", manifest_path, "provision/state/") if rc: + _ssh_retry(profile, "remove linux provision environment after upload failure", + "rm -f ~/provision/state/env") return rc print("[provision] running bootstrap.sh") rc = _ssh_retry(profile, "start linux provisioning runner", @@ -349,11 +395,15 @@ def provision_ubuntu(profile: str, env: dict[str, str], root: Path) -> int: " ~/provision/scripts/steps/*.sh ~/provision/scripts/lib/*.sh" " && ~/provision/scripts/bootstrap.sh") if rc: + _ssh_retry(profile, "remove linux provision environment after bootstrap failure", + "rm -f ~/provision/state/env") return rc poll_interval = os.environ.get("EVE_PROVISION_WAIT_POLL_INTERVAL", "5") print(f"[provision] waiting for linux provisioning to finish (polling remote log every {poll_interval}s)") rc = _run([str(root / "scripts/wait-for-provision"), profile]).returncode - return rc + cleanup_rc = _ssh_retry(profile, "remove linux provision environment", + "rm -f ~/provision/state/env") + return rc if rc else cleanup_rc finally: import shutil shutil.rmtree(stage, ignore_errors=True) @@ -385,20 +435,26 @@ def provision_windows(profile: str, env: dict[str, str], root: Path) -> int: ) return 1 - tmp_dir = root / "tmp" - tmp_dir.mkdir(parents=True, exist_ok=True) - env_json = ConfigEnv.provision_env_payload(windows_password) - - (tmp_dir / "env.json").write_text(json.dumps(env_json, indent=2) + "\n", encoding="utf-8") - + plugin_doc = json.loads(subprocess.run( + [str(root / "scripts/plugin-list"), "--kind", "package", "--json"], + cwd=root, capture_output=True, text=True, check=False, + ).stdout) + _selected_plugins, package_env = _selected_package_environment( + plugin_doc, bundle_packages, profile + ) stage = Path(tempfile.mkdtemp()) try: copytree(os_provision_dir, stage / "provision") + env_names = package_environment_names(_selected_plugins) + env_json = ConfigEnv.provision_env_payload( + windows_password, package_env, env_names + ) + state_dir = stage / "provision" / "state" + state_dir.mkdir(parents=True, exist_ok=True) + _write_private_text( + state_dir / "env.json", json.dumps(env_json, indent=2) + "\n" + ) - plugin_doc = json.loads(subprocess.run( - [str(root / "scripts/plugin-list"), "--kind", "package", "--json"], - cwd=root, capture_output=True, text=True, check=False, - ).stdout) for plugin_path, step in _plugin_windows_steps(plugin_doc, bundle_packages): if not plugin_path or not step: continue @@ -442,16 +498,33 @@ def provision_windows(profile: str, env: dict[str, str], root: Path) -> int: 'Remove-Item -Recurse -Force "C:\\Users\\Administrator\\provision\\state" }; ' 'if (Test-Path "C:\\Users\\Administrator\\provision\\logs") { ' 'Remove-Item -Recurse -Force "C:\\Users\\Administrator\\provision\\logs" }; ' - 'New-Item -ItemType Directory -Force C:\\Users\\Administrator\\provision\\state | Out-Null') + 'New-Item -ItemType Directory -Force C:\\Users\\Administrator\\provision\\state | Out-Null; ' + 'icacls "C:\\Users\\Administrator\\provision\\state" /inheritance:r ' + '/grant:r "Administrator:(OI)(CI)F" "SYSTEM:(OI)(CI)F" | Out-Null; ' + 'if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }') if rc: return rc rc = _ssh_retry_scp(profile, "upload windows provision tree", str(stage / "provision"), "/C:/Users/Administrator/") if rc: + _ssh_retry( + profile, "remove windows provision environment after upload failure", + windows_secret_cleanup(r"C:\Users\Administrator\provision\state\env.json"), + ) return rc - rc = _ssh_retry_scp(profile, "upload windows env.json", - str(tmp_dir / "env.json"), "/C:/Users/Administrator/provision/state/env.json") + rc = _ssh_retry( + profile, + "protect windows provision environment", + 'icacls "C:\\Users\\Administrator\\provision\\state\\env.json" ' + '/inheritance:r /grant:r "Administrator:F" "SYSTEM:F" | Out-Null; ' + 'if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }', + ) if rc: + _ssh_retry( + profile, + "remove unprotected windows provision environment", + windows_secret_cleanup(r"C:\Users\Administrator\provision\state\env.json"), + ) return rc print("[provision] running bootstrap.ps1") instance_ssh = str(root / "scripts/instance-ssh") @@ -460,11 +533,21 @@ def provision_windows(profile: str, env: dict[str, str], root: Path) -> int: cwd=root, check=False, ) if result.returncode != 0 and result.returncode != 255: + _ssh_retry( + profile, + "remove windows provision environment after bootstrap failure", + windows_secret_cleanup(r"C:\Users\Administrator\provision\state\env.json"), + ) return result.returncode poll_interval = os.environ.get("EVE_PROVISION_WAIT_POLL_INTERVAL", "5") print(f"[provision] waiting for windows provisioning to finish (polling remote log every {poll_interval}s)") rc = _run([str(root / "scripts/wait-for-provision"), profile]).returncode - return rc + cleanup_rc = _ssh_retry( + profile, + "remove windows provision environment", + windows_secret_cleanup(r"C:\Users\Administrator\provision\state\env.json"), + ) + return rc if rc else cleanup_rc finally: import shutil shutil.rmtree(stage, ignore_errors=True) diff --git a/scripts/test b/scripts/test index c6447f61..155c5e3c 100755 --- a/scripts/test +++ b/scripts/test @@ -46,6 +46,14 @@ SUITE_SCRIPTS: dict[str, list[str]] = { def main(argv: list[str]) -> int: + # Match bin/eve: child scripts use the project environment, so repository + # tests never depend on or mutate a separate host-Python installation. + venv_bin = ROOT / ".venv" / "bin" + if not (venv_bin / "python").is_file(): + print("scripts/test: project environment missing; run poetry install", file=sys.stderr) + return 1 + os.environ["PATH"] = f"{venv_bin}{os.pathsep}{os.environ['PATH']}" + # Set hermetic plugin discovery for all test subprocesses. os.environ["EVE_PLUGIN_ROOTS"] = str(ROOT / "tests/fixtures/hermetic") os.environ["EVE_PLUGIN_ROOTS_EXCLUSIVE"] = "1" diff --git a/scripts/test-ci-pipeline b/scripts/test-ci-pipeline index f37f5d44..8a9d8398 100755 --- a/scripts/test-ci-pipeline +++ b/scripts/test-ci-pipeline @@ -110,6 +110,12 @@ def main() -> int: print("\n== ci-pipeline workflow contract ==") + workflow_env = workflow.get("env", {}) + checks.assert_check( + "Poetry creates the project environment required by scripts/test", + isinstance(workflow_env, dict) and workflow_env.get("POETRY_VIRTUALENVS_IN_PROJECT") == "true", + ) + triggers = workflow.get("on", workflow.get(True)) checks.assert_check( "workflow runs on pull_request", diff --git a/scripts/test-cross-repo b/scripts/test-cross-repo new file mode 100755 index 00000000..151434fe --- /dev/null +++ b/scripts/test-cross-repo @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Validate tracked sibling manifests, dual-OS schemas, and the combined catalog.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from eve_sdk.catalog import load_catalog # noqa: E402 +from eve_sdk.plugin_manifest import PluginManifest # noqa: E402 +from eve_sdk.plugin_test import run_plugin_test # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plugin-root", action="append", required=True, type=Path) + args = parser.parse_args() + roots = [path.resolve() for path in args.plugin_root] + os.environ["EVE_PLUGIN_ROOTS"] = os.pathsep.join(map(str, roots)) + os.environ["EVE_PLUGIN_ROOTS_EXCLUSIVE"] = "1" + with tempfile.TemporaryDirectory(prefix="eve-cross-repo.") as home: + os.environ["EVE_HOME"] = home + halves: dict[str, list[dict[str, Any]]] = defaultdict(list) + failures = 0 + for root in roots: + tracked = subprocess.run( + ["git", "ls-files", "-z", "*eve-plugin.yaml"], + cwd=root, check=True, capture_output=True, text=True, + ).stdout + for name in sorted(filter(None, tracked.split("\0"))): + path = root / name + result = run_plugin_test(path) + print(f"{'PASS' if result.passed else 'FAIL'} {root.name}/{name}") + for check in result.failures: + print(f" {check.name}: {check.message}") + failures += not result.passed + plugin = PluginManifest.load(path) + if plugin.get("kind") == "package": + halves[str(plugin["id"])].append(plugin) + for package_id, plugins in sorted(halves.items()): + if len(plugins) > 1: + schemas = [plugin.get("config_schema") or {} for plugin in plugins] + if any(schema != schemas[0] for schema in schemas[1:]): + print(f"FAIL dual-OS configuration schema parity: {package_id}") + failures += 1 + plugins = PluginManifest.load_all() + for plugin in plugins: + PluginManifest.validate(plugin) + catalog = load_catalog(plugins=plugins) + available = {package["id"] for package in catalog["packages"]} + for bundle in catalog["bundles"]: + missing = set(bundle.get("packages") or []) - available + if missing: + print(f"FAIL bundle {bundle['id']} has missing packages: {sorted(missing)}") + failures += 1 + print(json.dumps({ + "plugins": len(plugins), "kinds": dict(sorted(Counter(p["kind"] for p in plugins).items())), + "catalog": {key: len(value) for key, value in catalog.items()}, + "failures": failures, + }, indent=2)) + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test-provision-runner b/scripts/test-provision-runner index b60d827b..af3d4e56 100755 --- a/scripts/test-provision-runner +++ b/scripts/test-provision-runner @@ -3,6 +3,7 @@ from __future__ import annotations +import argparse import json import os import shutil @@ -12,7 +13,12 @@ import tempfile from pathlib import Path from typing import Any -os.environ["EVE_PLUGIN_ROOTS"] = str(Path(__file__).resolve().parents[1] / "tests/fixtures/hermetic") +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--plugin-roots", help="Explicit OS plugin roots for cross-repository runner tests") +args = parser.parse_args() +os.environ["EVE_PLUGIN_ROOTS"] = args.plugin_roots or str( + Path(__file__).resolve().parents[1] / "tests/fixtures/hermetic" +) os.environ["EVE_PLUGIN_ROOTS_EXCLUSIVE"] = "1" os.environ.setdefault("EVE_HOME", tempfile.mkdtemp(prefix="eve-prov.")) @@ -237,8 +243,10 @@ print("\n== provision-runner: windows (powershell) ==") pwsh = shutil.which("pwsh") if pwsh: for name, body, expected_status in [ - ("ok", "Write-Host 'ok'; exit 0", "done"), ("fail", "Write-Error 'fail'; exit 42", "failed"), + ("native-info", "& (Get-Process -Id $PID).Path -NoProfile -Command 'exit 7'", "done"), + ("ok", "Write-Host 'ok'; exit 0", "done"), + ("throw", "throw 'failed step'", "failed"), ]: with tempfile.TemporaryDirectory(prefix=f"eve-runner-pwsh-{name}-") as tmp: root = Path(tmp) @@ -248,7 +256,7 @@ if pwsh: logs_dir = prov_dir / "logs" for path in [steps_dir, state_dir, logs_dir]: path.mkdir(parents=True, exist_ok=True) - step_name = f"00_{name}.ps1" if name == "ok" else "00_fail.ps1" + step_name = f"00_{name}.ps1" (steps_dir / step_name).write_text(body, encoding="utf-8") (state_dir / "state.json").write_text('{"currentStep":0}', encoding="utf-8") (state_dir / "steps.list").write_text(step_name + "\n", encoding="utf-8") @@ -269,7 +277,7 @@ if pwsh: else: actual_status = status.get("status") if status else "nil" fail_check(f"Windows runner status: expected={expected_status} actual={actual_status}") - if expected_status == "failed": + if name == "fail": actual_exit = status["steps"][0]["exit_code"] if status else None pass_check("Windows runner captures exit_code=42") if actual_exit == 42 else fail_check( f"Windows runner exit_code: expected=42 actual={actual_exit}" diff --git a/scripts/test-python b/scripts/test-python index a24e7883..819c9441 100755 --- a/scripts/test-python +++ b/scripts/test-python @@ -37,7 +37,7 @@ def main() -> int: return code code = subprocess.run( - ["poetry", "run", "pytest", "tests/python"], + ["poetry", "run", "pytest", "tests"], cwd=str(ROOT), check=False, ).returncode if code != 0: diff --git a/tests/python/commands/test_package_action_launch.py b/tests/python/commands/test_package_action_launch.py index 0919dde3..93fc1f42 100644 --- a/tests/python/commands/test_package_action_launch.py +++ b/tests/python/commands/test_package_action_launch.py @@ -4,7 +4,6 @@ import importlib.machinery import importlib.util -import os from pathlib import Path from typing import Any @@ -13,16 +12,6 @@ ROOT = Path(__file__).resolve().parents[3] -@pytest.fixture(autouse=True) -def _restore_environ() -> Any: - """export_remote_context mutates os.environ; restore it between tests so the - EVE_REMOTE_* leak does not break byte-identical env tests elsewhere.""" - snapshot = dict(os.environ) - yield - os.environ.clear() - os.environ.update(snapshot) - - def _load_package_action() -> Any: path = ROOT / "scripts" / "package-action" loader = importlib.machinery.SourceFileLoader("package_action_v44", str(path)) @@ -39,14 +28,22 @@ def pa() -> Any: return _load_package_action() -def _stub_spine(monkeypatch: pytest.MonkeyPatch, pa: Any, ip: str = "1.2.3.4") -> dict[str, list[list[str]]]: - monkeypatch.setattr(pa, "resolve_env", lambda root, inst: {"VM_USER_NAME": "alice", "OS_FAMILY": "ubuntu", "ENGINE": "terraform"}) +def _stub_spine(monkeypatch: pytest.MonkeyPatch, pa: Any, ip: str = "1.2.3.4") -> dict[str, Any]: + monkeypatch.setattr( + pa, + "resolve_env", + lambda root, inst: { + "ENGINE": "terraform", + "OS_FAMILY": "ubuntu", + "VM_USER_NAME": "alice", + }, + ) monkeypatch.setattr(pa, "instance_ip", lambda root, inst: ip) - monkeypatch.setattr(pa, "resolve_private_key", lambda: "/key") - captured: dict[str, list[list[str]]] = {"cmds": []} + captured: dict[str, Any] = {"cmds": [], "envs": []} - def fake_exec(cmd: list[str]) -> None: + def fake_exec(cmd: list[str], env: dict[str, str]) -> None: captured["cmds"].append(cmd) + captured["envs"].append(env) monkeypatch.setattr(pa, "exec_cmd", fake_exec) return captured @@ -61,10 +58,14 @@ def test_exec_action_exports_remote_context_and_runs( action = {"id": "open", "label": "Open", "target": "pkg.open", "exec": "remote-open"} captured = _stub_spine(monkeypatch, pa) - pa.run_action_target(action, package, "inst", {}, tmp_path) + resolved = {"instance": {"name": "inst"}, "package_config": {}} + operation_env = {"SSH_PUBLIC_KEY_FILE": "/key.pub"} + pa.run_action_target( + action, package, "inst", {}, tmp_path, resolved, operation_env + ) assert captured["cmds"] == [[str(tmp_path / "remote-open")]] - env = os.environ + env = captured["envs"][0] assert env["EVE_REMOTE_IP"] == "1.2.3.4" assert env["EVE_REMOTE_USER"] == "alice" assert env["EVE_REMOTE_OS_FAMILY"] == "ubuntu" @@ -83,7 +84,10 @@ def test_exec_action_wait_for_runs_target_first(pa: Any, monkeypatch: pytest.Mon target_map = {"pkg.wait": {"package": package, "action": wait_action}} captured = _stub_spine(monkeypatch, pa) - pa.run_action_target(open_action, package, "inst", target_map, tmp_path) + resolved = {"instance": {"name": "inst"}, "package_config": {}} + pa.run_action_target( + open_action, package, "inst", target_map, tmp_path, resolved, {} + ) assert captured["cmds"] == [[str(tmp_path / "wait")], [str(tmp_path / "open")]] @@ -93,4 +97,69 @@ def test_exec_action_missing_exec_fails_loudly(pa: Any, monkeypatch: pytest.Monk action = {"id": "open", "label": "Open", "target": "pkg.open", "exec": "no-such-file"} _stub_spine(monkeypatch, pa) with pytest.raises(pa.DispatchError, match="launcher exec not found"): - pa.run_action_target(action, package, "inst", {}, tmp_path) + pa.run_action_target( + action, + package, + "inst", + {}, + tmp_path, + {"instance": {"name": "inst"}, "package_config": {}}, + {}, + ) + + +def test_main_uses_full_manifest_for_action_environment( + pa: Any, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + action = { + "exec": "remote-open", + "id": "open", + "label": "Open", + "target": "pkg.open", + } + manifest = { + "_path": str(tmp_path / "eve-plugin.yaml"), + "_source": "test", + "actions": [action], + "config_schema": { + "secrets": {"token": {"env_var": "PKG_TOKEN"}}, + }, + "id": "pkg", + } + monkeypatch.setattr(pa, "process_environment", lambda: {}) + monkeypatch.setattr( + pa, + "resolve_instance", + lambda instance, registry: { + "instance": {"name": instance}, + "package_config": {}, + }, + ) + monkeypatch.setattr( + pa, + "run_json", + lambda *args: { + "packages": [{"actions": [action], "id": "pkg", "path": "reduced"}] + }, + ) + monkeypatch.setattr( + pa.PluginManifest, + "load_all", + classmethod(lambda cls, kind: [manifest]), + ) + monkeypatch.setattr(pa, "prepare_overlay", lambda instance, registry: "/overlay") + captured: dict[str, Any] = {} + + def fake_run( + selected_action: dict[str, Any], + package: dict[str, Any], + *args: Any, + ) -> None: + captured["action"] = selected_action + captured["package"] = package + + monkeypatch.setattr(pa, "run_action_target", fake_run) + + assert pa.main(["--instance", "inst", "--package", "pkg", "--action", "open"]) == 0 + assert captured["action"] == action + assert captured["package"]["config_schema"] == manifest["config_schema"] diff --git a/tests/python/commands/test_provision_secrets.py b/tests/python/commands/test_provision_secrets.py new file mode 100644 index 00000000..a300711f --- /dev/null +++ b/tests/python/commands/test_provision_secrets.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import importlib.machinery +import importlib.util +import json +import os +import stat +import subprocess +from pathlib import Path +from types import ModuleType + +import pytest + +ROOT = Path(__file__).resolve().parents[3] + + +def _load_script(name: str) -> ModuleType: + path = ROOT / "scripts" / name + loader = importlib.machinery.SourceFileLoader(name.replace("-", "_"), str(path)) + spec = importlib.util.spec_from_loader(loader.name, loader) + assert spec is not None + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +@pytest.mark.parametrize("script", ["package-provision", "provision"]) +def test_shell_environment_values_round_trip_without_expansion(script: str) -> None: + module = _load_script(script) + hostile = 'cash$HOME $(printf injected) `printf injected` "quote" \\ slash' + source = module._env_line("TEST_SECRET", hostile) + + result = subprocess.run( + ["/bin/sh", "-c", f"{source}; printf %s \"$TEST_SECRET\""], + text=True, + capture_output=True, + check=True, + ) + + assert result.stdout == hostile + + +@pytest.mark.parametrize("script", ["package-provision", "provision"]) +def test_secret_environment_file_is_owner_only(script: str, tmp_path: Path) -> None: + module = _load_script(script) + path = tmp_path / "env" + + module._write_private_text(path, "secret") + + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + + +@pytest.mark.parametrize("script", ["package-provision", "provision"]) +def test_staged_path_expands_only_remote_home(script: str) -> None: + module = _load_script(script) + filename = 'installer $(printf injected) `printf injected` "quote".deb' + relative = "provision/downloads/installer/" + filename + source = module._env_line("INSTALLER", "$HOME/" + relative, remote_home=True) + result = subprocess.run( + ["/bin/sh", "-c", source + '; printf %s "$INSTALLER"'], + env={"HOME": "/remote/home"}, text=True, capture_output=True, check=True, + ) + assert result.stdout == "/remote/home/" + relative + + +@pytest.mark.parametrize("existing_mode", [None, 0o644]) +def test_private_payload_permissions_are_set_before_writing( + existing_mode: int | None, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +) -> None: + from eve_sdk.secrets import write_private_text + + path = tmp_path / "env.json" + if existing_mode is not None: + path.write_text("old") + path.chmod(existing_mode) + original_fdopen = os.fdopen + + def checked_fdopen(fd: int, *args: object, **kwargs: object) -> object: + # Existing files are secured inside the writer before handle.write. + handle = original_fdopen(fd, *args, **kwargs) + + class CheckedWriter: + def __enter__(self) -> object: + return self + + def __exit__(self, *exc: object) -> None: + handle.close() + + def fileno(self) -> int: + return handle.fileno() + + def write(self, text: str) -> int: + assert stat.S_IMODE(os.fstat(fd).st_mode) == 0o600 + return handle.write(text) + + return CheckedWriter() + + monkeypatch.setattr(os, "fdopen", checked_fdopen) + write_private_text(path, '{"secret":"fake"}') + assert path.read_text() == '{"secret":"fake"}' + + +def test_windows_partial_package_payload_upload_is_cleaned( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +) -> None: + module = _load_script("package-provision-windows") + tree = tmp_path / "oses" / "test-windows" / "provision" + (tree / "scripts" / "lib").mkdir(parents=True) + (tree / "scripts" / "steps").mkdir() + (tree / "scripts" / "steps" / "install.ps1").write_text("exit 0") + entry = { + "id": "test-package", "path": str(tmp_path / "eve-plugin.yaml"), + "install": {"windows": {"steps": ["install.ps1"], "state_files": ["env.json"]}}, + "config_schema": {"secrets": {"password": {"env_var": "TEST_PASSWORD"}}}, + } + monkeypatch.setattr(module, "repo_root", lambda: tmp_path) + monkeypatch.setattr(module, "resolve_env", lambda *args: {"OS_FAMILY": "windows", "OS_ID": "test-windows"}) + monkeypatch.setattr(module, "_plugin_list_json", lambda: {"plugins": [entry]}) + monkeypatch.setattr(module, "instance_package_config", lambda *args: {}) + monkeypatch.setattr(module, "resolved_package_set_environment", lambda *args: {"TEST_PASSWORD": "fake-secret"}) + monkeypatch.setenv("EPHEMERAL_WINDOWS_PASSWORD", "fake-login") + monkeypatch.setattr(module.subprocess, "run", lambda *args, **kwargs: subprocess.CompletedProcess(args, 0)) + cleanups: list[str] = [] + monkeypatch.setattr(module, "_instance_ssh", lambda *args: cleanups.append(args[-1]) or 0) + + def upload(label: str, cmd: list[str]) -> int: + if label == "upload windows package env.json": + payload = Path(cmd[-2]) + assert stat.S_IMODE(payload.stat().st_mode) == 0o600 + assert "fake-secret" in json.dumps(json.loads(payload.read_text())) + return 23 + return 0 + + monkeypatch.setattr(module, "_ssh_retry", upload) + assert module.main(["--instance", "test", "--package", "test-package"]) == 23 + assert any("Remove-Item" in cmd and "env.json" in cmd for cmd in cleanups) + + +@pytest.mark.parametrize("family", ["ubuntu", "windows"]) +def test_full_provision_cleans_partial_secret_uploads( + family: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +) -> None: + module = _load_script("provision") + tree = tmp_path / "provision" + steps = tree / "scripts" / "steps" + steps.mkdir(parents=True) + for name in ["base.sh", "finish.sh", "human-user.sh", "timezone.sh"]: + (steps / name).write_text("exit 0") + monkeypatch.setattr(module.PluginManifest, "os_provision_dir", lambda *args: tree) + monkeypatch.setattr(module, "_selected_package_environment", lambda *args: ([], {"TIMEZONE": "UTC"})) + monkeypatch.setenv("EPHEMERAL_WINDOWS_PASSWORD", "fake-login") + monkeypatch.setattr( + module.subprocess, "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args, 0, stdout='{"plugins": []}'), + ) + cleanups: list[str] = [] + monkeypatch.setattr(module, "_ssh_retry", lambda *args: cleanups.append(args[1]) or 0) + + def upload(profile: str, label: str, local: str, remote: str) -> int: + if label in {"upload linux environment", "upload windows provision tree"}: + path = Path(local) if family == "ubuntu" else Path(local) / "state" / "env.json" + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + return 23 + return 0 + + monkeypatch.setattr(module, "_ssh_retry_scp", upload) + action = module.provision_ubuntu if family == "ubuntu" else module.provision_windows + assert action("test", {"OS_FAMILY": family, "OS_ID": "test-os"}, tmp_path) == 23 + assert any("environment after upload failure" in label for label in cleanups) + + +def test_provision_scripts_remove_remote_secret_files() -> None: + linux = (ROOT / "scripts" / "package-provision").read_text(encoding="utf-8") + full = (ROOT / "scripts" / "provision").read_text(encoding="utf-8") + windows = (ROOT / "scripts" / "package-provision-windows").read_text( + encoding="utf-8" + ) + + assert "trap \\'rm -f" in linux + assert "$HOME/provision/state/env" in linux + assert "remove linux provision environment" in full + assert "remove windows provision environment" in full + assert "windows_secret_cleanup" in windows diff --git a/tests/python/sdk/test_command_observations.py b/tests/python/sdk/test_command_observations.py new file mode 100644 index 00000000..46cf9447 --- /dev/null +++ b/tests/python/sdk/test_command_observations.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import json + +import pytest + +from eve_sdk.dispatch import package_status_from_output + + +@pytest.mark.parametrize("indent", [None, 2]) +def test_root_package_status_wins_over_nested_observations(indent: int | None) -> None: + output = json.dumps({"status": "missing", "packages": [{"status": "installed"}]}, indent=indent) + assert package_status_from_output(output) == "missing" + + +@pytest.mark.parametrize("indent", [None, 2]) +def test_nested_observations_do_not_supply_a_missing_root_status(indent: int | None) -> None: + output = json.dumps({"packages": [{"status": "installed"}]}, indent=indent) + assert package_status_from_output(output) is None + + +def test_last_root_status_survives_surrounding_command_logs() -> None: + output = 'starting\n{"status":"missing"}\n{"status":"installed"}\nfinished\n' + assert package_status_from_output(output) == "installed" diff --git a/tests/python/sdk/test_config_provider_paths.py b/tests/python/sdk/test_config_provider_paths.py index 02fdea24..e603532b 100644 --- a/tests/python/sdk/test_config_provider_paths.py +++ b/tests/python/sdk/test_config_provider_paths.py @@ -1,6 +1,9 @@ from __future__ import annotations + from pathlib import Path + import pytest + from eve_sdk.config import ConfigEnv @@ -22,6 +25,7 @@ def test_config_env_expands_provider_path_field(monkeypatch: pytest.MonkeyPatch, def test_config_env_has_no_provider_env_names_hardcoded() -> None: """Core no longer carries provider env-var literals (they live in manifests).""" import inspect + from eve_sdk import config as cfg src = inspect.getsource(cfg) for provider_env in ( @@ -39,18 +43,27 @@ def test_bootstrap_sudo_password_reads_declared_env(monkeypatch: pytest.MonkeyPa from eve_sdk import plugin_manifest fake = [ - {"id": "p1", "bootstrap": {"sudo_password_env": "P1_PW"}}, + { + "id": "p1", + "bootstrap": {"sudo_password_env": "P1_PW"}, + "config_schema": { + "secrets": { + "password": {"env_var": "P1_PW", "type": "string"} + } + }, + }, {"id": "p2"}, # declares no bootstrap ] monkeypatch.setattr( plugin_manifest.PluginManifest, "load_all", classmethod(lambda cls, kind=None: fake), ) - monkeypatch.setenv("P1_PW", "s3cret") + monkeypatch.setattr( + "eve_sdk.provider_command.Secrets.read", + staticmethod(lambda name: {"password": "s3cret"} if name == "p1" else {}), + ) assert ConfigEnv.bootstrap_sudo_password("p1") == "s3cret" # declared + set - monkeypatch.delenv("P1_PW") - assert ConfigEnv.bootstrap_sudo_password("p1") == "" # declared, unset assert ConfigEnv.bootstrap_sudo_password("p2") == "" # no bootstrap block assert ConfigEnv.bootstrap_sudo_password("missing") == "" # unknown provider @@ -84,6 +97,24 @@ def test_plugin_provision_env_names_aggregates_config_and_secrets( assert names == ["PKG_KEY_FILE", "PKG_PASSWORD", "PKG_VERSION"] +def test_provision_env_payload_includes_only_selected_package_names() -> None: + environment = { + "EPHEMERAL_DISPLAY_RESOLUTION": "1920x1080", + "SELECTED_PASSWORD": "secret-one", + "UNRELATED_PASSWORD": "secret-two", + } + + payload = ConfigEnv.provision_env_payload( + "windows-secret", environment, ["SELECTED_PASSWORD"] + ) + + assert payload == { + "windows_password": "windows-secret", + "display_resolution": "1920x1080", + "selected_password": "secret-one", + } + + def test_package_stage_env_names_only_package_type_path( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -138,3 +169,101 @@ def test_instance_package_env_maps_overrides_via_manifest( }) assert out == {"STREAMER_VERSION": "9.9", "STREAMER_BITRATE": "20000"} assert ConfigEnv.instance_package_env({}) == {} + + +def test_package_environment_is_operation_local( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from eve_sdk import plugin_manifest + from eve_sdk.package_dispatch import resolved_package_environment + + plugin = { + "id": "streamer", + "config_schema": { + "config": {"version": {"env_var": "STREAMER_VERSION"}}, + "secrets": {"token": {"env_var": "STREAMER_TOKEN"}}, + }, + } + other_plugin = { + "id": "other", + "config_schema": { + "config": {"version": {"env_var": "OTHER_VERSION"}}, + }, + } + monkeypatch.setattr( + plugin_manifest.PluginManifest, + "load_all", + classmethod( + lambda cls, kind=None: [plugin, other_plugin] + if kind == "package" + else [] + ), + ) + monkeypatch.setattr( + ConfigEnv, + "environment", + classmethod( + lambda cls, *a, **k: { + "OTHER_PLUGIN_TOKEN": "must-not-leak", + "STREAMER_VERSION": "1.0", + } + ), + ) + monkeypatch.setattr( + "eve_sdk.provider_command.Secrets.read", + staticmethod(lambda name: {"token": "secret"}), + ) + monkeypatch.setenv("UNDECLARED_AMBIENT_TOKEN", "must-not-leak") + + env = resolved_package_environment( + plugin, + { + "instance": {"name": "test"}, + "package_config": { + "other": {"version": "must-not-leak"}, + "streamer": {"version": "2.0"}, + }, + }, + ) + + assert env["STREAMER_TOKEN"] == "secret" + assert env["STREAMER_VERSION"] == "2.0" + assert "OTHER_PLUGIN_TOKEN" not in env + assert "OTHER_VERSION" not in env + assert "UNDECLARED_AMBIENT_TOKEN" not in env + + +def test_provider_environment_is_operation_local( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from eve_sdk.provider_command import resolved_provider_environment + + plugin = { + "id": "cloud", + "config_schema": { + "config": {"region": {"env_var": "CLOUD_REGION"}}, + "secrets": {"token": {"env_var": "CLOUD_TOKEN"}}, + }, + } + monkeypatch.setattr( + ConfigEnv, + "environment", + classmethod( + lambda cls, *a, **k: { + "CLOUD_REGION": "r1", + "OTHER_PROVIDER_TOKEN": "must-not-leak", + } + ), + ) + monkeypatch.setattr( + "eve_sdk.provider_command.Secrets.read", + staticmethod(lambda name: {"token": "secret"}), + ) + monkeypatch.setenv("UNDECLARED_AMBIENT_TOKEN", "must-not-leak") + + env = resolved_provider_environment("cloud", plugin) + + assert env["CLOUD_REGION"] == "r1" + assert env["CLOUD_TOKEN"] == "secret" + assert "OTHER_PROVIDER_TOKEN" not in env + assert "UNDECLARED_AMBIENT_TOKEN" not in env diff --git a/tests/python/sdk/test_plugin_manifest_contract.py b/tests/python/sdk/test_plugin_manifest_contract.py index b66401e3..6a64c43c 100644 --- a/tests/python/sdk/test_plugin_manifest_contract.py +++ b/tests/python/sdk/test_plugin_manifest_contract.py @@ -100,6 +100,58 @@ def test_core_version_is_4_0(self) -> None: # --------------------------------------------------------------------------- class TestPackageConfigSchema: + def test_required_any_accepts_declared_alternatives(self, tmp_path: Path) -> None: + manifest = _provider_manifest( + tmp_path, + config_schema={ + "config": {"profile": {"env_var": "AWS_PROFILE", "type": "string"}}, + "required_any": [ + ["config.profile"], + ["secrets.access_key", "secrets.secret_key"], + ], + "secrets": { + "access_key": {"env_var": "AWS_ACCESS_KEY_ID", "type": "string"}, + "secret_key": {"env_var": "AWS_SECRET_ACCESS_KEY", "type": "string"}, + }, + }, + ) + PluginManifest.validate(manifest) + + def test_required_any_rejects_undeclared_reference(self, tmp_path: Path) -> None: + manifest = _provider_manifest( + tmp_path, + config_schema={ + "config": {"profile": {"env_var": "AWS_PROFILE", "type": "string"}}, + "required_any": [["secrets.missing"]], + }, + ) + with pytest.raises(ValueError, match=r"undeclared field secrets\.missing"): + PluginManifest.validate(manifest) + + def test_required_any_rejects_field_without_env_var(self, tmp_path: Path) -> None: + manifest = _provider_manifest( + tmp_path, + config_schema={ + "config": {"profile": {"type": "string"}}, + "required_any": [["config.profile"]], + }, + ) + with pytest.raises(ValueError, match=r"config\.profile must declare env_var"): + PluginManifest.validate(manifest) + + def test_required_any_rejects_package_manifest(self, tmp_path: Path) -> None: + manifest = _package_manifest( + tmp_path, + config_schema={ + "config": { + "profile": {"env_var": "PROFILE", "type": "string"} + }, + "required_any": [["config.profile"]], + }, + ) + with pytest.raises(ValueError, match="only valid on provider plugins"): + PluginManifest.validate(manifest) + def test_package_config_schema_valid(self, tmp_path: Path) -> None: manifest = _package_manifest( tmp_path, diff --git a/tests/python/sdk/test_state.py b/tests/python/sdk/test_state.py index bfd264dc..78cd70a5 100644 --- a/tests/python/sdk/test_state.py +++ b/tests/python/sdk/test_state.py @@ -86,6 +86,75 @@ def test_provider_event_drives_state_through_the_machine( assert State.read("demo")["provider_state"] == "error" +def test_provision_operation_drives_state_through_the_machine( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("EVE_STATE_DIR", str(tmp_path)) + + assert ( + State.record_operation("demo", "provision", "running")["provision_state"] + == "provisioning" + ) + recovered = State.recover_running("demo") + assert recovered["provision_state"] == "error" + assert recovered["last_operation"]["status"] == "failed" + assert State.record_operation("demo", "provision", "failed")["provision_state"] == "error" + assert ( + State.record_operation("demo", "provision", "running")["provision_state"] + == "provisioning" + ) + assert ( + State.record_operation("demo", "provision", "succeeded")["provision_state"] + == "provisioned" + ) + assert ( + State.record_operation("demo", "provision", "running")["provision_state"] + == "provisioning" + ) + + +def test_package_operation_and_provision_observations_are_machine_driven( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("EVE_STATE_DIR", str(tmp_path)) + + installing = State.record_operation( + "demo", "package.install", "running", package="mock-app" + ) + assert installing["package_state"]["mock-app"]["status"] == "installing" + assert installing["last_operation"]["package"] == "mock-app" + + recovered = State.recover_running("demo") + assert recovered["package_state"]["mock-app"]["status"] == "failed" + + State.record_operation("demo", "provision", "running") + completed = State.record_operation( + "demo", + "provision", + "succeeded", + package_observations={"mock-app": "installed", "mock-tool": "installed"}, + ) + assert completed["provision_state"] == "provisioned" + assert completed["package_state"]["mock-app"]["status"] == "installed" + assert completed["package_state"]["mock-tool"]["status"] == "installed" + assert completed["last_operation"]["name"] == "provision" + + +def test_operation_ids_remain_unique_after_history_is_trimmed( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("EVE_STATE_DIR", str(tmp_path)) + + for _ in range(State.DEFAULT_HISTORY_LIMIT + 5): + State.record_operation("demo", "provider.status", "succeeded") + + history = State.read("demo")["operation_history"] + ids = [entry["id"] for entry in history] + assert len(ids) == State.DEFAULT_HISTORY_LIMIT + assert len(ids) == len(set(ids)) + assert ids[-1] == State.DEFAULT_HISTORY_LIMIT + 5 + + def test_state_concurrent_writers_preserve_history(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("EVE_STATE_DIR", str(tmp_path)) State.write("concurrency-test", State.default_state("concurrency-test", "2026-01-01T00:00:00Z")) diff --git a/tests/test_fsm_machines.py b/tests/test_fsm_machines.py index 190e6e93..7a8cbf50 100644 --- a/tests/test_fsm_machines.py +++ b/tests/test_fsm_machines.py @@ -38,7 +38,28 @@ def test_package_lifecycle() -> None: fsm.fire("package", "x", "install_ok") assert fsm.leaves("package", "x") == ["installed"] - fsm.fire("package", "x", "absent") # drift: disappeared out of band + fsm.fire("package", "x", "install") # reinstall + assert fsm.leaves("package", "x") == ["installing"] + fsm.fire("package", "x", "install_ok") + + fsm.fire("package", "x", "remove") + assert fsm.leaves("package", "x") == ["removing"] + fsm.fire("package", "x", "remove_fail") + assert fsm.leaves("package", "x") == ["failed"] + + fsm.fire("package", "x", "remove") # retry + fsm.fire("package", "x", "remove_ok") + assert fsm.leaves("package", "x") == ["removed"] + + fsm.fire("package", "x", "found") # appeared out of band + assert fsm.leaves("package", "x") == ["installed"] + + fsm.fire("package", "x", "install") + fsm.fire("package", "x", "found") # authoritative observation + assert fsm.leaves("package", "x") == ["installed"] + + fsm.fire("package", "x", "remove") + fsm.fire("package", "x", "absent") # authoritative observation assert fsm.leaves("package", "x") == ["missing"] @@ -68,7 +89,7 @@ def test_instance_provider_lifecycle() -> None: def test_instance_provision_lifecycle() -> None: fsm = _fsm("instance", machine_id="instance_provision") - assert fsm.leaves("instance", "x") == ["unprovisioned"] + assert fsm.leaves("instance", "x") == ["unknown"] fsm.fire("instance", "x", "provision") fsm.fire("instance", "x", "provision_ok") @@ -76,7 +97,7 @@ def test_instance_provision_lifecycle() -> None: fsm.fire("instance", "x", "provision") # re-converge fsm.fire("instance", "x", "provision_fail") - assert fsm.leaves("instance", "x") == ["provision_error"] + assert fsm.leaves("instance", "x") == ["error"] fsm.fire("instance", "x", "provision") # retry fsm.fire("instance", "x", "provision_ok") diff --git a/tests/test_fsm_platform.py b/tests/test_fsm_platform.py index c3607e6e..e95cbfb6 100644 --- a/tests/test_fsm_platform.py +++ b/tests/test_fsm_platform.py @@ -77,7 +77,10 @@ def test_provider_context_resolver_uses_shared_config_secret_path(monkeypatch: A lambda kind, plugin_id: { "id": plugin_id, "kind": "provider", - "config_schema": {"secrets": {"api_key": {"env_var": "VULTR_API_KEY"}}}, + "config_schema": { + "config": {"region": {"env_var": "VULTR_REGION"}}, + "secrets": {"api_key": {"env_var": "VULTR_API_KEY"}}, + }, }, ) monkeypatch.setattr("eve_sdk.provider_command.Secrets.read", staticmethod(lambda name: {"api_key": "SECRET"})) @@ -121,3 +124,97 @@ def test_core_provider_machine_configured_from_required_secret(monkeypatch: Any) # the connectivity probe succeeds -> reachable fsm.fire("provider", "vultr", "probe_ok") assert _leaves(fsm) == ["reachable"] + + +def test_core_provider_machine_honors_required_config(monkeypatch: Any) -> None: + from eve_sdk.fsm import EveFsm, provider_configured_resolver + + monkeypatch.delenv("AWS_PROFILE", raising=False) + box = {"env": {}} + monkeypatch.setattr( + "eve_sdk.config.ConfigEnv.environment", + classmethod(lambda cls, *a, **k: dict(box["env"])), + ) + monkeypatch.setattr( + "eve_sdk.provider_command._load_public_plugin", + lambda kind, plugin_id: { + "id": plugin_id, + "kind": "provider", + "config_schema": { + "config": {"profile": {"required": True, "env_var": "AWS_PROFILE"}} + }, + }, + ) + monkeypatch.setattr( + "eve_sdk.provider_command.Secrets.read", staticmethod(lambda name: {}) + ) + + fsm = EveFsm(provider_configured_resolver()) + core = (Path(__file__).resolve().parents[1] / "core" / "fsm" / "provider.yaml").read_text() + fsm.load("provider", core) + fsm.fire("provider", "aws", "resolve") + assert fsm.state("provider", "aws") == "unconfigured" + + box["env"] = {"AWS_PROFILE": "eve"} + fsm.refresh("provider", "aws") + fsm.fire("provider", "aws", "resolve") + assert fsm.state("provider", "aws") == "unreachable" + + +def test_core_provider_machine_honors_required_any(monkeypatch: Any) -> None: + from eve_sdk.fsm import EveFsm, provider_configured_resolver + + box = {"env": {}} + monkeypatch.setattr( + "eve_sdk.config.ConfigEnv.environment", + classmethod(lambda cls, *a, **k: dict(box["env"])), + ) + plugin = { + "id": "aws", + "kind": "provider", + "config_schema": { + "config": {"profile": {"env_var": "AWS_PROFILE"}}, + "required_any": [ + ["config.profile"], + ["secrets.access_key", "secrets.secret_key"], + ], + "secrets": { + "access_key": {"env_var": "AWS_ACCESS_KEY_ID"}, + "secret_key": {"env_var": "AWS_SECRET_ACCESS_KEY"}, + }, + }, + } + monkeypatch.setattr( + "eve_sdk.provider_command._load_public_plugin", + lambda kind, plugin_id: plugin, + ) + box_secrets: dict[str, str] = {} + monkeypatch.setattr( + "eve_sdk.provider_command.Secrets.read", + staticmethod(lambda name: dict(box_secrets)), + ) + + fsm = EveFsm(provider_configured_resolver()) + core = ( + Path(__file__).resolve().parents[1] / "core" / "fsm" / "provider.yaml" + ).read_text() + fsm.load("provider", core) + + fsm.fire("provider", "aws", "resolve") + assert fsm.state("provider", "aws") == "unconfigured" + + box_secrets["access_key"] = "ACCESS" + fsm.refresh("provider", "aws") + fsm.fire("provider", "aws", "resolve") + assert fsm.state("provider", "aws") == "unconfigured" + + box_secrets["secret_key"] = "SECRET" + fsm.refresh("provider", "aws") + fsm.fire("provider", "aws", "resolve") + assert fsm.state("provider", "aws") == "unreachable" + + box_secrets.clear() + box["env"] = {"AWS_PROFILE": "eve"} + fsm.refresh("provider", "aws") + fsm.fire("provider", "aws", "resolve") + assert fsm.state("provider", "aws") == "unreachable" From 13341e8fcdb4acfa3530055f144e003b8f5fb3c2 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Sun, 4 Oct 2026 04:20:52 +0000 Subject: [PATCH 30/32] refactor(v4.5): remove unused alternative auth schema --- .github/workflows/test.yml | 1 - core/schema/plugin-manifest.schema.json | 14 --- docs/recovery-cloud-validation.md | 27 +++++- docs/v3.2-roadmap.md | 41 ++++----- docs/v4.5-fsm-design.md | 9 +- eve_sdk/fsm.py | 24 +---- eve_sdk/plugin_manifest.py | 26 ------ eve_sdk/profile_resolve.py | 4 + eve_sdk/provider_command.py | 4 +- eve_sdk/tf_dispatch.py | 8 +- .../providers/mock-cloud/commands/tf-env | 6 ++ .../providers/mock-cloud/eve-plugin.yaml | 1 + tests/python/commands/test_tf_dispatch.py | 18 ++++ .../python/sdk/test_aws_profile_isolation.py | 92 +++++++++++++++++++ .../sdk/test_plugin_manifest_contract.py | 52 ----------- tests/test_fsm_platform.py | 59 ------------ tui/settings.py | 9 +- 17 files changed, 185 insertions(+), 210 deletions(-) create mode 100755 tests/fixtures/hermetic/providers/mock-cloud/commands/tf-env create mode 100644 tests/python/sdk/test_aws_profile_isolation.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index df203c06..74437c78 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,6 @@ env: jobs: test: - if: github.event_name != 'pull_request' || github.event.pull_request.draft == false strategy: fail-fast: false matrix: diff --git a/core/schema/plugin-manifest.schema.json b/core/schema/plugin-manifest.schema.json index 1021b001..e24e27e8 100644 --- a/core/schema/plugin-manifest.schema.json +++ b/core/schema/plugin-manifest.schema.json @@ -193,20 +193,6 @@ "$ref": "#/$defs/field_spec" } }, - "required_any": { - "description": "Provider-only alternative complete credential/configuration groups. At least one group must have every referenced field set. References use config. or secrets..", - "type": "array", - "minItems": 1, - "items": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "pattern": "^(config|secrets)\\.[a-z][a-z0-9_]*$" - }, - "uniqueItems": true - } - }, "secrets": { "type": "object", "additionalProperties": { diff --git a/docs/recovery-cloud-validation.md b/docs/recovery-cloud-validation.md index 7e19b9d3..cc61e322 100644 --- a/docs/recovery-cloud-validation.md +++ b/docs/recovery-cloud-validation.md @@ -79,8 +79,9 @@ accept explicit roots and never need live providers. Determa remains the transition authority for provider, provisioning, package, and interrupted-operation recovery. Operation IDs are allocated under the state lock and remain monotonic when history is trimmed. Authentication evaluates all -universal requirements plus at least one complete alternative group -(`required_any` is OR-of-AND). Provider and package command environments are +ordinary `required: true` configuration and secret fields. AWS requires an +explicit named profile; Eve does not manage AWS key material or inspect profile +credentials. Provider and package command environments are operation-local and inject only declared target credentials. Fresh checks identified and fixed these additional problems: @@ -106,6 +107,28 @@ Fresh checks identified and fixed these additional problems: ## GitHub and remaining rollout dependency +### Profile-only AWS revision — 2026-10-04 (Asia/Tokyo) + +AWS now requires an explicitly configured named profile. Eve no longer declares, +reads, injects, or edits AWS key material. AWS tooling owns profile credentials +and login/SSO sessions; Eve does not inspect the profile's contents. Without a +profile, connectivity returns unconfigured without invoking AWS. With a profile, +the STS probe always supplies `--profile` and reports a failed login/session as +configured but unreachable. + +No other shipped manifest used alternative authentication groups, so their +schema, validator, FSM evaluation, tests, and documentation were removed. +The operation-local environment allowlist remains unchanged. Regression tests +cover ambient AWS credential isolation and skipped secret-store reads in +dispatch, legacy profile resolution, and settings. Terraform environment +failures now abort before backend access instead of applying partial exports. + +Revised offline validation: 529 Python tests passed with one intentional skip; +49 focused schema/configuration/isolation checks passed. All six AWS connectivity +tests, eight provider backend/profile checks, provider environment contracts, +actual Linux/PowerShell runners, and combined 47-plugin conformance passed. +Core CI now runs on draft PRs so readiness can be checked while #55 remains draft. + Core recovery CI skipped because its PR was a draft; the cross-repository job is scheduled/manual only. Provider and Linux/Windows recovery CI was absent because those workflows filtered PR bases to `main`. Companion workflows now include diff --git a/docs/v3.2-roadmap.md b/docs/v3.2-roadmap.md index 2a3e6f8c..76e625c1 100644 --- a/docs/v3.2-roadmap.md +++ b/docs/v3.2-roadmap.md @@ -67,12 +67,13 @@ Extend `core/schema/` with: `.eve/secrets/.yaml` (mode 0600, gitignored). Format: ```yaml -aws: - access_key_id: - secret_access_key: - profile: +vultr: + api_key: ``` +AWS uses an explicitly configured named profile in `config.yaml`, not an Eve +secret store. AWS tooling owns profile credentials and SSO sessions. + Secrets never appear in TUI plaintext display (masked). Secret values flow into the resolved instance JSON via the same env_or() path already used by `scripts/profile-resolve`. @@ -102,12 +103,10 @@ screen per provider, listing the fields declared by that provider's ``` AWS Configuration ───────────────── -profile: default [config.yaml] +profile: my-sso-profile [config.yaml] region: us-east-1 [config.yaml] config_file: ~/.aws/config [env] shared_credentials_file: ~/.aws/credentials [env] -access_key_id: ●●●●●●●● (masked) [secret store] -secret_access_key: ●●●●●●●● (masked) [secret store] [Test connection] [Save] [Cancel] ``` @@ -197,8 +196,6 @@ its new home and classification. Coding-agent reference. | `aws.profile` | `AWS_PROFILE` | string | no | | `aws.region` | `AWS_REGION` | string | no | | `aws.shared_credentials_file` | `AWS_SHARED_CREDENTIALS_FILE` | path | no | -| `secrets/aws.access_key_id` | `AWS_ACCESS_KEY_ID` | string | **yes** | -| `secrets/aws.secret_access_key` | `AWS_SECRET_ACCESS_KEY` | string | **yes** | **gcp:** @@ -289,14 +286,13 @@ end Storage format (mode 0600, gitignored): ```yaml -# .eve/secrets/aws.yaml -aws: - access_key_id: "" - secret_access_key: "" +# .eve/secrets/vultr.yaml +vultr: + api_key: "" ``` Top-level key matches the provider id so consumers grep'd -`aws.access_key_id` resolve unambiguously. Multi-provider secrets +`vultr.api_key` resolve unambiguously. Multi-provider secrets (e.g. a shared SSH key) get one secret entry per provider that references them; we do not de-dupe. @@ -306,8 +302,8 @@ references them; we do not de-dupe. default)` for each emitted env var. v3.2 adds a step before that which: -1. Reads `.eve/secrets/.yaml` for the resolved instance's - provider. +1. Reads `.eve/secrets/.yaml` only when the resolved instance's + provider declares secret fields. AWS declares none. 2. Populates a process-scoped hash, preferring `ENV[NAME]` when set (env wins over secrets store; matches the precedence table). 3. Exports the merged env block so `env_or()` sees the secrets store @@ -328,23 +324,18 @@ config_schema: config: profile: type: string - description: "AWS named profile to use" + env_var: AWS_PROFILE + required: true + description: "An explicit named AWS profile is required" region: type: string - enum: [us-east-1, us-west-2, eu-west-1, ap-northeast-1, ...] + enum: [ap-northeast-1, eu-west-1, us-east-1, us-west-2] config_file: type: path default: "~/.aws/config" shared_credentials_file: type: path default: "~/.aws/credentials" - secrets: - access_key_id: - type: string - description: "AWS_ACCESS_KEY_ID" - secret_access_key: - type: string - description: "AWS_SECRET_ACCESS_KEY" ``` Each field gains: `type` (string/integer/boolean/path/url/host/enum), diff --git a/docs/v4.5-fsm-design.md b/docs/v4.5-fsm-design.md index da669699..cf395d31 100644 --- a/docs/v4.5-fsm-design.md +++ b/docs/v4.5-fsm-design.md @@ -237,8 +237,7 @@ Eve now tracks the compatible 0.2.x line.) - ☑ **Core provider machine** — `core/fsm/provider.yaml`: one shared provider lifecycle (unconfigured → configured → reachable) for *every* provider. `is_configured` is derived by `provider_configured_resolver()` from the plugin's `config_schema` - universally `required` secrets/config fields plus the optional provider-only - `required_any` alternative groups (via the shared path), so "configured" is + ordinary `required: true` secrets/config fields (via the shared path), so "configured" is eve-resolved, not a plugin's ambient self-report. The lifecycle is identical across providers, so it lives in `core/`, not per-provider. Tested end-to-end. - ☑ **Dependency** — `determa-state (>=0.2.0,<0.3.0)` from PyPI, locked; the FSM tests @@ -247,9 +246,9 @@ Eve now tracks the compatible 0.2.x line.) + `provider_configured_resolver`); the `connectivity` command runs **only for `reachable`** (its self-reported `configured` is ignored) and only when configured. Live-verified (vultr yes/yes, truenas no, docker/qemu yes); full suite green. - - AWS uses `required_any` to express a profile **or** a complete static access-key pair; - all static credential fields map to their AWS environment names. A partial key pair - remains unconfigured, while connectivity remains a separate live probe. + - AWS requires an explicitly configured named profile. AWS tooling owns the profile's + credentials and login/SSO session; Eve does not read or manage them. Ambient AWS + credentials do not configure the provider. Connectivity remains a separate probe. - ☑ **Instance & package machines (declarative)** — `core/fsm/instance.yaml` (two machines in one bundle: `instance_provider` + `instance_provision`, since format 1 has no orthogonal regions; `desired` as external context) + `core/fsm/package.yaml`, diff --git a/eve_sdk/fsm.py b/eve_sdk/fsm.py index 09d6a673..d4b6ea66 100644 --- a/eve_sdk/fsm.py +++ b/eve_sdk/fsm.py @@ -173,9 +173,8 @@ def provider_context_resolver() -> ContextResolver: FSM context and what dispatch runs with are identical (v4.5 bug M: status ≡ dispatch). ``key`` is the provider id. - (Systemic follow-up: drop ambient ``os.environ`` from *both* this resolver and - dispatch so no config reaches a plugin behind eve's back; kept here for now to - preserve exact parity.) + Both the resolver and dispatch use the same operation-local environment; + ambient credentials cannot configure a provider. """ from eve_sdk.provider_command import _load_public_plugin, resolved_provider_environment @@ -190,9 +189,8 @@ def resolve(scope: str, key: str, names: list[str]) -> dict[str, Any]: def provider_configured_resolver() -> ContextResolver: """The :data:`ContextResolver` for the core provider machine (``core/fsm/provider.yaml``). - Derives ``is_configured`` from both universally ``required`` fields and the - optional ``required_any`` alternatives. Every universal field must be set, - and at least one alternative group must be complete. So "configured" is + Derives ``is_configured`` from ordinary ``required`` config and secret fields. + Every required field must be set. So "configured" is eve-resolved, never a plugin's connectivity self-report. ``key`` is the provider id; ``names`` is typically ``["is_configured"]``. """ @@ -218,20 +216,8 @@ def present(field: str, spec: Mapping[str, Any]) -> bool: candidates.append(str(env_var)) return any(env.get(candidate) for candidate in candidates) - alternatives = schema.get("required_any") or [] - - def referenced_present(reference: str) -> bool: - section, field = reference.split(".", 1) - spec = (schema.get(section) or {}).get(field) - return isinstance(spec, Mapping) and present(field, spec) - - required_ok = all(present(field, spec) for field, spec in required) - alternatives_ok = not alternatives or any( - all(referenced_present(str(reference)) for reference in group) - for group in alternatives - ) values: dict[str, Any] = { - "is_configured": required_ok and alternatives_ok + "is_configured": all(present(field, spec) for field, spec in required) } return {name: values.get(name, "") for name in names} diff --git a/eve_sdk/plugin_manifest.py b/eve_sdk/plugin_manifest.py index 381a16e6..5d81e6bb 100644 --- a/eve_sdk/plugin_manifest.py +++ b/eve_sdk/plugin_manifest.py @@ -282,34 +282,8 @@ def validate(cls, plugin: dict[str, Any]) -> None: if not isinstance(config_schema, dict): raise ValueError(f"{path}: config_schema must be a map") validate_json_schema_fragment(config_schema, f"{path}: config_schema") - cls._validate_config_required_any(config_schema, path, str(kind)) cls._validate_requires(plugin, path) - @staticmethod - def _validate_config_required_any( - config_schema: dict[str, Any], path: str, kind: str - ) -> None: - alternatives = config_schema.get("required_any") or [] - if alternatives and kind != "provider": - raise ValueError( - f"{path}: config_schema.required_any is only valid on provider plugins" - ) - for group in alternatives: - for reference in group: - section, field = str(reference).split(".", 1) - fields = config_schema.get(section) or {} - if field not in fields: - raise ValueError( - f"{path}: config_schema.required_any references " - f"undeclared field {reference}" - ) - spec = fields[field] - if not isinstance(spec, dict) or not spec.get("env_var"): - raise ValueError( - f"{path}: config_schema.required_any field {reference} " - "must declare env_var" - ) - @staticmethod def public(plugin: dict[str, Any]) -> dict[str, Any]: output = {key: value for key, value in plugin.items() if not key.startswith("_")} diff --git a/eve_sdk/profile_resolve.py b/eve_sdk/profile_resolve.py index 58fd4a3d..c6742155 100644 --- a/eve_sdk/profile_resolve.py +++ b/eve_sdk/profile_resolve.py @@ -19,6 +19,7 @@ import yaml from eve_sdk.catalog import load_catalog +from eve_sdk.plugin_manifest import PluginManifest from eve_sdk.secrets import Secrets from eve_sdk.workdir import Workdir @@ -223,6 +224,9 @@ def apply_provider_secrets(resolved: dict[str, Any]) -> None: provider = resolved.get("machine", {}).get("provider") or "" if not provider: return + plugin = next((p for p in PluginManifest.load_all("provider") if p["id"] == provider), {}) + if not (plugin.get("config_schema") or {}).get("secrets"): + return for key, value in Secrets.read(provider).items(): if value is None or key in os.environ: continue diff --git a/eve_sdk/provider_command.py b/eve_sdk/provider_command.py index 50e48aae..d79df549 100644 --- a/eve_sdk/provider_command.py +++ b/eve_sdk/provider_command.py @@ -124,10 +124,12 @@ def dispatch(argv: list[str]) -> int: def _inject_secrets( provider_name: str, plugin: dict[str, Any], env: dict[str, str] ) -> dict[str, str]: + schema_secrets = (plugin.get("config_schema") or {}).get("secrets") or {} + if not schema_secrets: + return env secrets = Secrets.read(provider_name) if not secrets: return env - schema_secrets = (plugin.get("config_schema") or {}).get("secrets") or {} for secret_key, value in secrets.items(): schema_entry = schema_secrets.get(secret_key) if not isinstance(schema_entry, dict): diff --git a/eve_sdk/tf_dispatch.py b/eve_sdk/tf_dispatch.py index 597b88d2..943c2bbd 100644 --- a/eve_sdk/tf_dispatch.py +++ b/eve_sdk/tf_dispatch.py @@ -70,16 +70,16 @@ def resolve_env(root: Path, profile: str) -> dict[str, str]: def apply_tf_env(root: Path, profile: str) -> None: """Run scripts/tf-env and apply its `export KEY=value;` lines to os.environ. - Mirrors bash `eval "$(./scripts/tf-env "$PROFILE")"`: tf-env's stdout is - consumed (parsed into env exports), its stderr passes through, and its exit - status is not propagated — a partial failure leaves whatever exports were - emitted applied, exactly as the command-substitution+eval form did. + Abort on a failed provider environment so authentication requirements cannot + be bypassed by continuing with partial exports or ambient backend defaults. """ result = subprocess.run( [str(root / "scripts/tf-env"), profile], cwd=root, text=True, capture_output=True, check=False, ) sys.stderr.write(result.stderr) + if result.returncode != 0: + raise SystemExit(result.returncode) for line in result.stdout.splitlines(): stripped = line.rstrip(";").strip() if stripped.startswith("export "): diff --git a/tests/fixtures/hermetic/providers/mock-cloud/commands/tf-env b/tests/fixtures/hermetic/providers/mock-cloud/commands/tf-env new file mode 100755 index 00000000..1c4247f8 --- /dev/null +++ b/tests/fixtures/hermetic/providers/mock-cloud/commands/tf-env @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +"""Hermetic Terraform provider environment hook.""" + +from eve_sdk.tf_env import load_env, print_kv + +print_kv("TF_VAR_mock_region", load_env().get("LOCATION_REGION", "")) diff --git a/tests/fixtures/hermetic/providers/mock-cloud/eve-plugin.yaml b/tests/fixtures/hermetic/providers/mock-cloud/eve-plugin.yaml index 5028f095..0725ddc8 100644 --- a/tests/fixtures/hermetic/providers/mock-cloud/eve-plugin.yaml +++ b/tests/fixtures/hermetic/providers/mock-cloud/eve-plugin.yaml @@ -11,6 +11,7 @@ commands: start: {exec: commands/provider-command, args: [start]} stop: {exec: commands/provider-command, args: [stop]} status: {exec: commands/provider-command, args: [status]} + tf-env: {exec: commands/tf-env} ip: {exec: commands/provider-command, args: [ip]} ssh: {exec: commands/provider-command, args: [ssh]} validate: {exec: ../_common/commands/validate-os, args: [all, cloud image URL, cloud_image_url]} diff --git a/tests/python/commands/test_tf_dispatch.py b/tests/python/commands/test_tf_dispatch.py index 1427d52a..2cd46c36 100644 --- a/tests/python/commands/test_tf_dispatch.py +++ b/tests/python/commands/test_tf_dispatch.py @@ -66,6 +66,24 @@ def _run(script: str, *args: str, env: dict[str, str]) -> subprocess.CompletedPr ) +def test_failed_tf_env_aborts_before_applying_partial_exports( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +) -> None: + from eve_sdk.tf_dispatch import apply_tf_env + + monkeypatch.delenv("TF_VAR_partial", raising=False) + monkeypatch.setattr( + "eve_sdk.tf_dispatch.subprocess.run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 23, "export TF_VAR_partial='unsafe';\n", "configuration is required\n", + ), + ) + with pytest.raises(SystemExit) as failure: + apply_tf_env(tmp_path, "test") + assert failure.value.code == 23 + assert "TF_VAR_partial" not in os.environ + + # --------------------------- usage / missing arg --------------------------- # @pytest.mark.parametrize("script", ["tf-init", "tf-plan", "tf-apply", "tf-destroy"]) def test_missing_arg_prints_usage_and_exits_nonzero(script: str, tf_env: dict[str, str]) -> None: diff --git a/tests/python/sdk/test_aws_profile_isolation.py b/tests/python/sdk/test_aws_profile_isolation.py new file mode 100644 index 00000000..4aa791cd --- /dev/null +++ b/tests/python/sdk/test_aws_profile_isolation.py @@ -0,0 +1,92 @@ +"""A profile-only provider cannot consume ambient or Eve-managed credentials.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import pytest + +from eve_sdk.config import ConfigEnv +from eve_sdk.dispatch import process_environment +from eve_sdk.fsm import EveFsm, provider_configured_resolver +from eve_sdk.profile_resolve import apply_provider_secrets +from eve_sdk.provider_command import resolved_provider_environment + + +@pytest.fixture +def profile_plugin() -> dict[str, Any]: + return { + "id": "aws", + "kind": "provider", + "config_schema": {"config": {"profile": {"env_var": "AWS_PROFILE", "required": True}}}, + } + + +def reject_secret_read(*args: Any) -> None: + raise AssertionError("A provider without declared secrets must not read a secret store") + + +@pytest.mark.parametrize("profile", ["", "named-profile"]) +def test_aws_profile_is_the_only_configuration_path( + monkeypatch: pytest.MonkeyPatch, profile_plugin: dict[str, Any], profile: str, +) -> None: + ambient = { + "AWS_ACCESS_KEY_ID": "ignored-access", + "AWS_PROFILE": "ignored-ambient-profile", + "AWS_SECRET_ACCESS_KEY": "ignored-secret", + "AWS_SESSION_TOKEN": "ignored-token", + } + for name, value in ambient.items(): + monkeypatch.setenv(name, value) + monkeypatch.setattr(ConfigEnv, "environment", classmethod(lambda cls: {"AWS_PROFILE": profile})) + monkeypatch.setattr("eve_sdk.provider_command.Secrets.read", reject_secret_read) + monkeypatch.setattr("eve_sdk.provider_command._load_public_plugin", lambda *args: profile_plugin) + + assert not set(ambient) & process_environment().keys() + before = dict(os.environ) + env = resolved_provider_environment("aws", profile_plugin) + assert env["AWS_PROFILE"] == profile + assert not (set(ambient) - {"AWS_PROFILE"}) & env.keys() + assert dict(os.environ) == before + + fsm = EveFsm(provider_configured_resolver()) + fsm.load("provider", (Path(__file__).resolve().parents[3] / "core/fsm/provider.yaml").read_text()) + fsm.fire("provider", "aws", "resolve") + assert fsm.state("provider", "aws") == ("unreachable" if profile else "unconfigured") + + +def test_legacy_profile_resolution_does_not_read_aws_secrets( + monkeypatch: pytest.MonkeyPatch, profile_plugin: dict[str, Any], +) -> None: + monkeypatch.setattr("eve_sdk.profile_resolve.PluginManifest.load_all", lambda *args: [profile_plugin]) + monkeypatch.setattr("eve_sdk.profile_resolve.Secrets.read", reject_secret_read) + before = dict(os.environ) + apply_provider_secrets({"machine": {"provider": "aws"}}) + assert dict(os.environ) == before + + +def test_legacy_profile_resolution_preserves_other_provider_secrets(monkeypatch: pytest.MonkeyPatch) -> None: + plugin = {"id": "cloud", "config_schema": {"secrets": {"token": {"env_var": "CLOUD_TOKEN"}}}} + monkeypatch.setattr("eve_sdk.profile_resolve.PluginManifest.load_all", lambda *args: [plugin]) + monkeypatch.setattr("eve_sdk.profile_resolve.Secrets.read", lambda *args: {"CLOUD_TOKEN": "test-token"}) + monkeypatch.delenv("CLOUD_TOKEN", raising=False) + apply_provider_secrets({"machine": {"provider": "cloud"}}) + assert os.environ["CLOUD_TOKEN"] == "test-token" + monkeypatch.delenv("CLOUD_TOKEN") + + +def test_profile_only_settings_do_not_read_or_write_secrets( + monkeypatch: pytest.MonkeyPatch, profile_plugin: dict[str, Any], +) -> None: + from tui import settings + + monkeypatch.setattr(settings, "load_provider_schema", lambda *args: profile_plugin["config_schema"]) + monkeypatch.setattr(settings.Secrets, "read", reject_secret_read) + monkeypatch.setattr(settings.Secrets, "keys_set", reject_secret_read) + monkeypatch.setattr(settings.Secrets, "update", reject_secret_read) + assert settings.load_provider_secrets("aws") == {} + assert settings.load_provider_secret_keys("aws") == [] + with pytest.raises(RuntimeError, match="not declared"): + settings.save_provider_secret("aws", "access_key_id", "ignored-value") diff --git a/tests/python/sdk/test_plugin_manifest_contract.py b/tests/python/sdk/test_plugin_manifest_contract.py index 6a64c43c..b66401e3 100644 --- a/tests/python/sdk/test_plugin_manifest_contract.py +++ b/tests/python/sdk/test_plugin_manifest_contract.py @@ -100,58 +100,6 @@ def test_core_version_is_4_0(self) -> None: # --------------------------------------------------------------------------- class TestPackageConfigSchema: - def test_required_any_accepts_declared_alternatives(self, tmp_path: Path) -> None: - manifest = _provider_manifest( - tmp_path, - config_schema={ - "config": {"profile": {"env_var": "AWS_PROFILE", "type": "string"}}, - "required_any": [ - ["config.profile"], - ["secrets.access_key", "secrets.secret_key"], - ], - "secrets": { - "access_key": {"env_var": "AWS_ACCESS_KEY_ID", "type": "string"}, - "secret_key": {"env_var": "AWS_SECRET_ACCESS_KEY", "type": "string"}, - }, - }, - ) - PluginManifest.validate(manifest) - - def test_required_any_rejects_undeclared_reference(self, tmp_path: Path) -> None: - manifest = _provider_manifest( - tmp_path, - config_schema={ - "config": {"profile": {"env_var": "AWS_PROFILE", "type": "string"}}, - "required_any": [["secrets.missing"]], - }, - ) - with pytest.raises(ValueError, match=r"undeclared field secrets\.missing"): - PluginManifest.validate(manifest) - - def test_required_any_rejects_field_without_env_var(self, tmp_path: Path) -> None: - manifest = _provider_manifest( - tmp_path, - config_schema={ - "config": {"profile": {"type": "string"}}, - "required_any": [["config.profile"]], - }, - ) - with pytest.raises(ValueError, match=r"config\.profile must declare env_var"): - PluginManifest.validate(manifest) - - def test_required_any_rejects_package_manifest(self, tmp_path: Path) -> None: - manifest = _package_manifest( - tmp_path, - config_schema={ - "config": { - "profile": {"env_var": "PROFILE", "type": "string"} - }, - "required_any": [["config.profile"]], - }, - ) - with pytest.raises(ValueError, match="only valid on provider plugins"): - PluginManifest.validate(manifest) - def test_package_config_schema_valid(self, tmp_path: Path) -> None: manifest = _package_manifest( tmp_path, diff --git a/tests/test_fsm_platform.py b/tests/test_fsm_platform.py index e95cbfb6..9ec9a94a 100644 --- a/tests/test_fsm_platform.py +++ b/tests/test_fsm_platform.py @@ -159,62 +159,3 @@ def test_core_provider_machine_honors_required_config(monkeypatch: Any) -> None: fsm.refresh("provider", "aws") fsm.fire("provider", "aws", "resolve") assert fsm.state("provider", "aws") == "unreachable" - - -def test_core_provider_machine_honors_required_any(monkeypatch: Any) -> None: - from eve_sdk.fsm import EveFsm, provider_configured_resolver - - box = {"env": {}} - monkeypatch.setattr( - "eve_sdk.config.ConfigEnv.environment", - classmethod(lambda cls, *a, **k: dict(box["env"])), - ) - plugin = { - "id": "aws", - "kind": "provider", - "config_schema": { - "config": {"profile": {"env_var": "AWS_PROFILE"}}, - "required_any": [ - ["config.profile"], - ["secrets.access_key", "secrets.secret_key"], - ], - "secrets": { - "access_key": {"env_var": "AWS_ACCESS_KEY_ID"}, - "secret_key": {"env_var": "AWS_SECRET_ACCESS_KEY"}, - }, - }, - } - monkeypatch.setattr( - "eve_sdk.provider_command._load_public_plugin", - lambda kind, plugin_id: plugin, - ) - box_secrets: dict[str, str] = {} - monkeypatch.setattr( - "eve_sdk.provider_command.Secrets.read", - staticmethod(lambda name: dict(box_secrets)), - ) - - fsm = EveFsm(provider_configured_resolver()) - core = ( - Path(__file__).resolve().parents[1] / "core" / "fsm" / "provider.yaml" - ).read_text() - fsm.load("provider", core) - - fsm.fire("provider", "aws", "resolve") - assert fsm.state("provider", "aws") == "unconfigured" - - box_secrets["access_key"] = "ACCESS" - fsm.refresh("provider", "aws") - fsm.fire("provider", "aws", "resolve") - assert fsm.state("provider", "aws") == "unconfigured" - - box_secrets["secret_key"] = "SECRET" - fsm.refresh("provider", "aws") - fsm.fire("provider", "aws", "resolve") - assert fsm.state("provider", "aws") == "unreachable" - - box_secrets.clear() - box["env"] = {"AWS_PROFILE": "eve"} - fsm.refresh("provider", "aws") - fsm.fire("provider", "aws", "resolve") - assert fsm.state("provider", "aws") == "unreachable" diff --git a/tui/settings.py b/tui/settings.py index 5b5b97ae..80eab378 100644 --- a/tui/settings.py +++ b/tui/settings.py @@ -10,7 +10,6 @@ from eve_sdk.engine import default_engine from eve_sdk.secrets import Secrets, SecretsError - _CACHE_TTL = 5.0 _cache: dict[tuple[str, tuple[str, ...]], tuple[float, Any]] = {} @@ -84,6 +83,8 @@ def load_provider_schema(provider_id: str) -> dict[str, Any]: def load_provider_secrets(provider_id: str) -> dict[str, str]: + if not load_provider_schema(provider_id).get("secrets"): + return {} try: return Secrets.read(provider_id) except SecretsError: @@ -91,6 +92,8 @@ def load_provider_secrets(provider_id: str) -> dict[str, str]: def load_provider_secret_keys(provider_id: str) -> list[str]: + if not load_provider_schema(provider_id).get("secrets"): + return [] try: return Secrets.keys_set(provider_id) except SecretsError: @@ -98,6 +101,8 @@ def load_provider_secret_keys(provider_id: str) -> list[str]: def save_provider_secret(provider_id: str, key: str, value: str) -> None: + if key not in (load_provider_schema(provider_id).get("secrets") or {}): + raise RuntimeError(f"Secret field is not declared: {provider_id}.{key}") try: Secrets.update(provider_id, {key: value}) except SecretsError as error: @@ -120,7 +125,7 @@ def unset_value(section: str, field: str) -> None: def load_missing_fields() -> list[dict[str, Any]]: - code, stdout, _ = _run(["./scripts/check-required", "--json"]) + _code, stdout, _ = _run(["./scripts/check-required", "--json"]) if not stdout.strip(): return [] try: From 4282c0ebe3542402d3e79baee6a74f2c5e797690 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 02:30:42 +0000 Subject: [PATCH 31/32] feat(v4.5): add container and typed guest access contracts --- AGENTS.md | 3 +- core/schema/command-io.schema.json | 153 ++++++++++++++++-- core/schema/observed-state.schema.json | 113 +++++++++++++ core/schema/plugin-manifest.schema.json | 19 ++- core/schema/resolved-instance.schema.json | 6 +- eve_sdk/resolve.py | 3 +- eve_sdk/schema.py | 3 +- scripts/instance-observe | 12 ++ .../sdk/test_container_access_contract.py | 92 +++++++++++ 9 files changed, 380 insertions(+), 24 deletions(-) create mode 100644 tests/python/sdk/test_container_access_contract.py diff --git a/AGENTS.md b/AGENTS.md index 9576c3a4..c2c0f531 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,10 +110,11 @@ The Windows SSH server default shell is **PowerShell** (not `cmd.exe`). All scri This project provisions environments from scratch — we control the OS, installed packages, shell, and runtime versions. **Never add fallback logic** (e.g. "try PowerShell, fall back to cmd.exe" or "check both Program Files and AppData just in case"). Always use the single correct path for the environment we build. If the environment changes, change the script — don't layer on fallbacks that mask the real requirement. -## Catalog kinds: vm vs metal +## Catalog machine kinds Machine entries declare a `kind:` field. Current supported kinds: +- `kind: container` — disposable system-container lifecycle (`incus`), with provider-owned cloud-init SSH bootstrap. This is not a VM or a guest package. - `kind: vm` — disposable VM lifecycle (`aws`, `gcp`, `vultr`, `truenas`, `local-qemu`). `up` creates, `down` deletes. - `kind: metal` — persistent hardware (`raspberry-pi`). `down` tears down managed workloads, **not** the machine. Don't force metal targets through VM lifecycle assumptions. diff --git a/core/schema/command-io.schema.json b/core/schema/command-io.schema.json index 51c6688d..9326cb58 100644 --- a/core/schema/command-io.schema.json +++ b/core/schema/command-io.schema.json @@ -4,7 +4,6 @@ "title": "Plugin Command I/O", "description": "Documents the JSON in/out contract for plugin commands. The input shape is resolved-instance.schema.json (passed via stdin or EVE_RESOLVED_JSON). The $defs below describe the output shapes. This schema is documentation-only -- no data fixtures validate against it because provider commands emit mixed text/JSON on stdout.", "$comment": "Documentation-only: defines $defs for reference but has no root type constraint. Do not add a root type or oneOf without also adding corresponding command-output fixtures.", - "$defs": { "provider_command_output": { "title": "Provider Command Output", @@ -15,16 +14,28 @@ "status": { "description": "Provider status value from the status command.", "type": "string", - "enum": ["running", "stopped", "absent", "unknown"] + "enum": [ + "absent", + "creating", + "failed", + "running", + "stopped", + "unknown" + ] }, "ip": { "description": "IP address from the ip command.", "type": "string", "minLength": 1 + }, + "guest_access": { + "$ref": "#/$defs/guest_access" + }, + "provider_identity": { + "$ref": "#/$defs/provider_identity" } } }, - "package_command_output": { "title": "Package Command Output", "description": "JSON emitted by package commands on stdout. The dispatcher scans the last JSON object in stdout for a status field.", @@ -34,28 +45,150 @@ "status": { "description": "Package status value.", "type": "string", - "enum": ["installed", "missing", "unknown", "failed"] + "enum": [ + "installed", + "missing", + "unknown", + "failed" + ] } } }, - "desired_state_mapping": { "title": "Desired State Mapping", "description": "Maps provider command to the desired_state the dispatcher records. up/start -> running, stop -> stopped, down -> absent.", "type": "object", "additionalProperties": false, "properties": { - "up": { "const": "running" }, - "start": { "const": "running" }, - "stop": { "const": "stopped" }, - "down": { "const": "absent" } + "up": { + "const": "running" + }, + "start": { + "const": "running" + }, + "stop": { + "const": "stopped" + }, + "down": { + "const": "absent" + } } }, - "provider_state_mapping": { "title": "Provider State Mapping", "description": "Maps (command, dispatch_status) to the provider_state recorded by the dispatcher (the core/fsm/instance.yaml vocabulary). On success: init/plan -> creating, up/start -> running, stop -> stopped, down -> absent. On running (in flight): up -> creating, start -> starting, stop -> stopping, down -> destroying. On failure: -> error (except resolve/status/ip/ssh which set null).", "type": "object" + }, + "provider_identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "endpoint", + "instance_name", + "owner_id", + "project", + "provider", + "remote" + ], + "properties": { + "endpoint": { + "type": "string", + "pattern": "^https://[^/]+$" + }, + "instance_name": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]{0,61}[a-z0-9]$" + }, + "owner_id": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "project": { + "type": "string", + "minLength": 1 + }, + "provider": { + "type": "string", + "minLength": 1 + }, + "remote": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]*$" + } + } + }, + "guest_access": { + "title": "Authenticated SSH guest access", + "type": "object", + "additionalProperties": false, + "required": [ + "address", + "credential_reference", + "host_identity", + "port", + "protocol", + "provider_identity", + "username", + "version" + ], + "properties": { + "address": { + "type": "string", + "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])$" + }, + "credential_reference": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "type" + ], + "properties": { + "path": { + "type": "string", + "pattern": "^/" + }, + "type": { + "const": "ssh-private-key-file" + } + } + }, + "host_identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "fingerprint", + "mechanism" + ], + "properties": { + "fingerprint": { + "type": "string", + "pattern": "^SHA256:[A-Za-z0-9+/]{43}$" + }, + "mechanism": { + "const": "provider-authenticated-exec" + } + } + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "protocol": { + "const": "ssh" + }, + "provider_identity": { + "$ref": "#/$defs/provider_identity" + }, + "username": { + "type": "string", + "pattern": "^[a-z_][a-z0-9_-]*$" + }, + "version": { + "const": 2 + } + } } } } diff --git a/core/schema/observed-state.schema.json b/core/schema/observed-state.schema.json index 8b9b814a..1745fc86 100644 --- a/core/schema/observed-state.schema.json +++ b/core/schema/observed-state.schema.json @@ -53,6 +53,8 @@ "type": "object", "additionalProperties": true, "properties": { + "guest_access": { "$ref": "#/$defs/guest_access" }, + "provider_identity": { "$ref": "#/$defs/provider_identity" }, "observed_at": { "description": "ISO 8601 timestamp of this observation.", "type": "string" @@ -123,6 +125,117 @@ }, "$defs": { + "guest_access": { + "title": "Authenticated SSH guest access", + "type": "object", + "additionalProperties": false, + "required": [ + "address", + "credential_reference", + "host_identity", + "port", + "protocol", + "provider_identity", + "username", + "version" + ], + "properties": { + "address": { + "type": "string", + "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])$" + }, + "credential_reference": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "type" + ], + "properties": { + "path": { + "type": "string", + "pattern": "^/" + }, + "type": { + "const": "ssh-private-key-file" + } + } + }, + "host_identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "fingerprint", + "mechanism" + ], + "properties": { + "fingerprint": { + "type": "string", + "pattern": "^SHA256:[A-Za-z0-9+/]{43}$" + }, + "mechanism": { + "const": "provider-authenticated-exec" + } + } + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "protocol": { + "const": "ssh" + }, + "provider_identity": { + "$ref": "#/$defs/provider_identity" + }, + "username": { + "type": "string", + "pattern": "^[a-z_][a-z0-9_-]*$" + }, + "version": { + "const": 2 + } + } + }, + "provider_identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "endpoint", + "instance_name", + "owner_id", + "project", + "provider", + "remote" + ], + "properties": { + "endpoint": { + "type": "string", + "pattern": "^https://[^/]+$" + }, + "instance_name": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]{0,61}[a-z0-9]$" + }, + "owner_id": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "project": { + "type": "string", + "minLength": 1 + }, + "provider": { + "type": "string", + "minLength": 1 + }, + "remote": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]*$" + } + } + }, "package_state_entry": { "description": "Per-package state record.", "type": "object", diff --git a/core/schema/plugin-manifest.schema.json b/core/schema/plugin-manifest.schema.json index e24e27e8..dc8d7084 100644 --- a/core/schema/plugin-manifest.schema.json +++ b/core/schema/plugin-manifest.schema.json @@ -525,11 +525,12 @@ "items": { "type": "string", "enum": [ - "terraform", - "vagrant", - "qemu", + "docker", + "incus", "metal", - "docker" + "qemu", + "terraform", + "vagrant" ] } }, @@ -538,8 +539,9 @@ "items": { "type": "string", "enum": [ - "vm", - "metal" + "container", + "metal", + "vm" ] } } @@ -1017,8 +1019,9 @@ "kind": { "type": "string", "enum": [ - "vm", - "metal" + "container", + "metal", + "vm" ] }, "supports": { diff --git a/core/schema/resolved-instance.schema.json b/core/schema/resolved-instance.schema.json index f17a0c2f..d3594057 100644 --- a/core/schema/resolved-instance.schema.json +++ b/core/schema/resolved-instance.schema.json @@ -25,7 +25,7 @@ "properties": { "access": { - "description": "Resolved VM access identities. bootstrap_user is the initial login user, provision_user runs automation, and human_user owns desktop/session state.", + "description": "Resolved guest access identities. bootstrap_user is the initial login user, provision_user runs automation, and human_user owns desktop/session state.", "type": "object", "required": ["bootstrap_user", "provision_user", "human_user"], "additionalProperties": false, @@ -141,7 +141,7 @@ "kind": { "type": "string", "description": "Machine lifecycle kind.", - "enum": ["vm", "metal"] + "enum": ["container", "metal", "vm"] }, "supports": { "type": "object", @@ -276,7 +276,7 @@ "engine": { "description": "The execution engine determined from the machine kind and provider.", "type": "string", - "enum": ["terraform", "vagrant", "qemu", "metal"] + "enum": ["incus", "metal", "qemu", "terraform", "vagrant"] }, "stack_tags": { diff --git a/eve_sdk/resolve.py b/eve_sdk/resolve.py index f7d2ca88..c4db2c44 100644 --- a/eve_sdk/resolve.py +++ b/eve_sdk/resolve.py @@ -292,7 +292,8 @@ def validate_catalog_selection( f"got machine provider {machine['provider']}" ) provider = machine["provider"] - if not location.get(provider): + # An explicit empty mapping is valid for providers without region/zone data. + if not isinstance(location.get(provider), dict): raise ResolveError(f"Location {location['name']} has no mapping for provider {provider}") supports_raw = machine.get("supports") supports = supports_raw if isinstance(supports_raw, dict) else {} diff --git a/eve_sdk/schema.py b/eve_sdk/schema.py index adf8c02f..0960d92b 100644 --- a/eve_sdk/schema.py +++ b/eve_sdk/schema.py @@ -58,7 +58,8 @@ def validate_def(schema_name: str, def_name: str, data: Any, label: str) -> None defs = raw.get("$defs") or raw.get("defs") or {} if not isinstance(defs, dict) or def_name not in defs: raise SchemaValidationError(f"Unknown $defs entry: {def_name}") - schema = defs[def_name] + # Keep sibling definitions available to typed command-output references. + schema = {"$defs": defs, "$ref": f"#/$defs/{def_name}"} try: Draft202012Validator.check_schema(schema) except SchemaError as error: diff --git a/scripts/instance-observe b/scripts/instance-observe index 9316c4ab..e05be5ea 100755 --- a/scripts/instance-observe +++ b/scripts/instance-observe @@ -10,6 +10,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) +from eve_sdk.dispatch import last_json_object, validate_provider_output # noqa: E402 from eve_sdk.state import State, StateError # noqa: E402 DEFAULT_TTL_SECONDS = 30 @@ -43,6 +44,17 @@ def normalize_provider_status(value: str) -> str: def parse_status_output(output: str) -> dict[str, object]: observed: dict[str, object] = {} + parsed = last_json_object(output, {"status"}) + if parsed is not None: + validate_provider_output(output) + # Structured providers already speak the validated observation vocabulary. + observed["provider_status"] = parsed["status"] + for field in ("guest_access", "ip", "provider_identity"): + if field in parsed: + observed[field] = parsed[field] + if "guest_access" in parsed: + observed["ip"] = parsed["guest_access"]["address"] + return observed for line in output.splitlines(): parts = [part.strip() for part in line.split(":", 1)] if len(parts) != 2: diff --git a/tests/python/sdk/test_container_access_contract.py b/tests/python/sdk/test_container_access_contract.py new file mode 100644 index 00000000..571b647a --- /dev/null +++ b/tests/python/sdk/test_container_access_contract.py @@ -0,0 +1,92 @@ +"""Container and authenticated guest-access contract regressions.""" + +from __future__ import annotations + +import copy +import importlib.machinery +import importlib.util +from pathlib import Path +from typing import Any + +import pytest + +from eve_sdk.resolve import engine_for, validate_provider_plugin +from eve_sdk.schema import SchemaValidationError, validate_output, validate_schema + + +def binding() -> dict[str, Any]: + return { + "address": "10.201.153.42", + "credential_reference": {"path": "/explicit/controller-key", "type": "ssh-private-key-file"}, + "host_identity": {"fingerprint": "SHA256:" + "A" * 43, "mechanism": "provider-authenticated-exec"}, + "port": 22, + "protocol": "ssh", + "provider_identity": { + "endpoint": "https://pool:8443", + "instance_name": "eve-test", + "owner_id": "a" * 64, + "project": "eve-ci", + "provider": "incus", + "remote": "eve-incus-pool", + }, + "username": "ubuntu", + "version": 2, + } + + +def observer() -> Any: + path = Path(__file__).resolve().parents[3] / "scripts/instance-observe" + loader = importlib.machinery.SourceFileLoader("container_observer", str(path)) + spec = importlib.util.spec_from_loader(loader.name, loader) + assert spec + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +def test_container_engine_is_manifest_driven() -> None: + plugin = {"id": "incus", "kind": "provider", "supports": {"engines": ["incus"], "kinds": ["container"]}} + machine = {"provider": "incus", "kind": "container"} + assert engine_for(machine, [plugin]) == "incus" + assert validate_provider_plugin(machine, "incus", [plugin]) == plugin + + +@pytest.mark.parametrize("status", ["absent", "creating", "failed", "running", "stopped", "unknown"]) +def test_typed_observations_preserve_provider_states(status: str) -> None: + import json + + access = binding() + output = {"status": status, "provider_identity": access["provider_identity"]} + if status == "running": + output["guest_access"] = access + validate_output(output, "provider_command_output") + result = observer().parse_status_output(json.dumps(output)) + assert result["provider_status"] == status + assert result["provider_identity"] == access["provider_identity"] + if status == "running": + assert result["guest_access"] == access + assert result["ip"] == access["address"] + + +@pytest.mark.parametrize("field", sorted(binding())) +def test_guest_access_requires_every_typed_field(field: str) -> None: + access = copy.deepcopy(binding()) + del access[field] + with pytest.raises(SchemaValidationError): + validate_output({"status": "running", "guest_access": access}, "provider_command_output") + + +def test_observed_state_validates_binding_in_cache() -> None: + state = { + "instance": "test", + "desired_state": "running", + "provider_state": "running", + "provision_state": "unknown", + "package_state": {}, + "operation_history": [], + "observed_state": {"guest_access": binding()}, + } + validate_schema("observed-state.schema.json", state, "Observation") + state["observed_state"]["guest_access"]["credential_reference"] = {"raw_private_key": "must-not-be-accepted"} # type: ignore[index] + with pytest.raises(SchemaValidationError): + validate_schema("observed-state.schema.json", state, "Observation") From 04919109ee398118d6e8cf5b450f3962ed9754d3 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 05:06:51 +0000 Subject: [PATCH 32/32] fix(v4.5): package runtime assets and repair provider observations --- .github/workflows/test.yml | 3 + core/schema/command-io.schema.json | 96 +++++++++++-------- core/schema/observed-state.schema.json | 96 +++++++++++-------- docs/provider-access-contract.md | 36 +++++++ eve_sdk/cli.py | 2 +- eve_sdk/state.py | 14 ++- eve_sdk/state_machine.py | 6 ++ pyproject.toml | 11 ++- scripts/instance-observe | 14 ++- scripts/test-wheel | 66 +++++++++++++ .../sdk/test_container_access_contract.py | 76 +++++++++++++-- 11 files changed, 330 insertions(+), 90 deletions(-) create mode 100644 docs/provider-access-contract.md create mode 100755 scripts/test-wheel diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 74437c78..09a082ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -119,6 +119,9 @@ jobs: python -m pip install --upgrade poetry poetry install --no-interaction --no-root + - name: Test isolated installed wheel + run: python scripts/test-wheel + - name: Generate Terramate code if: runner.os == 'Linux' run: | diff --git a/core/schema/command-io.schema.json b/core/schema/command-io.schema.json index 9326cb58..cc037c63 100644 --- a/core/schema/command-io.schema.json +++ b/core/schema/command-io.schema.json @@ -5,6 +5,39 @@ "description": "Documents the JSON in/out contract for plugin commands. The input shape is resolved-instance.schema.json (passed via stdin or EVE_RESOLVED_JSON). The $defs below describe the output shapes. This schema is documentation-only -- no data fixtures validate against it because provider commands emit mixed text/JSON on stdout.", "$comment": "Documentation-only: defines $defs for reference but has no root type constraint. Do not add a root type or oneOf without also adding corresponding command-output fixtures.", "$defs": { + "incus_identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "endpoint", + "instance_name", + "owner_id", + "project", + "remote" + ], + "properties": { + "endpoint": { + "type": "string", + "pattern": "^https://[^/]+$" + }, + "instance_name": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]{0,61}[a-z0-9]$" + }, + "owner_id": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "project": { + "type": "string", + "minLength": 1 + }, + "remote": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]*$" + } + } + }, "provider_command_output": { "title": "Provider Command Output", "description": "JSON emitted by provider commands on stdout. Most commands stream text; lifecycle commands record state through instance-state rather than structured output. When a provider command emits JSON containing a status or ip field, those fields are validated against this contract.", @@ -80,42 +113,25 @@ "type": "object" }, "provider_identity": { - "type": "object", - "additionalProperties": false, - "required": [ - "endpoint", - "instance_name", - "owner_id", - "project", - "provider", - "remote" - ], - "properties": { - "endpoint": { - "type": "string", - "pattern": "^https://[^/]+$" - }, - "instance_name": { - "type": "string", - "pattern": "^[a-z][a-z0-9-]{0,61}[a-z0-9]$" - }, - "owner_id": { - "type": "string", - "pattern": "^[0-9a-f]{64}$" - }, - "project": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "remote": { - "type": "string", - "pattern": "^[a-z][a-z0-9-]*$" + "description": "Closed provider-discriminated envelope. Each provider implements a namespaced identity variant; new providers add a schema variant, never arbitrary fields.", + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "identity", + "provider" + ], + "properties": { + "provider": { + "const": "incus" + }, + "identity": { + "$ref": "#/$defs/incus_identity" + } + } } - } + ] }, "guest_access": { "title": "Authenticated SSH guest access", @@ -166,9 +182,13 @@ "pattern": "^SHA256:[A-Za-z0-9+/]{43}$" }, "mechanism": { - "const": "provider-authenticated-exec" + "enum": [ + "configured-fingerprint", + "incus-authenticated-exec" + ] } - } + }, + "description": "SSH host-key fingerprint established by explicit controller configuration or a provider-specific authenticated mechanism; no trust-on-first-use." }, "port": { "type": "integer", @@ -183,7 +203,7 @@ }, "username": { "type": "string", - "pattern": "^[a-z_][a-z0-9_-]*$" + "pattern": "^[A-Za-z_][A-Za-z0-9_.-]*$" }, "version": { "const": 2 diff --git a/core/schema/observed-state.schema.json b/core/schema/observed-state.schema.json index 1745fc86..3c738215 100644 --- a/core/schema/observed-state.schema.json +++ b/core/schema/observed-state.schema.json @@ -125,6 +125,39 @@ }, "$defs": { + "incus_identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "endpoint", + "instance_name", + "owner_id", + "project", + "remote" + ], + "properties": { + "endpoint": { + "type": "string", + "pattern": "^https://[^/]+$" + }, + "instance_name": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]{0,61}[a-z0-9]$" + }, + "owner_id": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "project": { + "type": "string", + "minLength": 1 + }, + "remote": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]*$" + } + } + }, "guest_access": { "title": "Authenticated SSH guest access", "type": "object", @@ -174,9 +207,13 @@ "pattern": "^SHA256:[A-Za-z0-9+/]{43}$" }, "mechanism": { - "const": "provider-authenticated-exec" + "enum": [ + "configured-fingerprint", + "incus-authenticated-exec" + ] } - } + }, + "description": "SSH host-key fingerprint established by explicit controller configuration or a provider-specific authenticated mechanism; no trust-on-first-use." }, "port": { "type": "integer", @@ -191,7 +228,7 @@ }, "username": { "type": "string", - "pattern": "^[a-z_][a-z0-9_-]*$" + "pattern": "^[A-Za-z_][A-Za-z0-9_.-]*$" }, "version": { "const": 2 @@ -199,42 +236,25 @@ } }, "provider_identity": { - "type": "object", - "additionalProperties": false, - "required": [ - "endpoint", - "instance_name", - "owner_id", - "project", - "provider", - "remote" - ], - "properties": { - "endpoint": { - "type": "string", - "pattern": "^https://[^/]+$" - }, - "instance_name": { - "type": "string", - "pattern": "^[a-z][a-z0-9-]{0,61}[a-z0-9]$" - }, - "owner_id": { - "type": "string", - "pattern": "^[0-9a-f]{64}$" - }, - "project": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "remote": { - "type": "string", - "pattern": "^[a-z][a-z0-9-]*$" + "description": "Closed provider-discriminated envelope. Each provider implements a namespaced identity variant; new providers add a schema variant, never arbitrary fields.", + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "identity", + "provider" + ], + "properties": { + "provider": { + "const": "incus" + }, + "identity": { + "$ref": "#/$defs/incus_identity" + } + } } - } + ] }, "package_state_entry": { "description": "Per-package state record.", diff --git a/docs/provider-access-contract.md b/docs/provider-access-contract.md new file mode 100644 index 00000000..f71ef69f --- /dev/null +++ b/docs/provider-access-contract.md @@ -0,0 +1,36 @@ +# Typed guest access and provider identity + +`command-io.schema.json` and `observed-state.schema.json` validate the same closed +SSH guest-access object. Required fields are `address` (IPv4), `port` (1–65535), +`protocol` (`ssh`), `version` (`2`), `username` (SSH account name, including Linux +and Windows account names), `credential_reference` (absolute file path and type +`ssh-private-key-file`), `host_identity`, and `provider_identity`. Credential +references are selectors, never private-key contents. + +Host identity contains only a SHA-256 OpenSSH fingerprint and a mechanism: +`configured-fingerprint` means an explicit controller pin; `incus-authenticated-exec` +means the guest public host key was retrieved through the authenticated Incus +API. Neither permits trust-on-first-use or bypassing SSH host verification. + +Provider identity is a closed, provider-discriminated envelope, not an arbitrary +configuration object. The current variant is `{provider: incus, identity: {...}}`. +Only the Incus-specific `identity` requires HTTPS endpoint, remote, project, +instance name, and a 64-hex ownership digest. Future providers add their own +closed schema variants under the discriminator; they do not inherit these Incus +concepts. Every variant and nested object rejects undeclared fields. The access +object itself contains no OS or provider lifecycle assumptions. + +Normal Incus running status includes authenticated guest access. Core validates +the exact JSON status object it consumes, normalizes failed observations through +its existing error presentation, and preserves the raw observation for diagnosis. +Every refresh replaces the access binding; non-running states, invalid output, +or refresh errors remove cached binding/address. Confirmed observed failures +disable running-only actions even if an earlier provisioned state was running. +These are observation/presentation changes, not new Determa transition or +restoration semantics. + +Runtime schema, FSM, scripts, default config, TUI, and shipped OS assets are +explicitly included in installed wheels. `python scripts/test-wheel` builds and +installs a wheel in a fresh venv outside the repository, checks `eve --help`, +loads runtime schemas/assets, and accepts/rejects command outputs. CI executes +this artifact check independently of source/Poetry test execution. diff --git a/eve_sdk/cli.py b/eve_sdk/cli.py index 8614770c..a8d905e7 100644 --- a/eve_sdk/cli.py +++ b/eve_sdk/cli.py @@ -16,7 +16,7 @@ def main() -> int: sys.path.insert(0, str(root)) import runpy - module = runpy.run_path(str(root / "scripts" / "eve-cli"), run_name="__main__") + module = runpy.run_path(str(root / "scripts" / "eve-cli"), run_name="eve_cli") return int(module["main"]()) diff --git a/eve_sdk/state.py b/eve_sdk/state.py index 618da481..d675e132 100644 --- a/eve_sdk/state.py +++ b/eve_sdk/state.py @@ -265,7 +265,19 @@ def update_observed(cls, instance_name: str, observed: dict[str, Any]) -> dict[s now = datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") def update(state: dict[str, Any]) -> dict[str, Any]: - state["observed_state"] = state.get("observed_state", {}) | observed + cached = state.get("observed_state", {}).copy() + # Every refresh replaces access. A merge must never retain an old + # authenticated binding after a non-running observation or error. + if "provider_status" in observed or "refresh_error" in observed: + cached.pop("guest_access", None) + if observed.get("provider_status") != "running" or observed.get("refresh_error"): + cached.pop("ip", None) + observed_without_access = {k: v for k, v in observed.items() if k not in {"guest_access", "ip"}} + state["observed_state"] = cached | observed_without_access + else: + state["observed_state"] = cached | observed + else: + state["observed_state"] = cached | observed state["updated_at"] = now return state diff --git a/eve_sdk/state_machine.py b/eve_sdk/state_machine.py index 035a65bb..f71f2fa1 100644 --- a/eve_sdk/state_machine.py +++ b/eve_sdk/state_machine.py @@ -18,6 +18,9 @@ def normalize_provider_state(status: Any) -> str: def effective_provider_state(state: dict[str, Any]) -> str: + observed = state.get("observed_state", {}) + if isinstance(observed, dict) and normalize_provider_state(observed.get("provider_status", "")) == "error": + return "error" provider_state = str(state.get("provider_state", "unknown")) desired_state = str(state.get("desired_state", "unknown")) provision_state = str(state.get("provision_state", "unknown")) @@ -80,6 +83,9 @@ def provider_actions_available(state: dict[str, Any]) -> bool: eps = effective_provider_state(state) desired_state = str(state.get("desired_state", "unknown")) provision_state = str(state.get("provision_state", "unknown")) + observed = state.get("observed_state", {}) + if isinstance(observed, dict) and normalize_provider_state(observed.get("provider_status", "")) == "error": + return False return eps == "running" or (eps == "error" and desired_state == "running" and provision_state == "provisioned") diff --git a/pyproject.toml b/pyproject.toml index 2afa26e3..2b8bb84c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,16 @@ pythonpath = ["."] [tool.poetry] packages = [{include = "eve_sdk"}] -include = ["scripts", "core", "config", "oses"] +include = [ + {path = ".env", format = ["sdist", "wheel"]}, + {path = "config/catalog.yaml", format = ["sdist", "wheel"]}, + {path = "config/defaults.yaml", format = ["sdist", "wheel"]}, + {path = "core", format = ["sdist", "wheel"]}, + {path = "oses", format = ["sdist", "wheel"]}, + {path = "scripts", format = ["sdist", "wheel"]}, + {path = "tui", format = ["sdist", "wheel"]}, +] +exclude = ["**/__pycache__/**", "**/*.pyc", "**/*.local.*", ".env.local"] [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] diff --git a/scripts/instance-observe b/scripts/instance-observe index e05be5ea..763be9fd 100755 --- a/scripts/instance-observe +++ b/scripts/instance-observe @@ -10,7 +10,8 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from eve_sdk.dispatch import last_json_object, validate_provider_output # noqa: E402 +from eve_sdk.dispatch import last_json_object # noqa: E402 +from eve_sdk.schema import validate_output # noqa: E402 from eve_sdk.state import State, StateError # noqa: E402 DEFAULT_TTL_SECONDS = 30 @@ -46,9 +47,9 @@ def parse_status_output(output: str) -> dict[str, object]: observed: dict[str, object] = {} parsed = last_json_object(output, {"status"}) if parsed is not None: - validate_provider_output(output) + validate_output(parsed, "provider_command_output") # Structured providers already speak the validated observation vocabulary. - observed["provider_status"] = parsed["status"] + observed["provider_status"] = normalize_provider_status(parsed["status"]) for field in ("guest_access", "ip", "provider_identity"): if field in parsed: observed[field] = parsed[field] @@ -100,7 +101,12 @@ def main(argv: list[str]) -> int: env, str(ROOT / "scripts/provider-dispatch"), "--instance", instance, "--command", "status" ) if status.returncode == 0: - observed.update(parse_status_output(status.stdout)) + try: + observed.update(parse_status_output(status.stdout)) + except Exception as error: + observed.update(provider_status="unknown", refresh_error="Invalid provider observation") + State.update_observed(instance, observed) + raise ValueError("Invalid provider observation") from error observed["provider_status_raw"] = status.stdout.strip() observed["refresh_error"] = None if observed.get("provider_status") == "running" and not observed.get("ip"): diff --git a/scripts/test-wheel b/scripts/test-wheel new file mode 100755 index 00000000..eab85ddb --- /dev/null +++ b/scripts/test-wheel @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Build and test a wheel in a clean venv outside the source checkout.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import venv +from pathlib import Path + + +def main() -> int: + root = Path(__file__).resolve().parents[1] + env = {k: v for k, v in os.environ.items() if k not in {"PYTHONHOME", "PYTHONPATH"} and not k.startswith("EVE_")} + with tempfile.TemporaryDirectory(prefix="eve-wheel-") as scratch: + work = Path(scratch) + subprocess.run( + [sys.executable, "-m", "pip", "wheel", "--no-deps", "--wheel-dir", str(work), str(root)], + cwd=work, + env=env, + check=True, + ) + venv.EnvBuilder(with_pip=True).create(work / "venv") + python = work / "venv/bin/python" + subprocess.run( + [str(python), "-m", "pip", "install", str(next(work.glob("eve-*.whl")))], cwd=work, env=env, check=True + ) + subprocess.run([str(work / "venv/bin/eve"), "--help"], cwd=work, env=env, check=True) + subprocess.run( + [ + str(python), + "-I", + "-c", + """ +from eve_sdk.schema import load_schema, validate_output, SchemaValidationError +from eve_sdk.workdir import Workdir +assert 'site-packages' in str(Workdir.repo_root()) +for schema in ('command-io.schema.json', 'observed-state.schema.json', + 'plugin-manifest.schema.json', 'resolved-instance.schema.json'): + load_schema(schema) +validate_output({'status': 'running'}, 'provider_command_output') +try: + validate_output({'status': 'bogus'}, 'provider_command_output') +except SchemaValidationError: + pass +else: + raise AssertionError('invalid output accepted') +for asset in ('config/defaults.yaml', 'core/fsm/instance.yaml', 'scripts/provision', + 'oses/mockos-1.0-amd64/provision/scripts/bootstrap.sh', 'tui/app.py'): + assert (Workdir.repo_root() / asset).is_file(), asset +assert not (Workdir.repo_root() / '.env.local').exists() +assert not (Workdir.repo_root() / 'config/catalog.local.yaml').exists() +""", + ], + cwd=work, + env=env, + check=True, + ) + print("[OK] isolated installed wheel: CLI, schemas, command validation, runtime assets") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/python/sdk/test_container_access_contract.py b/tests/python/sdk/test_container_access_contract.py index 571b647a..c63cbd19 100644 --- a/tests/python/sdk/test_container_access_contract.py +++ b/tests/python/sdk/test_container_access_contract.py @@ -18,16 +18,18 @@ def binding() -> dict[str, Any]: return { "address": "10.201.153.42", "credential_reference": {"path": "/explicit/controller-key", "type": "ssh-private-key-file"}, - "host_identity": {"fingerprint": "SHA256:" + "A" * 43, "mechanism": "provider-authenticated-exec"}, + "host_identity": {"fingerprint": "SHA256:" + "A" * 43, "mechanism": "incus-authenticated-exec"}, "port": 22, "protocol": "ssh", "provider_identity": { - "endpoint": "https://pool:8443", - "instance_name": "eve-test", - "owner_id": "a" * 64, - "project": "eve-ci", "provider": "incus", - "remote": "eve-incus-pool", + "identity": { + "endpoint": "https://pool:8443", + "instance_name": "eve-test", + "owner_id": "a" * 64, + "project": "eve-ci", + "remote": "eve-incus-pool", + }, }, "username": "ubuntu", "version": 2, @@ -61,7 +63,7 @@ def test_typed_observations_preserve_provider_states(status: str) -> None: output["guest_access"] = access validate_output(output, "provider_command_output") result = observer().parse_status_output(json.dumps(output)) - assert result["provider_status"] == status + assert result["provider_status"] == ("unreachable" if status == "failed" else status) assert result["provider_identity"] == access["provider_identity"] if status == "running": assert result["guest_access"] == access @@ -90,3 +92,63 @@ def test_observed_state_validates_binding_in_cache() -> None: state["observed_state"]["guest_access"]["credential_reference"] = {"raw_private_key": "must-not-be-accepted"} # type: ignore[index] with pytest.raises(SchemaValidationError): validate_schema("observed-state.schema.json", state, "Observation") + + +def test_invalid_status_cannot_hide_behind_later_valid_ip() -> None: + with pytest.raises(SchemaValidationError): + observer().parse_status_output('{"status":"invalid"}\n{"ip":"10.201.153.42"}') + + +@pytest.mark.parametrize("status", ["absent", "creating", "failed", "stopped", "unknown", "unreachable"]) +def test_refresh_replaces_stale_access(tmp_path: Path, monkeypatch: Any, status: str) -> None: + from eve_sdk.state import State + from eve_sdk.workdir import Workdir + + monkeypatch.setenv("EVE_HOME", str(tmp_path)) + Workdir.reset_overrides() + State.update_observed("test-one", {"provider_status": "running", "guest_access": binding(), "ip": "10.201.153.42"}) + state = State.update_observed("test-one", {"provider_status": status}) + assert "guest_access" not in state["observed_state"] + assert "ip" not in state["observed_state"] + + +def test_failed_observation_disables_running_provisioned_actions() -> None: + from eve_sdk.state_machine import effective_provider_state, provider_actions_available, status_with_observed_state + + parsed = observer().parse_status_output('{"status":"failed"}') + result = status_with_observed_state( + {"state": {"provider_state": "running", "desired_state": "running", "provision_state": "provisioned"}}, + {"observed_state": parsed}, + ) + assert result["state"]["provider_state"] == "error" + assert effective_provider_state(result["state"]) == "error" + assert not provider_actions_available(result["state"]) + + +def test_provider_envelope_and_host_identity_variants_are_closed() -> None: + access = binding() + access["username"] = "Administrator" + access["host_identity"]["mechanism"] = "configured-fingerprint" + validate_output(access, "guest_access") + access["provider_identity"]["identity"]["opaque"] = "not allowed" + with pytest.raises(SchemaValidationError): + validate_output(access, "guest_access") + + +def test_shared_access_definitions_match() -> None: + from eve_sdk.schema import load_schema + + output = load_schema("command-io.schema.json")["$defs"] + cache = load_schema("observed-state.schema.json")["$defs"] + for definition in ("guest_access", "incus_identity", "provider_identity"): + assert output[definition] == cache[definition] + + +def test_failed_refresh_removes_binding_even_with_running_status(tmp_path: Path, monkeypatch: Any) -> None: + from eve_sdk.state import State + + monkeypatch.setenv("EVE_HOME", str(tmp_path)) + State.update_observed("test-one", {"provider_status": "running", "guest_access": binding(), "ip": "10.201.153.42"}) + state = State.update_observed("test-one", {"provider_status": "running", "refresh_error": "timeout"}) + assert "guest_access" not in state["observed_state"] + assert "ip" not in state["observed_state"]