From f85df8a91795ad27fca5d50528975fa770de3843 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 30 Sep 2026 00:34:00 +0300 Subject: [PATCH] fix: expose Qwen vision and simplify signed Spark updates --- Makefile | 6 +- README.md | 4 + .../sparkplane/engines/vllm-qwen38-mmap.toml | 3 +- configs/sparkplane/models.toml | 2 +- docs/how-to/develop-spark.md | 203 ++++++++- docs/how-to/install-spark.md | 19 +- docs/reference/spark.md | 23 +- scripts/update-spark.py | 418 ++++++++++++++++++ src/generated_files.rs | 85 +++- src/spark/agent.rs | 119 ++++- src/spark/cli.rs | 1 + src/spark/launch.rs | 55 ++- src/spark/state.rs | 1 + src/spark/wire.rs | 7 + tests/generated_files.rs | 45 ++ tests/spark_qwen_vllm_optimization.rs | 75 ++++ tests/spark_update.py | 298 +++++++++++++ 17 files changed, 1333 insertions(+), 31 deletions(-) create mode 100755 scripts/update-spark.py create mode 100644 tests/spark_update.py diff --git a/Makefile b/Makefile index 5c48068..9aff265 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ -.PHONY: build test lint lint-spark test-client fmt-check audit +.PHONY: build test lint lint-spark test-client fmt-check audit update build: cargo build --locked --release --features appliance test: python3 tests/standalone_contract.py + python3 tests/spark_update.py cargo test --locked --workspace --all-targets --features appliance test-client: cargo test --locked --no-default-features --all-targets @@ -15,3 +16,6 @@ fmt-check: cargo fmt --all -- --check audit: cargo deny --all-features check +HOST ?= dgx-spark +update: + @python3 scripts/update-spark.py "$(HOST)" diff --git a/README.md b/README.md index b71db0f..060614d 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,10 @@ hermetic and do not contact a Spark or allocate a GPU. - [CLI and runtime reference](docs/reference/spark.md) - [Security and release authority](SECURITY.md) +To build and preview a signed source update, run `make update HOST=dgx-spark`. +Apply it with `python3 scripts/update-spark.py dgx-spark --apply`; see the +[workstation update workflow](docs/how-to/develop-spark.md#update-a-spark-from-this-checkout). + Inference URLs use `/openai//v1` and `/anthropic//v1`. The control API uses `/api/sparkplane/v1`. Runtime paths, units, schemas, environment variables and Docker ownership use the Sparkplane namespace. diff --git a/configs/sparkplane/engines/vllm-qwen38-mmap.toml b/configs/sparkplane/engines/vllm-qwen38-mmap.toml index 6eeabe9..a9ae53a 100644 --- a/configs/sparkplane/engines/vllm-qwen38-mmap.toml +++ b/configs/sparkplane/engines/vllm-qwen38-mmap.toml @@ -109,6 +109,7 @@ arguments = [ "--reasoning-parser", "qwen3", "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", ] -capabilities = ["text_generation", "tool_calling"] +capabilities = ["text_generation", "tool_calling", "vision"] +vision = { processor_sha256 = "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", media_types = ["image/jpeg", "image/png", "image/webp"], max_bytes = 524288, max_total_bytes = 524288, max_count = 1, max_width = 4096, max_height = 4096, health_media_type = "image/png", health_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAOAAAADgAQAAAAAyVldMAAAAHUlEQVRYw+3BMQEAAADCoPVPbQZ/oAAAAAAAAHgMGWAAAct2tnAAAAAASUVORK5CYII=", health_image_sha256 = "1b8ab2918e9fb346d8ff5b7579372510b0d8b1566d590dac31aea311fff15fc6", health_prompt = "Name the single dominant color using one lowercase word.", health_expected_text = "black", health_max_tokens = 32, health_disable_thinking = true } startup_protocol_probe = true sampling = { temperature = 1.0, top_p = 0.95, top_k = 20, min_p = 0.0, presence_penalty = 0.0, repetition_penalty = 1.0, thinking_token_budget = 8192, reasoning_effort_map = { low = "low", medium = "medium", high = "xhigh", max = "xhigh" } } diff --git a/configs/sparkplane/models.toml b/configs/sparkplane/models.toml index 258d71b..ba5e2f6 100644 --- a/configs/sparkplane/models.toml +++ b/configs/sparkplane/models.toml @@ -619,7 +619,7 @@ revision = "7b719225242aacd3dbd3f9407468c2ee9a9d2594" [models.artifact] format = "safetensors" quantization = "NVFP4" -capabilities = ["reasoning", "text_generation", "tool_calling"] +capabilities = ["reasoning", "text_generation", "tool_calling", "vision"] engine_profile = "qwen3.8-flash-next-nvfp4" [models.artifact.primary] diff --git a/docs/how-to/develop-spark.md b/docs/how-to/develop-spark.md index d774604..fd72d0a 100644 --- a/docs/how-to/develop-spark.md +++ b/docs/how-to/develop-spark.md @@ -21,8 +21,8 @@ commit revision. The laptop is the control plane. `sparkplane ...` talks to the remote agent over its pinned TLS endpoint; it does not run Docker commands or copy -arbitrary files over SSH. SSH/SFTP is used only by the signed bootstrap and -upgrade protocol. +arbitrary files over SSH. SSH/SFTP handles signed bootstrap, upgrade and recovery; +the workstation update helper also reads the installed public release key over SSH. ## Repository map @@ -30,6 +30,7 @@ upgrade protocol. | --- | --- | | `.github/workflows/spark-release.yml` | CI contract, ARM64 build, packaging, and tagged signing | | `scripts/package-spark-release.sh` | Creates the exact release bundle and checksum manifest | +| `scripts/update-spark.py` | Builds, signs, previews and applies a local workstation update | | `configs/sparkplane/agent.toml`, `executor.toml` | Listener, paths, resource reserves, and pinned host policy | | `configs/sparkplane/engines/*.toml` | Digest-pinned runtime recipes, matchers, routes, and profiles | | `configs/sparkplane/models.toml` | Reproducible model identities and artifact metadata | @@ -61,6 +62,82 @@ The tests parse and validate the real files under `configs/`; do not replace them with a second test fixture. A new workload or protocol path also needs a black-box test against the running daemon where practical. +## Update a Spark from this checkout + +Use the workstation helper for a source update. It requires Linux x86-64 or +ARM64, Python 3.11+, the development dependencies above, Cargo Auditable, +Cargo Zigbuild, Zig, the Rust ARM64 target, Minisign, and a working OpenSSH alias +with an existing Sparkplane installation and authenticated client configuration. +The [workstation signing authority](#reuse-the-workstation-signing-credential) +must already exist; the helper does not provision or replace keys. + +```bash +# Build, sign and preview the exact candidate; engines continue serving. +python3 scripts/update-spark.py dgx-spark +# Equivalent convenience target: +make update HOST=dgx-spark + +# Build, preview and apply, then install the verified workstation client. +python3 scripts/update-spark.py dgx-spark --apply +``` + +Preparation runs `make lint`, `make test`, `make test-client` and `make audit`, +builds an auditable native client and ARM64 appliance, and packages the shipped +catalogs using the existing release packager. Each run uses fresh directories +under ignored `target/spark-updates/`; it never edits an earlier signed bundle. +The outer signed manifest covers the native client, build metadata, public key, +and signed appliance manifest. Both inventories and signatures are verified +against the independently pinned key before the bundled client executes. + +By default the key is `release.key`, the pinned public key is `release.pub`, and +the encrypted user credential is `password.cred`, all under +`${XDG_CONFIG_HOME:-$HOME/.config}/sparkplane-release-signing/`. Use +`--signing-dir` and `--public-key` for other established paths. When +`password.cred` exists, the helper decrypts it with the user-scoped credential +name `sparkplane-release-password` and feeds Minisign through stdin. The password +stays in process memory and stdin; it is never an argument, environment variable, +log or release file. Without that credential, Minisign prompts in your terminal. +SSH and remote sudo authentication use their existing interactive prompts. + +The helper compares the pinned authority with +`/opt/sparkplane/current/minisign.pub` over authenticated SSH and rejects a +mismatch. `--apply` performs the managed signed `upgrade` only after the exact +candidate dry-run succeeds. Afterwards it checks agent/executor versions, +semantic health, `doctor`, and the running instances' generation, model, +artifacts, engine fingerprints, resources and context. It then atomically +updates `~/.local/bin/sparkplane` to a verified client in a fresh digest-named +release directory. The preceding client remains available, and the published +client updater's `current` pointer is unchanged. An existing regular executable +at that path is a conflict; use `--no-install-client` to retain it. + +The preview prints its signed release path. Reuse that exact release to apply +without rebuilding or accessing the signing secret; the helper verifies it and +performs a fresh host preview: + +```bash +python3 scripts/update-spark.py dgx-spark \ + --prepared target/spark-updates/release- --apply +``` + +Use `--json` for one `sparkplane.workstation-update/v1` result on stdout; progress +and authentication prompts go to stderr. Check `state`, `appliance_applied`, +`release` and `evidence`. Every run retains private check logs, before/after +status and instance inventories, the dry-run, activation result and final +result. `--evidence-dir` changes the parent of that fresh private run directory; +`--config-dir` selects an existing Sparkplane client configuration directory. +If interrupted or if verification fails after activation, inspect the evidence +and `sparkplane status --json` before choosing the documented recovery +operation. A failed result does not imply the appliance was unchanged. + +This helper updates the control plane and workstation client while preserving +running engines. For changes requiring a new engine generation or refreshed +model traits, follow [capability activation](#activate-a-capability-change-on-an-existing-instance) +and use the helper's signed `appliance/` bundle at the upgrade step. After a +healthy instance already exposes its new capabilities, add +`--codex-model qwen3.8:flash-next` to an apply to regenerate the managed Codex +profile and model catalog. Ownership conflicts still fail through the normal +launch guard. + ## Build the ARM64 executable Install the target and the tools used by CI (`cargo-zigbuild` is preferred for @@ -75,7 +152,7 @@ python3 -m pip install --user ziglang==0.16.0 Build locally with the same feature boundary as Spark: ```bash -cargo zigbuild --release \ +cargo auditable zigbuild --locked --release \ --target aarch64-unknown-linux-gnu \ --no-default-features --features appliance --bin sparkplane ``` @@ -125,6 +202,35 @@ is pinned in the operator's local Spark configuration. A manually dispatched workflow builds and uploads an artifact but deliberately does not sign it; only a `v*` tag invokes the CI signing step. +### Reuse the workstation signing credential + +The existing workstation authority may already be provisioned under +`~/.config/sparkplane-release-signing/`. Its encrypted `release.key` uses the +user-scoped systemd credential `password.cred`, whose embedded name is +`sparkplane-release-password`. Reuse this authority; do not run key generation +again for a local activation. Confirm its public key matches the installed, +trusted authority before signing. + +Feed the decrypted credential directly to Minisign, with shell tracing disabled +and pipeline failures enabled: + +```bash +set +x +set -o pipefail +sparkplane_signing_dir="${XDG_CONFIG_HOME:-$HOME/.config}/sparkplane-release-signing" +systemd-creds --user --name=sparkplane-release-password \ + decrypt "$sparkplane_signing_dir/password.cred" - | + minisign -Sm target/spark-release/SHA256SUMS \ + -s "$sparkplane_signing_dir/release.key" +minisign -Vm target/spark-release/SHA256SUMS \ + -p "$sparkplane_signing_dir/release.pub" +``` + +This requires the same workstation user and credential encryption key used at +provisioning. The plaintext password stays in the pipe. SSH and remote sudo +authentication remain separate and use the existing OpenSSH transport; do not +put those credentials in command arguments, documentation, or release assets. + ### Provision the release authority Generate the dedicated encrypted key in your own terminal. Store its password @@ -194,7 +300,9 @@ sparkplane dgx-spark install --yes \ For an installed host, use the side-by-side upgrade protocol: ```bash -sparkplane dgx-spark upgrade --dry-run --json +sparkplane dgx-spark upgrade --dry-run --json \ + --probe release/sparkplane-aarch64 \ + --release-manifest release/SHA256SUMS sparkplane dgx-spark upgrade --yes \ --probe release/sparkplane-aarch64 \ --release-manifest release/SHA256SUMS \ @@ -237,6 +345,93 @@ Certificate rotation is separate from a software release: `sparkplane dgx-spark cert rotate --dry-run --json`, followed by `--yes` after review. Use `--ca` only when replacing the certificate authority. +### Activate a capability change on an existing instance + +A control-plane-only upgrade can preserve a running engine. Changing an engine +profile or stored model capability requires a managed stop and a new serving +generation. Existing model records retain their verified artifact traits; +deploying a new `models.toml` does not update those records automatically. + +For the Qwen3.8 image-input correction, build both the appliance and the +workstation client from the tested source. The client generates Codex's model +catalog, while the appliance publishes the live route's supported modalities: + +```bash +cargo auditable build --locked --release --bin sparkplane +``` + +Use `target/release/sparkplane` for the following commands if the client on +`PATH` has not been updated. Package, sign, verify, and dry-run the ARM64 release +before stopping the model. Save `ps --json`, including the instance name, +generation, immutable model, image/profile fingerprints, resources and context. +Confirm the workload has no running or queued requests before the restart. + +```bash +sparkplane dgx-spark ps --json +sparkplane dgx-spark stop qwen38-vllm --json +sparkplane dgx-spark upgrade --yes --json \ + --probe release/sparkplane-aarch64 \ + --release-manifest release/SHA256SUMS \ + --release-signature release/SHA256SUMS.minisig \ + --release-public-key "$sparkplane_signing_dir/release.pub" +sparkplane dgx-spark download qwen3.8:flash-next --dry-run --json +sparkplane dgx-spark download qwen3.8:flash-next --update-alias --detach --json +sparkplane dgx-spark operations --follow --json +sparkplane dgx-spark serve qwen3.8:flash-next --name qwen38-vllm --dry-run --json +sparkplane dgx-spark serve qwen3.8:flash-next --name qwen38-vllm --detach --json +sparkplane dgx-spark operations --follow --json +sparkplane dgx-spark ps --json +``` + +The download operation resolves the signed catalog's immutable revision and +reverifies the existing native cache. With a complete cache, the dry-run reports +`unique_bytes: 0` and no model weight transfer is needed. Wait for each managed +operation to succeed before advancing. Retain the previous signed release and +image for rollback; do not edit the live database, catalogs, or container. + +Hub metadata resolution can exceed the workstation host profile's default +`request_timeout_seconds = 30`, even when all weights are cached. A dry-run +timeout does not prove the cache is missing. Increase that host's request timeout +for a bounded retry of the plan. Detached refresh and serving operations return +their identifiers immediately. Following uses the same request timeout; if it +reports that the operation is still running, reconnect with the same identifier +instead of submitting another download or serve request. Advance only after +`state: "succeeded"`. A complete cache reports zero transfer bytes, but its +existing files still undergo checksum verification, which can take minutes. +Refreshing capability traits also changes the artifact fingerprint and selects +a separate compile-cache namespace. Allow the normal cold start, compilation, +and semantic probes to finish; retain the previous namespace for rollback. + +For this correction, the engine image, launch arguments, resource envelope, +262,144-token context, BF16 KV and MTP settings stay unchanged. The new profile +adds a bounded image policy and semantic image health probe. After the new +generation becomes healthy, `ps --json` must advertise +`input_modalities: ["text", "image"]`. Verify an inline-image request through the +authenticated public gateway, then regenerate the owned client configuration: + +```bash +sparkplane dgx-spark launch codex --model qwen3.8:flash-next --config --json +sparkplane dgx-spark launch codex --model qwen3.8:flash-next +``` + +The generated `sparkplane-launch-models.json` must include both `text` and +`image`. Relaunch Codex so it reads that catalog. Do not hand-edit the catalog +or its ownership receipt to make an old session accept attachments. + +If regeneration reports an ownership conflict, preserve a private backup of the +affected profile and its receipt, move the indicated profile aside, and rerun +`launch ... --config --json`. Additional tables such as `[tui]` in an earlier +profile can cause this conflict. The launcher regenerates both owned files and +their receipt through the normal publication path. + +Verify an actual attachment with a small test image: + +```bash +sparkplane dgx-spark launch codex --model qwen3.8:flash-next --mode inherit -- \ + exec --sandbox read-only --image ./test-image.png -- \ + "Describe the attached image." +``` + ## Add an engine Add a complete TOML file under `configs/sparkplane/engines/`. Start from the diff --git a/docs/how-to/install-spark.md b/docs/how-to/install-spark.md index fe5b636..6a57350 100644 --- a/docs/how-to/install-spark.md +++ b/docs/how-to/install-spark.md @@ -73,11 +73,21 @@ You can now [serve a model](serve-a-model-on-spark.md). ## Upgrade or recover the control plane +For a workstation source checkout, the [update helper](develop-spark.md#update-a-spark-from-this-checkout) +automates the checks, build, signing, preview, activation and local client update: + +```bash +python3 scripts/update-spark.py dgx-spark # build and preview +python3 scripts/update-spark.py dgx-spark --apply # build, preview and apply +``` + Preview and approve the same signed ARM64 artifact transaction used at first install: ```bash -sparkplane dgx-spark upgrade --dry-run --json +sparkplane dgx-spark upgrade --dry-run --json \ + --probe release/sparkplane-aarch64 \ + --release-manifest release/SHA256SUMS sparkplane dgx-spark upgrade --yes \ --probe release/sparkplane-aarch64 \ --release-manifest release/SHA256SUMS \ @@ -91,6 +101,13 @@ and the protected DGX fingerprint. It changes only the control-plane release and keeps healthy engine containers running. Failed semantic health requests automatic rollback. +An engine profile or model capability change also needs a managed stop and +re-serve. Existing model records keep their original artifact traits, so refresh +them through the managed download operation using the new signed catalog. +Follow [activating a capability change](develop-spark.md#activate-a-capability-change-on-an-existing-instance) +for the complete sequence, including reuse of cached weights and regeneration +of Codex's image-capable model catalog. + Use the SSH-only recovery path even when HTTPS is unavailable: ```bash diff --git a/docs/reference/spark.md b/docs/reference/spark.md index 928ec4c..f30a07a 100644 --- a/docs/reference/spark.md +++ b/docs/reference/spark.md @@ -239,9 +239,12 @@ host/integration selection is reused; an interactive terminal can select from installed models. `--config` writes only launch-owned state/config and exits. `--dry-run` performs no local or remote mutation. `--json` is valid with either of those non-agent modes. Generated files have a private digest-based ownership -receipt. Launch and restore verify both files before changing either; a user edit, -symlink or unowned destination is a conflict, not permission to overwrite it. -Move a conflicting file to your own backup location before regenerating it. +receipt. Launch and restore verify both files before changing either. Codex may +write its own settings, such as `approvals_reviewer`, into the profile; those +extra lines do not block the next launch, and the profile is rewritten from the +managed route. A change to a managed line, a symlink, or an unowned destination +is a conflict. The error prints the path; move that file aside before retrying. +There is no `--force`. Interrupted publication can be retried safely. `--restore` removes only Sparkplane-owned Codex profile/catalog files. `-y` permits the fixed Claude or OpenCode installer when the executable is absent. Only arguments after `--` are forwarded, without a @@ -253,6 +256,20 @@ inference-only token and pinned CA, never the administrator credential. Claude uses the native Anthropic route, Codex uses a Sparkplane-owned Responses profile and catalog, and OpenCode uses process-local `OPENCODE_CONFIG_CONTENT`. +Codex image attachments follow `input_modalities` reported by the healthy +instance's published gateway route. Image support requires the verified model +and engine vision policy; a vision-capable checkpoint on a text-only route +still accepts only text. Older agents that omit this field default to text. +Relaunch Codex after changing the qualified instance so its catalog is refreshed. +Existing model records retain their verified artifact traits across upgrades. +When a catalog adds vision support, refresh the record through the managed +`download --update-alias` operation and re-serve it with the updated engine +policy before relaunching Codex. +The Qwen3.8 vLLM policy accepts one inline PNG, JPEG, or WebP image per request, +at most 512 KiB of image data, with width and height at most 4096 pixels. +See the [live capability activation procedure](../how-to/develop-spark.md#activate-a-capability-change-on-an-existing-instance) +for signing, cached-model refresh, managed restart, and attachment verification. + ### Internet access for development Launches default to `--mode auto` (`SPARKPLANE_LAUNCH_MODE=auto`): full diff --git a/scripts/update-spark.py b/scripts/update-spark.py new file mode 100755 index 0000000..78e87d4 --- /dev/null +++ b/scripts/update-spark.py @@ -0,0 +1,418 @@ +#!/usr/bin/env python3 +"""Build, sign, preview and apply a workstation's Sparkplane update.""" + +import argparse +import fcntl +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import tomllib + +ROOT = Path(__file__).resolve().parents[1] +CHECKS = ["lint", "test", "test-client", "audit"] +OUTER_FILES = {"sparkplane-client", "update.json", "minisign.pub", + "appliance/SHA256SUMS", "appliance/SHA256SUMS.minisig"} +INSTANCE_FIELDS = ["id", "name", "model_id", "model", "model_commit", "engine_id", + "engine_fingerprint", "artifact_fingerprint", "artifacts", "resources", + "context_window", "generation", "desired"] + + +class UpdateError(Exception): + pass + + +def progress(message): + print(f"spark-update: {message}", file=sys.stderr, flush=True) + + +def digest(path): + with path.open("rb") as handle: + return hashlib.file_digest(handle, "sha256").hexdigest() + + +def regular(path): + if not stat.S_ISREG(path.lstat().st_mode): + raise UpdateError(f"expected a regular file: {path}") + + +def private_file(path): + regular(path) + info = path.stat() + if info.st_uid != os.getuid() or info.st_mode & 0o077: + raise UpdateError(f"signing secret must be owned by this user with mode 0600: {path}") + + +def run(argv, *, cwd=ROOT, output=None, input_bytes=None, timeout=1800): + """Inherit authentication stdin; keep all subprocess output off JSON stdout.""" + with output.open("xb") if output else open(os.devnull, "wb") as handle: + result = subprocess.run(argv, cwd=cwd, input=input_bytes, + stdout=handle if output else sys.stderr, + stderr=sys.stderr, timeout=timeout) + if result.returncode: + raise UpdateError(f"{Path(argv[0]).name} failed (exit {result.returncode}); see stderr") + + +def write_json(path, value): + with path.open("x", encoding="utf-8") as handle: + json.dump(value, handle, indent=2) + handle.write("\n") + + +def manifest_entries(root): + entries = {} + for line in (root / "SHA256SUMS").read_text().splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) ([A-Za-z0-9_./-]+)", line) + if not match: + raise UpdateError("checksum inventory has an invalid entry") + sha, name = match.groups() + path = Path(name) + if path.is_absolute() or ".." in path.parts or name in entries: + raise UpdateError("checksum inventory has an escaping or duplicate path") + candidate = root / path + if not candidate.resolve(strict=True).is_relative_to(root.resolve(strict=True)): + raise UpdateError("checksum inventory escapes the release") + # Reject symlinks, including directory components within the bundle. + for parent in [candidate, *candidate.parents]: + if parent == root: + break + if parent.is_symlink(): + raise UpdateError("checksum inventory contains a symlink") + regular(candidate) + if digest(candidate) != sha: + raise UpdateError(f"checksum mismatch: {name}") + entries[name] = sha + if not entries: + raise UpdateError("checksum inventory is empty") + return entries + + +def public_identity(value): + lines = [line.strip() for line in value.splitlines() if line.strip()] + if len(lines) == 2 and lines[0].startswith("untrusted comment: minisign public key"): + lines = lines[1:] + if len(lines) != 1 or not re.fullmatch(r"[A-Za-z0-9+/=]{56}", lines[0]): + raise UpdateError("invalid release public key") + return lines[0] + + +def verify_signature(manifest, public_key): + regular(manifest) + regular(Path(str(manifest) + ".minisig")) + run(["minisign", "-V", "-m", str(manifest), "-p", str(public_key)]) + + +def verify_release(release, public_key): + # Verify the external authority before executing any bundled client. + verify_signature(release / "SHA256SUMS", public_key) + if set(manifest_entries(release)) != OUTER_FILES: + raise UpdateError("workstation release has an unexpected signed inventory") + if public_identity((release / "minisign.pub").read_text()) != public_identity(public_key.read_text()): + raise UpdateError("bundled public key differs from the pinned authority") + appliance = release / "appliance" + verify_signature(appliance / "SHA256SUMS", public_key) + entries = manifest_entries(appliance) + engine_names = {f"configs/sparkplane/engines/{path.name}" + for path in (appliance / "configs/sparkplane/engines").glob("*.toml")} + if not engine_names or set(entries) != {"sparkplane-aarch64", "configs/sparkplane/models.toml", *engine_names}: + raise UpdateError("appliance release has an unexpected signed inventory") + metadata = json.loads((release / "update.json").read_text()) + if (metadata.get("schema") != "sparkplane.workstation-release/v1" + or metadata.get("machine") != platform.machine() + or metadata.get("checks") != CHECKS + or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:[-+][A-Za-z0-9.-]+)?", metadata.get("version", ""))): + raise UpdateError("release metadata, native architecture or required checks do not match") + return metadata + + +def installed_authority(host, public_key, evidence): + # This fixed read needs no sudo. OpenSSH supplies transport authentication. + path = evidence / "installed-minisign.pub" + run(["ssh", host, "cat /opt/sparkplane/current/minisign.pub"], output=path, timeout=120) + if public_identity(path.read_text()) != public_identity(public_key.read_text()): + raise UpdateError("installed release authority differs from the pinned key; refusing to replace it") + + +def sign_manifest(manifest, key, credential): + private_file(key) + password = None + if credential: + private_file(credential) + result = subprocess.run([ + "systemd-creds", "--user", "--name=sparkplane-release-password", + "decrypt", str(credential), "-"], capture_output=True, timeout=30) + if result.returncode or not result.stdout or len(result.stdout) > 65536: + raise UpdateError("cannot decrypt the workstation signing credential") + # Plaintext remains in memory and Minisign's stdin, never argv or logs. + password = result.stdout.rstrip(b"\n") + b"\n" + elif not sys.stdin.isatty(): + raise UpdateError("signing needs a terminal or the encrypted password.cred credential") + run(["minisign", "-S", "-m", str(manifest), "-s", str(key)], input_bytes=password) + + +def write_manifest(root, names): + with (root / "SHA256SUMS").open("x", encoding="utf-8") as handle: + for name in sorted(names): + handle.write(f"{digest(root / name)} {name}\n") + + +def prepare(args, public_key, evidence): + signing = args.signing_dir + key = signing / "release.key" + credential = signing / "password.cred" + credential = credential if credential.exists() else None + private_file(key) + installed_authority(args.host, public_key, evidence) + for check in CHECKS: + progress(f"running make {check}") + log = evidence / f"check-{check}.log" + with log.open("xb") as handle: + result = subprocess.run(["make", check], cwd=ROOT, stdout=handle, + stderr=subprocess.STDOUT, timeout=1800) + if result.returncode: + raise UpdateError(f"make {check} failed; inspect {log}") + release = Path(tempfile.mkdtemp(prefix="release-", dir=ROOT / "target/spark-updates")) + progress(f"building native client and ARM64 appliance into {release}") + target = ROOT / "target" + run(["cargo", "auditable", "build", "--locked", "--release", "--no-default-features", + "--bin", "sparkplane", "--target-dir", str(target)]) + shutil.copyfile(target / "release/sparkplane", release / "sparkplane-client") + (release / "sparkplane-client").chmod(0o555) + run(["cargo", "auditable", "zigbuild", "--locked", "--release", + "--target", "aarch64-unknown-linux-gnu", "--no-default-features", "--features", + "appliance", "--bin", "sparkplane", "--target-dir", str(target)]) + run(["bash", str(ROOT / "scripts/package-spark-release.sh"), + str(target / "aarch64-unknown-linux-gnu/release/sparkplane"), str(release / "appliance")]) + manifest_entries(release / "appliance") + sign_manifest(release / "appliance/SHA256SUMS", key, credential) + shutil.copyfile(public_key, release / "minisign.pub") + cargo = tomllib.loads((ROOT / "Cargo.toml").read_text()) + revision = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, + check=True, capture_output=True, text=True).stdout.strip() + dirty = subprocess.run(["git", "status", "--porcelain"], cwd=ROOT, + check=True, capture_output=True).stdout != b"" + write_json(release / "update.json", { + "schema":"sparkplane.workstation-release/v1", "version":cargo["workspace"]["package"]["version"], + "machine":platform.machine(), "checks":CHECKS, "revision":revision, "dirty":dirty, + }) + write_manifest(release, OUTER_FILES) + sign_manifest(release / "SHA256SUMS", key, credential) + return release + + +def client_json(release, args, evidence, name, command): + output = evidence / f"{name}.json" + argv = [str(release / "sparkplane-client"), args.host, *command, "--json"] + if args.config_dir: + argv.extend(["--config-dir", str(args.config_dir)]) + progress(name) + run(argv, output=output) + value = json.loads(output.read_text()) + if not isinstance(value, dict): + raise UpdateError(f"{name} returned an invalid document") + return value + + +def healthy_status(status, version=None): + if (status.get("schema") != "sparkplane.status/v1" or status.get("read_only") is not False + or status.get("degraded_reasons") != []): + raise UpdateError("control plane is not healthy and writable") + if version and (status.get("agent") != version or status.get("executor") != version): + raise UpdateError("agent and executor versions differ from the signed candidate") + + +def preserve_instances(before, after): + if not isinstance(before.get("instances"), list) or not isinstance(after.get("instances"), list): + raise UpdateError("instance inventory is invalid") + observed = {instance["id"]: instance for instance in after["instances"]} + for instance in before["instances"]: + if instance.get("desired") != "running": + continue + current = observed.get(instance["id"], {}) + if (any(instance.get(field) != current.get(field) for field in INSTANCE_FIELDS) + or (instance.get("healthy") is True and current.get("healthy") is not True)): + raise UpdateError(f"running instance changed during the update: {instance['name']}") + + +def client_destination(): + data = Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local/share")) + return data / "sparkplane/client/releases", Path.home() / ".local/bin/sparkplane" + + +def check_client_destination(): + _, link = client_destination() + if link.exists() and not link.is_symlink(): + raise UpdateError(f"local client is not a managed symlink: {link}; use --no-install-client") + + +def publish_client(release, public_key): + # Retain the prior target and signed evidence; do not change the release updater's `current`. + releases, link = client_destination() + releases.mkdir(parents=True, exist_ok=True) + destination = releases / f"workstation-{digest(release / 'SHA256SUMS')}" + if destination.exists(): + verify_signature(destination / "SHA256SUMS", public_key) + if (set(manifest_entries(destination)) != OUTER_FILES + or digest(destination / "SHA256SUMS") != digest(release / "SHA256SUMS")): + raise UpdateError("existing content-addressed client release differs from this update") + else: + staged = Path(tempfile.mkdtemp(prefix=".update-", dir=releases)) + try: + for name in [*OUTER_FILES, "SHA256SUMS", "SHA256SUMS.minisig"]: + target = staged / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(release / name, target) + target.chmod(0o555 if name == "sparkplane-client" else 0o444) + staged.rename(destination) + finally: + if staged.exists(): + shutil.rmtree(staged) + check_client_destination() + link.parent.mkdir(parents=True, exist_ok=True) + staged_link = link.parent / f".sparkplane-update-{os.getpid()}" + try: + staged_link.symlink_to(destination / "sparkplane-client") + staged_link.replace(link) + finally: + staged_link.unlink(missing_ok=True) + return str(link) + + +def update(args, evidence, result): + public_key = (args.public_key or args.signing_dir / "release.pub").resolve(strict=True) + regular(public_key) + if public_key.stat().st_mode & 0o022: + raise UpdateError("pinned public key is writable by another user") + if args.prepared: + release = args.prepared.resolve(strict=True) + else: + release = prepare(args, public_key, evidence) + result["release"] = str(release) + metadata = verify_release(release, public_key) + if args.prepared: + installed_authority(args.host, public_key, evidence) + before_status = client_json(release, args, evidence, "status-before", ["status"]) + healthy_status(before_status) + before = client_json(release, args, evidence, "instances-before", ["ps"]) + command = ["upgrade", "--probe", str(release / "appliance/sparkplane-aarch64"), + "--release-manifest", str(release / "appliance/SHA256SUMS")] + plan = client_json(release, args, evidence, "upgrade-dry-run", [*command, "--dry-run"]) + sha = digest(release / "appliance/sparkplane-aarch64") + if (plan.get("schema") != "sparkplane.install-manifest/v1" + or plan.get("host_alias") != args.host or plan.get("operation") != "upgrade" + or plan.get("dry_run") is not True + or plan.get("inventory", {}).get("existing_installation", {}).get("present") is not True + or plan.get("probe", {}).get("local_sha256") != sha + or plan.get("probe", {}).get("reported_sha256") != sha + or plan.get("probe", {}).get("removed") is not True): + raise UpdateError("dry-run did not inspect and clean up the exact candidate") + result["state"] = "previewed" + if not args.apply: + return + if not args.no_install_client: + check_client_destination() + # Recheck signed bytes immediately before the managed activation. + verify_release(release, public_key) + result["state"] = "applying" + applied = client_json(release, args, evidence, "upgrade", [*command, "--yes", + "--release-signature", str(release / "appliance/SHA256SUMS.minisig"), + "--release-public-key", str(public_key)]) + if applied.get("execution", {}).get("state") != "applied": + raise UpdateError("upgrade did not report an applied release") + result["appliance_applied"] = True + if applied.get("protected_before") != plan.get("protected_before"): + raise UpdateError("protected DGX fingerprint changed between preview and activation") + after_status = client_json(release, args, evidence, "status-after", ["status"]) + healthy_status(after_status, metadata["version"]) + after = client_json(release, args, evidence, "instances-after", ["ps"]) + preserve_instances(before, after) + client_json(release, args, evidence, "doctor-after", ["doctor"]) + if not args.no_install_client: + result["client"] = publish_client(release, public_key) + if args.codex_model: + client_json(release, args, evidence, "codex-config", [ + "launch", "codex", "--model", args.codex_model, "--config"]) + result["state"] = "applied" + + +def main(): + config = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config")) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("host", help="existing OpenSSH/Sparkplane host alias") + parser.add_argument("--apply", action="store_true", help="apply after a successful preview") + parser.add_argument("--prepared", type=Path, help="reuse a signed release; no rebuild or signing") + parser.add_argument("--signing-dir", type=Path, default=config / "sparkplane-release-signing") + parser.add_argument("--public-key", type=Path, help="independently pinned public key (default: signing-dir/release.pub)") + parser.add_argument("--config-dir", type=Path, help="existing Sparkplane client configuration directory") + parser.add_argument("--evidence-dir", type=Path, help="parent directory for a fresh private run directory") + parser.add_argument("--no-install-client", action="store_true", help="leave the workstation client symlink unchanged") + parser.add_argument("--codex-model", help="regenerate managed Codex config after a successful apply") + parser.add_argument("--json", action="store_true", help="emit one result document; progress goes to stderr") + args = parser.parse_args() + for name in ["prepared", "signing_dir", "public_key", "config_dir", "evidence_dir"]: + value = getattr(args, name) + if value is not None: + setattr(args, name, value.resolve()) + os.umask(0o077) + result = {"schema":"sparkplane.workstation-update/v1", "state":"preparing", "host":args.host, + "appliance_applied":False, "release":None, "evidence":None, "client":None} + code = 0 + try: + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", args.host) or args.host == "bootstrap": + raise UpdateError("host must be an existing OpenSSH alias") + if args.codex_model and not args.apply: + raise UpdateError("--codex-model requires --apply") + if platform.system() != "Linux" or platform.machine() not in {"x86_64", "aarch64"}: + raise UpdateError("workstation updates support Linux x86-64 and ARM64") + # Ignore ambient CLI action flags; the helper chooses every typed operation. + for key in list(os.environ): + if key.startswith("SPARKPLANE_") and key != "SPARKPLANE_CONFIG_DIR": + del os.environ[key] + updates = ROOT / "target/spark-updates" + updates.mkdir(parents=True, exist_ok=True) + parent = (args.evidence_dir or updates).resolve() + parent.mkdir(parents=True, exist_ok=True) + evidence = Path(tempfile.mkdtemp(prefix="run-", dir=parent)) + result["evidence"] = str(evidence) + progress(f"evidence: {evidence}") + with (updates / "update.lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise UpdateError("another workstation update is running in this checkout") from error + update(args, evidence, result) + except (UpdateError, OSError, ValueError, KeyError, subprocess.SubprocessError) as error: + result["failed_during"] = result["state"] + result["state"] = "failed" + result["error"] = str(error) + progress(str(error)) + code = 1 + except KeyboardInterrupt: + result["failed_during"] = result["state"] + result["state"] = "interrupted" + progress("interrupted; inspect saved evidence and Spark status before retrying an apply") + code = 130 + if result["evidence"]: + write_json(Path(result["evidence"]) / "result.json", result) + if args.json: + print(json.dumps(result)) + else: + print(f"Spark update: {result['state']}") + if result["release"]: + print(f"Signed release: {result['release']}") + if result["evidence"]: + print(f"Evidence: {result['evidence']}") + if result["state"] == "previewed": + print("Apply this signed release with --prepared --apply.") + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/generated_files.rs b/src/generated_files.rs index 617dd5a..9b14865 100644 --- a/src/generated_files.rs +++ b/src/generated_files.rs @@ -14,9 +14,24 @@ const FILES: [&str; 2] = [ "sparkplane-launch.config.toml", "sparkplane-launch-models.json", ]; +const PROFILE: &str = FILES[0]; const RECEIPT: &str = "sparkplane-launch.ownership.json"; const SCHEMA: &str = "sparkplane.generated-files/v1"; const MAX_BYTES: u64 = 1024 * 1024; +/// Assignments written by `codex_client_config`. Codex persists other settings, +/// such as `approvals_reviewer`, into the active profile; those lines are not +/// a conflict. Edits to these keys still are. +const MANAGED_KEYS: [&str; 9] = [ + "model", + "model_provider", + "web_search", + "name", + "base_url", + "env_key", + "wire_api", + "supports_standalone_web_search", + "supports_websockets", +]; #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -131,13 +146,11 @@ pub fn publish(home: &Path, profile: &[u8], catalog: &[u8]) -> Result<()> { let mut owned = vec![hash.clone()]; if let Some(current) = read(home, name)? { let current_hash = format!("{:x}", Sha256::digest(¤t)); + let hashes = pending.files.get(name).map(Vec::as_slice).unwrap_or(&[]); ensure!( - current == bytes - || pending - .files - .get(name) - .is_some_and(|hashes| hashes.contains(¤t_hash)), - "refusing to overwrite modified or unowned generated file {name}" + current == bytes || still_owned(name, ¤t, hashes), + "{}", + conflict(home, name, "overwrite") ); if current_hash != hash { owned.push(current_hash); @@ -160,13 +173,11 @@ pub fn remove(home: &Path) -> Result<()> { let mut present = Vec::new(); for name in FILES { if let Some(bytes) = read(home, name)? { - let hash = format!("{:x}", Sha256::digest(bytes)); + let hashes = receipt.files.get(name).map(Vec::as_slice).unwrap_or(&[]); ensure!( - receipt - .files - .get(name) - .is_some_and(|hashes| hashes.contains(&hash)), - "refusing to remove modified or unowned generated file {name}" + still_owned(name, &bytes, hashes), + "{}", + conflict(home, name, "remove") ); present.push(name); } @@ -183,3 +194,53 @@ pub fn remove(home: &Path) -> Result<()> { })?, ) } + +fn still_owned(name: &str, bytes: &[u8], hashes: &[String]) -> bool { + if owned_digest(bytes, hashes) { + return true; + } + // Codex rewrites the live profile with settings Sparkplane does not generate. + // The managed lines must still be exactly the bytes Sparkplane published. + name == PROFILE && owned_digest(&without_foreign_assignments(bytes), hashes) +} + +fn owned_digest(bytes: &[u8], hashes: &[String]) -> bool { + let digest = format!("{:x}", Sha256::digest(bytes)); + hashes.iter().any(|hash| hash == &digest) +} + +fn without_foreign_assignments(bytes: &[u8]) -> Vec { + let Ok(text) = std::str::from_utf8(bytes) else { + return bytes.to_vec(); + }; + let mut kept = String::with_capacity(text.len()); + for line in text.split_inclusive('\n') { + if !foreign_assignment(line) { + kept.push_str(line); + } + } + kept.into_bytes() +} + +fn foreign_assignment(line: &str) -> bool { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with(['#', '[']) { + return false; + } + let Some((key, _)) = trimmed.split_once('=') else { + return false; + }; + let key = key.trim(); + !key.is_empty() + && key + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + && !MANAGED_KEYS.contains(&key) +} + +fn conflict(home: &Path, name: &str, action: &str) -> String { + format!( + "refusing to {action} {}: it no longer matches the Sparkplane ownership receipt. Move it aside and retry", + home.join(name).display() + ) +} diff --git a/src/spark/agent.rs b/src/spark/agent.rs index 6db96e4..8a1840e 100644 --- a/src/spark/agent.rs +++ b/src/spark/agent.rs @@ -1699,6 +1699,7 @@ async fn serve_instance( objective: "inference".into(), resources, context_window, + input_modalities: vec!["text".into()], default_reasoning_effort, generation: 0, desired: InstanceDesiredState::Running, @@ -2751,9 +2752,18 @@ async fn list_instances(State(state): State, RawQuery(query): RawQue } fn project_route_health(routes: &RouteRegistry, instances: &mut [InstanceDocument]) { - for instance in instances.iter_mut().filter(|instance| instance.healthy) { - if !matches!(routes.lookup(&instance.name), RouteLookup::Healthy(route) if route.generation == instance.generation) + for instance in instances { + instance.input_modalities = vec!["text".into()]; + if !instance.healthy { + continue; + } + if let RouteLookup::Healthy(route) = routes.lookup(&instance.name) + && route.generation == instance.generation { + if route.profile.capabilities.contains("vision") && route.profile.vision.is_some() { + instance.input_modalities.push("image".into()); + } + } else { instance.observed = InstanceObservedState::Degraded; instance.healthy = false; instance.endpoint = None; @@ -6460,6 +6470,7 @@ mod tests { compile_cache_bytes: 1, }, context_window: 65_536, + input_modalities: vec!["text".into()], default_reasoning_effort: None, generation: 0, desired: crate::spark::wire::InstanceDesiredState::Running, @@ -6545,6 +6556,109 @@ mod tests { assert!(!super::reconcile_expects_container(&instance)); } + #[tokio::test] + async fn instance_image_support_follows_the_published_generation_over_https() { + let (state, database, _root) = durable_state().await; + let model = ornith_model(); + database.promote_model(model.clone(), false).await.unwrap(); + let instance = database + .begin_serve(creating_instance(&model)) + .await + .unwrap() + .instance; + database + .set_instance_observed( + &instance.id, + instance.generation, + crate::spark::wire::InstanceObservedState::Healthy, + Some("/openai/ornith/v1".into()), + None, + None, + ) + .await + .unwrap(); + let routes = state.routes.clone(); + let rcgen::CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let certificate = cert.pem(); + let tls = super::tls13_config( + certificate.as_bytes().to_vec(), + signing_key.serialize_pem().into_bytes(), + ) + .await + .unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let handle = axum_server::Handle::new(); + let server_handle = handle.clone(); + let server = tokio::spawn(async move { + axum_server::from_tcp_rustls(listener, tls) + .unwrap() + .handle(server_handle) + .serve(router(state).into_make_service_with_connect_info::()) + .await + .unwrap(); + }); + let client = reqwest::Client::builder() + .tls_built_in_root_certs(false) + .add_root_certificate(reqwest::Certificate::from_pem(certificate.as_bytes()).unwrap()) + .build() + .unwrap(); + let vision = crate::spark::engine::EnginePolicy::parse(include_str!( + "../../configs/sparkplane/engines/llama-cpp.toml" + )) + .unwrap() + .gateway_profile(None); + let mut missing_policy = vision.clone(); + missing_policy.vision = None; + for (generation, profile, expected) in [ + ( + instance.generation, + crate::spark::gateway::GatewayProfile::text(), + vec!["text"], + ), + (instance.generation, vision.clone(), vec!["text", "image"]), + (instance.generation, missing_policy, vec!["text"]), + (instance.generation + 1, vision, vec!["text"]), + ] { + let upstream = crate::spark::upstream::ObservedRoute::new( + &instance.id, + generation, + "172.30.0.2".parse().unwrap(), + 8000, + [("POST", "/v1/chat/completions")], + ) + .unwrap(); + routes.publish_with_profile( + &instance.name, + instance.model.clone(), + "Ornith-1.5-9B".into(), + profile, + upstream, + ); + let list: crate::spark::wire::InstanceListDocument = client + .get(format!( + "https://localhost:{}{API_BASE}/instances", + address.port() + )) + .bearer_auth(TOKEN) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(list.instances[0].input_modalities, expected); + assert_eq!(list.instances[0].healthy, generation == instance.generation); + } + handle.graceful_shutdown(None); + server.await.unwrap(); + database.shutdown().unwrap(); + } + #[tokio::test] async fn reconcile_restarts_the_same_engine_and_artifact_generation() { let (state, database, root) = durable_state().await; @@ -6785,6 +6899,7 @@ mod tests { compile_cache_bytes: 1, }, context_window: 65_536, + input_modalities: vec!["text".into()], default_reasoning_effort: None, generation: 0, desired: crate::spark::wire::InstanceDesiredState::Running, diff --git a/src/spark/cli.rs b/src/spark/cli.rs index 43ea0a3..b5d1247 100644 --- a/src/spark/cli.rs +++ b/src/spark/cli.rs @@ -2101,6 +2101,7 @@ mod tests { compile_cache_bytes: 4, }, context_window: 65_536, + input_modalities: vec!["text".into()], default_reasoning_effort: None, generation: 1, desired: InstanceDesiredState::Running, diff --git a/src/spark/launch.rs b/src/spark/launch.rs index 638269b..dfcae32 100644 --- a/src/spark/launch.rs +++ b/src/spark/launch.rs @@ -620,22 +620,22 @@ fn configure_integration( "Spark instance has no declared context window; restart it with the current engine configuration", )); } - let catalog_value = codex_catalog(model, instance.context_window); + let catalog_value = codex_catalog(model, instance); let catalog_text = serde_json::to_vec_pretty(&catalog_value) .map_err(|_| failure(EXIT_INTERNAL, "could not encode Codex model catalog"))?; crate::generated_files::publish(&home, profile_text.as_bytes(), &catalog_text) .map_err(|error| usage(format!("generated client files: {error:#}"))) } -fn codex_catalog(model: &ModelDocument, context_window: u64) -> Value { +fn codex_catalog(model: &ModelDocument, instance: &InstanceDocument) -> Value { serde_json::json!({ "owned_by": OWNED_MARKER, "models": [{ "slug": model.canonical, "display_name": model.aliases.first().unwrap_or(&model.repository), "description": model.canonical, - "context_window": context_window, - "max_context_window": context_window, + "context_window": instance.context_window, + "max_context_window": instance.context_window, "effective_context_window_percent": 100, "shell_type": "default", "visibility": "list", @@ -644,7 +644,7 @@ fn codex_catalog(model: &ModelDocument, context_window: u64) -> Value { "additional_speed_tiers": [], "service_tiers": [], "truncation_policy": { "mode": "bytes", "limit": 10000 }, - "input_modalities": ["text"], + "input_modalities": instance.input_modalities, "base_instructions": "", "default_reasoning_summary": "none", "support_verbosity": true, @@ -1348,6 +1348,7 @@ mod tests { compile_cache_bytes: 0, }, context_window: 65_536, + input_modalities: vec!["text".into()], default_reasoning_effort: None, generation: 1, desired: InstanceDesiredState::Running, @@ -1688,9 +1689,51 @@ mod tests { assert!(!content.contains("inference-secret")); } + #[test] + fn codex_catalog_advertises_the_active_instances_image_support() { + let model = model("m_one", "vision"); + let mut instance = instance("vision", &model.id); + instance.input_modalities = vec!["text".into(), "image".into()]; + let catalog = codex_catalog(&model, &instance); + assert_eq!( + catalog["models"][0]["input_modalities"], + serde_json::json!(["text", "image"]) + ); + assert_eq!( + catalog["models"][0]["context_window"], + instance.context_window + ); + } + + #[test] + fn codex_catalog_keeps_a_vision_checkpoint_text_only_on_a_text_route() { + let mut model = model("m_one", "vision"); + let mut artifacts = artifacts(); + artifacts.capabilities.push("vision".into()); + model.artifacts = Some(artifacts); + let instance = instance("vision", &model.id); + let catalog = codex_catalog(&model, &instance); + assert_eq!( + catalog["models"][0]["input_modalities"], + serde_json::json!(["text"]) + ); + } + + #[test] + fn legacy_instance_documents_default_to_text_input() { + let instance = instance("vision", "m_one"); + let mut document = serde_json::to_value(instance).unwrap(); + document.as_object_mut().unwrap().remove("input_modalities"); + let instance: InstanceDocument = serde_json::from_value(document).unwrap(); + assert_eq!(instance.input_modalities, ["text"]); + } + #[test] fn codex_catalog_matches_current_required_shape_without_secret() { - let catalog = codex_catalog(&model("m_one", "ornith"), 262_144); + let model = model("m_one", "ornith"); + let mut instance = instance("ornith", &model.id); + instance.context_window = 262_144; + let catalog = codex_catalog(&model, &instance); assert_eq!(catalog["models"][0]["context_window"], 262_144); assert_eq!(catalog["models"][0]["max_context_window"], 262_144); assert_eq!( diff --git a/src/spark/state.rs b/src/spark/state.rs index c86fda5..daf8e2a 100644 --- a/src/spark/state.rs +++ b/src/spark/state.rs @@ -2468,6 +2468,7 @@ mod tests { compile_cache_bytes: 1, }, context_window: 65_536, + input_modalities: vec!["text".into()], default_reasoning_effort: None, generation: 0, desired: InstanceDesiredState::Running, diff --git a/src/spark/wire.rs b/src/spark/wire.rs index c92d8f2..911aebc 100644 --- a/src/spark/wire.rs +++ b/src/spark/wire.rs @@ -580,6 +580,9 @@ pub struct InstanceDocument { pub resources: RecipeResourceEnvelopeDocument, #[serde(default)] pub context_window: u64, + /// Input supported by the published route for this exact generation. + #[serde(default = "default_instance_input_modalities")] + pub input_modalities: Vec, #[serde(default)] pub default_reasoning_effort: Option, pub generation: u64, @@ -607,6 +610,10 @@ pub struct InstanceListDocument { pub instances: Vec, } +fn default_instance_input_modalities() -> Vec { + vec!["text".into()] +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] #[serde(deny_unknown_fields)] diff --git a/tests/generated_files.rs b/tests/generated_files.rs index 863665d..b17a411 100644 --- a/tests/generated_files.rs +++ b/tests/generated_files.rs @@ -32,6 +32,51 @@ fn unowned_marker_and_symlink_do_not_grant_permission_to_overwrite() { assert!(sparkplane::generated_files::publish(root.path(), b"next", b"catalog").is_err()); } +#[test] +fn codex_profile_settings_do_not_block_republication_or_restore() { + let root = tempfile::tempdir().unwrap(); + let profile = b"# owned-by: sparkplane launch\nmodel = \"m\"\nmodel_provider = \"p\"\nweb_search = \"disabled\"\n\n[model_providers.p]\nname = \"n\"\nbase_url = \"https://example/v1\"\nenv_key = \"SPARKPLANE_INFERENCE_TOKEN\"\nwire_api = \"responses\"\nsupports_standalone_web_search = false\nsupports_websockets = false\n"; + sparkplane::generated_files::publish(root.path(), profile, b"catalog").unwrap(); + let edited = std::str::from_utf8(profile).unwrap().replace( + "web_search = \"disabled\"\n", + "web_search = \"disabled\"\napprovals_reviewer = \"user\"\n", + ); + let path = root.path().join("sparkplane-launch.config.toml"); + std::fs::write(&path, &edited).unwrap(); + let next = b"# owned-by: sparkplane launch\nmodel = \"next\"\nmodel_provider = \"p\"\nweb_search = \"disabled\"\n\n[model_providers.p]\nname = \"n\"\nbase_url = \"https://example/v1\"\nenv_key = \"SPARKPLANE_INFERENCE_TOKEN\"\nwire_api = \"responses\"\nsupports_standalone_web_search = false\nsupports_websockets = false\n"; + sparkplane::generated_files::publish(root.path(), next, b"next catalog").unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), next); + std::fs::write( + &path, + std::str::from_utf8(next).unwrap().replace( + "web_search = \"disabled\"\n", + "web_search = \"disabled\"\napprovals_reviewer = \"user\"\n", + ), + ) + .unwrap(); + sparkplane::generated_files::remove(root.path()).unwrap(); + assert!(!path.exists()); +} + +#[test] +fn a_changed_managed_line_names_the_file_and_leaves_it_in_place() { + let root = tempfile::tempdir().unwrap(); + let profile = b"model = \"m\"\nweb_search = \"disabled\"\n"; + sparkplane::generated_files::publish(root.path(), profile, b"catalog").unwrap(); + let path = root.path().join("sparkplane-launch.config.toml"); + std::fs::write(&path, b"model = \"edited\"\nweb_search = \"disabled\"\n").unwrap(); + let error = + sparkplane::generated_files::publish(root.path(), b"model = \"next\"\n", b"catalog") + .unwrap_err() + .to_string(); + assert!(error.contains(&path.display().to_string()), "{error}"); + assert!(error.contains("Move it aside and retry"), "{error}"); + assert_eq!( + std::fs::read(&path).unwrap(), + b"model = \"edited\"\nweb_search = \"disabled\"\n" + ); +} + #[test] fn an_interrupted_pair_update_can_be_completed_without_losing_ownership() { use sha2::{Digest, Sha256}; diff --git a/tests/spark_qwen_vllm_optimization.rs b/tests/spark_qwen_vllm_optimization.rs index 497d4da..3d28d01 100644 --- a/tests/spark_qwen_vllm_optimization.rs +++ b/tests/spark_qwen_vllm_optimization.rs @@ -1,3 +1,78 @@ +#[cfg(feature = "appliance")] +#[test] +fn qwen38_checkpoint_and_selected_engine_accept_inline_images() { + use sparkplane::spark::{engine::EngineCatalog, gateway, model_catalog::ModelCatalog}; + let models = ModelCatalog::parse(include_str!("../configs/sparkplane/models.toml")).unwrap(); + let artifacts = models.resolve("qwen3.8:flash-next").unwrap().artifacts(); + let directory = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("configs/sparkplane/engines"); + let files = std::fs::read_dir(directory) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "toml") + }) + .map(|path| { + ( + path.display().to_string(), + std::fs::read_to_string(path).unwrap(), + ) + }); + let engines = EngineCatalog::parse_files(files).unwrap(); + let engine = engines.select(artifacts).unwrap(); + let profile = engine + .gateway_profile_for(Some("qwen4_exp"), artifacts, None) + .unwrap(); + let vision = profile + .vision + .as_ref() + .expect("Qwen image input must have a verified vision policy"); + assert_eq!( + vision.processor_sha256, + "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516" + ); + let image = serde_json::json!({"type":"input_image", "image_url":format!("data:image/png;base64,{}", vision.health_image_base64)}); + let mut request = serde_json::json!({"input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"Name the color."},image]}]}); + let rewrite = gateway::rewrite_responses_request_with_profile( + &serde_json::to_vec(&request).unwrap(), + "served-model", + &profile, + ) + .unwrap(); + let body: serde_json::Value = serde_json::from_slice(&rewrite.body).unwrap(); + assert_eq!( + body["messages"][0]["content"][1]["image_url"]["url"], + image["image_url"] + ); + + let mut text_only = artifacts.clone(); + text_only.capabilities.retain(|value| value != "vision"); + let text_profile = engine + .gateway_profile_for(Some("qwen4_exp"), &text_only, None) + .unwrap(); + assert!( + gateway::rewrite_responses_request_with_profile( + &serde_json::to_vec(&request).unwrap(), + "served-model", + &text_profile + ) + .is_err() + ); + request["input"][0]["content"] + .as_array_mut() + .unwrap() + .push(image); + assert!( + gateway::rewrite_responses_request_with_profile( + &serde_json::to_vec(&request).unwrap(), + "served-model", + &profile + ) + .is_err() + ); +} + #[test] fn optimized_runtime_keeps_graphs_without_global_eager_module_loading() { let profile: toml::Value = toml::from_str(include_str!( diff --git a/tests/spark_update.py b/tests/spark_update.py new file mode 100644 index 0000000..705b7d2 --- /dev/null +++ b/tests/spark_update.py @@ -0,0 +1,298 @@ +"""Signed workstation updates, exercised through the operator entrypoint.""" + +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts/update-spark.py" + +CLIENT = '''#!/usr/bin/env python3 +import json, os, pathlib, sys +args = sys.argv[1:] +state = pathlib.Path(os.environ["UPDATE_FIXTURE"]) +with (state / "calls.jsonl").open("a") as out: + out.write(json.dumps(args) + "\\n") +command = args[1] +changed = (state / "applied").exists() +failure = os.environ.get("UPDATE_FAILURE") +if command == "status": + result = {"schema":"sparkplane.status/v1", "agent":"0.1.6", + "executor":"0.1.6", "read_only":False, "degraded_reasons":[]} + if changed and failure == "health": result["read_only"] = True + if changed and failure == "version": result["executor"] = "old-version" +elif command == "ps": + instance = {"id":"i_fixture", "name":"model", "model_id":"m_fixture", + "model":"immutable", "model_commit":"commit", "engine_id":"engine", + "engine_fingerprint":"engine-sha", "artifact_fingerprint":"artifact-sha", + "artifacts":{}, "resources":{}, "context_window":262144, "generation":10, + "desired":"running", "observed":"healthy", "healthy":True} + if changed and failure == "generation": instance["generation"] += 1 + if changed and failure == "context": instance["context_window"] = 32768 + if changed and failure == "engine": instance["engine_fingerprint"] = "changed" + result = {"schema":"sparkplane.instance-list/v1", "instances":[instance]} +elif command == "upgrade": + applied = "--yes" in args + if not applied and failure == "preview": sys.exit(3) + probe = pathlib.Path(args[args.index("--probe") + 1]) + import hashlib + digest = hashlib.sha256(probe.read_bytes()).hexdigest() + result = {"schema":"sparkplane.install-manifest/v1", "operation":"upgrade", + "host_alias":args[0], "dry_run":not applied, + "inventory":{"existing_installation":{"present":True}}, + "probe":{"local_sha256":digest, "reported_sha256":digest, "removed":True}, + "protected_before":{"sha256":"protected"}, + "execution":{"state":"applied" if applied else "planned"}} + if applied: (state / "applied").touch() +elif command == "doctor": result = {"schema":"sparkplane.doctor/v1", "checks":[]} +elif command == "launch": result = {"schema":"sparkplane.launch/v1"} +else: sys.exit(2) +print(json.dumps(result)) +''' + + +class UpdateWorkflow(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + checkout = self.root / "checkout/scripts" + checkout.mkdir(parents=True) + self.script = checkout / "update-spark.py" + shutil.copyfile(SCRIPT, self.script) + self.release = self.root / "release" + self.release.mkdir() + self.home = self.root / "home" + self.home.mkdir() + self.tools = self.root / "tools" + self.tools.mkdir() + self.secret = self.root / "release.key" + self.public = self.root / "release.pub" + subprocess.run(["minisign", "-G", "-W", "-s", str(self.secret), "-p", str(self.public)], + check=True, capture_output=True) + ssh = self.tools / "ssh" + ssh.write_text('#!/usr/bin/env python3\nimport os,pathlib\nprint(pathlib.Path(os.environ["UPDATE_AUTHORITY"]).read_text(),end="")\n') + ssh.chmod(0o755) + self.env = dict(os.environ, HOME=str(self.home), + XDG_CONFIG_HOME=str(self.home / ".config"), + XDG_DATA_HOME=str(self.home / ".local/share"), + UPDATE_FIXTURE=str(self.root), UPDATE_AUTHORITY=str(self.public), + PATH=f"{self.tools}:{os.environ['PATH']}") + for key in list(self.env): + if key.startswith("SPARKPLANE_") or key.startswith("UPDATE_FAILURE"): + del self.env[key] + client = self.release / "sparkplane-client" + client.write_text(CLIENT) + client.chmod(0o555) + appliance = self.release / "appliance" + engines = appliance / "configs/sparkplane/engines" + engines.mkdir(parents=True) + (appliance / "sparkplane-aarch64").write_bytes(b"arm64 appliance") + (appliance / "configs/sparkplane/models.toml").write_text("models = []\n") + (engines / "engine.toml").write_text("engine = 'fixture'\n") + self.manifest(appliance, ["sparkplane-aarch64", "configs/sparkplane/models.toml", + "configs/sparkplane/engines/engine.toml"]) + self.sign(appliance / "SHA256SUMS") + shutil.copyfile(self.public, self.release / "minisign.pub") + (self.release / "update.json").write_text(json.dumps({ + "schema":"sparkplane.workstation-release/v1", "version":"0.1.6", + "machine":os.uname().machine, "checks":["lint", "test", "test-client", "audit"], + })) + self.seal() + + def manifest(self, directory, files): + (directory / "SHA256SUMS").write_text("".join( + f"{hashlib.sha256((directory / name).read_bytes()).hexdigest()} {name}\n" + for name in files)) + + def sign(self, path): + subprocess.run(["minisign", "-S", "-s", str(self.secret), "-m", str(path)], + check=True, capture_output=True) + + def seal(self): + self.manifest(self.release, ["sparkplane-client", "update.json", "minisign.pub", + "appliance/SHA256SUMS", "appliance/SHA256SUMS.minisig"]) + self.sign(self.release / "SHA256SUMS") + + def run_update(self, *args, failure=None): + env = dict(self.env) + if failure: + env["UPDATE_FAILURE"] = failure + output = subprocess.run([ + "python3", str(self.script), "fixture-spark", "--prepared", str(self.release), + "--public-key", str(self.public), "--evidence-dir", str(self.root / "evidence"), + "--json", *args], env=env, capture_output=True, text=True) + result = json.loads(output.stdout) + self.assertNotIn("Traceback", output.stderr) + return output, result + + def calls(self): + path = self.root / "calls.jsonl" + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + def test_preview_never_activates_or_installs_client(self): + output, result = self.run_update() + self.assertEqual(output.returncode, 0, output.stderr) + self.assertEqual(result["state"], "previewed") + self.assertFalse((self.root / "applied").exists()) + self.assertFalse((self.home / ".local/bin/sparkplane").exists()) + calls = self.calls() + self.assertTrue(any("--dry-run" in call for call in calls)) + self.assertFalse(any("--yes" in call for call in calls)) + for call in calls: + if call[1] == "upgrade": + self.assertEqual(call[call.index("--probe") + 1], + str(self.release / "appliance/sparkplane-aarch64")) + evidence = Path(result["evidence"]) + self.assertEqual(evidence.stat().st_mode & 0o777, 0o700) + self.assertEqual((evidence / "upgrade-dry-run.json").stat().st_mode & 0o777, 0o600) + + def test_apply_preserves_engine_and_publishes_verified_native_client(self): + output, result = self.run_update("--apply", "--codex-model", "model:tag") + self.assertEqual(output.returncode, 0, output.stderr) + self.assertEqual(result["state"], "applied") + calls = self.calls() + commands = [call[1] for call in calls] + self.assertFalse(set(commands) & {"stop", "serve", "download"}) + upgrades = [call for call in calls if call[1] == "upgrade"] + self.assertEqual(len(upgrades), 2) + self.assertIn("--dry-run", upgrades[0]) + self.assertIn("--yes", upgrades[1]) + self.assertIn("--release-public-key", upgrades[1]) + link = self.home / ".local/bin/sparkplane" + self.assertTrue(link.is_symlink()) + self.assertEqual(link.read_bytes(), (self.release / "sparkplane-client").read_bytes()) + self.assertEqual(link.stat().st_mode & 0o777, 0o555) + self.assertIn("launch", commands) + self.assertIn("--config", calls[-1]) + + def test_corrupt_client_is_rejected_before_it_can_execute(self): + (self.release / "sparkplane-client").chmod(0o755) + (self.release / "sparkplane-client").write_text(CLIENT + "\n# corruption\n") + output, result = self.run_update("--apply") + self.assertNotEqual(output.returncode, 0) + self.assertEqual(result["state"], "failed") + self.assertEqual(self.calls(), []) + + def test_corrupt_appliance_is_rejected_before_activation(self): + (self.release / "appliance/sparkplane-aarch64").write_bytes(b"corruption") + output, _ = self.run_update("--apply") + self.assertNotEqual(output.returncode, 0) + self.assertEqual(self.calls(), []) + + def test_signature_cannot_be_replaced_with_bundle_authority(self): + other_key, other_pub = self.root / "other.key", self.root / "other.pub" + subprocess.run(["minisign", "-G", "-W", "-s", str(other_key), "-p", str(other_pub)], + check=True, capture_output=True) + shutil.copyfile(other_pub, self.release / "minisign.pub") + self.secret = other_key + self.seal() + output, _ = self.run_update("--apply") + self.assertNotEqual(output.returncode, 0) + self.assertEqual(self.calls(), []) + + def test_installed_authority_mismatch_rejects_even_a_locally_signed_release(self): + foreign = self.root / "foreign.pub" + foreign.write_text("untrusted comment: minisign public key\n" + "A" * 56 + "\n") + self.env["UPDATE_AUTHORITY"] = str(foreign) + output, _ = self.run_update("--apply") + self.assertNotEqual(output.returncode, 0) + self.assertFalse((self.root / "applied").exists()) + + def test_failed_preview_prevents_activation(self): + output, _ = self.run_update("--apply", failure="preview") + self.assertNotEqual(output.returncode, 0) + self.assertFalse((self.root / "applied").exists()) + self.assertFalse(any("--yes" in call for call in self.calls())) + + def test_failed_health_or_changed_engine_prevents_client_publication(self): + for failure in ["health", "context", "generation", "engine", "version"]: + with self.subTest(failure=failure): + (self.root / "applied").unlink(missing_ok=True) + output, _ = self.run_update("--apply", failure=failure) + self.assertNotEqual(output.returncode, 0) + self.assertFalse((self.home / ".local/bin/sparkplane").exists()) + + def test_symlink_in_signed_inventory_is_rejected(self): + payload = self.release / "appliance/sparkplane-aarch64" + outside = self.root / "outside" + payload.rename(outside) + payload.symlink_to(outside) + output, _ = self.run_update("--apply") + self.assertNotEqual(output.returncode, 0) + self.assertEqual(self.calls(), []) + + def test_wrong_native_architecture_is_rejected_without_execution(self): + metadata = self.release / "update.json" + data = json.loads(metadata.read_text()) + data["machine"] = "unsupported" + metadata.write_text(json.dumps(data)) + self.seal() + output, _ = self.run_update("--apply") + self.assertNotEqual(output.returncode, 0) + self.assertEqual(self.calls(), []) + + def test_no_install_client_leaves_existing_executable_unchanged(self): + link = self.home / ".local/bin/sparkplane" + link.parent.mkdir(parents=True) + link.write_text("unmanaged client") + output, _ = self.run_update("--apply", "--no-install-client") + self.assertEqual(output.returncode, 0, output.stderr) + self.assertEqual(link.read_text(), "unmanaged client") + + def test_encrypted_credential_signs_without_exposing_password(self): + password = b"fixture-only signing password" + key = self.root / "encrypted.key" + public = self.root / "encrypted.pub" + subprocess.run(["minisign", "-G", "-s", str(key), "-p", str(public)], + input=password + b"\n" + password + b"\n", check=True, capture_output=True) + key.chmod(0o600) + credential = self.root / "password.cred" + credential.write_bytes(b"encrypted fixture credential") + credential.chmod(0o600) + decrypt = self.tools / "systemd-creds" + decrypt.write_text('#!/usr/bin/env python3\nimport sys\n' + 'assert sys.argv[1:4] == ["--user", "--name=sparkplane-release-password", "decrypt"]\n' + 'sys.stdout.write("fixture-only signing password")\n') + decrypt.chmod(0o755) + manifest = self.root / "to-sign" + manifest.write_text("fixture manifest\n") + code = '''import importlib.util, pathlib, sys +spec = importlib.util.spec_from_file_location("updater", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +module.sign_manifest(*map(pathlib.Path, sys.argv[2:])) +''' + output = subprocess.run(["python3", "-c", code, str(SCRIPT), str(manifest), str(key), + str(credential)], env=self.env, capture_output=True) + self.assertEqual(output.returncode, 0, output.stderr) + self.assertNotIn(password, output.stdout + output.stderr) + subprocess.run(["minisign", "-V", "-m", str(manifest), "-p", str(public)], + check=True, capture_output=True) + + def test_path_traversal_in_signed_inventory_is_rejected(self): + manifest = self.release / "appliance/SHA256SUMS" + manifest.write_text(manifest.read_text() + "0" * 64 + " ../../outside\n") + self.sign(manifest) + self.seal() + output, _ = self.run_update("--apply") + self.assertNotEqual(output.returncode, 0) + self.assertEqual(self.calls(), []) + + def test_conflicting_local_executable_prevents_remote_apply(self): + link = self.home / ".local/bin/sparkplane" + link.parent.mkdir(parents=True) + link.write_text("unmanaged client") + output, _ = self.run_update("--apply") + self.assertNotEqual(output.returncode, 0) + self.assertFalse((self.root / "applied").exists()) + self.assertEqual(link.read_text(), "unmanaged client") + + +if __name__ == "__main__": + unittest.main()