From 68c0d1e9a2a7658eb0b6cb7b63dfe63658cf0a6c Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Thu, 1 Oct 2026 01:36:30 +0300 Subject: [PATCH] feat: embed RTK, report session economics and retain image history Enable embedded RTK by default with per-session usage accounting and public cloud price comparisons. Normalize bounded vision uploads and prune inspected image history through an authenticated launch adapter while preserving tool pairing, observations and archived originals. Keep temporary reconciliation admission deferrals out of the engine restart budget and close stale missing-engine routes. Validation: make lint, make test, make test-client, make audit and warning-denying rustdoc passed. Live Codex image calls and a 500-frame, 38.4 MiB history completed on Spark with matching token accounting. --- Cargo.lock | 12 + Cargo.toml | 14 +- README.md | 11 + configs/sparkplane/agent.toml | 8 + .../sparkplane/engines/vllm-qwen38-mmap.toml | 3 +- crates/sparkplane-rtk/ADAPTATION.patch | 280 + crates/sparkplane-rtk/Cargo.toml | 12 + crates/sparkplane-rtk/LICENSE | 190 + crates/sparkplane-rtk/PROVENANCE.json | 96 + crates/sparkplane-rtk/src/cargo.rs | 666 ++ .../src/core/builtin_filters.toml | 2493 ++++++ crates/sparkplane-rtk/src/core/constants.rs | 38 + crates/sparkplane-rtk/src/core/guard.rs | 71 + crates/sparkplane-rtk/src/core/stream.rs | 13 + crates/sparkplane-rtk/src/core/toml_filter.rs | 2129 +++++ crates/sparkplane-rtk/src/core/truncate.rs | 60 + crates/sparkplane-rtk/src/core/utils.rs | 26 + crates/sparkplane-rtk/src/git.rs | 35 + crates/sparkplane-rtk/src/lib.rs | 132 + crates/sparkplane-rtk/src/pytest.rs | 290 + crates/sparkplane-rtk/upstream/Cargo.toml | 89 + crates/sparkplane-rtk/upstream/LICENSE | 190 + .../upstream/src/cmds/git/git_cmd.rs | 6833 +++++++++++++++++ .../upstream/src/cmds/python/pytest_cmd.rs | 604 ++ .../upstream/src/cmds/rust/cargo_cmd.rs | 2899 +++++++ .../upstream/src/core/constants.rs | 38 + .../sparkplane-rtk/upstream/src/core/guard.rs | 71 + .../upstream/src/core/stream.rs | 1469 ++++ .../upstream/src/core/toml_filter.rs | 2229 ++++++ .../upstream/src/core/truncate.rs | 60 + .../sparkplane-rtk/upstream/src/core/utils.rs | 1842 +++++ .../upstream/src/filters/README.md | 142 + .../src/filters/ansible-playbook.toml | 34 + .../upstream/src/filters/basedpyright.toml | 47 + .../upstream/src/filters/biome.toml | 45 + .../upstream/src/filters/brew-install.toml | 37 + .../upstream/src/filters/bundle-install.toml | 61 + .../src/filters/composer-install.toml | 40 + .../upstream/src/filters/df.toml | 16 + .../upstream/src/filters/dotnet-build.toml | 64 + .../upstream/src/filters/du.toml | 16 + .../upstream/src/filters/fail2ban-client.toml | 15 + .../upstream/src/filters/gcc.toml | 49 + .../upstream/src/filters/gcloud.toml | 22 + .../upstream/src/filters/hadolint.toml | 24 + .../upstream/src/filters/helm.toml | 103 + .../upstream/src/filters/iptables.toml | 27 + .../upstream/src/filters/jira.toml | 20 + .../upstream/src/filters/jj.toml | 28 + .../upstream/src/filters/jq.toml | 24 + .../upstream/src/filters/just.toml | 26 + .../upstream/src/filters/liquibase.toml | 84 + .../upstream/src/filters/make.toml | 41 + .../upstream/src/filters/markdownlint.toml | 24 + .../upstream/src/filters/mise.toml | 30 + .../upstream/src/filters/mix-compile.toml | 27 + .../upstream/src/filters/mix-format.toml | 15 + .../upstream/src/filters/nx.toml | 25 + .../upstream/src/filters/ollama.toml | 23 + .../upstream/src/filters/oxlint.toml | 43 + .../upstream/src/filters/ping.toml | 63 + .../upstream/src/filters/pio-run.toml | 40 + .../upstream/src/filters/poetry-install.toml | 50 + .../upstream/src/filters/pre-commit.toml | 35 + .../upstream/src/filters/ps.toml | 16 + .../upstream/src/filters/pulumi-destroy.toml | 60 + .../upstream/src/filters/pulumi-preview.toml | 59 + .../upstream/src/filters/pulumi-refresh.toml | 57 + .../upstream/src/filters/pulumi-stack.toml | 48 + .../upstream/src/filters/pulumi-up.toml | 64 + .../upstream/src/filters/quarto-render.toml | 41 + .../upstream/src/filters/rsync.toml | 48 + .../upstream/src/filters/shellcheck.toml | 27 + .../upstream/src/filters/shopify-theme.toml | 29 + .../upstream/src/filters/skopeo.toml | 45 + .../upstream/src/filters/sops.toml | 16 + .../upstream/src/filters/spring-boot.toml | 28 + .../upstream/src/filters/ssh.toml | 44 + .../upstream/src/filters/stat.toml | 44 + .../upstream/src/filters/swift-build.toml | 41 + .../src/filters/systemctl-status.toml | 33 + .../upstream/src/filters/task.toml | 27 + .../upstream/src/filters/terraform-plan.toml | 35 + .../upstream/src/filters/tofu-fmt.toml | 16 + .../upstream/src/filters/tofu-init.toml | 38 + .../upstream/src/filters/tofu-plan.toml | 35 + .../upstream/src/filters/tofu-validate.toml | 17 + .../upstream/src/filters/trunk-build.toml | 39 + .../upstream/src/filters/turbo.toml | 30 + .../upstream/src/filters/ty.toml | 50 + .../upstream/src/filters/uv-sync.toml | 37 + .../upstream/src/filters/xcodebuild.toml | 99 + .../upstream/src/filters/yadm.toml | 21 + .../upstream/src/filters/yamllint.toml | 25 + data/economics-prices.json | 40 + docs/how-to/develop-spark.md | 9 + docs/reference/spark.md | 147 +- specs/openapi/sparkplane-control-v1.json | 7 + src/main.rs | 10 + src/spark/agent.rs | 1545 +++- src/spark/cli.rs | 28 + src/spark/client.rs | 83 +- src/spark/eco.rs | 568 ++ src/spark/economics.rs | 507 ++ src/spark/gateway.rs | 167 +- src/spark/gateway_images.rs | 394 + src/spark/image_history.rs | 435 ++ src/spark/inference_adapter.rs | 670 ++ src/spark/launch.rs | 216 +- src/spark/mod.rs | 6 + src/spark/sessions.rs | 460 ++ src/spark/state.rs | 150 +- src/spark/upstream.rs | 361 +- src/spark/wire.rs | 47 + tests/eco_launch.rs | 205 + tests/gateway_image_upload.rs | 545 ++ tests/image_history.rs | 104 + tests/spark_anthropic_e2e.rs | 2 + tests/spark_modalities_e2e.rs | 2 + tests/spark_openai_e2e.rs | 2 + tests/spark_qwen_vllm_optimization.rs | 11 + tests/spark_vllm_e2e.rs | 2 + tests/standalone_contract.py | 9 + 123 files changed, 31965 insertions(+), 179 deletions(-) create mode 100644 crates/sparkplane-rtk/ADAPTATION.patch create mode 100644 crates/sparkplane-rtk/Cargo.toml create mode 100644 crates/sparkplane-rtk/LICENSE create mode 100644 crates/sparkplane-rtk/PROVENANCE.json create mode 100644 crates/sparkplane-rtk/src/cargo.rs create mode 100644 crates/sparkplane-rtk/src/core/builtin_filters.toml create mode 100644 crates/sparkplane-rtk/src/core/constants.rs create mode 100644 crates/sparkplane-rtk/src/core/guard.rs create mode 100644 crates/sparkplane-rtk/src/core/stream.rs create mode 100644 crates/sparkplane-rtk/src/core/toml_filter.rs create mode 100644 crates/sparkplane-rtk/src/core/truncate.rs create mode 100644 crates/sparkplane-rtk/src/core/utils.rs create mode 100644 crates/sparkplane-rtk/src/git.rs create mode 100644 crates/sparkplane-rtk/src/lib.rs create mode 100644 crates/sparkplane-rtk/src/pytest.rs create mode 100644 crates/sparkplane-rtk/upstream/Cargo.toml create mode 100644 crates/sparkplane-rtk/upstream/LICENSE create mode 100644 crates/sparkplane-rtk/upstream/src/cmds/git/git_cmd.rs create mode 100644 crates/sparkplane-rtk/upstream/src/cmds/python/pytest_cmd.rs create mode 100644 crates/sparkplane-rtk/upstream/src/cmds/rust/cargo_cmd.rs create mode 100644 crates/sparkplane-rtk/upstream/src/core/constants.rs create mode 100644 crates/sparkplane-rtk/upstream/src/core/guard.rs create mode 100644 crates/sparkplane-rtk/upstream/src/core/stream.rs create mode 100644 crates/sparkplane-rtk/upstream/src/core/toml_filter.rs create mode 100644 crates/sparkplane-rtk/upstream/src/core/truncate.rs create mode 100644 crates/sparkplane-rtk/upstream/src/core/utils.rs create mode 100644 crates/sparkplane-rtk/upstream/src/filters/README.md create mode 100644 crates/sparkplane-rtk/upstream/src/filters/ansible-playbook.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/basedpyright.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/biome.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/brew-install.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/bundle-install.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/composer-install.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/df.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/dotnet-build.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/du.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/fail2ban-client.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/gcc.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/gcloud.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/hadolint.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/helm.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/iptables.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/jira.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/jj.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/jq.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/just.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/liquibase.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/make.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/markdownlint.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/mise.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/mix-compile.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/mix-format.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/nx.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/ollama.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/oxlint.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/ping.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/pio-run.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/poetry-install.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/pre-commit.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/ps.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/pulumi-destroy.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/pulumi-preview.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/pulumi-refresh.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/pulumi-stack.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/pulumi-up.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/quarto-render.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/rsync.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/shellcheck.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/shopify-theme.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/skopeo.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/sops.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/spring-boot.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/ssh.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/stat.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/swift-build.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/systemctl-status.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/task.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/terraform-plan.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/tofu-fmt.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/tofu-init.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/tofu-plan.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/tofu-validate.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/trunk-build.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/turbo.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/ty.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/uv-sync.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/xcodebuild.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/yadm.toml create mode 100644 crates/sparkplane-rtk/upstream/src/filters/yamllint.toml create mode 100644 data/economics-prices.json create mode 100644 src/spark/eco.rs create mode 100644 src/spark/economics.rs create mode 100644 src/spark/gateway_images.rs create mode 100644 src/spark/image_history.rs create mode 100644 src/spark/inference_adapter.rs create mode 100644 src/spark/sessions.rs create mode 100644 tests/eco_launch.rs create mode 100644 tests/gateway_image_upload.rs create mode 100644 tests/image_history.rs diff --git a/Cargo.lock b/Cargo.lock index 48a851e..472cd96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3097,8 +3097,10 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "shlex", "sparkplane-core", "sparkplane-ipc", + "sparkplane-rtk", "tempfile", "tokio", "tokio-util", @@ -3150,6 +3152,16 @@ dependencies = [ "ulid", ] +[[package]] +name = "sparkplane-rtk" +version = "0.49.0" +dependencies = [ + "regex", + "serde", + "serde_json", + "toml", +] + [[package]] name = "spinning_top" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 6dfc6b5..f4a9902 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "crates/sparkplane-core", "crates/sparkplane-ipc"] +members = [".", "crates/sparkplane-core", "crates/sparkplane-ipc", "crates/sparkplane-rtk"] resolver = "3" [workspace.package] @@ -37,7 +37,7 @@ publish = false [features] default = [] -appliance = ["dep:axum", "dep:axum-server", "dep:hmac", "dep:governor", "dep:rcgen", "dep:rusqlite", "dep:rusqlite_migration", "dep:rustls", "dep:secrecy", "dep:tower", "dep:utoipa", "dep:x509-parser", "dep:bollard", "dep:http-body-util", "dep:hyper", "dep:hyper-util", "dep:hf-hub", "dep:image", "dep:arc-swap", "dep:sparkplane-ipc", "dep:tempfile"] +appliance = ["dep:axum", "dep:axum-server", "dep:hmac", "dep:governor", "dep:rcgen", "dep:rusqlite", "dep:rusqlite_migration", "dep:rustls", "dep:secrecy", "dep:tower", "dep:utoipa", "dep:x509-parser", "dep:bollard", "dep:hf-hub", "dep:image", "dep:arc-swap", "dep:sparkplane-ipc", "dep:tempfile"] [dependencies] anyhow.workspace = true @@ -52,6 +52,8 @@ futures-util.workspace = true tracing.workspace = true tracing-subscriber.workspace = true sparkplane-core.workspace = true +sparkplane-rtk = { path = "crates/sparkplane-rtk", version = "=0.49.0" } +shlex = "1" sparkplane-ipc = { workspace = true, optional = true } tempfile = { workspace = true, optional = true } clap = { version = "4", features = ["derive", "env"] } @@ -62,7 +64,7 @@ reqwest = { version = "0.12", default-features = false, features = ["blocking", sha2 = "0.10" base64 = "0.22" minisign-verify = "0.2" -nix = { version = "0.29", features = ["fs"] } +nix = { version = "0.29", features = ["fs", "signal", "process"] } uuid = { version = "1", features = ["v4"] } zstd = "0.13" walkdir = "2" @@ -81,9 +83,9 @@ rusqlite = { version = "0.40.0", default-features = false, features = ["backup", rusqlite_migration = { version = "2.6.0", default-features = false, optional = true } governor = { version = "0.10.4", default-features = false, features = ["dashmap", "std"], optional = true } bollard = { version = "0.21.1", default-features = false, features = ["pipe"], optional = true } -http-body-util = { version = "0.1", optional = true } -hyper = { version = "1", default-features = false, features = ["client", "http1"], optional = true } -hyper-util = { version = "0.1", default-features = false, features = ["tokio"], optional = true } +http-body-util = { version = "0.1" } +hyper = { version = "1", default-features = false, features = ["client", "server", "http1"] } +hyper-util = { version = "0.1", default-features = false, features = ["tokio"] } hf-hub = { version = "1.0.0", default-features = false, features = ["rustls-tls"], optional = true } image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"], optional = true } arc-swap = { version = "1", optional = true } diff --git a/README.md b/README.md index 060614d..e1f3d55 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,17 @@ without action approval prompts. Use `--mode inherit` to keep the agent's own permission settings. In inherit mode, `--allow-network` grants network access within a supported sandbox. See [launch permissions](docs/reference/spark.md#internet-access-for-development). +Launches also default to `--eco-mode=max`, using RTK filters embedded in the +Sparkplane executable. Use `--eco-mode=none` to disable compression. When the +coding session exits, Sparkplane prints token usage and public cloud price +comparisons; `sparkplane economics --json` reads the latest retained report. +See [session economics](docs/reference/spark.md#eco-mode-and-session-economics). + +Vision launches also retain recent images automatically and archive older +originals for reopening. A private local adapter prunes historical image payloads +before upload, while the gateway enforces the qualified model limits. See +[image history and gateway limits](docs/reference/spark.md#image-history-and-upload-limits). + ## Architecture The unprivileged HTTPS agent owns state and protocol translation. A separate, diff --git a/configs/sparkplane/agent.toml b/configs/sparkplane/agent.toml index d6d5766..3081d6b 100644 --- a/configs/sparkplane/agent.toml +++ b/configs/sparkplane/agent.toml @@ -6,6 +6,14 @@ allowed_client_cidrs = ["10.1.30.0/24"] plain_http_loopback_only = true executor_socket = "/run/sparkplane/executor.sock" +[gateway] +# Public uploads are normalized to the selected engine's stricter image policy. +request_body_bytes = 33554432 +image_bytes = 16777216 +image_pixels = 32000000 # Total source pixels across the request. +image_dimension = 16384 +max_parallel_requests = 2 + [models] cache_root = "/var/lib/sparkplane/huggingface" endpoint = "https://huggingface.co" diff --git a/configs/sparkplane/engines/vllm-qwen38-mmap.toml b/configs/sparkplane/engines/vllm-qwen38-mmap.toml index a9ae53a..dc7d7c0 100644 --- a/configs/sparkplane/engines/vllm-qwen38-mmap.toml +++ b/configs/sparkplane/engines/vllm-qwen38-mmap.toml @@ -110,6 +110,7 @@ arguments = [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", ] capabilities = ["text_generation", "tool_calling", "vision"] -vision = { processor_sha256 = "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", media_types = ["image/jpeg", "image/png", "image/webp"], max_bytes = 524288, max_total_bytes = 524288, max_count = 1, max_width = 4096, max_height = 4096, health_media_type = "image/png", health_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAOAAAADgAQAAAAAyVldMAAAAHUlEQVRYw+3BMQEAAADCoPVPbQZ/oAAAAAAAAHgMGWAAAct2tnAAAAAASUVORK5CYII=", health_image_sha256 = "1b8ab2918e9fb346d8ff5b7579372510b0d8b1566d590dac31aea311fff15fc6", health_prompt = "Name the single dominant color using one lowercase word.", health_expected_text = "black", health_max_tokens = 32, health_disable_thinking = true } +# Multi-frame requests retain the same total byte and dimension budgets. +vision = { processor_sha256 = "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", media_types = ["image/jpeg", "image/png", "image/webp"], max_bytes = 524288, max_total_bytes = 524288, max_count = 16, max_width = 4096, max_height = 4096, health_media_type = "image/png", health_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAOAAAADgAQAAAAAyVldMAAAAHUlEQVRYw+3BMQEAAADCoPVPbQZ/oAAAAAAAAHgMGWAAAct2tnAAAAAASUVORK5CYII=", health_image_sha256 = "1b8ab2918e9fb346d8ff5b7579372510b0d8b1566d590dac31aea311fff15fc6", health_prompt = "Name the single dominant color using one lowercase word.", health_expected_text = "black", health_max_tokens = 32, health_disable_thinking = true } startup_protocol_probe = true sampling = { temperature = 1.0, top_p = 0.95, top_k = 20, min_p = 0.0, presence_penalty = 0.0, repetition_penalty = 1.0, thinking_token_budget = 8192, reasoning_effort_map = { low = "low", medium = "medium", high = "xhigh", max = "xhigh" } } diff --git a/crates/sparkplane-rtk/ADAPTATION.patch b/crates/sparkplane-rtk/ADAPTATION.patch new file mode 100644 index 0000000..e7c5b5a --- /dev/null +++ b/crates/sparkplane-rtk/ADAPTATION.patch @@ -0,0 +1,280 @@ +--- upstream/src/core/toml_filter.rs ++++ src/core/toml_filter.rs +@@ -29,7 +29,7 @@ + use std::sync::LazyLock; + + // Built-in filters: concatenated from src/filters/*.toml by build.rs at compile time. +-const BUILTIN_TOML: &str = include_str!(concat!(env!("OUT_DIR"), "/builtin_filters.toml")); ++const BUILTIN_TOML: &str = include_str!("builtin_filters.toml"); + + // --------------------------------------------------------------------------- + // Deserialization types (TOML schema) +@@ -188,38 +188,12 @@ + fn load() -> Self { + let mut filters = Vec::new(); + +- for path in crate::hooks::trust::gated_filter_paths() { +- Self::extend_with_trusted(&mut filters, &path); +- } +- + match Self::parse_and_compile(BUILTIN_TOML, "builtin") { + Ok(f) => filters.extend(f), + Err(e) => eprintln!("[rtk] warning: builtin filters: {}", e), + } + + TomlFilterRegistry { filters } +- } +- +- fn extend_with_trusted(filters: &mut Vec, path: &std::path::Path) { +- if !path.exists() { +- return; +- } +- let label = path.display().to_string(); +- let (status, content) = crate::hooks::trust::check_trust_with_content(path) +- .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); +- match status { +- crate::hooks::trust::TrustStatus::Trusted +- | crate::hooks::trust::TrustStatus::EnvOverride => { +- if let Some(content) = content { +- match Self::parse_and_compile(&content, &label) { +- Ok(f) => filters.extend(f), +- Err(e) => eprintln!("[rtk] warning: {}: {}", label, e), +- } +- } +- } +- crate::hooks::trust::TrustStatus::Untrusted +- | crate::hooks::trust::TrustStatus::ContentChanged { .. } => {} +- } + } + + fn parse_and_compile(content: &str, source: &str) -> Result, String> { +@@ -447,24 +421,7 @@ + } + + fn collect_match_patterns() -> Vec { +- let mut patterns: Vec = Vec::new(); +- for path in crate::hooks::trust::gated_filter_paths() { +- if !path.exists() { +- continue; +- } +- let (status, content) = crate::hooks::trust::check_trust_with_content(&path) +- .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); +- if matches!( +- status, +- crate::hooks::trust::TrustStatus::Trusted +- | crate::hooks::trust::TrustStatus::EnvOverride +- ) && let Some(content) = content +- { +- patterns.extend(match_patterns_in(&content)); +- } +- } +- patterns.extend(match_patterns_in(BUILTIN_TOML)); +- patterns ++ match_patterns_in(BUILTIN_TOML) + } + + /// Strip a leading inline-flag group such as `(?i)` so the anchor check sees the +@@ -722,131 +679,6 @@ + /// + /// - `filter_name_opt`: if `Some`, only run tests for that filter name. + /// - Returns `VerifyResults` with all outcomes and filters that have no tests. +-pub fn run_filter_tests(filter_name_opt: Option<&str>) -> VerifyResults { +- let mut outcomes = Vec::new(); +- let mut all_filter_names: Vec = Vec::new(); +- let mut tested_filter_names: std::collections::HashSet = +- std::collections::HashSet::new(); +- +- let builtin = BUILTIN_TOML; +- collect_test_outcomes( +- builtin, +- filter_name_opt, +- &mut outcomes, +- &mut all_filter_names, +- &mut tested_filter_names, +- ); +- +- // Trust-gated: only verify project-local filters if trusted (SA-2025-RTK-002) +- let project_path = std::path::Path::new(".rtk/filters.toml"); +- if project_path.exists() { +- let (trust_status, verified_content) = +- crate::hooks::trust::check_trust_with_content(project_path) +- .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); +- match trust_status { +- crate::hooks::trust::TrustStatus::Trusted +- | crate::hooks::trust::TrustStatus::EnvOverride => { +- if let Some(content) = verified_content { +- collect_test_outcomes( +- &content, +- filter_name_opt, +- &mut outcomes, +- &mut all_filter_names, +- &mut tested_filter_names, +- ); +- } +- } +- _ => { +- eprintln!("[rtk] WARNING: untrusted project filters skipped in verify"); +- } +- } +- } +- +- let filters_without_tests = all_filter_names +- .into_iter() +- .filter(|name| { +- // When a specific filter is requested, only report that one as missing tests +- filter_name_opt.is_none_or(|f| name == f) +- }) +- .filter(|name| !tested_filter_names.contains(name)) +- .collect(); +- +- VerifyResults { +- outcomes, +- filters_without_tests, +- } +-} +- +-fn collect_test_outcomes( +- content: &str, +- filter_name_opt: Option<&str>, +- outcomes: &mut Vec, +- all_filter_names: &mut Vec, +- tested_filter_names: &mut std::collections::HashSet, +-) { +- let file: TomlFilterFile = match toml::from_str(content) { +- Ok(f) => f, +- Err(e) => { +- eprintln!("[rtk] warning: TOML parse error during verify: {}", e); +- return; +- } +- }; +- +- // Compile all filters and track their names +- let mut compiled_filters: BTreeMap = BTreeMap::new(); +- for (name, def) in file.filters { +- all_filter_names.push(name.clone()); +- match compile_filter(name.clone(), def) { +- Ok(f) => { +- compiled_filters.insert(name, f); +- } +- Err(e) => eprintln!("[rtk] warning: filter '{}' compilation error: {}", name, e), +- } +- } +- +- // Run tests +- for (filter_name, tests) in file.tests { +- if let Some(name) = filter_name_opt +- && filter_name != name +- { +- continue; +- } +- +- tested_filter_names.insert(filter_name.clone()); +- +- let compiled = match compiled_filters.get(&filter_name) { +- Some(f) => f, +- None => { +- eprintln!( +- "[rtk] warning: [[tests.{}]] references unknown filter", +- filter_name +- ); +- continue; +- } +- }; +- +- for test in tests { +- let actual = apply_filter(compiled, &test.input); +- // Trim trailing newlines: TOML multiline strings end with a newline +- let actual_cmp = actual.trim_end_matches('\n').to_string(); +- let expected_cmp = test.expected.trim_end_matches('\n').to_string(); +- outcomes.push(TestOutcome { +- filter_name: filter_name.clone(), +- test_name: test.name, +- passed: actual_cmp == expected_cmp, +- actual: actual_cmp, +- expected: expected_cmp, +- }); +- } +- } +-} +- +-// --------------------------------------------------------------------------- +-// Convenience wrapper (uses singleton — for run_fallback) +-// --------------------------------------------------------------------------- +- +-/// Find a matching filter from the global registry. Initialises the registry +-/// lazily on first call. Returns `None` if no filter matches. + pub fn find_matching_filter(command: &str) -> Option<&'static CompiledFilter> { + if std::env::var("RTK_TOML_DEBUG").is_ok() { + eprintln!( +@@ -2227,3 +2059,71 @@ + assert_eq!(found.unwrap().name, "my-new-tool"); + } + } ++ ++pub fn registry() -> &'static TomlFilterRegistry { ++ ®ISTRY ++} ++ ++fn collect_test_outcomes( ++ content: &str, ++ filter_name_opt: Option<&str>, ++ outcomes: &mut Vec, ++ all_filter_names: &mut Vec, ++ tested_filter_names: &mut std::collections::HashSet, ++) { ++ let file: TomlFilterFile = match toml::from_str(content) { ++ Ok(f) => f, ++ Err(e) => { ++ eprintln!("[rtk] warning: TOML parse error during verify: {}", e); ++ return; ++ } ++ }; ++ ++ // Compile all filters and track their names ++ let mut compiled_filters: BTreeMap = BTreeMap::new(); ++ for (name, def) in file.filters { ++ all_filter_names.push(name.clone()); ++ match compile_filter(name.clone(), def) { ++ Ok(f) => { ++ compiled_filters.insert(name, f); ++ } ++ Err(e) => eprintln!("[rtk] warning: filter '{}' compilation error: {}", name, e), ++ } ++ } ++ ++ // Run tests ++ for (filter_name, tests) in file.tests { ++ if let Some(name) = filter_name_opt ++ && filter_name != name ++ { ++ continue; ++ } ++ ++ tested_filter_names.insert(filter_name.clone()); ++ ++ let compiled = match compiled_filters.get(&filter_name) { ++ Some(f) => f, ++ None => { ++ eprintln!( ++ "[rtk] warning: [[tests.{}]] references unknown filter", ++ filter_name ++ ); ++ continue; ++ } ++ }; ++ ++ for test in tests { ++ let actual = apply_filter(compiled, &test.input); ++ // Trim trailing newlines: TOML multiline strings end with a newline ++ let actual_cmp = actual.trim_end_matches('\n').to_string(); ++ let expected_cmp = test.expected.trim_end_matches('\n').to_string(); ++ outcomes.push(TestOutcome { ++ filter_name: filter_name.clone(), ++ test_name: test.name, ++ passed: actual_cmp == expected_cmp, ++ actual: actual_cmp, ++ expected: expected_cmp, ++ }); ++ } ++ } ++} diff --git a/crates/sparkplane-rtk/Cargo.toml b/crates/sparkplane-rtk/Cargo.toml new file mode 100644 index 0000000..210deed --- /dev/null +++ b/crates/sparkplane-rtk/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "sparkplane-rtk" +version = "0.49.0" +edition = "2024" +license = "Apache-2.0" +publish = false + +[dependencies] +regex = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +toml = "0.8" diff --git a/crates/sparkplane-rtk/LICENSE b/crates/sparkplane-rtk/LICENSE new file mode 100644 index 0000000..0afaf4b --- /dev/null +++ b/crates/sparkplane-rtk/LICENSE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 rtk-ai and rtk-ai Labs + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/sparkplane-rtk/PROVENANCE.json b/crates/sparkplane-rtk/PROVENANCE.json new file mode 100644 index 0000000..d30a693 --- /dev/null +++ b/crates/sparkplane-rtk/PROVENANCE.json @@ -0,0 +1,96 @@ +{ + "repository": "https://github.com/rtk-ai/rtk", + "revision": "6d4b77eadee1c66dc1f68466ad77e96d1b6e4989", + "archive_sha256": "400abfa429aa974604ba9b0223871d82aad59443cb2ec6af7d37fad8f0cb8de0", + "files": { + "Cargo.toml": "681d327949ac960367c13fc14f6df8ce3fcc2500acde370c37b8f1612ecdf967", + "LICENSE": "4044ade9c21d8b084d3d16a03375cf3b7e166b946a327bb37a3fbbdb53287cfd", + "src/cmds/git/git_cmd.rs": "7ae499835723e3715c48d91d8fa5121e3c20d74054719b9b5f535c102984dbaa", + "src/cmds/python/pytest_cmd.rs": "497e9a12174a4fcf28fad2bb62a470c0c2431e7c6be2431cc76087f3db76ec53", + "src/cmds/rust/cargo_cmd.rs": "87aac95895e4418fd6abcee0a4f99a2504e830e0e5da02b9e92a65bb570c559c", + "src/core/constants.rs": "1b65b0fc6ef9e221fe10ca30092fe239e9c1237f8f38c55ff80dc2422f3c37d1", + "src/core/guard.rs": "64cbff90d714e97de74767fc26770779ebeb25705cabb5dc28e98af01b287dce", + "src/core/stream.rs": "3953c190b2fe586374355c622adc57f3c7312836443f56a846e68f855c13fc61", + "src/core/toml_filter.rs": "da4313afd8e33715760b7512c17b5991c81d64ace500cfa33710debed097d17d", + "src/core/truncate.rs": "6c3ea9994eb0dbdd2f5de231f19cb375e23dbcb0695a9c046f4849e70173ff7d", + "src/core/utils.rs": "0ed65369a95bde40d717445bfc64ecbdabbc3755696a9bba110de02e5eefccff", + "src/filters/README.md": "92e3c23bfa488dc03a72632e57a3c868e18845657b152a38c99d3b60624152fc", + "src/filters/ansible-playbook.toml": "5a6c880f1baf60399b4afece9eb2f0761e599bb1cb050dbf8c4cc4a57527c7e2", + "src/filters/basedpyright.toml": "f6df2dfffb311a193194d74f8a536dd006913c038b97af20d1aa301400eeb410", + "src/filters/biome.toml": "b476a13b4cf253c4aa4968cf5192ef3e61b352ee2db0a4f054bdc09ec5d4b04b", + "src/filters/brew-install.toml": "9ce35afac31248db32e2033a88bfb32ae9d64ce4700a4c9faa0effb490bbafdf", + "src/filters/bundle-install.toml": "f9bdea56d32f4cb94388226ec53a49ffec458ac3f887895ca2aebd2da81d37a3", + "src/filters/composer-install.toml": "e3c53ea5d13c8eb06bfcec2952f55346dce4a41732bb85bd5665aa24c2a25d31", + "src/filters/df.toml": "6c496a1559cd22ce8227f6ba5e88c48ba9e06e92fccdbb3ca5c128bd3dd1d532", + "src/filters/dotnet-build.toml": "32941a4c4b21e6f54f4f2fbe012b61ac491f69e5cbba142fe77579851f52a4aa", + "src/filters/du.toml": "cb363ec3e163c6fed016339d3adeb9b3c36bedb3b7d1ae3501a78b049e8ef595", + "src/filters/fail2ban-client.toml": "66a46d62819254bc9e26db95dc51a403e54fc6e089168a6c5ef0bb514e559fd9", + "src/filters/gcc.toml": "1d0d719ee01fa0f5e23ddfcbd3b02b47ece743cf61af335e26f313b3faca617c", + "src/filters/gcloud.toml": "30bb8ccb7b5a642a17c68545612707ddec801e9c0cb980e190c40fcf0bbaff91", + "src/filters/hadolint.toml": "51575715ef665ea55e57219d784e38336d2c6924829a0abb52062f2a78aa8a22", + "src/filters/helm.toml": "a29171e30f1e2a41f9f82616bf00a3430b2c66052dbb5c897342406777bf76c0", + "src/filters/iptables.toml": "0b2802aab90ccd03e9c1d674e8cd49f04861bb5b7bc34aef4a92ea659222563e", + "src/filters/jira.toml": "9557fa46444e0308c36ec1cb5e735ddb043cc841330ba0ac0cb58f6ef2d1e454", + "src/filters/jj.toml": "da5cc860cd564a38169b404a4ab3d55c4e91154c90083da364cacc7549f48a8a", + "src/filters/jq.toml": "fec22286464e3c66229d784febc3ba51d0d6f98f907af18f28930fa997980b98", + "src/filters/just.toml": "3ab46b21148ebf4af210200d408515c12ebb5f6590b14048db8af31ba5b2923e", + "src/filters/liquibase.toml": "5fac9073de0e1e9c897eed522ffe487bba66afa5d37fbfd2b06e35ebed75bf94", + "src/filters/make.toml": "0d2f67f2e2fd4820853e998f7ca2bf9445ffa6632642d10b15a681e9f5c36614", + "src/filters/markdownlint.toml": "827cf11614d4dca4d64563060e6665ce135b01d61ebb86621ba8af12ea04dbd9", + "src/filters/mise.toml": "5a416be11bdb8ea5ea1d06c47e9c96920ecea1e57d8fb17956f6a2cbc1e0d192", + "src/filters/mix-compile.toml": "4167205058fecc76f43d26bf71a7b904b5d553ed81216732fa4dcac51763cd6c", + "src/filters/mix-format.toml": "f5be87d80dd5d8a0276a101bc5c8bad16662d4b58c4c691f6e3561e1db427cc6", + "src/filters/nx.toml": "dde44afaf6026c8b4ce912e7c8f61a0f2890eb52b8245169a2766306910ae562", + "src/filters/ollama.toml": "bf2898bffa2fbbf8a7945c344abf52d3a760dce67f28113bd26bba031836088c", + "src/filters/oxlint.toml": "4cdd71f51128e588f0abb100e4e61c8ff4f2a7906383b90cf3b821d9c0532488", + "src/filters/ping.toml": "a22fd6e62016c3846eed832bfb0213d4a91f4ae976b204ac03817c99acc6c970", + "src/filters/pio-run.toml": "ccc98f26a35f76e7cf10befa13d95179cd96bec7184d10fc2468c255620c1c57", + "src/filters/poetry-install.toml": "ebff2dac219dc7b86b8ddeeb9dd37d12fa91c9b81368e58ba06aa25242019b3b", + "src/filters/pre-commit.toml": "02446f5d004d999f762f791c468dc8b2808e3bdcb5bfdd03dd7e07bc6c695305", + "src/filters/ps.toml": "325d04ccee15075c02feb2ee63ecc08763bc253a2fb55b48bb08432bd527bcb5", + "src/filters/pulumi-destroy.toml": "115a118816320f93fec2f976a277e47f731f618f5faa6a2ba44f92a3040a7b7b", + "src/filters/pulumi-preview.toml": "b72c409df7103acac6a8cb881c96820a1370bc2fc048cfc8b3aefa85b0b025ed", + "src/filters/pulumi-refresh.toml": "47d215de95ea56d14e21028cbbe582da6d65a1a0e7464e69f45d0d705b2b7e13", + "src/filters/pulumi-stack.toml": "4e0034b23a74b9660f11c307fca35de21b306b6a8af498867ca8905eca641993", + "src/filters/pulumi-up.toml": "09afaf8881c6cd44b6d09f36897c9eba90fbff075242db00b0d1824a3cc51a2a", + "src/filters/quarto-render.toml": "1894954417a930b25b7f8463b6301d178680dd17fe6a7c56e933e8961f170dcf", + "src/filters/rsync.toml": "abf8979dcef021d69b6ff2fc49df55127b80232bc93ee1c5bda862f4aa851d39", + "src/filters/shellcheck.toml": "aacca530bb613df62fe73cf01a428b1a0e7828c7098939ee6aec1428bea7f4e4", + "src/filters/shopify-theme.toml": "b2193351d19ccefe433b4dedca792fa3e4e9208a6ffce57790adffaba497ae9f", + "src/filters/skopeo.toml": "a9347ce2a4d0f43aa2b19eaae43f93a3d69a951c7657210e1a0547c14847f032", + "src/filters/sops.toml": "5c400d57fda6db0f7a69d567aa0aaa28be3a680c2266da79b9f7e6f46b60905a", + "src/filters/spring-boot.toml": "41c8ef73ae1336d79e317219f8c6c1d6b271be17f9523e85f28c97271e489eca", + "src/filters/ssh.toml": "f80964f50ff7345e622060c34ee1772dd3f8d468335a0f4a964a02a4067b7e05", + "src/filters/stat.toml": "f3a9814c2bf2eded442bd6e66fedde4ae60dc0d3139bb96dd36ce360d5d1f189", + "src/filters/swift-build.toml": "1c397ddd29292ceef5a4615c0b744c10be77febcbb270767a9a9d9d7dd76eeab", + "src/filters/systemctl-status.toml": "6299897c067c39972712421d30c8faaa965673543360570c52f67bb2a0bbfe7a", + "src/filters/task.toml": "2151fdb559927059edd19c6fbc4311739faf4493f2b27817de9b38150e306e2f", + "src/filters/terraform-plan.toml": "05b7014a38de8781a41778a775ef19ad0474d1f304018483b682b9c0c0dc5a50", + "src/filters/tofu-fmt.toml": "3c8731e0c1f764cc2fed3b4d0a4c2101813ec64d03ff8c68f8728ea5ebb976a1", + "src/filters/tofu-init.toml": "127abb8fbedd19f2b0f860c9733d44788a8ed11f423d0077cda447976f14df8f", + "src/filters/tofu-plan.toml": "e444466da52ce184f3ff351ff9acdf71b4bbd1e230aea94e010c05dfb5c196a7", + "src/filters/tofu-validate.toml": "419c8acf53ee7160f43960ef4e7be10f1ecac72ad3f7490bab7743d923d4f909", + "src/filters/trunk-build.toml": "0d587fc0403985326813b6a3daffc4a4b9889e9488d37b319c599bfc7575d17b", + "src/filters/turbo.toml": "1a3c521e4b9d3ec68c5d5da746453de7d64ab671827101f510f00f7adc2a4353", + "src/filters/ty.toml": "233607e246349ca4ef50b73b081bf2138936381a6394f72eaa495cc9885b00c9", + "src/filters/uv-sync.toml": "1b78722bdca34d821dd7d721ffdeaf67c9d2e407d01dc78f48ab88e09c245d04", + "src/filters/xcodebuild.toml": "0fc3dc2d11ff554cdf2e58cefed56c4d9f2428b271dad328ecdd69d985412eec", + "src/filters/yadm.toml": "1dc264e305d5ddac99aa803d34a5074702c154b02f5cf178ba57f8e3c10919ed", + "src/filters/yamllint.toml": "3230ef732858c5a6d830d4f689501695ba5056f37e238c89096b19e4715705df" + }, + "adaptation": { + "scope": "Pure output reducers and bundled declarative filters; Sparkplane owns execution, hooks and per-session metrics. No RTK installer, binary, SQLite telemetry, project filters, trust override or API-key access.", + "modules": { + "cargo.rs": "src/cmds/rust/cargo_cmd.rs: CargoBuildHandler, CargoTestHandler, diagnostics, filter_cargo_build_labeled, filter_cargo_test", + "pytest.rs": "src/cmds/python/pytest_cmd.rs: filter_pytest_output and ParseState", + "git.rs": "src/cmds/git/git_cmd.rs: filter_status_with_args", + "core/utils.rs": "src/core/utils.rs: truncate, strip_ansi, join_with_overflow", + "core/stream.rs": "src/core/stream.rs: BlockHandler only", + "core/toml_filter.rs": "src/core/toml_filter.rs: bundled registry only; filesystem trust/custom rules and CLI verify removed", + "core/constants.rs": "src/core/constants.rs: unchanged", + "core/guard.rs": "src/core/guard.rs: unchanged", + "core/truncate.rs": "src/core/truncate.rs: unchanged" + }, + "filter_bundle": "schema_version = 1 plus upstream/src/filters/*.toml in lexical filename order, with each schema_version header removed" + } +} diff --git a/crates/sparkplane-rtk/src/cargo.rs b/crates/sparkplane-rtk/src/cargo.rs new file mode 100644 index 0000000..cdf7981 --- /dev/null +++ b/crates/sparkplane-rtk/src/cargo.rs @@ -0,0 +1,666 @@ +use crate::core::stream::BlockHandler; +use crate::core::truncate::{CAP_ERRORS, CAP_WARNINGS}; +use crate::core::utils::{join_with_overflow, truncate}; +use serde::Deserialize; +use std::sync::LazyLock; +struct CargoBuildHandler { + compiled: usize, + warnings: usize, + error_count: usize, + finished_line: Option, + label: &'static str, +} + +impl CargoBuildHandler { + fn with_label(label: &'static str) -> Self { + Self { + compiled: 0, + warnings: 0, + error_count: 0, + finished_line: None, + label, + } + } +} + +impl BlockHandler for CargoBuildHandler { + fn should_skip(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if trimmed.starts_with("Compiling") || trimmed.starts_with("Checking") { + self.compiled += 1; + return true; + } + if trimmed.starts_with("Downloading") || trimmed.starts_with("Downloaded") { + return true; + } + if trimmed.starts_with("Finished") { + self.finished_line = Some(trimmed.to_string()); + return true; + } + if line.starts_with("warning:") && line.contains("generated") && line.contains("warning") { + return true; + } + if (line.starts_with("error:") || line.starts_with("error[")) + && (line.contains("aborting due to") || line.contains("could not compile")) + { + return true; + } + false + } + + fn is_block_start(&mut self, line: &str) -> bool { + if line.starts_with("error[") || line.starts_with("error:") { + self.error_count += 1; + return true; + } + if line.starts_with("warning:") || line.starts_with("warning[") { + self.warnings += 1; + return true; + } + false + } + + fn is_block_continuation(&mut self, line: &str, block: &[String]) -> bool { + !(line.trim().is_empty() && block.len() > 3) + } + + fn format_summary(&self, exit_code: i32, raw: &str) -> Option { + if self.error_count == 0 && self.warnings == 0 && exit_code == 0 { + let summary = + cargo_build_success_line(self.compiled, self.finished_line.as_deref(), self.label); + return Some(crate::core::guard::never_worse(raw, &summary).to_string()); + } + // The streamed path only runs for non-json build/check; error blocks are + // emitted live, so the summary carries no rendered diagnostics. + let empty = JsonDiagnostics { + errors: Vec::new(), + warnings: Vec::new(), + }; + Some(cargo_build_failure_summary( + self.compiled, + self.error_count, + self.warnings, + &empty, + self.label, + exit_code, + )) + } +} + +struct CargoTestHandler { + in_failure_section: bool, + in_failure_names: bool, + summary_lines: Vec, + has_compile_errors: bool, +} + +impl CargoTestHandler { + fn new() -> Self { + Self { + in_failure_section: false, + in_failure_names: false, + summary_lines: Vec::new(), + has_compile_errors: false, + } + } + + /// Compacted `cargo test` summary, before the never-worse guard. + fn compute_test_summary(&self, raw: &str) -> Option { + if self.summary_lines.is_empty() { + let json = extract_json_diagnostics(raw); + if self.has_compile_errors || !json.errors.is_empty() { + // Content-based (exit 0): a real compile error yields "cargo test: N + // errors"; a bare "could not compile" leaves the raw tail fallback. + let build_filtered = filter_cargo_build_labeled(raw, "test", 0); + if build_filtered.contains("cargo test:") { + return Some(format!("{}\n", build_filtered)); + } + // Fallback: last 5 meaningful lines + let meaningful: Vec<&str> = raw + .lines() + .filter(|l| !l.trim().is_empty() && !l.trim_start().starts_with("Compiling")) + .collect(); + let last5: Vec<&str> = meaningful.iter().rev().take(5).rev().copied().collect(); + return Some(format!("{}\n", last5.join("\n"))); + } + } + + // No failures emitted — aggregate pass results + let mut aggregated: Option = None; + let mut all_parsed = true; + + for line in &self.summary_lines { + if let Some(parsed) = AggregatedTestResult::parse_line(line) { + if let Some(ref mut agg) = aggregated { + agg.merge(&parsed); + } else { + aggregated = Some(parsed); + } + } else { + all_parsed = false; + break; + } + } + + if all_parsed + && let Some(agg) = aggregated + && agg.suites > 0 + { + return Some(format!("{}\n", agg.format_compact())); + } + + // Fallback: show raw summary lines + if !self.summary_lines.is_empty() { + let mut s = String::new(); + for line in &self.summary_lines { + s.push_str(line); + s.push('\n'); + } + return Some(s); + } + + None + } +} + +impl BlockHandler for CargoTestHandler { + fn should_skip(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if trimmed.starts_with("Compiling") + || trimmed.starts_with("Downloading") + || trimmed.starts_with("Downloaded") + || trimmed.starts_with("Finished") + { + return true; + } + if line.starts_with("running ") { + return true; + } + if line.starts_with("test ") && line.ends_with("... ok") { + return true; + } + // Track compile errors for fallback + if trimmed.starts_with("error[") || trimmed.starts_with("error:") { + self.has_compile_errors = true; + } + // "failures:" toggles section state + if line == "failures:" { + if self.in_failure_section { + // Second "failures:" = list of failure names — skip them + self.in_failure_names = true; + } + self.in_failure_section = true; + return true; + } + // Skip the failure name listing section + if self.in_failure_names { + if line.starts_with("test result:") { + self.in_failure_names = false; + self.in_failure_section = false; + self.summary_lines.push(line.to_string()); + return true; + } + return true; + } + if line.starts_with("test result:") { + self.summary_lines.push(line.to_string()); + self.in_failure_section = false; + return true; + } + false + } + + fn is_block_start(&mut self, line: &str) -> bool { + self.in_failure_section && line.starts_with("---- ") + } + + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + self.in_failure_section && !line.starts_with("---- ") + } + + fn format_summary(&self, _exit_code: i32, raw: &str) -> Option { + // Same never-worse guard as CargoBuildHandler (#3430 review): if the + // compacted summary ends up larger than the raw output (e.g. a tiny + // `cargo test` run), keep the raw output instead of "compacting" it + // into something bigger. + let summary = self.compute_test_summary(raw)?; + Some(crate::core::guard::never_worse(raw, &summary).to_string()) + } +} + +/// Generic cargo command runner with filtering. +/// Builds the Command with restored `--` separator, then delegates to shared runner. + +struct JsonDiagnostics { + errors: Vec, + warnings: Vec, +} + +#[derive(Deserialize)] +struct CargoJsonLine { + reason: String, + message: Option, +} + +#[derive(Deserialize)] +struct CargoDiagnostic { + level: String, + #[serde(default)] + message: String, + rendered: Option, +} + +fn extract_json_diagnostics(raw: &str) -> JsonDiagnostics { + let mut errors = Vec::new(); + let mut warnings = Vec::new(); + for line in raw.lines() { + let line = line.trim_start(); + if !line.starts_with('{') { + continue; + } + let Ok(entry) = serde_json::from_str::(line) else { + continue; + }; + if entry.reason != "compiler-message" { + continue; + } + let Some(msg) = entry.message else { + continue; + }; + let bucket = match msg.level.as_str() { + "error" | "error: internal compiler error" => &mut errors, + "warning" => &mut warnings, + _ => continue, + }; + if msg.message.starts_with("aborting due to") + || msg.message.starts_with("could not compile") + || (msg.message.contains("warning") && msg.message.contains("generated")) + { + continue; + } + if let Some(rendered) = msg.rendered { + bucket.push( + crate::core::utils::strip_ansi(&rendered) + .trim_end() + .to_string(), + ); + } else if !msg.message.is_empty() { + bucket.push(msg.message); + } + } + JsonDiagnostics { errors, warnings } +} + +fn merge_diag_counts( + error_count: usize, + warnings: usize, + json: &JsonDiagnostics, +) -> (usize, usize) { + ( + error_count.max(json.errors.len()), + warnings.max(json.warnings.len()), + ) +} + +fn cargo_build_success_line(compiled: usize, finished: Option<&str>, label: &str) -> String { + match finished { + Some(f) => format!("cargo {} ({} crates compiled)\n{}\n", label, compiled, f), + None => format!("cargo {} ({} crates compiled)\n", label, compiled), + } +} + +fn cargo_build_failure_summary( + compiled: usize, + errors: usize, + warnings: usize, + json: &JsonDiagnostics, + label: &str, + exit_code: i32, +) -> String { + let mut out = if errors == 0 && warnings == 0 { + format!("cargo {}: failed (exit {})\n", label, exit_code) + } else { + format!( + "cargo {}: {} errors, {} warnings ({} crates)\n", + label, errors, warnings, compiled + ) + }; + if !json.errors.is_empty() { + let shown: Vec = json.errors.iter().take(CAP_ERRORS).cloned().collect(); + out.push_str(&join_with_overflow( + &shown, + json.errors.len(), + CAP_ERRORS, + "errors", + )); + out.push('\n'); + } + if !json.warnings.is_empty() { + let shown: Vec = json.warnings.iter().take(CAP_WARNINGS).cloned().collect(); + out.push_str(&join_with_overflow( + &shown, + json.warnings.len(), + CAP_WARNINGS, + "warnings", + )); + out.push('\n'); + } + if json.errors.len() > CAP_ERRORS || json.warnings.len() > CAP_WARNINGS { + let full = json + .errors + .iter() + .chain(json.warnings.iter()) + .map(String::as_str) + .collect::>() + .join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&full, "cargo-json-issues") { + out.push_str(&format!(" {}\n", hint)); + } + } + out +} + +fn filter_cargo_build(output: &str) -> String { + filter_cargo_build_labeled(output, "build", 0) +} + +pub fn filter_cargo_build_labeled(output: &str, label: &'static str, exit_code: i32) -> String { + let mut handler = CargoBuildHandler::with_label(label); + let mut blocks: Vec> = Vec::new(); + let mut current_block: Vec = Vec::new(); + let mut in_block = false; + + for line in output.lines() { + if handler.should_skip(line) { + continue; + } + if handler.is_block_start(line) { + if in_block && !current_block.is_empty() { + blocks.push(std::mem::take(&mut current_block)); + } + in_block = true; + current_block.push(line.to_string()); + } else if in_block { + if handler.is_block_continuation(line, ¤t_block) { + current_block.push(line.to_string()); + } else { + blocks.push(std::mem::take(&mut current_block)); + in_block = false; + } + } + } + if !current_block.is_empty() { + blocks.push(current_block); + } + + let json = extract_json_diagnostics(output); + let (errors, warnings) = merge_diag_counts(handler.error_count, handler.warnings, &json); + + if errors == 0 && warnings == 0 && exit_code == 0 { + let summary = + cargo_build_success_line(handler.compiled, handler.finished_line.as_deref(), label); + return crate::core::guard::never_worse(output, &summary).to_string(); + } + + let mut result = + cargo_build_failure_summary(handler.compiled, errors, warnings, &json, label, exit_code); + const MAX_CHECK_BLOCKS: usize = CAP_ERRORS; + for (i, blk) in blocks.iter().enumerate().take(MAX_CHECK_BLOCKS) { + result.push_str(&blk.join("\n")); + result.push('\n'); + if i < blocks.len() - 1 { + result.push('\n'); + } + } + if blocks.len() > MAX_CHECK_BLOCKS { + result.push_str(&format!( + "\n… +{} more issues\n", + blocks.len() - MAX_CHECK_BLOCKS + )); + let all_blocks: String = blocks + .iter() + .map(|b| b.join("\n")) + .collect::>() + .join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_blocks, "cargo-check-issues") { + result.push_str(&format!(" {}\n", hint)); + } + } + result.trim().to_string() +} + +/// Aggregated test results for compact display +#[derive(Debug, Default, Clone)] +struct AggregatedTestResult { + passed: usize, + failed: usize, + ignored: usize, + measured: usize, + filtered_out: usize, + suites: usize, + duration_secs: f64, + has_duration: bool, +} + +impl AggregatedTestResult { + /// Parse a test result summary line + /// Format: "test result: ok. 15 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s" + fn parse_line(line: &str) -> Option { + static RE: LazyLock = LazyLock::new(|| { + regex::Regex::new( + r"test result: (\w+)\.\s+(\d+) passed;\s+(\d+) failed;\s+(\d+) ignored;\s+(\d+) measured;\s+(\d+) filtered out(?:;\s+finished in ([\d.]+)s)?" + ).unwrap() + }); + + let caps = RE.captures(line)?; + let status = caps.get(1)?.as_str(); + + // Only aggregate if status is "ok" (all tests passed) + if status != "ok" { + return None; + } + + let passed = caps.get(2)?.as_str().parse().ok()?; + let failed = caps.get(3)?.as_str().parse().ok()?; + let ignored = caps.get(4)?.as_str().parse().ok()?; + let measured = caps.get(5)?.as_str().parse().ok()?; + let filtered_out = caps.get(6)?.as_str().parse().ok()?; + + let (duration_secs, has_duration) = if let Some(duration_match) = caps.get(7) { + (duration_match.as_str().parse().unwrap_or(0.0), true) + } else { + (0.0, false) + }; + + Some(Self { + passed, + failed, + ignored, + measured, + filtered_out, + suites: 1, + duration_secs, + has_duration, + }) + } + + /// Merge another test result into this one + fn merge(&mut self, other: &Self) { + self.passed += other.passed; + self.failed += other.failed; + self.ignored += other.ignored; + self.measured += other.measured; + self.filtered_out += other.filtered_out; + self.suites += other.suites; + self.duration_secs += other.duration_secs; + self.has_duration = self.has_duration && other.has_duration; + } + + /// Format as compact single line + fn format_compact(&self) -> String { + let mut parts = vec![format!("{} passed", self.passed)]; + + if self.ignored > 0 { + parts.push(format!("{} ignored", self.ignored)); + } + if self.filtered_out > 0 { + parts.push(format!("{} filtered out", self.filtered_out)); + } + + let counts = parts.join(", "); + + let suite_text = if self.suites == 1 { + "1 suite".to_string() + } else { + format!("{} suites", self.suites) + }; + + if self.has_duration { + format!( + "cargo test: {} ({}, {:.2}s)", + counts, suite_text, self.duration_secs + ) + } else { + format!("cargo test: {} ({})", counts, suite_text) + } + } +} + +pub(crate) fn filter_cargo_test(output: &str) -> String { + let mut failures: Vec = Vec::new(); + let mut summary_lines: Vec = Vec::new(); + let mut in_failure_section = false; + let mut current_failure = Vec::new(); + + for line in output.lines() { + // Skip compilation lines + if line.trim_start().starts_with("Compiling") + || line.trim_start().starts_with("Downloading") + || line.trim_start().starts_with("Downloaded") + || line.trim_start().starts_with("Finished") + { + continue; + } + + // Skip "running N tests" and individual "test ... ok" lines + if line.starts_with("running ") || (line.starts_with("test ") && line.ends_with("... ok")) { + continue; + } + + // Detect failures section + if line == "failures:" { + in_failure_section = true; + continue; + } + + if in_failure_section { + if line.starts_with("test result:") { + in_failure_section = false; + summary_lines.push(line.to_string()); + } else if line.starts_with(" ") || line.starts_with("---- ") { + current_failure.push(line.to_string()); + } else if line.trim().is_empty() && !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } else if !line.trim().is_empty() { + current_failure.push(line.to_string()); + } + } + + // Capture test result summary + if !in_failure_section && line.starts_with("test result:") { + summary_lines.push(line.to_string()); + } + } + + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + } + + let mut result = String::new(); + + if failures.is_empty() && !summary_lines.is_empty() { + // All passed - try to aggregate + let mut aggregated: Option = None; + let mut all_parsed = true; + + for line in &summary_lines { + if let Some(parsed) = AggregatedTestResult::parse_line(line) { + if let Some(ref mut agg) = aggregated { + agg.merge(&parsed); + } else { + aggregated = Some(parsed); + } + } else { + all_parsed = false; + break; + } + } + + // If all lines parsed successfully and we have at least one suite, return compact format + if all_parsed + && let Some(agg) = aggregated + && agg.suites > 0 + { + return agg.format_compact(); + } + + // Fallback: use original behavior if regex failed + for line in &summary_lines { + result.push_str(&format!("{}\n", line)); + } + return result.trim().to_string(); + } + + if !failures.is_empty() { + result.push_str(&format!("FAILURES ({}):\n", failures.len())); + const MAX_FAILURES: usize = CAP_WARNINGS; + for (i, failure) in failures.iter().enumerate().take(MAX_FAILURES) { + result.push_str(&format!("{}. {}\n", i + 1, truncate(failure, 200))); + } + if failures.len() > MAX_FAILURES { + result.push_str(&format!( + "\n… +{} more failures\n", + failures.len() - MAX_FAILURES + )); + let all_failures = failures.join("\n\n"); + if let Some(hint) = + crate::core::tee::force_tee_hint(&all_failures, "cargo-test-failures") + { + result.push_str(&format!(" {}\n", hint)); + } + } + result.push('\n'); + } + + for line in &summary_lines { + result.push_str(&format!("{}\n", line)); + } + + if result.trim().is_empty() { + let json = extract_json_diagnostics(output); + let has_compile_errors = !json.errors.is_empty() + || output.lines().any(|line| { + let trimmed = line.trim_start(); + trimmed.starts_with("error[") || trimmed.starts_with("error:") + }); + + if has_compile_errors { + let build_filtered = filter_cargo_build_labeled(output, "test", 0); + if build_filtered.contains("cargo test:") { + return build_filtered; + } + } + + // Fallback: show last meaningful lines + let meaningful: Vec<&str> = output + .lines() + .filter(|l| !l.trim().is_empty() && !l.trim_start().starts_with("Compiling")) + .collect(); + for line in meaningful.iter().rev().take(5).rev() { + result.push_str(&format!("{}\n", line)); + } + } + + result.trim().to_string() +} diff --git a/crates/sparkplane-rtk/src/core/builtin_filters.toml b/crates/sparkplane-rtk/src/core/builtin_filters.toml new file mode 100644 index 0000000..b381b4e --- /dev/null +++ b/crates/sparkplane-rtk/src/core/builtin_filters.toml @@ -0,0 +1,2493 @@ +schema_version = 1 + +[filters.ansible-playbook] +description = "Compact ansible-playbook output" +match_command = "^ansible-playbook\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ok: \\[", + "^skipping: \\[", +] +max_lines = 60 + +[[tests.ansible-playbook]] +name = "strips ok and skipping lines, keeps changed and failures" +input = """ +PLAY [all] ********************************************************************* + +TASK [Gathering Facts] ********************************************************* +ok: [web01] +ok: [web02] + +TASK [Install nginx] *********************************************************** +changed: [web01] +skipping: [web02] + +PLAY RECAP ********************************************************************* +web01 : ok=2 changed=1 unreachable=0 failed=0 +web02 : ok=1 changed=0 unreachable=0 failed=0 +""" +expected = "PLAY [all] *********************************************************************\nTASK [Gathering Facts] *********************************************************\nTASK [Install nginx] ***********************************************************\nchanged: [web01]\nPLAY RECAP *********************************************************************\nweb01 : ok=2 changed=1 unreachable=0 failed=0\nweb02 : ok=1 changed=0 unreachable=0 failed=0" + +[[tests.ansible-playbook]] +name = "failed task preserved" +input = "TASK [Start service] ***\nfailed: [web01] => {\"msg\": \"Service not found\"}\nPLAY RECAP ***\nweb01 : ok=1 failed=1" +expected = "TASK [Start service] ***\nfailed: [web01] => {\"msg\": \"Service not found\"}\nPLAY RECAP ***\nweb01 : ok=1 failed=1" + + +[filters.basedpyright] +description = "Compact basedpyright type checker output — strip blank lines, keep errors" +match_command = "^basedpyright\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Searching for source files", + "^Found \\d+ source file", + "^Pyright \\d+\\.\\d+", + "^basedpyright \\d+\\.\\d+", +] +max_lines = 50 +on_empty = "basedpyright: ok" + +[[tests.basedpyright]] +name = "strips noise, keeps errors and summary" +input = """ +basedpyright 1.22.0 +Searching for source files +Found 42 source files + +/home/user/app/main.py + /home/user/app/main.py:10:5 - error: "foo" is not defined (reportUndefinedVariable) + /home/user/app/main.py:25:1 - error: Type "str" is not assignable to type "int" (reportAssignmentType) + +/home/user/app/utils.py + /home/user/app/utils.py:8:9 - warning: Variable "x" is not accessed (reportUnusedVariable) + +3 errors, 1 warning, 0 informations +""" +expected = "/home/user/app/main.py\n /home/user/app/main.py:10:5 - error: \"foo\" is not defined (reportUndefinedVariable)\n /home/user/app/main.py:25:1 - error: Type \"str\" is not assignable to type \"int\" (reportAssignmentType)\n/home/user/app/utils.py\n /home/user/app/utils.py:8:9 - warning: Variable \"x\" is not accessed (reportUnusedVariable)\n3 errors, 1 warning, 0 informations" + +[[tests.basedpyright]] +name = "clean output" +input = """ +basedpyright 1.22.0 +Searching for source files +Found 10 source files + +0 errors, 0 warnings, 0 informations +""" +expected = "0 errors, 0 warnings, 0 informations" + +[[tests.basedpyright]] +name = "empty input returns on_empty message" +input = "" +expected = "basedpyright: ok" + + +[filters.biome] +description = "Compact Biome lint/format output — strip blank lines, keep diagnostics" +match_command = "^biome\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Checked \\d+ file", + "^Fixed \\d+ file", + "^The following command", + "^Run it with", +] +max_lines = 50 +on_empty = "biome: ok" + +[[tests.biome]] +name = "lint strips noise, keeps diagnostics" +input = """ +Checked 42 files in 0.5s + +src/app.tsx:5:3 lint/suspicious/noExplicitAny ━━━━━━━━━━━━━━━━━━━━ + × Unexpected any. Specify a different type. + 3 │ interface Props { + 4 │ data: any; + 5 │ ^^^ + +src/utils.ts:12:1 lint/complexity/noForEach ━━━━━━━━━━━━━━━━━━━━ + × Prefer for...of instead of forEach. + 12 │ items.forEach(item => process(item)); + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Found 2 errors. +""" +expected = "src/app.tsx:5:3 lint/suspicious/noExplicitAny ━━━━━━━━━━━━━━━━━━━━\n × Unexpected any. Specify a different type.\n 3 │ interface Props {\n 4 │ data: any;\n 5 │ ^^^\nsrc/utils.ts:12:1 lint/complexity/noForEach ━━━━━━━━━━━━━━━━━━━━\n × Prefer for...of instead of forEach.\n 12 │ items.forEach(item => process(item));\n │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nFound 2 errors." + +[[tests.biome]] +name = "clean check" +input = """ +Checked 42 files in 0.3s +""" +expected = "biome: ok" + +[[tests.biome]] +name = "empty input returns on_empty message" +input = "" +expected = "biome: ok" + + +[filters.brew-install] +description = "Compact brew install/upgrade output — strip downloads, short-circuit when already installed" +match_command = "^brew\\s+(install|upgrade)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^==> Downloading", + "^==> Pouring", + "^Already downloaded:", + "^###", + "^==> Fetching", +] +match_output = [ + { pattern = "already installed", message = "ok (already installed)" }, +] +max_lines = 20 + +[[tests.brew-install]] +name = "already installed short-circuits" +input = """ +Warning: rtk 0.27.1 is already installed and up-to-date. +To reinstall 0.27.1, run: + brew reinstall rtk +""" +expected = "ok (already installed)" + +[[tests.brew-install]] +name = "install strips download lines" +input = """ +==> Fetching jq +==> Downloading https://homebrew.bintray.com/bottles/jq-1.7.1.arm64_sonoma.bottle.tar.gz +######################################################################## 100.0% +==> Pouring jq-1.7.1.arm64_sonoma.bottle.tar.gz +==> Summary +/opt/homebrew/Cellar/jq/1.7.1: 18 files, 1.2MB +""" +expected = "==> Summary\n/opt/homebrew/Cellar/jq/1.7.1: 18 files, 1.2MB" + + +[filters.bundle-install] +description = "Compact bundle install/update — strip 'Using' lines, keep installs and errors" +match_command = "^bundle\\s+(install|update)\\b" +strip_ansi = true +strip_lines_matching = [ + "^Using ", + "^\\s*$", + "^Fetching gem metadata", + "^Resolving dependencies", +] +match_output = [ + { pattern = "Bundle complete!", message = "ok bundle: complete" }, + { pattern = "Bundle updated!", message = "ok bundle: updated" }, +] +max_lines = 30 + +[[tests.bundle-install]] +name = "all cached short-circuits" +input = """ +Using bundler 2.5.6 +Using rake 13.1.0 +Using ast 2.4.2 +Using base64 0.2.0 +Using minitest 5.22.2 +Bundle complete! 85 Gemfile dependencies, 200 gems now installed. +Use `bundle info [gemname]` to see where a bundled gem is installed. +""" +expected = "ok bundle: complete" + +[[tests.bundle-install]] +name = "mixed install keeps Fetching and Installing lines" +input = """ +Fetching gem metadata from https://rubygems.org/......... +Resolving dependencies... +Using rake 13.1.0 +Using ast 2.4.2 +Fetching rspec 3.13.0 +Installing rspec 3.13.0 +Using rubocop 1.62.0 +Fetching simplecov 0.22.0 +Installing simplecov 0.22.0 +Bundle complete! 85 Gemfile dependencies, 202 gems now installed. +""" +expected = "ok bundle: complete" + +[[tests.bundle-install]] +name = "update output" +input = """ +Fetching gem metadata from https://rubygems.org/......... +Resolving dependencies... +Using rake 13.1.0 +Fetching rspec 3.14.0 (was 3.13.0) +Installing rspec 3.14.0 (was 3.13.0) +Bundle updated! +""" +expected = "ok bundle: updated" + +[[tests.bundle-install]] +name = "empty output" +input = "" +expected = "" + + +[filters.composer-install] +description = "Compact composer install/update/require output — strip downloads, short-circuit when up-to-date" +match_command = "^composer\\s+(install|update|require)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ - Downloading ", + "^ - Installing ", + "^Loading composer", + "^Updating dependencies", +] +match_output = [ + { pattern = "Nothing to install, update or remove", message = "ok (up to date)" }, +] +max_lines = 30 + +[[tests.composer-install]] +name = "nothing to do short-circuits" +input = """ +Loading composer repositories with package information +Updating dependencies +Lock file operations: 0 installs, 0 updates, 0 removals +Nothing to install, update or remove +Generating autoload files +""" +expected = "ok (up to date)" + +[[tests.composer-install]] +name = "install strips download lines" +input = """ +Loading composer repositories with package information +Updating dependencies + - Downloading symfony/console (v6.4.0) + - Installing symfony/console (v6.4.0): Extracting archive + - Downloading psr/log (3.0.0) + - Installing psr/log (3.0.0): Extracting archive +Writing lock file +Generating autoload files +""" +expected = "Writing lock file\nGenerating autoload files" + + +[filters.df] +description = "Compact df output — truncate wide columns, limit rows" +match_command = "^df(\\s|$)" +strip_ansi = true +truncate_lines_at = 80 +max_lines = 20 + +[[tests.df]] +name = "short output passes through unchanged" +input = "Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/sda1 4096000 123456 3972544 4% /" +expected = "Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/sda1 4096000 123456 3972544 4% /" + +[[tests.df]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.dotnet-build] +description = "Compact dotnet build output — short-circuit on success, strip banners" +match_command = "^dotnet\\s+build\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Microsoft \\(R\\)", + "^Copyright \\(C\\)", + "^ Determining projects", +] +match_output = [ + { pattern = "0 Warning\\(s\\)\\n\\s+0 Error\\(s\\)", message = "ok (build succeeded)" }, +] +max_lines = 40 + +[[tests.dotnet-build]] +name = "successful build short-circuits to ok" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... + All projects are up-to-date for restore. + MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll + +Build succeeded. + 0 Warning(s) + 0 Error(s) + +Time Elapsed 00:00:02.34 +""" +expected = "ok (build succeeded)" + +[[tests.dotnet-build]] +name = "build with warnings not short-circuited" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... + MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll + +Build succeeded. + 3 Warning(s) + 0 Error(s) + +Time Elapsed 00:00:01.87 +""" +expected = " MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll\nBuild succeeded.\n 3 Warning(s)\n 0 Error(s)\nTime Elapsed 00:00:01.87" + +[[tests.dotnet-build]] +name = "build errors pass through" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... +src/Program.cs(10,5): error CS1002: ; expected [/home/user/MyApp/MyApp.csproj] + +Build FAILED. + 0 Warning(s) + 1 Error(s) +""" +expected = "src/Program.cs(10,5): error CS1002: ; expected [/home/user/MyApp/MyApp.csproj]\nBuild FAILED.\n 0 Warning(s)\n 1 Error(s)" + + +[filters.du] +description = "Compact du output" +match_command = "^du\\b" +strip_lines_matching = ["^\\s*$"] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.du]] +name = "preserves sizes, strips blank lines" +input = "4.0K\t./src\n\n8.0K\t./tests\n16K\t." +expected = "4.0K\t./src\n8.0K\t./tests\n16K\t." + +[[tests.du]] +name = "single line passthrough" +input = "128K\t." +expected = "128K\t." + + +[filters.fail2ban-client] +description = "Compact fail2ban-client output" +match_command = "^fail2ban-client\\b" +strip_lines_matching = ["^\\s*$"] +max_lines = 30 + +[[tests.fail2ban-client]] +name = "strips blank lines" +input = "Status for the jail: sshd\n|- Filter\n| |- Currently failed: 3\n\n|- Actions\n `- Total banned: 42" +expected = "Status for the jail: sshd\n|- Filter\n| |- Currently failed: 3\n|- Actions\n `- Total banned: 42" + +[[tests.fail2ban-client]] +name = "single line passthrough" +input = "Shutdown successful" +expected = "Shutdown successful" + + +[filters.gcc] +description = "Compact gcc/g++ compiler output — strip notes, keep errors and warnings" +match_command = "^g(cc|\\+\\+)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s+\\|\\s*$", + "^In file included from", + "^\\s+from\\s", + "^\\d+ warnings? generated", + "^\\d+ errors? generated", +] +max_lines = 50 +on_empty = "gcc: ok" + +[[tests.gcc]] +name = "strips include chain, keeps errors and warnings" +input = """ +In file included from /usr/include/stdio.h:42: + from main.c:1: +main.c:10:5: error: use of undeclared identifier 'foo' + foo(); + ^ +main.c:15:12: warning: unused variable 'x' [-Wunused-variable] + int x = 42; + ^ +2 warnings generated. +1 error generated. +""" +expected = "main.c:10:5: error: use of undeclared identifier 'foo'\n foo();\n ^\nmain.c:15:12: warning: unused variable 'x' [-Wunused-variable]\n int x = 42;\n ^" + +[[tests.gcc]] +name = "clean compilation" +input = """ +""" +expected = "gcc: ok" + +[[tests.gcc]] +name = "linker error kept" +input = """ +/usr/bin/ld: /tmp/main.o: undefined reference to 'missing_func' +collect2: error: ld returned 1 exit status +""" +expected = "/usr/bin/ld: /tmp/main.o: undefined reference to 'missing_func'\ncollect2: error: ld returned 1 exit status" + +[[tests.gcc]] +name = "empty input returns on_empty message" +input = "" +expected = "gcc: ok" + + +[filters.gcloud] +description = "Compact gcloud output" +match_command = "^gcloud\\b" +strip_ansi = true +strip_lines_matching = ["^\\s*$"] +truncate_lines_at = 120 +max_lines = 30 + +[[tests.gcloud]] +name = "strips blank lines, preserves output" +input = """ +Updated property [core/project]. + +NAME REGION STATUS +my-cluster us-central1 RUNNING +""" +expected = "Updated property [core/project].\nNAME REGION STATUS\nmy-cluster us-central1 RUNNING" + +[[tests.gcloud]] +name = "single line passthrough" +input = "Listed 0 items." +expected = "Listed 0 items." + + +[filters.hadolint] +description = "Compact hadolint Dockerfile linting output" +match_command = "^hadolint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.hadolint]] +name = "Dockerfile warnings kept, blank lines stripped" +input = """ +Dockerfile:3 DL3008 warning: Pin versions in apt-get install +Dockerfile:5 DL3009 info: Delete apt-get lists after installing + +Dockerfile:8 DL4006 warning: Set SHELL option -o pipefail before RUN with pipe +""" +expected = "Dockerfile:3 DL3008 warning: Pin versions in apt-get install\nDockerfile:5 DL3009 info: Delete apt-get lists after installing\nDockerfile:8 DL4006 warning: Set SHELL option -o pipefail before RUN with pipe" + +[[tests.hadolint]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.helm] +description = "Compact helm output" +match_command = "^helm\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^W\\d{4}", +] +truncate_lines_at = 120 + +[[tests.helm]] +name = "strips blank lines, preserves release info" +input = """ +NAME: my-release +LAST DEPLOYED: Mon Jan 15 10:30:00 2024 +NAMESPACE: default +STATUS: deployed +REVISION: 3 + +NOTES: +Application is running. +""" +expected = "NAME: my-release\nLAST DEPLOYED: Mon Jan 15 10:30:00 2024\nNAMESPACE: default\nSTATUS: deployed\nREVISION: 3\nNOTES:\nApplication is running." + +[[tests.helm]] +name = "strips glog W-prefix warnings" +input = "W0115 10:30:00 warning message from internal\nNAME: my-chart\nSTATUS: deployed" +expected = "NAME: my-chart\nSTATUS: deployed" + +[[tests.helm]] +name = "helm template output is not truncated" +input = """ +# Source: my-chart/templates/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: my-ns +--- +# Source: my-chart/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-secret +--- +# Source: my-chart/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-cm +--- +# Source: my-chart/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc +--- +# Source: my-chart/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: my-svc +--- +# Source: my-chart/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-deploy +""" +expected = """# Source: my-chart/templates/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: my-ns +--- +# Source: my-chart/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-secret +--- +# Source: my-chart/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-cm +--- +# Source: my-chart/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc +--- +# Source: my-chart/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: my-svc +--- +# Source: my-chart/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-deploy""" + + +[filters.iptables] +description = "Compact iptables output" +match_command = "^iptables\\b" +strip_lines_matching = [ + "^\\s*$", + "^Chain DOCKER", + "^Chain BR-", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.iptables]] +name = "strips Docker chains, preserves real rules" +input = """ +Chain INPUT (policy ACCEPT) +num target prot opt source destination +1 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 +Chain DOCKER (1 references) + DOCKER all -- 0.0.0.0/0 0.0.0.0/0 +Chain BR-abcdef (0 references) +""" +expected = "Chain INPUT (policy ACCEPT)\nnum target prot opt source destination\n1 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0\n DOCKER all -- 0.0.0.0/0 0.0.0.0/0" + +[[tests.iptables]] +name = "preserves FORWARD and OUTPUT chains" +input = "Chain FORWARD (policy DROP)\n1 ACCEPT tcp\nChain OUTPUT (policy ACCEPT)\n1 ACCEPT all" +expected = "Chain FORWARD (policy DROP)\n1 ACCEPT tcp\nChain OUTPUT (policy ACCEPT)\n1 ACCEPT all" + + +[filters.jira] +description = "Compact Jira CLI output — strip verbose metadata, keep essentials" +match_command = "^jira\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*--", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.jira]] +name = "strips blank lines from issue list" +input = "TYPE\tKEY\tSUMMARY\tSTATUS\n\nStory\tPROJ-123\tAdd login feature\tIn Progress\n\nBug\tPROJ-456\tFix crash on startup\tOpen" +expected = "TYPE\tKEY\tSUMMARY\tSTATUS\nStory\tPROJ-123\tAdd login feature\tIn Progress\nBug\tPROJ-456\tFix crash on startup\tOpen" + +[[tests.jira]] +name = "single issue view" +input = "KEY: PROJ-123\nSummary: Add login feature\nStatus: In Progress\nAssignee: john@example.com" +expected = "KEY: PROJ-123\nSummary: Add login feature\nStatus: In Progress\nAssignee: john@example.com" + + +[filters.jj] +description = "Compact Jujutsu (jj) output — strip blank lines, truncate" +match_command = "^jj\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Hint:", + "^Working copy now at:", +] +max_lines = 30 +truncate_lines_at = 120 + +[[tests.jj]] +name = "log output stripped of hints" +input = """ +@ qpvuntsm patrick@example.com 2026-03-10 12:00 abc123 +│ feat: add new feature +◉ zzzzzzzz root() + +Working copy now at: qpvuntsm abc123 feat: add new feature +Hint: use `jj log` to see the full history +""" +expected = "@ qpvuntsm patrick@example.com 2026-03-10 12:00 abc123\n│ feat: add new feature\n◉ zzzzzzzz root()" + +[[tests.jj]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.jq] +description = "Compact jq output — truncate large JSON results" +match_command = "^jq\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 40 +truncate_lines_at = 120 + +[[tests.jq]] +name = "short output passes through" +input = """ +{ + "name": "test", + "version": "1.0" +} +""" +expected = "{\n \"name\": \"test\",\n \"version\": \"1.0\"\n}" + +[[tests.jq]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.just] +description = "Compact just task runner output — strip recipe headers, keep command output" +match_command = "^just\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Available recipes:", + "^\\s*just --list", +] +truncate_lines_at = 150 +max_lines = 50 + +[[tests.just]] +name = "preserves command output" +input = "cargo test\n\ntest result: ok. 42 passed; 0 failed\n" +expected = "cargo test\ntest result: ok. 42 passed; 0 failed" + +[[tests.just]] +name = "preserves error output" +input = "error: Compilation failed\nsrc/main.rs:10: expected `;`" +expected = "error: Compilation failed\nsrc/main.rs:10: expected `;`" + +[[tests.just]] +name = "empty input" +input = "" +expected = "" + + +[filters.liquibase] +description = "Compact liquibase output — strip headers and generic info" +match_command = "^liquibase(?:\\s|$)" +strip_ansi = true +filter_stderr = true +strip_lines_matching = [ + "^\\s*$", + "^Starting Liquibase at", + "^Liquibase (?:Community|Open Source)", + "^Liquibase Home:", + "^Java Home", + "^Libraries:", + "^\\s*-\\s+\\S+\\.jar", + "^INFO \\[liquibase\\.integration\\]", + "^INFO \\[liquibase\\.core\\] Reading resource", + "^INFO \\[liquibase\\.core\\] Parsing", + "^(?:\\[?INFO\\]?\\s*)?#+$", + "^\\s*##" +] +on_empty = "liquibase: ok" +max_lines = 200 + +[[tests.liquibase]] +name = "strip ascii banner and info logs from subcommand" +input = ''' +#################################################### +## _ _ _ _ ## +## | | (_) (_) | ## +#################################################### +Starting Liquibase at 10:12:11 (version 4.29.1) +Liquibase Version: 4.29.1 +Liquibase Open Source 4.29.1 by Liquibase +INFO [liquibase.integration] Starting command +INFO [liquibase.core] Reading resource db/changelog.xml +INFO [liquibase.core] Parsing db/changelog.xml +Running Changeset: filepath::id::author +Changeset filepath::id::author ran successfully +''' +expected = ''' +Liquibase Version: 4.29.1 +Running Changeset: filepath::id::author +Changeset filepath::id::author ran successfully''' + +[[tests.liquibase]] +name = "strip --version noise, keep only version line" +input = ''' +#################################################### +## _ _ _ _ ## +#################################################### +Starting Liquibase at 13:45:24 using Java 17.0.15 (version 4.30.0 #4943 built at 2024-10-31 17:00+0000) +Liquibase Home: D:\mcp\bash\lbr\third-party +Java Home C:\Program Files\Java\jdk-17.0.15 (Version 17.0.15) +Libraries: + - internal\lib\commons-io.jar: Apache Commons IO 2.17.0 By The Apache Software Foundation + - internal\lib\picocli.jar: picocli 4.7.6 By Remko Popma + - lib\ojdbc10-19.30.0.0.jar: JDBC 19.30.0.0.0 By Oracle Corporation + +Liquibase Version: 4.30.0 +Liquibase Open Source 4.30.0 by Liquibase +''' +expected = ''' +Liquibase Version: 4.30.0''' + +[[tests.liquibase]] +name = "keep status and error lines" +input = ''' +#################################################### +## _ _ _ _ ## +#################################################### +Starting Liquibase at 10:00:00 (version 4.30.0) +Liquibase Version: 4.30.0 +Liquibase Open Source 4.30.0 by Liquibase +HR@jdbc:oracle:thin:@localhost:1523:XE is up to date +Liquibase command 'status' was executed successfully. +''' +expected = ''' +Liquibase Version: 4.30.0 +HR@jdbc:oracle:thin:@localhost:1523:XE is up to date +Liquibase command 'status' was executed successfully.''' + +[[tests.liquibase]] +name = "empty input" +input = "" +expected = "liquibase: ok" + + +[filters.make] +description = "Compact make output" +match_command = "^make\\b" +strip_lines_matching = [ + "^make\\[\\d+\\]:", + "^\\s*$", + "^Nothing to be done", +] +max_lines = 50 +on_empty = "make: ok" + +[[tests.make]] +name = "strips entering/leaving lines" +input = """ +make[1]: Entering directory '/home/user' +gcc -O2 foo.c +make[1]: Leaving directory '/home/user' +""" +expected = """ +gcc -O2 foo.c +""" + +[[tests.make]] +name = "strips blank lines" +input = """ +gcc -O2 foo.c + +gcc -O2 bar.c +""" +expected = """ +gcc -O2 foo.c +gcc -O2 bar.c +""" + +[[tests.make]] +name = "on_empty when all stripped" +input = """ +make[1]: Entering directory '/home/user' +make[1]: Leaving directory '/home/user' +""" +expected = "make: ok" + + +[filters.markdownlint] +description = "Compact markdownlint output — strip blank lines, limit rows" +match_command = "^markdownlint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.markdownlint]] +name = "linting errors stripped of blank lines" +input = """ +README.md:1:1 MD041/first-line-heading/first-line-h1 First line in file should be a top level heading +README.md:10:1 MD022/blanks-around-headings Headings should be surrounded by blank lines + +README.md:15:80 MD013/line-length Line length [Expected: 80; Actual: 95] +""" +expected = "README.md:1:1 MD041/first-line-heading/first-line-h1 First line in file should be a top level heading\nREADME.md:10:1 MD022/blanks-around-headings Headings should be surrounded by blank lines\nREADME.md:15:80 MD013/line-length Line length [Expected: 80; Actual: 95]" + +[[tests.markdownlint]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.mise] +description = "Compact mise task runner output — strip status lines, keep task results" +match_command = "^mise\\s+(run|exec|install|upgrade)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^mise\\s+(trust|install|upgrade).*✓", + "^mise\\s+Installing\\s", + "^mise\\s+Downloading\\s", + "^mise\\s+Extracting\\s", + "^mise\\s+\\w+@[\\d.]+ installed", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "mise: ok" + +[[tests.mise]] +name = "strips install noise, keeps task output" +input = "mise Installing node@20.0.0\nmise Downloading node@20.0.0\nmise Extracting node@20.0.0\nmise node@20.0.0 installed\n\nlint check passed\n2 warnings found" +expected = "lint check passed\n2 warnings found" + +[[tests.mise]] +name = "preserves error output" +input = "mise run lint\nError: biome check failed\nsrc/index.ts:5 — unused variable" +expected = "mise run lint\nError: biome check failed\nsrc/index.ts:5 — unused variable" + +[[tests.mise]] +name = "empty after stripping" +input = "mise trust ~/dev/.mise.toml ✓\nmise install node@20 ✓\n" +expected = "mise: ok" + + +[filters.mix-compile] +description = "Compact mix compile output" +match_command = "^mix\\s+compile(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^Compiling \\d+ file", + "^\\s*$", + "^Generated\\s", +] +max_lines = 40 +on_empty = "mix compile: ok" + +[[tests.mix-compile]] +name = "strips compile noise, preserves warnings" +input = """ +Compiling 12 files (.ex) +Generated my_app app + +warning: variable "conn" is unused + lib/router.ex:42 +""" +expected = "warning: variable \"conn\" is unused\n lib/router.ex:42" + +[[tests.mix-compile]] +name = "on_empty when only noise" +input = "Compiling 3 files (.ex)\nGenerated my_app app\n" +expected = "mix compile: ok" + + +[filters.mix-format] +description = "Compact mix format output" +match_command = "^mix\\s+format(\\s|$)" +on_empty = "mix format: ok" +max_lines = 20 + +[[tests.mix-format]] +name = "empty output returns ok" +input = "" +expected = "mix format: ok" + +[[tests.mix-format]] +name = "changed files pass through" +input = "lib/my_app.ex\ntest/my_app_test.exs" +expected = "lib/my_app.ex\ntest/my_app_test.exs" + + +[filters.nx] +description = "Compact Nx monorepo output — strip task graph noise, keep results" +match_command = "^(pnpm\\s+)?nx\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*>\\s*NX\\s+Running target", + "^\\s*>\\s*NX\\s+Nx read the output", + "^\\s*>\\s*NX\\s+View logs", + "^———————", + "^—————————", + "^\\s+Nx \\(powered by", +] +truncate_lines_at = 150 +max_lines = 60 + +[[tests.nx]] +name = "strips Nx noise, keeps build output" +input = "\n > NX Running target build for project myapp\n\n———————————————————————————————————————\nCompiled successfully.\nOutput: dist/apps/myapp\n\n > NX View logs at /tmp/.nx/runs/abc123\n\n Nx (powered by computation caching)\n" +expected = "Compiled successfully.\nOutput: dist/apps/myapp" + +[[tests.nx]] +name = "preserves error output" +input = "ERROR: Cannot find module '@myapp/shared'\n\n > NX Running target build for project myapp\n\nFailed at step: build" +expected = "ERROR: Cannot find module '@myapp/shared'\nFailed at step: build" + + +[filters.ollama] +description = "Strip ANSI spinners and cursor control from ollama output, keep final text" +match_command = "^ollama\\s+run\\b" +strip_ansi = true +strip_lines_matching = [ + "^[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏\\s]*$", + "^\\s*$", +] + +[[tests.ollama]] +name = "strips spinner lines, keeps response" +input = "⠋ \n⠙ \n⠹ \nHello! How can I help you today?" +expected = "Hello! How can I help you today?" + +[[tests.ollama]] +name = "preserves multi-line response" +input = "⠋ \n⠙ \nLine one of the response.\nLine two of the response." +expected = "Line one of the response.\nLine two of the response." + +[[tests.ollama]] +name = "empty input" +input = "" +expected = "" + + +[filters.oxlint] +description = "Compact oxlint output — strip blank lines, keep diagnostics" +match_command = "^oxlint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Finished in \\d+", + "^Found \\d+ warning", +] +max_lines = 50 +on_empty = "oxlint: ok" + +[[tests.oxlint]] +name = "strips noise, keeps diagnostics" +input = """ + × eslint(no-console): Unexpected console statement. + ╭─[src/app.ts:5:3] + 5 │ console.log("debug"); + │ ^^^^^^^^^^^ + ╰──── + + × eslint(no-unused-vars): 'x' is defined but never used. + ╭─[src/utils.ts:2:7] + 2 │ let x = 42; + │ ^ + ╰──── + +Found 2 warnings on 2 files. +Finished in 12ms on 100 files. +""" +expected = " × eslint(no-console): Unexpected console statement.\n ╭─[src/app.ts:5:3]\n 5 │ console.log(\"debug\");\n │ ^^^^^^^^^^^\n ╰────\n × eslint(no-unused-vars): 'x' is defined but never used.\n ╭─[src/utils.ts:2:7]\n 2 │ let x = 42;\n │ ^\n ╰────" + +[[tests.oxlint]] +name = "clean output" +input = """ +Finished in 5ms on 100 files. +""" +expected = "oxlint: ok" + +[[tests.oxlint]] +name = "empty input returns on_empty message" +input = "" +expected = "oxlint: ok" + + +[filters.ping] +description = "Compact ping output — strip per-packet lines, keep summary" +match_command = "^ping\\b" +strip_ansi = true +strip_lines_matching = [ + "^PING ", + "^Pinging ", + "^\\d+ bytes from ", + "^Reply from .+: bytes=", + "^\\s*$", +] +tail_lines = 4 + +[[tests.ping]] +name = "success keeps summary only" +input = """ +PING example.com (93.184.216.34): 56 data bytes +64 bytes from 93.184.216.34: icmp_seq=0 ttl=56 time=14.2 ms +64 bytes from 93.184.216.34: icmp_seq=1 ttl=56 time=13.8 ms +64 bytes from 93.184.216.34: icmp_seq=2 ttl=56 time=14.1 ms +64 bytes from 93.184.216.34: icmp_seq=3 ttl=56 time=13.9 ms + +--- example.com ping statistics --- +4 packets transmitted, 4 packets received, 0.0% packet loss +round-trip min/avg/max/stddev = 13.8/14.0/14.2/0.2 ms +""" +expected = """--- example.com ping statistics --- +4 packets transmitted, 4 packets received, 0.0% packet loss +round-trip min/avg/max/stddev = 13.8/14.0/14.2/0.2 ms""" + +[[tests.ping]] +name = "windows format keeps stats block only" +input = """ +Pinging 192.0.2.1 with 32 bytes of data: +Reply from 192.0.2.1: bytes=32 time=14ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=13ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=14ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=13ms TTL=56 + +Ping statistics for 192.0.2.1: + Packets: Sent = 4, Received = 4, Lost = 0 (0% loss), +Approximate round trip times in milli-seconds: + Minimum = 13ms, Maximum = 14ms, Average = 13ms +""" +expected = """Ping statistics for 192.0.2.1: + Packets: Sent = 4, Received = 4, Lost = 0 (0% loss), +Approximate round trip times in milli-seconds: + Minimum = 13ms, Maximum = 14ms, Average = 13ms""" + +[[tests.ping]] +name = "unreachable host passes error through" +input = """ +PING unreachable.example.com (192.0.2.1): 56 data bytes +Request timeout for icmp_seq 0 +Request timeout for icmp_seq 1 + +--- unreachable.example.com ping statistics --- +2 packets transmitted, 0 packets received, 100.0% packet loss +""" +expected = """Request timeout for icmp_seq 0 +Request timeout for icmp_seq 1 +--- unreachable.example.com ping statistics --- +2 packets transmitted, 0 packets received, 100.0% packet loss""" + + +[filters.pio-run] +description = "Compact PlatformIO build output" +match_command = "^pio\\s+run(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Verbose mode", + "^CONFIGURATION:", + "^LDF:", + "^Library Manager:", + "^Compiling\\s", + "^Linking\\s", + "^Building\\s", + "^Checking size", +] +max_lines = 30 +on_empty = "pio run: ok" + +[[tests.pio-run]] +name = "strips build noise, preserves errors" +input = """ +Verbose mode can be enabled via `-v, --verbose` option +CONFIGURATION: https://docs.platformio.org/page/boards/espressif32/esp32dev.html +LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf +Compiling .pio/build/esp32dev/src/main.cpp.o +Building .pio/build/esp32dev/firmware.elf +Linking .pio/build/esp32dev/firmware.elf +Checking size .pio/build/esp32dev/firmware.elf +src/main.cpp:10:3: error: 'LED_BUILTINN' was not declared +""" +expected = "src/main.cpp:10:3: error: 'LED_BUILTINN' was not declared" + +[[tests.pio-run]] +name = "on_empty when clean build with only noise" +input = """ +Verbose mode can be enabled via `-v, --verbose` option +Compiling .pio/build/esp32dev/src/main.cpp.o +Linking .pio/build/esp32dev/firmware.elf +""" +expected = "pio run: ok" + + +[filters.poetry-install] +description = "Compact poetry install/lock/update output — strip downloads, short-circuit when up-to-date" +match_command = "^poetry\\s+(install|lock|update)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ [-•] Downloading ", + "^ [-•] Installing .* \\(", + "^Creating virtualenv", + "^Using virtualenv", +] +match_output = [ + { pattern = "No dependencies to install or update|No changes\\.", message = "ok (up to date)" }, +] +max_lines = 30 + +[[tests.poetry-install]] +name = "up to date short-circuits" +input = """ +Installing dependencies from lock file + +No dependencies to install or update +""" +expected = "ok (up to date)" + +[[tests.poetry-install]] +name = "poetry 2.x bullet syntax short-circuits to ok" +input = """ +• Installing requests (2.31.0) +• Installing certifi (2023.11.17) + +No changes. +""" +expected = "ok (up to date)" + +[[tests.poetry-install]] +name = "install strips download lines" +input = """ +Installing dependencies from lock file + + - Downloading requests-2.31.0-py3-none-any.whl (62.6 kB) + - Installing certifi (2023.11.17) + - Installing charset-normalizer (3.3.2) + - Installing idna (3.6) + - Installing urllib3 (2.1.0) + - Installing requests (2.31.0) + +Writing lock file +""" +expected = "Installing dependencies from lock file\nWriting lock file" + + +[filters.pre-commit] +description = "Compact pre-commit output" +match_command = "^pre-commit\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\[INFO\\] Installing environment", + "^\\[INFO\\] Once installed this environment will be reused", + "^\\[INFO\\] This may take a few minutes", + "^\\s*$", +] +max_lines = 40 + +[[tests.pre-commit]] +name = "strips INFO install noise, keeps hook results" +input = """ +[INFO] Installing environment for https://github.com/psf/black. +[INFO] Once installed this environment will be reused. +[INFO] This may take a few minutes... +Trim Trailing Whitespace.................................................Passed +Fix End of Files.........................................................Passed +Check Yaml...............................................................Failed +- hook id: check-yaml +- exit code: 1 +""" +expected = "Trim Trailing Whitespace.................................................Passed\nFix End of Files.........................................................Passed\nCheck Yaml...............................................................Failed\n- hook id: check-yaml\n- exit code: 1" + +[[tests.pre-commit]] +name = "all passed — no INFO noise" +input = """ +[INFO] Installing environment for https://github.com/pre-commit/mirrors-isort. +[INFO] Once installed this environment will be reused. +isort....................................................................Passed +black....................................................................Passed +""" +expected = "isort....................................................................Passed\nblack....................................................................Passed" + + +[filters.ps] +description = "Compact ps output — truncate wide lines, limit rows" +match_command = "^ps(\\s|$)" +strip_ansi = true +truncate_lines_at = 120 +max_lines = 30 + +[[tests.ps]] +name = "short process list passes through unchanged" +input = "USER PID %CPU %MEM COMMAND\nroot 1 0.0 0.0 /sbin/launchd\nflorian 42 0.1 0.2 bash" +expected = "USER PID %CPU %MEM COMMAND\nroot 1 0.0 0.0 /sbin/launchd\nflorian 42 0.1 0.2 bash" + +[[tests.ps]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.pulumi-destroy] +description = "Compact Pulumi destroy output" +match_command = "^pulumi\\s+destroy(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Destroying \\(", + "^Previewing (update|destroy)", + "^View in Browser", + "^View Live:", + "^Duration:", + "^Permalink:", + "^\\s*Type\\s+Name\\s+", + "^Loading policy packs", + "^@ (Previewing (update|destroy)|Destroying)", + "^More information at:", + "^Use `pulumi ", + "^The resources in the stack have been deleted", + "^If you want to remove the stack completely", + "^\\s+-\\s+.*\\bdeleting\\s+\\(", + "^\\s{4,}at\\s+\\S+\\s*\\(", + "^\\s{4,}at\\s+/", + "^\\s+at\\s+processTicksAndRejections", + "^\\s+promise:\\s+Promise", + "^\\s+\\[Circular", + "^\\s*-.jar name, and jars under a path containing spaces, are deliberately left to full passthrough rather than risk misclassifying an unrelated tool." +match_command = "^(mvn\\s+spring-boot:run|java\\s+-jar\\s+(?:\\S*[/\\\\])?[^/\\\\\\s]*(?i:spring)[^/\\\\\\s]*\\.jar|gradle\\s+.*bootRun)" +strip_ansi = true +keep_lines_matching = [ + "Started\\s.*\\sin\\s", + "Tomcat started on port", + "ERROR", + "WARN", + "Exception", + "Caused by:", + "Application run failed", + "BUILD\\s", + "Tests run:", + "FAILURE", + "listening on port", +] +max_lines = 30 + +[[tests.spring-boot]] +name = "keeps startup summary and errors" +input = " . ____ _ \n /\\\\ / ___'_ __ _ _(_)_ __ \n( ( )\\___ | '_ | '_| | '_ \\ \n \\/ ___)| |_)| | | | | || )\n ' |____| .__|_| |_|_| |_\\__|\n :: Spring Boot :: (v3.2.0)\n2024-01-01 INFO Initializing Spring\n2024-01-01 INFO Bean 'dataSource' created\n2024-01-01 INFO Tomcat started on port 8080\n2024-01-01 INFO Started MyApp in 3.2 seconds" +expected = "2024-01-01 INFO Tomcat started on port 8080\n2024-01-01 INFO Started MyApp in 3.2 seconds" + +[[tests.spring-boot]] +name = "preserves errors" +input = " :: Spring Boot :: (v3.2.0)\n2024-01-01 INFO Initializing Spring\n2024-01-01 ERROR Application run failed\nCaused by: java.lang.NullPointerException" +expected = "2024-01-01 ERROR Application run failed\nCaused by: java.lang.NullPointerException" + + +[filters.ssh] +description = "Compact ssh output — strip connection banners, keep command output" +match_command = "^ssh(?:\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Warning: Permanently added", + "^Connection to .+ closed", + "^Authenticated to", + "^debug1:", + "^OpenSSH_", + "^Pseudo-terminal", +] +max_lines = 200 +truncate_lines_at = 120 + +[[tests.ssh]] +name = "strips connection banners, keeps command output" +input = """ +Warning: Permanently added '192.168.1.10' (ED25519) to the list of known hosts. + +total 32 +drwxr-xr-x 4 user user 4096 Mar 10 12:00 app +-rw-r--r-- 1 user user 1234 Mar 10 11:00 config.yaml + +Connection to 192.168.1.10 closed. +""" +expected = "total 32\ndrwxr-xr-x 4 user user 4096 Mar 10 12:00 app\n-rw-r--r-- 1 user user 1234 Mar 10 11:00 config.yaml" + +[[tests.ssh]] +name = "verbose debug lines stripped" +input = """ +debug1: Connecting to host.example.com port 22. +debug1: Connection established. +Authenticated to host.example.com ([1.2.3.4]:22). +uptime: 12:00:00 up 42 days, load average: 0.10, 0.15, 0.12 +Connection to host.example.com closed. +""" +expected = "uptime: 12:00:00 up 42 days, load average: 0.10, 0.15, 0.12" + +[[tests.ssh]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.stat] +description = "Compact stat output — strip device/inode/birth noise" +match_command = "^stat\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Device:", + "^\\s*Birth:", +] +truncate_lines_at = 120 +max_lines = 20 + +[[tests.stat]] +name = "linux stat output strips device and birth" +input = """ + File: main.rs + Size: 12345 Blocks: 24 IO Block: 4096 regular file +Device: 801h/2049d Inode: 1234567 Links: 1 +Access: (0644/-rw-r--r--) Uid: ( 1000/ patrick) Gid: ( 1000/ patrick) +Access: 2026-03-10 12:00:00.000000000 +0100 +Modify: 2026-03-10 11:00:00.000000000 +0100 +Change: 2026-03-10 11:00:00.000000000 +0100 + Birth: 2026-03-09 10:00:00.000000000 +0100 +""" +expected = " File: main.rs\n Size: 12345 Blocks: 24 IO Block: 4096 regular file\nAccess: (0644/-rw-r--r--) Uid: ( 1000/ patrick) Gid: ( 1000/ patrick)\nAccess: 2026-03-10 12:00:00.000000000 +0100\nModify: 2026-03-10 11:00:00.000000000 +0100\nChange: 2026-03-10 11:00:00.000000000 +0100" + +[[tests.stat]] +name = "macOS stat -x strips device and birth" +input = """ + File: "main.rs" + Size: 82848 FileType: Regular File + Mode: (0644/-rw-r--r--) Uid: ( 501/ patrick) Gid: ( 20/ staff) +Device: 1,15 Inode: 66302332 Links: 1 +Access: Wed Mar 18 21:21:15 2026 +Modify: Wed Mar 18 20:56:11 2026 +Change: Wed Mar 18 20:56:11 2026 + Birth: Wed Mar 18 20:56:11 2026 +""" +expected = " File: \"main.rs\"\n Size: 82848 FileType: Regular File\n Mode: (0644/-rw-r--r--) Uid: ( 501/ patrick) Gid: ( 20/ staff)\nAccess: Wed Mar 18 21:21:15 2026\nModify: Wed Mar 18 20:56:11 2026\nChange: Wed Mar 18 20:56:11 2026" + +[[tests.stat]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.swift-build] +description = "Compact swift build output — short-circuit on success, strip Compiling/Linking" +match_command = "^swift\\s+build\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Compiling ", + "^Linking ", +] +match_output = [ + { pattern = "Build complete!", message = "ok (build complete)", unless = "warning:|error:" }, +] +max_lines = 40 + +[[tests.swift-build]] +name = "successful build short-circuits to ok" +input = """ +Build complete! +""" +expected = "ok (build complete)" + +[[tests.swift-build]] +name = "build errors pass through after stripping noise" +input = """ +Compiling MyApp MyApp.swift +/home/user/MyApp/Sources/MyApp/main.swift:5:1: error: use of unresolved identifier 'foo' +foo() +^~~ +Linking MyApp +error: build had 1 command failure +""" +expected = "/home/user/MyApp/Sources/MyApp/main.swift:5:1: error: use of unresolved identifier 'foo'\nfoo()\n^~~\nerror: build had 1 command failure" + +[[tests.swift-build]] +name = "warnings not swallowed when Build complete present" +input = """ +CompileSwift normal x86_64 MyFile.swift +/path/to/MyFile.swift:42:10: warning: unused variable 'x' +Build complete! (with warnings) +""" +expected = "CompileSwift normal x86_64 MyFile.swift\n/path/to/MyFile.swift:42:10: warning: unused variable 'x'\nBuild complete! (with warnings)" + + +[filters.systemctl-status] +description = "Compact systemctl status output — strip blank lines, limit to 20 lines" +match_command = "^systemctl\\s+status\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 20 + +[[tests.systemctl-status]] +name = "verbose unit status stripped of blank lines" +input = """ +● nginx.service - A high performance web server + Loaded: loaded (/lib/systemd/system/nginx.service; enabled) + Active: active (running) since Mon 2024-01-15 10:30:00 UTC; 2h ago + Docs: man:nginx(8) + Main PID: 1234 (nginx) + Tasks: 3 (limit: 4915) + Memory: 8.5M + + CGroup: /system.slice/nginx.service + ├─1234 nginx: master process /usr/sbin/nginx + └─1235 nginx: worker process + +Jan 15 10:30:00 host nginx[1234]: nginx/1.24.0 +Jan 15 10:30:00 host systemd[1]: Started nginx.service +""" +expected = "● nginx.service - A high performance web server\n Loaded: loaded (/lib/systemd/system/nginx.service; enabled)\n Active: active (running) since Mon 2024-01-15 10:30:00 UTC; 2h ago\n Docs: man:nginx(8)\n Main PID: 1234 (nginx)\n Tasks: 3 (limit: 4915)\n Memory: 8.5M\n CGroup: /system.slice/nginx.service\n ├─1234 nginx: master process /usr/sbin/nginx\n └─1235 nginx: worker process\nJan 15 10:30:00 host nginx[1234]: nginx/1.24.0\nJan 15 10:30:00 host systemd[1]: Started nginx.service" + +[[tests.systemctl-status]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.task] +description = "Compact go-task output — strip task headers, keep command results" +match_command = "^task\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^task: \\[.*\\] ", + "^task: Task .* is up to date", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "task: ok" + +[[tests.task]] +name = "strips task headers, keeps output" +input = "task: [build] go build ./...\n\ntask: [test] go test ./...\nok myapp 0.5s\n\ntask: Task \"lint\" is up to date" +expected = "ok myapp 0.5s" + +[[tests.task]] +name = "preserves error output" +input = "task: [build] go build ./...\n./main.go:10: undefined: foo\ntask: Failed to run task \"build\": exit status 1" +expected = "./main.go:10: undefined: foo\ntask: Failed to run task \"build\": exit status 1" + +[[tests.task]] +name = "all up to date" +input = "task: Task \"build\" is up to date\ntask: Task \"lint\" is up to date\n" +expected = "task: ok" + + +[filters.terraform-plan] +description = "Compact Terraform plan output" +match_command = "^terraform\\s+plan(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^Refreshing state", + "^\\s*#.*unchanged", + "^\\s*$", + "^Acquiring state lock", + "^Releasing state lock", +] +max_lines = 80 +on_empty = "terraform plan: no changes detected" + +[[tests.terraform-plan]] +name = "strips Refreshing state lines and blank lines" +input = """ +Acquiring state lock. This may take a few moments... +Refreshing state... [id=vpc-abc] +Refreshing state... [id=sg-123] +Releasing state lock. This may take a few moments... + +Terraform will perform the following actions: + + # aws_instance.web will be created + + resource "aws_instance" "web" {} + +Plan: 1 to add, 0 to change, 0 to destroy. +""" +expected = "Terraform will perform the following actions:\n # aws_instance.web will be created\n + resource \"aws_instance\" \"web\" {}\nPlan: 1 to add, 0 to change, 0 to destroy." + +[[tests.terraform-plan]] +name = "strips noise, preserves non-blank content" +input = "Refreshing state... [id=vpc-abc]\nNo changes. Your infrastructure matches the configuration." +expected = "No changes. Your infrastructure matches the configuration." + + +[filters.tofu-fmt] +description = "Compact OpenTofu fmt output" +match_command = "^tofu\\s+fmt(\\s|$)" +strip_ansi = true +on_empty = "tofu fmt: ok (no changes)" +max_lines = 30 + +[[tests.tofu-fmt]] +name = "empty output returns on_empty message" +input = "" +expected = "tofu fmt: ok (no changes)" + +[[tests.tofu-fmt]] +name = "changed files pass through" +input = "main.tf\nvariables.tf" +expected = "main.tf\nvariables.tf" + + +[filters.tofu-init] +description = "Compact OpenTofu init output" +match_command = "^tofu\\s+init(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^- Downloading", + "^- Installing", + "^- Using previously-installed", + "^\\s*$", + "^Initializing provider", + "^Initializing the backend", + "^Initializing modules", +] +max_lines = 20 +on_empty = "tofu init: ok" + +[[tests.tofu-init]] +name = "strips downloading/installing lines" +input = """ +Initializing the backend... +Initializing provider plugins... +- Downloading hashicorp/aws 5.0.0... +- Installing hashicorp/aws 5.0.0... +- Using previously-installed hashicorp/random 3.5.1 + +OpenTofu has been successfully initialized! +""" +expected = "OpenTofu has been successfully initialized!" + +[[tests.tofu-init]] +name = "on_empty when all noise stripped" +input = """ +Initializing the backend... +Initializing provider plugins... +- Using previously-installed hashicorp/aws 5.0.0 + +""" +expected = "tofu init: ok" + + +[filters.tofu-plan] +description = "Compact OpenTofu plan output" +match_command = "^tofu\\s+plan(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^Refreshing state", + "^\\s*#.*unchanged", + "^\\s*$", + "^Acquiring state lock", + "^Releasing state lock", +] +max_lines = 80 +on_empty = "tofu plan: no changes detected" + +[[tests.tofu-plan]] +name = "strips Refreshing state and lock lines" +input = """ +Acquiring state lock. This may take a few moments... +Refreshing state... [id=vpc-abc123] +Refreshing state... [id=sg-def456] +Releasing state lock. This may take a few moments... + +OpenTofu will perform the following actions: + + # aws_instance.web will be created + + resource "aws_instance" "web" {} + +Plan: 1 to add, 0 to change, 0 to destroy. +""" +expected = "OpenTofu will perform the following actions:\n # aws_instance.web will be created\n + resource \"aws_instance\" \"web\" {}\nPlan: 1 to add, 0 to change, 0 to destroy." + +[[tests.tofu-plan]] +name = "on_empty when all noise stripped" +input = "Refreshing state... [id=vpc-abc]\nAcquiring state lock. This may take a few moments...\nReleasing state lock. This may take a few moments..." +expected = "tofu plan: no changes detected" + + +[filters.tofu-validate] +description = "Compact OpenTofu validate output" +match_command = "^tofu\\s+validate(\\s|$)" +strip_ansi = true +match_output = [ + { pattern = "Success! The configuration is valid", message = "ok (valid)" }, +] + +[[tests.tofu-validate]] +name = "success short-circuits to ok" +input = "Success! The configuration is valid." +expected = "ok (valid)" + +[[tests.tofu-validate]] +name = "error passes through unchanged" +input = "Error: Invalid resource type\n on main.tf line 3: resource \"aws_instancee\" \"web\"" +expected = "Error: Invalid resource type\n on main.tf line 3: resource \"aws_instancee\" \"web\"" + + +[filters.trunk-build] +description = "Compact trunk build output" +match_command = "^trunk\\s+build(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Compiling\\s", + "^\\s*Downloading\\s", + "^\\s*Fetching\\s", + "^\\s*Fresh\\s", + "^\\s*Checking\\s", +] +tail_lines = 10 +max_lines = 30 +on_empty = "trunk build: ok" + +[[tests.trunk-build]] +name = "strips compile noise, keeps tail summary" +input = """ + Compiling tokio v1.35.0 + Compiling hyper v0.14.28 + Compiling my-crate v0.1.0 + Downloading serde v1.0.195 + Fresh regex v1.10.2 + + Finished release [optimized] target(s) in 45.23s + Binary: target/release/my-crate (5.2MB) +""" +expected = " Finished release [optimized] target(s) in 45.23s\n Binary: target/release/my-crate (5.2MB)" + +[[tests.trunk-build]] +name = "on_empty when all noise stripped" +input = """ + Compiling my-crate v0.1.0 + Fresh serde v1.0 + Checking tokio v1.35.0 + +""" +expected = "trunk build: ok" + + +[filters.turbo] +description = "Compact Turborepo output — strip cache status noise, keep task results" +match_command = "^turbo\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*cache (hit|miss|bypass)", + "^\\s*\\d+ packages in scope", + "^\\s*Tasks:\\s+\\d+", + "^\\s*Duration:\\s+", + "^\\s*Remote caching (enabled|disabled)", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "turbo: ok" + +[[tests.turbo]] +name = "strips cache noise, keeps task output" +input = " cache hit, replaying logs abc123\n cache miss, executing abc456\n\n3 packages in scope\n\n> myapp:build\n\nCompiled successfully.\n\nTasks: 2 successful, 2 total (1 cached)\nDuration: 3.2s" +expected = "> myapp:build\nCompiled successfully." + +[[tests.turbo]] +name = "preserves error output" +input = "> myapp:lint\n\nError: src/index.ts(5,1): error TS2304\n\nTasks: 0 successful, 1 total\nDuration: 1.1s" +expected = "> myapp:lint\nError: src/index.ts(5,1): error TS2304" + +[[tests.turbo]] +name = "empty after stripping" +input = " cache hit, replaying logs abc\n\n" +expected = "turbo: ok" + + +[filters.ty] +description = "Compact ty type checker output — strip blank lines, keep errors" +match_command = "^ty\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Checking \\d+ file", + "^ty \\d+\\.\\d+", +] +max_lines = 50 +on_empty = "ty: ok" + +[[tests.ty]] +name = "strips noise, keeps diagnostics" +input = """ +ty 0.1.0 +Checking 15 files + +error[unresolved-reference]: Name `foo` used when not defined + --> app/main.py:10:5 + | +10 | foo() + | ^^^ + | + +warning[unused-variable]: Variable `x` is not used + --> app/utils.py:8:9 + | + 8 | x = 42 + | ^ + | + +Found 1 error, 1 warning +""" +expected = "error[unresolved-reference]: Name `foo` used when not defined\n --> app/main.py:10:5\n |\n10 | foo()\n | ^^^\n |\nwarning[unused-variable]: Variable `x` is not used\n --> app/utils.py:8:9\n |\n 8 | x = 42\n | ^\n |\nFound 1 error, 1 warning" + +[[tests.ty]] +name = "clean output" +input = """ +ty 0.1.0 +Checking 10 files + +All checks passed! +""" +expected = "All checks passed!" + +[[tests.ty]] +name = "empty input returns on_empty message" +input = "" +expected = "ty: ok" + + +[filters.uv-sync] +description = "Compact uv sync/pip install output — strip downloads, short-circuit when up-to-date" +match_command = "^uv\\s+(sync|pip\\s+install)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s+Downloading ", + "^\\s+Using cached ", + "^\\s+Preparing ", +] +match_output = [ + { pattern = "Audited \\d+ package", message = "ok (up to date)" }, +] +max_lines = 20 + +[[tests.uv-sync]] +name = "audited packages short-circuits to ok" +input = """ +Resolved 42 packages in 123ms +Audited 42 packages in 0.05ms +""" +expected = "ok (up to date)" + +[[tests.uv-sync]] +name = "install strips download and cached lines" +input = """ + Downloading requests-2.31.0-py3-none-any.whl (62.6 kB) + Using cached certifi-2023.11.17-py3-none-any.whl (162 kB) + Preparing packages... +Installed 5 packages in 23ms + + certifi==2023.11.17 + + charset-normalizer==3.3.2 + + idna==3.6 + + requests==2.31.0 + + urllib3==2.1.0 +""" +expected = "Installed 5 packages in 23ms\n + certifi==2023.11.17\n + charset-normalizer==3.3.2\n + idna==3.6\n + requests==2.31.0\n + urllib3==2.1.0" + + +[filters.xcodebuild] +description = "Compact xcodebuild output — strip build phases, keep errors/warnings/summary" +match_command = "^xcodebuild\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^CompileC\\s", + "^CompileSwift\\s", + "^Ld\\s", + "^CreateBuildDirectory\\s", + "^MkDir\\s", + "^ProcessInfoPlistFile\\s", + "^CopySwiftLibs\\s", + "^CodeSign\\s", + "^Signing Identity:", + "^RegisterWithLaunchServices", + "^Validate\\s", + "^ProcessProductPackaging", + "^Touch\\s", + "^LinkStoryboards", + "^CompileStoryboard", + "^CompileAssetCatalog", + "^GenerateDSYMFile", + "^PhaseScriptExecution", + "^PBXCp\\s", + "^SetMode\\s", + "^SetOwnerAndGroup\\s", + "^Ditto\\s", + "^CpResource\\s", + "^CpHeader\\s", + "^\\s+cd\\s+/", + "^\\s+export\\s", + "^\\s+/Applications/Xcode", + "^\\s+/usr/bin/", + "^\\s+builtin-", + "^note: Using new build system", +] +max_lines = 60 +on_empty = "xcodebuild: ok" + +[[tests.xcodebuild]] +name = "strips build phases, keeps errors and summary" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/App/ViewController.swift + cd /Users/dev/App + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-frontend -c +CompileSwift normal arm64 /Users/dev/App/AppDelegate.swift + cd /Users/dev/App + export DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer +Ld /Users/dev/Build/Products/Debug/App normal arm64 + cd /Users/dev/App + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang +CodeSign /Users/dev/Build/Products/Debug/App.app + cd /Users/dev/App + builtin-codesign --force --sign + +/Users/dev/App/ViewController.swift:42:9: error: use of unresolved identifier 'foo' +/Users/dev/App/Model.swift:18:5: warning: variable 'x' was never used + +** BUILD FAILED ** +""" +expected = "/Users/dev/App/ViewController.swift:42:9: error: use of unresolved identifier 'foo'\n/Users/dev/App/Model.swift:18:5: warning: variable 'x' was never used\n** BUILD FAILED **" + +[[tests.xcodebuild]] +name = "clean build success" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/App/Main.swift + cd /Users/dev/App +Ld /Users/dev/Build/Products/Debug/App normal arm64 + cd /Users/dev/App +CodeSign /Users/dev/Build/Products/Debug/App.app + cd /Users/dev/App + builtin-codesign --force --sign + +** BUILD SUCCEEDED ** +""" +expected = "** BUILD SUCCEEDED **" + +[[tests.xcodebuild]] +name = "test output keeps test results" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/AppTests/Tests.swift + cd /Users/dev/App +Test Suite 'All tests' started at 2026-03-10 12:00:00 +Test Suite 'AppTests' started at 2026-03-10 12:00:00 +Test Case '-[AppTests testExample]' passed (0.001 seconds). +Test Case '-[AppTests testFailing]' failed (0.002 seconds). +Test Suite 'AppTests' passed at 2026-03-10 12:00:01 +Executed 2 tests, with 1 failure in 0.003 seconds +""" +expected = "Test Suite 'All tests' started at 2026-03-10 12:00:00\nTest Suite 'AppTests' started at 2026-03-10 12:00:00\nTest Case '-[AppTests testExample]' passed (0.001 seconds).\nTest Case '-[AppTests testFailing]' failed (0.002 seconds).\nTest Suite 'AppTests' passed at 2026-03-10 12:00:01\nExecuted 2 tests, with 1 failure in 0.003 seconds" + +[[tests.xcodebuild]] +name = "empty input returns on_empty message" +input = "" +expected = "xcodebuild: ok" + + +[filters.yadm] +description = "Compact yadm (git wrapper) output — same filtering as git" +match_command = "^yadm\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*\\(use \"git ", + "^\\s*\\(use \"yadm ", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.yadm]] +name = "strips hint lines" +input = "On branch main\nYour branch is up to date with 'origin/main'.\n\n (use \"yadm add\" to update what will be committed)\n\nChanges not staged for commit:\n modified: .bashrc" +expected = "On branch main\nYour branch is up to date with 'origin/main'.\nChanges not staged for commit:\n modified: .bashrc" + +[[tests.yadm]] +name = "short output preserved" +input = "Already up to date." +expected = "Already up to date." + + +[filters.yamllint] +description = "Compact yamllint output — strip blank lines, limit rows" +match_command = "^yamllint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.yamllint]] +name = "multi-warning output stripped of blank lines" +input = """ +config.yml + 3:1 warning missing document start "---" (document-start) + 5:12 error too many spaces inside braces (braces) + + 8:1 error wrong indentation: expected 2 but found 4 (indentation) +""" +expected = "config.yml\n 3:1 warning missing document start \"---\" (document-start)\n 5:12 error too many spaces inside braces (braces)\n 8:1 error wrong indentation: expected 2 but found 4 (indentation)" + +[[tests.yamllint]] +name = "empty input passes through" +input = "" +expected = "" + + diff --git a/crates/sparkplane-rtk/src/core/constants.rs b/crates/sparkplane-rtk/src/core/constants.rs new file mode 100644 index 0000000..e7cc757 --- /dev/null +++ b/crates/sparkplane-rtk/src/core/constants.rs @@ -0,0 +1,38 @@ +pub const RTK_DATA_DIR: &str = "rtk"; +pub const HISTORY_DB: &str = "history.db"; +pub const RECALL_DB: &str = "recall.db"; +pub const CONFIG_TOML: &str = "config.toml"; +pub const FILTERS_TOML: &str = "filters.toml"; +pub const TRUSTED_FILTERS_JSON: &str = "trusted_filters.json"; +pub const DEFAULT_HISTORY_DAYS: i64 = 90; + +/// RTK-only subcommands that should never fall back to raw execution. +/// When adding a new RTK-only subcommand to `Commands`, add its clap name here. +pub const RTK_META_COMMANDS: &[&str] = &[ + "gain", + "discover", + "learn", + "init", + "config", + "proxy", + "recall", + "run", + "hook", + "hook-audit", + "pipe", + "cc-economics", + "verify", + "trust", + "untrust", + "session", + "rewrite", + "telemetry", + "smart", + "deps", + "json", + // `err` and `summary` name no real binary, and since they grew `--shell` + // they have a flag to get wrong. Falling through would try to exec a + // program called `err` instead of reporting the flag error (#4125 review). + "err", + "summary", +]; diff --git a/crates/sparkplane-rtk/src/core/guard.rs b/crates/sparkplane-rtk/src/core/guard.rs new file mode 100644 index 0000000..3824d44 --- /dev/null +++ b/crates/sparkplane-rtk/src/core/guard.rs @@ -0,0 +1,71 @@ +//! Never-worse output guard: RTK never emits more tokens than the raw command. +//! +//! Two callers are allowed past it. `rtk diff` prints a one-line message for a +//! difference `str::lines()` cannot render — CRLF against LF, or a missing +//! final newline — where the raw fallback is two blobs that look identical and +//! answer the question worse at any size. The exception is bounded by the case +//! rather than by a token count: it fires only when the bytes differ and the +//! line vectors do not, so the message never competes with a change list. A +//! fixed allowance above raw was tried and could not be met by construction: +//! the shortest form of the message is ~20 tokens and a one-line pair is ~2, so +//! the ceiling sat under the message's own floor and dropped it on 90% of +//! one-line pairs. +//! +//! The other is a clean run of a command whose output format rtk injected +//! (`runner::RunOptions::clean_outputs`): ruff's `[]` answers a question the +//! user never asked, so the filter's summary is shown although it is longer. + +use crate::core::tracking::estimate_tokens; + +/// Returns `filtered`, or `raw` when `filtered` would emit more tokens. +pub fn never_worse<'a>(raw: &'a str, filtered: &'a str) -> &'a str { + if estimate_tokens(filtered) > estimate_tokens(raw) { + raw + } else { + filtered + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_filtered_when_smaller() { + let raw = "a".repeat(400); + assert_eq!(never_worse(&raw, "ok"), "ok"); + } + + #[test] + fn falls_back_to_raw_when_filtered_bigger() { + let raw = "{}"; + let filtered = "{\n \"pretty\": true\n}"; + assert_eq!(never_worse(raw, filtered), raw); + } + + #[test] + fn tie_keeps_filtered() { + assert_eq!(never_worse("abcd", "wxyz"), "wxyz"); + } + + #[test] + fn token_boundary_follows_estimate_tokens() { + assert_eq!(never_worse("abcd", "abcde"), "abcd"); + assert_eq!(never_worse("abcdefgh", "ijklmnop"), "ijklmnop"); + } + + #[test] + fn empty_raw_returns_raw() { + assert_eq!(never_worse("", "0 matches"), ""); + } + + #[test] + fn empty_filtered_returns_filtered() { + assert_eq!(never_worse("data", ""), ""); + } + + #[test] + fn both_empty_returns_filtered() { + assert_eq!(never_worse("", ""), ""); + } +} diff --git a/crates/sparkplane-rtk/src/core/stream.rs b/crates/sparkplane-rtk/src/core/stream.rs new file mode 100644 index 0000000..52f938f --- /dev/null +++ b/crates/sparkplane-rtk/src/core/stream.rs @@ -0,0 +1,13 @@ +use std::borrow::Cow; +pub trait BlockHandler { + /// Rewrite a raw line before it is matched or emitted — the place to + /// strip ANSI escapes for tools that colour by default. Identity unless + /// a handler opts in, so the rest of the pipeline sees raw bytes. + fn normalize_line<'a>(&self, line: &'a str) -> Cow<'a, str> { + Cow::Borrowed(line) + } + fn should_skip(&mut self, line: &str) -> bool; + fn is_block_start(&mut self, line: &str) -> bool; + fn is_block_continuation(&mut self, line: &str, block: &[String]) -> bool; + fn format_summary(&self, exit_code: i32, raw: &str) -> Option; +} diff --git a/crates/sparkplane-rtk/src/core/toml_filter.rs b/crates/sparkplane-rtk/src/core/toml_filter.rs new file mode 100644 index 0000000..fa15516 --- /dev/null +++ b/crates/sparkplane-rtk/src/core/toml_filter.rs @@ -0,0 +1,2129 @@ +//! Applies TOML-defined filter rules to command output. +/// +/// Provides a declarative pipeline of 8 stages that can be configured +/// via TOML files. Lookup priority (first match wins): +/// 1. `.rtk/filters.toml` — project-local, committable with the repo +/// 2. `~/.config/rtk/filters.toml` — user-global, applies to all projects +/// 3. Built-in TOML — `src/filters/*.toml`, concatenated by build.rs and embedded at compile time +/// 4. Passthrough — no match, handled by caller +/// +/// `rtk init` generates a commented template for both levels (project or global). +/// +/// Environment variables: +/// - `RTK_NO_TOML=1` — bypass TOML engine entirely +/// - `RTK_TOML_DEBUG=1` — print which filter matched and line counts to stderr +/// +/// Pipeline stages (applied in order): +/// 1. strip_ansi — remove ANSI escape codes +/// 2. replace — regex substitutions, line-by-line, chainable +/// 3. match_output — short-circuit: if blob matches a pattern, return message immediately +/// 4. strip/keep_lines — filter lines by regex +/// 5. truncate_lines_at — truncate each line to N chars +/// 6. head/tail_lines — keep first/last N lines +/// 7. max_lines — absolute line cap +/// 8. on_empty — message if result is empty +use super::constants::RTK_META_COMMANDS; +use regex::{Regex, RegexSet}; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::sync::LazyLock; + +// Built-in filters: concatenated from src/filters/*.toml by build.rs at compile time. +const BUILTIN_TOML: &str = include_str!("builtin_filters.toml"); + +// --------------------------------------------------------------------------- +// Deserialization types (TOML schema) +// --------------------------------------------------------------------------- + +/// A match-output rule: if `pattern` matches anywhere in the full output blob, +/// the filter short-circuits and returns `message` immediately. +/// First matching rule wins; remaining rules are not evaluated. +/// Optional `unless`: if this regex also matches the blob, the rule is skipped +/// (prevents short-circuiting when errors or warnings are present). +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct MatchOutputRule { + pattern: String, + message: String, + #[serde(default)] + unless: Option, +} + +/// A regex substitution applied line-by-line. Rules are chained sequentially: +/// rule N+1 operates on the output of rule N. +/// Backreferences (`$1`, `$2`, ...) are supported via the `regex` crate. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ReplaceRule { + pattern: String, + replacement: String, +} + +/// An inline test case attached to a filter in the TOML. +/// Lives in `[[tests.]]` sections, separate from `[filters.*]`. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TomlFilterTestDef { + pub name: String, + pub input: String, + pub expected: String, +} + +#[derive(Deserialize)] +struct TomlFilterFile { + schema_version: u32, + #[serde(default)] + filters: BTreeMap, + /// Inline tests keyed by filter name. Kept separate from `filters` so that + /// `TomlFilterDef` can keep `deny_unknown_fields` without touching test data. + #[serde(default)] + tests: BTreeMap>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TomlFilterDef { + description: Option, + match_command: String, + #[serde(default)] + strip_ansi: bool, + /// Regex substitutions, applied line-by-line before match_output (stage 2). + #[serde(default)] + replace: Vec, + /// Short-circuit rules: if the full output blob matches, return the message (stage 3). + #[serde(default)] + match_output: Vec, + #[serde(default)] + strip_lines_matching: Vec, + #[serde(default)] + keep_lines_matching: Vec, + truncate_lines_at: Option, + head_lines: Option, + tail_lines: Option, + max_lines: Option, + on_empty: Option, + /// When true, stderr is captured and merged with stdout before filtering. + /// Use for tools like liquibase that emit banners/logs to stderr. + #[serde(default)] + filter_stderr: bool, +} + +// --------------------------------------------------------------------------- +// Compiled types (post-validation, ready to use) +// --------------------------------------------------------------------------- + +#[derive(Debug)] +struct CompiledMatchOutputRule { + pattern: Regex, + message: String, + /// If set and matches the blob, this rule is skipped (prevents swallowing errors). + unless: Option, +} + +#[derive(Debug)] +struct CompiledReplaceRule { + pattern: Regex, + replacement: String, +} + +#[derive(Debug)] +enum LineFilter { + None, + Strip(RegexSet), + Keep(RegexSet), +} + +/// A filter that has been parsed and compiled — all regexes are ready. +#[derive(Debug)] +pub struct CompiledFilter { + pub name: String, + #[allow(dead_code)] + pub description: Option, + match_regex: Regex, + strip_ansi: bool, + replace: Vec, + match_output: Vec, + line_filter: LineFilter, + truncate_lines_at: Option, + head_lines: Option, + tail_lines: Option, + pub max_lines: Option, + on_empty: Option, + /// When true, the runner should capture stderr and merge it with stdout. + pub filter_stderr: bool, +} + +// --------------------------------------------------------------------------- +// Results for `rtk verify` +// --------------------------------------------------------------------------- + +/// Outcome of running a single inline test. +pub struct TestOutcome { + pub filter_name: String, + pub test_name: String, + pub passed: bool, + pub actual: String, + pub expected: String, +} + +/// Aggregated results from `run_filter_tests`. +pub struct VerifyResults { + /// Individual test outcomes (all filters, or just the requested one). + pub outcomes: Vec, + /// Filter names that have no inline tests (used by `--require-all`). + pub filters_without_tests: Vec, +} + +// --------------------------------------------------------------------------- +// Registry +// --------------------------------------------------------------------------- + +pub struct TomlFilterRegistry { + pub filters: Vec, +} + +impl TomlFilterRegistry { + /// Load registry from disk + built-in. Emits warnings to stderr on parse + /// errors but never panics — bad files are silently ignored. + fn load() -> Self { + let mut filters = Vec::new(); + + match Self::parse_and_compile(BUILTIN_TOML, "builtin") { + Ok(f) => filters.extend(f), + Err(e) => eprintln!("[rtk] warning: builtin filters: {}", e), + } + + TomlFilterRegistry { filters } + } + + fn parse_and_compile(content: &str, source: &str) -> Result, String> { + let file: TomlFilterFile = toml::from_str(content) + .map_err(|e| format!("TOML parse error in {}: {}", source, e))?; + + if file.schema_version != 1 { + return Err(format!( + "unsupported schema_version {} in {} (expected 1)", + file.schema_version, source + )); + } + + let mut compiled = Vec::new(); + for (name, def) in file.filters { + if !is_fully_anchored(&def.match_command) { + eprintln!( + "[rtk] warning: filter '{}' in {}: match_command '{}' has a top-level branch \ + that does not start with '^'; it would match a path component mid-command. \ + Filter ignored.", + name, source, def.match_command + ); + continue; + } + match compile_filter(name.clone(), def) { + Ok(f) => compiled.push(f), + Err(e) => eprintln!("[rtk] warning: filter '{}' in {}: {}", name, source, e), + } + } + Ok(compiled) + } +} + +/// Commands already handled by dedicated Rust modules (routed by Clap before TOML). +/// A TOML filter whose match_command matches one of these will never activate — +/// Clap routes the command before `run_fallback()` is reached. +const RUST_HANDLED_COMMANDS: &[&str] = &[ + "ls", + "tree", + "read", + "smart", + "git", + "gh", + "aws", + "psql", + "pnpm", + "err", + "test", + "json", + "deps", + "env", + "find", + "diff", + "log", + "docker", + "kubectl", + "summary", + "grep", + "init", + "wget", + "wc", + "gain", + "config", + "ctest", + "vitest", + "prisma", + "tsc", + "next", + "lint", + "prettier", + "format", + "playwright", + "cargo", + "npm", + "npx", + "curl", + "discover", + "ruff", + "pytest", + "mypy", + "pip", + "sqlfluff", + "go", + "golangci-lint", + "rewrite", + "proxy", + "verify", + "learn", +]; + +pub fn is_rtk_reserved_command(name: &str) -> bool { + RUST_HANDLED_COMMANDS.contains(&name) || RTK_META_COMMANDS.contains(&name) +} + +fn compile_filter(name: String, def: TomlFilterDef) -> Result { + // Mutual exclusion: strip and keep cannot both be set + if !def.strip_lines_matching.is_empty() && !def.keep_lines_matching.is_empty() { + return Err("strip_lines_matching and keep_lines_matching are mutually exclusive".into()); + } + + let match_regex = Regex::new(&def.match_command) + .map_err(|e| format!("invalid match_command regex: {}", e))?; + + // Shadow warning: if match_command matches a Rust-handled command, this filter + // will never activate (Clap routes before run_fallback). Warn the author. + for cmd in RUST_HANDLED_COMMANDS { + if match_regex.is_match(cmd) { + eprintln!( + "[rtk] warning: filter '{}' match_command matches '{}' which is already \ + handled by a Rust module — this filter will never activate for that command", + name, cmd + ); + break; + } + } + + let replace = def + .replace + .into_iter() + .map(|r| { + let pat = r.pattern.clone(); + Regex::new(&r.pattern) + .map(|pattern| CompiledReplaceRule { + pattern, + replacement: r.replacement, + }) + .map_err(|e| format!("invalid replace pattern '{}': {}", pat, e)) + }) + .collect::, _>>()?; + + let match_output = def + .match_output + .into_iter() + .map(|r| -> Result { + let pat = r.pattern.clone(); + let pattern = Regex::new(&r.pattern) + .map_err(|e| format!("invalid match_output pattern '{}': {}", pat, e))?; + let unless = r + .unless + .as_deref() + .map(|u| { + Regex::new(u) + .map_err(|e| format!("invalid match_output unless pattern '{}': {}", u, e)) + }) + .transpose()?; + Ok(CompiledMatchOutputRule { + pattern, + message: r.message, + unless, + }) + }) + .collect::, _>>()?; + + let line_filter = if !def.strip_lines_matching.is_empty() { + let set = RegexSet::new(&def.strip_lines_matching) + .map_err(|e| format!("invalid strip_lines_matching regex: {}", e))?; + LineFilter::Strip(set) + } else if !def.keep_lines_matching.is_empty() { + let set = RegexSet::new(&def.keep_lines_matching) + .map_err(|e| format!("invalid keep_lines_matching regex: {}", e))?; + LineFilter::Keep(set) + } else { + LineFilter::None + }; + + Ok(CompiledFilter { + name, + description: def.description, + match_regex, + strip_ansi: def.strip_ansi, + replace, + match_output, + line_filter, + truncate_lines_at: def.truncate_lines_at, + head_lines: def.head_lines, + tail_lines: def.tail_lines, + max_lines: def.max_lines, + on_empty: def.on_empty, + filter_stderr: def.filter_stderr, + }) +} + +// --------------------------------------------------------------------------- +// Singleton (lazy-loaded, one-time cost) +// --------------------------------------------------------------------------- + +static REGISTRY: LazyLock = LazyLock::new(TomlFilterRegistry::load); + +pub fn toml_disabled() -> bool { + std::env::var("RTK_NO_TOML").ok().as_deref() == Some("1") +} + +pub fn active_filter_summaries(content: &str) -> Vec<(String, String)> { + toml::from_str::(content) + .map(|f| { + f.filters + .into_iter() + .map(|(name, def)| (name, def.match_command)) + .collect() + }) + .unwrap_or_default() +} + +pub fn filter_parse_error(content: &str) -> Option { + toml::from_str::(content) + .err() + .map(|e| e.to_string()) +} + +static MATCH_SET: LazyLock = LazyLock::new(build_match_set); + +pub fn command_matches_filter(command: &str) -> bool { + MATCH_SET.is_match(command) +} + +fn build_match_set() -> RegexSet { + let patterns = collect_match_patterns(); + RegexSet::new(&patterns).unwrap_or_else(|_| { + let valid: Vec = patterns + .into_iter() + .filter(|p| Regex::new(p).is_ok()) + .collect(); + RegexSet::new(&valid).unwrap_or_else(|_| RegexSet::empty()) + }) +} + +fn collect_match_patterns() -> Vec { + match_patterns_in(BUILTIN_TOML) +} + +/// Strip a leading inline-flag group such as `(?i)` so the anchor check sees the +/// pattern body. `(?:` opens a non-capturing group and is left alone. +fn strip_inline_flags(branch: &str) -> &str { + let Some(rest) = branch.strip_prefix("(?") else { + return branch; + }; + let Some(end) = rest.find(')') else { + return branch; + }; + if rest[..end].chars().all(|c| "imsuxU-".contains(c)) { + &rest[end + 1..] + } else { + branch + } +} + +/// Split `pattern` on top-level `|`, ignoring alternations nested inside groups +/// or character classes. +fn top_level_branches(pattern: &str) -> Vec<&str> { + let mut branches = Vec::new(); + let mut depth = 0usize; + let mut escaped = false; + let mut in_class = false; + let mut start = 0usize; + + for (idx, character) in pattern.char_indices() { + if escaped { + escaped = false; + continue; + } + match character { + '\\' => escaped = true, + '[' if !in_class => in_class = true, + ']' if in_class => in_class = false, + '(' if !in_class => depth += 1, + ')' if !in_class => depth = depth.saturating_sub(1), + '|' if !in_class && depth == 0 => { + branches.push(&pattern[start..idx]); + start = idx + 1; + } + _ => {} + } + } + branches.push(&pattern[start..]); + branches +} + +/// A filter selects on argv[0], but its regex runs against the whole command +/// line. A top-level branch that does not start with `^` therefore matches a +/// path component or wrapper argument mid-line, so `timeout 5 /usr/bin/liquibase +/// update` activates the liquibase filter and rewrites the wrapper instead. +fn is_fully_anchored(pattern: &str) -> bool { + top_level_branches(pattern) + .iter() + .all(|branch| strip_inline_flags(branch).starts_with('^')) +} + +fn match_patterns_in(content: &str) -> Vec { + match toml::from_str::(content) { + Ok(file) if file.schema_version == 1 => file + .filters + .into_values() + .map(|def| def.match_command) + .filter(|pattern| is_fully_anchored(pattern)) + .collect(), + _ => Vec::new(), + } +} + +// --------------------------------------------------------------------------- +// Public API — pure functions (testable without global state) +// --------------------------------------------------------------------------- + +/// Find the first matching filter in a slice. O(N) on the number of filters. +/// Tests should call this directly with a local filter list. +pub fn find_filter_in<'a>( + command: &str, + filters: &'a [CompiledFilter], +) -> Option<&'a CompiledFilter> { + filters.iter().find(|f| f.match_regex.is_match(command)) +} + +/// Apply a compiled filter pipeline to raw stdout. Pure String -> String. +/// +/// Pipeline stages (in order): +/// 1. strip_ansi — remove ANSI escape codes +/// 2. replace — regex substitutions, line-by-line, chainable +/// 3. match_output — short-circuit if blob matches a pattern +/// 4. strip/keep_lines — filter lines by regex +/// 5. truncate_lines_at — truncate each line to N chars +/// 6. head/tail_lines — keep first/last N lines +/// 7. max_lines — absolute line cap +/// 8. on_empty — message if result is empty +pub fn apply_filter(filter: &CompiledFilter, stdout: &str) -> String { + apply_filter_with_info(filter, stdout).0 +} + +#[derive(Debug, PartialEq)] +pub enum Lossiness { + None, + /// `tail -n +{tail_offset}` over `tee_payload` reproduces the dropped lines, + /// up to the tee `max_file_size` cap (larger payloads are truncated in the tee file). + Tail { + tee_payload: String, + tail_offset: usize, + }, + Whole, +} + +pub fn apply_filter_with_info(filter: &CompiledFilter, stdout: &str) -> (String, Lossiness) { + let mut lines: Vec = stdout.lines().map(String::from).collect(); + + // 1. strip_ansi + if filter.strip_ansi { + lines = lines + .into_iter() + .map(|l| crate::core::utils::strip_ansi(&l)) + .collect(); + } + + // 2. replace — line-by-line, rules chained sequentially + if !filter.replace.is_empty() { + lines = lines + .into_iter() + .map(|mut line| { + for rule in &filter.replace { + line = rule + .pattern + .replace_all(&line, rule.replacement.as_str()) + .into_owned(); + } + line + }) + .collect(); + } + + // 3. match_output — short-circuit on full blob match (first rule wins) + // If `unless` is set and also matches the blob, the rule is skipped. + if !filter.match_output.is_empty() { + let blob = lines.join("\n"); + for rule in &filter.match_output { + if rule.pattern.is_match(&blob) { + if let Some(ref unless_re) = rule.unless + && unless_re.is_match(&blob) + { + continue; // errors/warnings present — skip this rule + } + return (rule.message.clone(), Lossiness::Whole); + } + } + } + + // 4. strip OR keep (mutually exclusive) + match &filter.line_filter { + LineFilter::Strip(set) => lines.retain(|l| !set.is_match(l)), + LineFilter::Keep(set) => lines.retain(|l| set.is_match(l)), + LineFilter::None => {} + } + + // 5. truncate_lines_at — uses utils::truncate (unicode-safe) + let mut intra_line_loss = false; + if let Some(max_chars) = filter.truncate_lines_at { + lines = lines + .into_iter() + .map(|line| { + let truncated = crate::core::utils::truncate(&line, max_chars); + if truncated != line { + intra_line_loss = true; + } + truncated + }) + .collect(); + } + + let snapshot_for_tail = !intra_line_loss + && filter.tail_lines.is_none() + && (filter.head_lines.is_some() || filter.max_lines.is_some()); + let pre_cut = snapshot_for_tail.then(|| lines.clone()); + + // 6. head + tail + let total = lines.len(); + let mut noncontiguous_drop = false; + let mut head_cut: Option = None; + if let (Some(head), Some(tail)) = (filter.head_lines, filter.tail_lines) { + if total > head + tail { + let mut result = lines[..head].to_vec(); + result.push(format!("... ({} lines omitted)", total - head - tail)); + result.extend_from_slice(&lines[total - tail..]); + lines = result; + noncontiguous_drop = true; + } + } else if let Some(head) = filter.head_lines { + if total > head { + lines.truncate(head); + lines.push(format!("... ({} lines omitted)", total - head)); + head_cut = Some(head); + } + } else if let Some(tail) = filter.tail_lines + && total > tail + { + let omitted = total - tail; + lines = lines[omitted..].to_vec(); + lines.insert(0, format!("... ({} lines omitted)", omitted)); + noncontiguous_drop = true; + } + + // 7. max_lines — absolute cap applied after head/tail (includes omit messages) + let mut max_cut: Option = None; + if let Some(max) = filter.max_lines + && lines.len() > max + { + let dropped = lines.len() - max; + lines.truncate(max); + lines.push(format!("... ({} lines truncated)", dropped)); + max_cut = Some(max); + } + + // 8. on_empty + let result = lines.join("\n"); + if result.trim().is_empty() + && let Some(ref msg) = filter.on_empty + { + return (msg.clone(), Lossiness::None); + } + + let loss = if let Some(snapshot) = pre_cut { + match (head_cut, max_cut) { + (Some(_), Some(_)) => Lossiness::Whole, + (Some(head), None) => Lossiness::Tail { + tee_payload: snapshot.join("\n"), + tail_offset: head + 1, + }, + (None, Some(max)) => Lossiness::Tail { + tee_payload: snapshot.join("\n"), + tail_offset: max + 1, + }, + (None, None) => Lossiness::None, + } + } else if noncontiguous_drop || intra_line_loss || head_cut.is_some() || max_cut.is_some() { + Lossiness::Whole + } else { + Lossiness::None + }; + + (result, loss) +} + +// --------------------------------------------------------------------------- +// rtk verify — inline test execution +// --------------------------------------------------------------------------- + +/// Run inline tests from loaded TOML files (builtin + project-local). +/// +/// - `filter_name_opt`: if `Some`, only run tests for that filter name. +/// - Returns `VerifyResults` with all outcomes and filters that have no tests. +pub fn find_matching_filter(command: &str) -> Option<&'static CompiledFilter> { + if std::env::var("RTK_TOML_DEBUG").is_ok() { + eprintln!( + "[rtk:toml] looking up filter for: {:?} ({} filters loaded)", + command, + REGISTRY.filters.len() + ); + } + let result = find_filter_in(command, ®ISTRY.filters); + if std::env::var("RTK_TOML_DEBUG").is_ok() { + match result { + Some(f) => eprintln!("[rtk:toml] matched filter: '{}'", f.name), + None => eprintln!("[rtk:toml] no filter matched — passthrough"), + } + } + result +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // Helper: build a CompiledFilter from inline TOML for tests. + // Never touches the lazy registry. + fn make_filters(toml: &str) -> Vec { + TomlFilterRegistry::parse_and_compile(toml, "test").expect("test TOML should be valid") + } + + fn first_filter(toml: &str) -> CompiledFilter { + make_filters(toml) + .into_iter() + .next() + .expect("expected at least one filter") + } + + #[test] + fn command_matches_filter_agrees_with_find_matching_filter() { + for cmd in ["jj log", "jq .", "frobnicate xyz", "cd /tmp"] { + assert_eq!( + command_matches_filter(cmd), + find_matching_filter(cmd).is_some(), + "match-set disagreed with registry for {cmd:?}" + ); + } + } + + #[test] + fn test_loss_tail_configured_unfired_max_fires_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntail_lines = 100\nmax_lines = 2\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc\nd\ne"); + assert!(out.contains("lines truncated")); + assert_ne!(loss, Lossiness::None); + } + + #[test] + fn test_match_patterns_in_honors_schema_version() { + let v1 = "schema_version = 1\n[filters.a]\nmatch_command = \"^aaa\"\n"; + assert_eq!(match_patterns_in(v1), vec!["^aaa".to_string()]); + + let v2 = "schema_version = 2\n[filters.a]\nmatch_command = \"^aaa\"\n"; + assert!(match_patterns_in(v2).is_empty()); + + assert!(match_patterns_in("not valid toml {{{").is_empty()); + } + + #[test] + fn test_filter_parse_error() { + assert!( + filter_parse_error("schema_version = 1\n[filters.a]\nmatch_command = \"^a\"\n") + .is_none() + ); + assert!(filter_parse_error("[filters.a]\nmatch_command = \"^a\"\n").is_some()); + assert!(filter_parse_error("this is { not toml").is_some()); + } + + #[test] + fn test_toml_parse_tolerates_utf8_bom() { + // Hand-edited filter files on Windows often carry a BOM. The toml + // crate tolerates it natively — this pin is why the TOML file-read + // sites (config.rs, toml_filter.rs) deliberately skip + // strip_leading_bom; if a crate upgrade regresses this, add it there. + let bom = "\u{feff}schema_version = 1\n[filters.a]\nmatch_command = \"^a\"\n"; + assert!(filter_parse_error(bom).is_none()); + assert_eq!(match_patterns_in(bom), vec!["^a".to_string()]); + } + + #[test] + fn test_is_rtk_reserved_command() { + assert!(is_rtk_reserved_command("git")); + assert!(is_rtk_reserved_command("cargo")); + assert!(is_rtk_reserved_command("json")); + assert!(is_rtk_reserved_command("rewrite")); + assert!(!is_rtk_reserved_command("jj")); + assert!(!is_rtk_reserved_command("jq")); + assert!(!is_rtk_reserved_command("just")); + assert!(!is_rtk_reserved_command("frobnicate")); + } + + fn loss_of(toml: &str, input: &str) -> Lossiness { + apply_filter_with_info(&first_filter(toml), input).1 + } + + #[test] + fn test_loss_head_lines_is_tail() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc\nd\ne"); + assert!(out.starts_with("a\nb\n")); + match loss { + Lossiness::Tail { + tee_payload, + tail_offset, + } => { + assert_eq!(tail_offset, 3); + let recovered: Vec<&str> = tee_payload.lines().skip(tail_offset - 1).collect(); + assert_eq!(recovered, vec!["c", "d", "e"]); + } + other => panic!("expected Tail, got {:?}", other), + } + } + + #[test] + fn test_loss_max_lines_is_tail() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nmax_lines = 2\n"; + match loss_of(toml, "a\nb\nc\nd\ne") { + Lossiness::Tail { + tee_payload, + tail_offset, + } => { + assert_eq!(tail_offset, 3); + let recovered: Vec<&str> = tee_payload.lines().skip(tail_offset - 1).collect(); + assert_eq!(recovered, vec!["c", "d", "e"]); + } + other => panic!("expected Tail, got {:?}", other), + } + } + + #[test] + fn test_loss_tail_lines_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntail_lines = 2\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_head_then_max_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\nmax_lines = 2\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_head_plus_tail_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 1\ntail_lines = 1\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_truncate_lines_at_is_whole() { + let toml = + "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntruncate_lines_at = 3\n"; + assert_eq!(loss_of(toml, "abcdefgh\nshort"), Lossiness::Whole); + } + + #[test] + fn test_loss_match_output_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\n[[filters.f.match_output]]\npattern = \"ok\"\nmessage = \"all good\"\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "everything ok here\nmore"); + assert_eq!(out, "all good"); + assert_eq!(loss, Lossiness::Whole); + } + + #[test] + fn test_loss_strip_only_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nstrip_lines_matching = [\"^noise\"]\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "keep\nnoise line\nkeep2"); + assert_eq!(out, "keep\nkeep2"); + assert_eq!(loss, Lossiness::None); + } + + #[test] + fn test_loss_on_empty_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nstrip_lines_matching = [\".\"]\non_empty = \"nothing\"\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc"); + assert_eq!(out, "nothing"); + assert_eq!(loss, Lossiness::None); + } + + #[test] + fn test_loss_no_truncation_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 10\n"; + assert_eq!(loss_of(toml, "a\nb\nc"), Lossiness::None); + } + + #[test] + fn test_apply_filter_wrapper_matches_with_info() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\n"; + let f = first_filter(toml); + let input = "a\nb\nc\nd"; + assert_eq!(apply_filter(&f, input), apply_filter_with_info(&f, input).0); + } + + // --- Pipeline primitives (existing) --- + + #[test] + fn test_strip_ansi_removes_codes() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +"#, + ); + let out = apply_filter(&f, "\x1b[31mError\x1b[0m\nnormal"); + assert_eq!(out, "Error\nnormal"); + } + + #[test] + fn test_strip_lines_matching_basic() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["^noise", "^verbose"] +"#, + ); + let input = "noise line\nkeep this\nverbose stuff\nalso keep"; + let out = apply_filter(&f, input); + assert_eq!(out, "keep this\nalso keep"); + } + + #[test] + fn test_keep_lines_matching_basic() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +keep_lines_matching = ["^PASS", "^FAIL"] +"#, + ); + let input = "PASS test_a\nsome noise\nFAIL test_b\nmore noise"; + let out = apply_filter(&f, input); + assert_eq!(out, "PASS test_a\nFAIL test_b"); + } + + #[test] + fn test_truncate_lines_at_unicode_safe() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +truncate_lines_at = 5 +"#, + ); + // utils::truncate(s, 5) takes 2 chars + "..." when len > 5 + // "hello" = 5 chars exactly, stays unchanged + // "日本語xyz" = 6 chars, truncated to "日本..." (take 2 + "...") + let out = apply_filter(&f, "hello\n日本語xyz"); + assert_eq!(out, "hello\n日本..."); + } + + #[test] + fn test_head_lines() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +head_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + assert!(out.starts_with("a\nb\n")); + assert!(out.contains("3 lines omitted")); + } + + #[test] + fn test_tail_lines() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +tail_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + assert!(out.contains("3 lines omitted")); + assert!(out.ends_with("d\ne")); + } + + #[test] + fn test_head_and_tail_combined() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +head_lines = 2 +tail_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne\nf"; + let out = apply_filter(&f, input); + assert!(out.starts_with("a\nb\n")); + assert!(out.contains("2 lines omitted")); + assert!(out.ends_with("e\nf")); + } + + #[test] + fn test_max_lines_counts_omit_message() { + // max_lines applied AFTER head — the "omitted" message counts as a line + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +max_lines = 3 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + let line_count = out.lines().count(); + // 3 content lines + 1 truncated message = 4 lines output + assert_eq!(line_count, 4); + assert!(out.contains("lines truncated")); + } + + #[test] + fn test_on_empty_when_all_filtered() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = [".*"] +on_empty = "nothing left" +"#, + ); + let out = apply_filter(&f, "line1\nline2"); + assert_eq!(out, "nothing left"); + } + + #[test] + fn test_on_empty_not_triggered_when_output_remains() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +keep_lines_matching = ["keep"] +on_empty = "nothing left" +"#, + ); + let out = apply_filter(&f, "keep this\nnoise"); + assert_eq!(out, "keep this"); + } + + #[test] + fn test_full_pipeline_order() { + // Verify all 8 stages fire in order on a single input + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +strip_lines_matching = ["^noise"] +truncate_lines_at = 10 +head_lines = 3 +max_lines = 4 +on_empty = "empty" +"#, + ); + let input = + "\x1b[31mred line\x1b[0m\nnoise skip\nkeep one\nkeep two\nkeep three\nkeep four"; + let out = apply_filter(&f, input); + // After strip_ansi: "red line", strip noise: removed, head 3 from remaining 4 lines + assert!(out.contains("red line")); + assert!(!out.contains("noise skip")); + assert!(out.contains("lines omitted") || out.contains("lines truncated")); + } + + // --- Validation --- + + #[test] + fn test_mutual_exclusion_strip_keep_errors() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["a"] +keep_lines_matching = ["b"] +"#, + ); + // The filter should be skipped (warning emitted), resulting in empty list + assert!(result.is_empty()); + } + + #[test] + fn test_invalid_regex_returns_err() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "[" +"#, + ); + assert!(result.is_empty()); + } + + #[test] + fn test_schema_version_mismatch_errors() { + let result = TomlFilterRegistry::parse_and_compile( + r#"schema_version = 99 +[filters.f] +match_command = "^cmd" +"#, + "test", + ); + assert!(result.is_err()); + } + + #[test] + fn test_unknown_field_typo_errors() { + // deny_unknown_fields should catch this + let result = TomlFilterRegistry::parse_and_compile( + r#"schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi_typo = true +"#, + "test", + ); + assert!(result.is_err()); + } + + #[test] + fn test_empty_filter_passthrough() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +"#, + ); + let input = "line1\nline2\nline3"; + let out = apply_filter(&f, input); + assert_eq!(out, input); + } + + // --- Registry / find --- + + #[test] + fn test_builtin_filters_compile() { + // Compile-time safety: panics if any src/filters/*.toml is broken + let builtin = BUILTIN_TOML; + let result = TomlFilterRegistry::parse_and_compile(builtin, "builtin"); + assert!( + result.is_ok(), + "builtin filters failed to compile: {:?}", + result + ); + assert!(!result.unwrap().is_empty()); + } + + #[test] + fn test_find_filter_matches_terraform() { + let filters = make_filters( + r#" +schema_version = 1 +[filters.terraform-plan] +match_command = "^terraform\\s+plan" +strip_ansi = true +"#, + ); + let found = find_filter_in("terraform plan -out=tfplan", &filters); + assert!(found.is_some()); + assert_eq!(found.unwrap().name, "terraform-plan"); + } + + #[test] + fn test_find_filter_no_match_returns_none() { + let filters = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^terraform" +"#, + ); + let found = find_filter_in("kubectl get pods", &filters); + assert!(found.is_none()); + } + + #[test] + fn test_project_filters_priority_over_builtin() { + // Project filter has same name but different max_lines — project wins + let project = make_filters( + r#" +schema_version = 1 +[filters.make] +match_command = "^make\\b" +max_lines = 999 +"#, + ); + let builtin = make_filters(BUILTIN_TOML); + + // Simulate the registry: project first + let mut all = project; + all.extend(builtin); + + let found = find_filter_in("make all", &all).expect("should match"); + assert_eq!(found.name, "make"); + // The first (project) match has max_lines=999 + assert_eq!(found.max_lines, Some(999)); + } + + // --- Token savings --- + + #[test] + fn test_terraform_savings_above_60pct() { + let filters = make_filters(BUILTIN_TOML); + let filter = find_filter_in("terraform plan", &filters).expect("terraform-plan built-in"); + + // Inline fixture: realistic terraform plan with many Refreshing state lines (noise). + // Real infra refreshes 30+ resources; the plan section is small. + // All Refreshing/lock/blank/unchanged lines are stripped -> >60% savings. + let input = concat!( + "Acquiring state lock. This may take a few moments...\n", + "Refreshing state... [id=vpc-0a1b2c3d]\n", + "Refreshing state... [id=subnet-11111111]\n", + "Refreshing state... [id=subnet-22222222]\n", + "Refreshing state... [id=subnet-33333333]\n", + "Refreshing state... [id=subnet-44444444]\n", + "Refreshing state... [id=igw-aabbccdd]\n", + "Refreshing state... [id=rtb-aabbccdd]\n", + "Refreshing state... [id=rtb-11223344]\n", + "Refreshing state... [id=sg-00112233]\n", + "Refreshing state... [id=sg-44556677]\n", + "Refreshing state... [id=sg-88990011]\n", + "Refreshing state... [id=nacl-00aabbcc]\n", + "Refreshing state... [id=acm-arn:us-east-1:cert/abc]\n", + "Refreshing state... [id=Z1234567890ABC]\n", + "Refreshing state... [id=alb-arn:my-alb]\n", + "Refreshing state... [id=tg-arn:my-tg]\n", + "Refreshing state... [id=db-ABCDEFGHIJKLMNO]\n", + "Refreshing state... [id=rds-cluster:my-aurora]\n", + "Refreshing state... [id=elasticache:my-cluster]\n", + "Refreshing state... [id=lambda:my-api-function]\n", + "Refreshing state... [id=lambda:my-worker]\n", + "Refreshing state... [id=iam-role:my-lambda-role]\n", + "Refreshing state... [id=iam-role:my-ecs-role]\n", + "Refreshing state... [id=s3:::my-app-assets]\n", + "Refreshing state... [id=s3:::my-app-logs]\n", + "Refreshing state... [id=cloudfront:ABCDEFGHIJK]\n", + "Refreshing state... [id=ssm:/my/app/db-url]\n", + "Refreshing state... [id=ssm:/my/app/api-key]\n", + "Refreshing state... [id=secretsmanager:my-secret]\n", + "Releasing state lock. This may take a few moments...\n", + "\n", + "Terraform will perform the following actions:\n", + "\n", + " # aws_instance.web will be created\n", + " + resource \"aws_instance\" \"web\" {\n", + " + ami = \"ami-0c55b159cbfafe1f0\"\n", + " + instance_type = \"t3.micro\"\n", + " }\n", + "\n", + "Plan: 1 to add, 0 to change, 0 to destroy.\n", + ); + let out = apply_filter(filter, input); + let input_words = input.split_whitespace().count(); + let out_words = out.split_whitespace().count(); + let savings = 100.0 - (out_words as f64 / input_words as f64 * 100.0); + assert!( + savings >= 60.0, + "terraform-plan filter: expected >=60% savings, got {:.1}% (in={} out={})", + savings, + input_words, + out_words + ); + } + + #[test] + fn test_make_savings_above_60pct() { + let filters = make_filters(BUILTIN_TOML); + let filter = find_filter_in("make all", &filters).expect("make built-in"); + + let input = r#"make[1]: Entering directory '/home/user/project' +make[2]: Entering directory '/home/user/project/src' +gcc -O2 -Wall -c foo.c -o foo.o + +make[2]: Nothing to be done for 'install'. +make[3]: Entering directory '/home/user/project/src/lib' +ar rcs libfoo.a foo.o bar.o baz.o +make[3]: Leaving directory '/home/user/project/src/lib' +make[2]: Leaving directory '/home/user/project/src' + +make[1]: Leaving directory '/home/user/project' +gcc -O2 -Wall -c bar.c -o bar.o + +gcc -O2 -Wall -c baz.c -o baz.o + +make[1]: Entering directory '/home/user/project/test' +make[2]: Entering directory '/home/user/project/test/unit' +./run_tests --verbose +make[2]: Nothing to be done for 'check'. +make[2]: Leaving directory '/home/user/project/test/unit' +make[1]: Leaving directory '/home/user/project/test' + +ld -o myapp foo.o bar.o baz.o -lfoo + +make[1]: Entering directory '/home/user/project/docs' +doxygen Doxyfile +make[1]: Leaving directory '/home/user/project/docs' +"#; + let out = apply_filter(filter, input); + let input_words = input.split_whitespace().count(); + let out_words = out.split_whitespace().count(); + let savings = 100.0 - (out_words as f64 / input_words as f64 * 100.0); + assert!( + savings >= 60.0, + "make filter: expected >=60% savings, got {:.1}% (in={} out={})", + savings, + input_words, + out_words + ); + } + + #[test] + fn test_spring_boot_match_command_requires_spring_named_jar() { + let filters = make_filters(BUILTIN_TOML); + let spring_boot = filters + .iter() + .find(|f| f.name == "spring-boot") + .expect("spring-boot filter must exist"); + + assert!( + !spring_boot + .match_regex + .is_match("java -jar build/libs/my-other-tool.jar"), + "a non-Spring jar must not activate the spring-boot filter" + ); + + let spring_jar = find_filter_in("java -jar build/libs/my-spring-app.jar", &filters) + .expect("a jar with 'spring' in its filename must still match"); + assert_eq!(spring_jar.name, "spring-boot"); + + let mvn_run = find_filter_in("mvn spring-boot:run", &filters) + .expect("mvn spring-boot:run must still match"); + assert_eq!(mvn_run.name, "spring-boot"); + + let capitalized = find_filter_in("java -jar build/libs/MySpringApp.jar", &filters) + .expect("a jar with 'Spring' capitalized in its filename must still match"); + assert_eq!(capitalized.name, "spring-boot"); + + assert!( + !spring_boot + .match_regex + .is_match("java -jar /opt/spring-cache/other-tool.jar"), + "'spring' appearing only in a directory segment (not the jar filename itself) must not activate the spring-boot filter" + ); + + // Windows paths: '\' must be treated as a path separator too, not swallowed + // into the filename-only match (the exact bug this guard exists to catch). + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\spring-cache\other.jar"), + "'spring' appearing only in a Windows directory segment must not activate the spring-boot filter" + ); + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\dev\spring-workspace\build\other-tool.jar"), + "'spring' appearing only in a nested Windows directory segment must not activate the spring-boot filter" + ); + assert!( + spring_boot + .match_regex + .is_match(r"java -jar C:\dev\my-spring-app.jar"), + "a Windows-path jar with 'spring' in its own filename must still match" + ); + + // argv reaches this regex as one space-joined string, so a path containing spaces + // is indistinguishable from a following argument. Widening the prefix across + // whitespace would let a later 'spring'-named jar argument re-trigger the filter, + // so jars under such a path stay on full passthrough instead. + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\Program Files\app\my-spring-app.jar"), + "a jar under a path containing spaces is deliberately left to full passthrough" + ); + } + + #[test] + fn test_liquibase_match_command_ignores_path_substring() { + let filters = make_filters(BUILTIN_TOML); + let liquibase = filters + .iter() + .find(|f| f.name == "liquibase") + .expect("liquibase filter must exist"); + + assert!( + !liquibase.match_regex.is_match("rm -rf /opt/liquibase"), + "'liquibase' appearing only as a path argument must not activate the liquibase filter" + ); + + let bare = find_filter_in("liquibase status", &filters) + .expect("bare liquibase invocation must still match"); + assert_eq!(bare.name, "liquibase"); + + // Production callers (run_fallback in src/main.rs, strip_absolute_path in + // src/discover/registry.rs) always basename argv[0] before this regex runs, + // so a raw path-qualified string must NOT match on its own — the regex has + // no path-prefix branch to fall back on. + assert!( + !liquibase + .match_regex + .is_match("/usr/local/bin/liquibase update"), + "a raw path-qualified invocation must not match — callers basename argv[0] before matching" + ); + + for wrapped in [ + "timeout 5 /usr/bin/liquibase update", + "nohup /opt/tools/liquibase update", + ] { + assert!( + !liquibase.match_regex.is_match(wrapped), + "a wrapper command must not activate the liquibase filter: {wrapped}" + ); + } + } + + #[test] + fn test_ssh_match_command_excludes_ssh_dash_utilities() { + let filters = make_filters(BUILTIN_TOML); + let ssh = filters + .iter() + .find(|f| f.name == "ssh") + .expect("ssh filter must exist"); + + assert!( + !ssh.match_regex.is_match("ssh-keygen -t ed25519"), + "ssh-keygen must not activate the plain ssh connection filter" + ); + assert!( + !ssh.match_regex.is_match("ssh-add id_ed25519"), + "ssh-add must not activate the plain ssh connection filter" + ); + + let plain = find_filter_in("ssh user@host", &filters) + .expect("plain ssh invocation must still match"); + assert_eq!(plain.name, "ssh"); + + let bare = find_filter_in("ssh", &filters).expect("bare ssh with no args must still match"); + assert_eq!(bare.name, "ssh"); + } + + // --- Edge cases --- + + #[test] + fn test_empty_input() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = [".*"] +"#, + ); + let out = apply_filter(&f, ""); + assert_eq!(out, ""); + } + + #[test] + fn test_unicode_preserved() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["^noise"] +"#, + ); + let out = apply_filter(&f, "日本語テスト\nnoise\n中文内容"); + assert_eq!(out, "日本語テスト\n中文内容"); + } + + // --- match_output tests (PR1) --- + + #[test] + fn test_match_output_basic_short_circuit() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +"#, + ); + let out = apply_filter(&f, "Switched to branch 'main'"); + assert_eq!(out, "ok"); + } + + #[test] + fn test_match_output_second_rule_matches() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "switched" }, + { pattern = "Already on", message = "already" }, +] +"#, + ); + let out = apply_filter(&f, "Already on 'main'"); + assert_eq!(out, "already"); + } + + #[test] + fn test_match_output_no_match_pipeline_continues() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +strip_lines_matching = ["^noise"] +"#, + ); + let out = apply_filter(&f, "noise\nkeep this"); + // No match_output match, pipeline continues and strips noise + assert_eq!(out, "keep this"); + } + + #[test] + fn test_match_output_strip_ansi_before_match() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +"#, + ); + // ANSI stripped before match_output check (stage 1 before stage 3) + let out = apply_filter(&f, "\x1b[32mSwitched to branch\x1b[0m 'main'"); + assert_eq!(out, "ok"); + } + + #[test] + fn test_match_output_no_match_then_on_empty() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched", message = "ok" }, +] +strip_lines_matching = [".*"] +on_empty = "nothing" +"#, + ); + // No match_output match; pipeline strips all lines; on_empty fires + let out = apply_filter(&f, "foo bar baz"); + assert_eq!(out, "nothing"); + } + + #[test] + fn test_match_output_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "[invalid", message = "ok" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- match_output unless tests (PR3) --- + + #[test] + fn test_match_output_unless_blocks_short_circuit_when_errors_present() { + // "total size is" matches, but "error" also matches — unless fires, rule is skipped. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^rsync" +match_output = [ + { pattern = "total size is", message = "ok (synced)", unless = "error|failed" }, +] +"#, + ); + let input = "rsync: [sender] error\ntotal size is 1000 speedup is 3.33\n"; + let out = apply_filter(&f, input); + // Should NOT return "ok (synced)" because "error" matches the unless pattern + assert_ne!( + out.trim(), + "ok (synced)", + "unless should have blocked short-circuit when errors are present" + ); + // The raw lines should pass through (no further strip rules in this filter) + assert!(out.contains("error")); + } + + #[test] + fn test_match_output_unless_allows_short_circuit_when_no_errors() { + // "total size is" matches and "error" does NOT appear — unless does not fire, rule wins. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^rsync" +match_output = [ + { pattern = "total size is", message = "ok (synced)", unless = "error|failed" }, +] +"#, + ); + let input = "file.txt\ntotal size is 98765 speedup is 77.31\n"; + let out = apply_filter(&f, input); + assert_eq!(out.trim(), "ok (synced)"); + } + + #[test] + fn test_match_output_unless_falls_through_to_next_rule() { + // First rule blocked by unless; second rule (no unless) should match. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "success", message = "ok", unless = "error" }, + { pattern = "success", message = "ok with warnings" }, +] +"#, + ); + let input = "success\nerror: something went wrong\n"; + let out = apply_filter(&f, input); + // First rule skipped (unless matched), second rule (no unless) fires + assert_eq!(out.trim(), "ok with warnings"); + } + + #[test] + fn test_match_output_unless_no_field_behaves_like_before() { + // When unless is absent, behaviour is identical to original (no regression). + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Build complete", message = "ok (build complete)" }, +] +"#, + ); + let out = apply_filter(&f, "Build complete!\n"); + assert_eq!(out.trim(), "ok (build complete)"); + } + + #[test] + fn test_match_output_unless_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "success", message = "ok", unless = "[invalid" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- replace tests (PR1) --- + + #[test] + fn test_replace_basic_all_occurrences() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "foo", replacement = "bar" }, +] +"#, + ); + let out = apply_filter(&f, "foo baz foo\nfoo"); + assert_eq!(out, "bar baz bar\nbar"); + } + + #[test] + fn test_replace_chaining_sequential() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "aaa", replacement = "bbb" }, + { pattern = "bbb", replacement = "ccc" }, +] +"#, + ); + // Rule 2 operates on the output of rule 1 + let out = apply_filter(&f, "aaa"); + assert_eq!(out, "ccc"); + } + + #[test] + fn test_replace_backreferences() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "(\\w+):(\\w+)", replacement = "$2:$1" }, +] +"#, + ); + let out = apply_filter(&f, "hello:world"); + assert_eq!(out, "world:hello"); + } + + #[test] + fn test_replace_then_strip_interaction() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "noise", replacement = "DROPPED" }, +] +strip_lines_matching = ["^DROPPED"] +"#, + ); + // replace transforms "noise line" -> "DROPPED line", strip removes it + let out = apply_filter(&f, "noise line\nkeep this"); + assert_eq!(out, "keep this"); + } + + #[test] + fn test_replace_empty_input_noop() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "foo", replacement = "bar" }, +] +"#, + ); + let out = apply_filter(&f, ""); + assert_eq!(out, ""); + } + + #[test] + fn test_replace_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "[invalid", replacement = "bar" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- verify (PR2) --- + + #[test] + fn test_run_filter_tests_passes_on_correct_expected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +strip_lines_matching = ["^make\\[\\d+\\]:"] + +[[tests.make]] +name = "strips entering/leaving lines" +input = """ +make[1]: Entering directory '/home/user' +gcc -O2 foo.c +make[1]: Leaving directory '/home/user' +""" +expected = """ +gcc -O2 foo.c +""" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + assert_eq!(outcomes.len(), 1); + assert!( + outcomes[0].passed, + "test should pass: {:?}", + outcomes[0].actual + ); + } + + #[test] + fn test_run_filter_tests_fails_on_wrong_expected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +strip_lines_matching = ["^make\\[\\d+\\]:"] + +[[tests.make]] +name = "wrong expected" +input = "make[1]: Entering\ngcc foo.c" +expected = "wrong output" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + assert_eq!(outcomes.len(), 1); + assert!(!outcomes[0].passed); + } + + #[test] + fn test_filters_without_tests_detected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + // No tests defined, but filter exists + assert_eq!(outcomes.len(), 0); + assert!(all_names.contains(&"make".to_string())); + assert!(!tested.contains("make")); + } + + // --- Multi-file architecture tests (build.rs) --- + + /// Verify BUILTIN_TOML was generated with the correct schema_version header. + /// build.rs injects it — if the const is somehow stale this fails immediately. + #[test] + fn test_builtin_toml_has_schema_version() { + assert!( + BUILTIN_TOML.contains("schema_version = 1"), + "BUILTIN_TOML must start with 'schema_version = 1' (injected by build.rs)" + ); + } + + /// Verify every expected filter name is present in BUILTIN_TOML. + /// This is the safeguard against accidentally deleting a filter file. + #[test] + fn test_builtin_all_expected_filters_present() { + let filters = make_filters(BUILTIN_TOML); + let names: std::collections::HashSet<&str> = + filters.iter().map(|f| f.name.as_str()).collect(); + + let expected = [ + "ansible-playbook", + "brew-install", + "composer-install", + "df", + "dotnet-build", + "du", + "fail2ban-client", + "gcloud", + "hadolint", + "helm", + "iptables", + "liquibase", + "make", + "markdownlint", + "mix-compile", + "mix-format", + "ping", + "pio-run", + "poetry-install", + "pre-commit", + "ps", + "pulumi-destroy", + "pulumi-preview", + "pulumi-refresh", + "pulumi-stack", + "pulumi-up", + "quarto-render", + "rsync", + "shellcheck", + "shopify-theme", + "sops", + "swift-build", + "systemctl-status", + "terraform-plan", + "tofu-fmt", + "tofu-init", + "tofu-plan", + "tofu-validate", + "trunk-build", + "uv-sync", + "yamllint", + ]; + + for name in &expected { + assert!( + names.contains(name), + "Built-in filter '{}' is missing — was its .toml file deleted from src/filters/?", + name + ); + } + } + + /// Verify the exact count of built-in filters. + /// Fails if a file is added/removed without updating this test. + #[test] + fn test_builtin_filter_count() { + let filters = make_filters(BUILTIN_TOML); + assert_eq!( + filters.len(), + 62, + "Expected exactly 62 built-in filters, got {}. \ + Update this count when adding/removing filters in src/filters/.", + filters.len() + ); + } + + /// Every built-in filter must be anchored on every top-level branch: the + /// match runs against the whole command line, so an unanchored branch + /// activates the filter on a path component or wrapper argument. + #[test] + fn test_builtin_all_filters_match_command_anchored() { + // Read the raw patterns rather than the compiled registry: both loaders + // drop unanchored filters, so a compiled view cannot observe one. + let file: TomlFilterFile = + toml::from_str(BUILTIN_TOML).expect("built-in filters should parse"); + assert!(!file.filters.is_empty()); + for (name, def) in &file.filters { + assert!( + is_fully_anchored(&def.match_command), + "Filter '{}' has match_command '{}' with a top-level branch that does not start with '^'", + name, + def.match_command + ); + } + + assert!(is_fully_anchored(r"^liquibase(?:\s|$)")); + assert!(is_fully_anchored(r"^(liquibase\b|/liquibase\b)")); + assert!(is_fully_anchored(r"^pnpm\b|^npm\b")); + assert!(is_fully_anchored(r"(?i)^liquibase\b")); + assert!(!is_fully_anchored(r"^liquibase\b|/liquibase\b")); + assert!(!is_fully_anchored(r"(?:^|/)liquibase(?:\s|$)")); + } + + /// The invariant above covers `BUILTIN_TOML`; project-local and global + /// filters are user-authored, so both loaders drop unanchored patterns + /// rather than letting them match mid-command. + #[test] + fn test_unanchored_user_filter_is_rejected_by_both_loaders() { + let unanchored = + "schema_version = 1\n[filters.mytool]\nmatch_command = \"(?:^|/)mytool\\\\b\"\n"; + assert!(match_patterns_in(unanchored).is_empty()); + assert!( + TomlFilterRegistry::parse_and_compile(unanchored, "test") + .expect("schema is valid") + .is_empty() + ); + + let anchored = "schema_version = 1\n[filters.mytool]\nmatch_command = \"^mytool\\\\b\"\n"; + assert_eq!(match_patterns_in(anchored), vec!["^mytool\\b".to_string()]); + assert_eq!( + TomlFilterRegistry::parse_and_compile(anchored, "test") + .expect("schema is valid") + .len(), + 1 + ); + } + + /// Verify that every built-in filter has at least one inline test. + /// Prevents shipping filters with zero test coverage. + #[test] + fn test_builtin_all_filters_have_inline_tests() { + let mut all_names: Vec = Vec::new(); + let mut tested: std::collections::HashSet = std::collections::HashSet::new(); + let mut outcomes = Vec::new(); + collect_test_outcomes( + BUILTIN_TOML, + None, + &mut outcomes, + &mut all_names, + &mut tested, + ); + + let untested: Vec<&str> = all_names + .iter() + .filter(|name| !tested.contains(name.as_str())) + .map(|s| s.as_str()) + .collect(); + + assert!( + untested.is_empty(), + "The following built-in filters have no inline tests: {:?}\n\ + Add [[tests.]] entries to the corresponding src/filters/.toml file.", + untested + ); + } + + /// Verify that adding a new filter entry to any TOML content makes it + /// immediately discoverable via find_filter_in — simulating how a new + /// src/filters/my-tool.toml would work after cargo build. + #[test] + fn test_new_filter_discoverable_after_concat() { + // Simulate build.rs: concat BUILTIN_TOML with a brand-new filter block + let new_filter = r#" +[filters.my-new-tool] +description = "Compact my-new-tool output" +match_command = "^my-new-tool\\b" +strip_lines_matching = ["^\\s*$"] +max_lines = 30 +on_empty = "my-new-tool: ok" + +[[tests.my-new-tool]] +name = "strips blank lines" +input = "output line 1\n\noutput line 2" +expected = "output line 1\noutput line 2" +"#; + let combined = format!("{}\n\n{}", BUILTIN_TOML, new_filter); + let filters = make_filters(&combined); + + // All 62 existing filters still present + 1 new = 63 + assert_eq!( + filters.len(), + 63, + "Expected 63 filters after concat (62 built-in + 1 new)" + ); + + // New filter is discoverable + let found = find_filter_in("my-new-tool --verbose", &filters); + assert!( + found.is_some(), + "Newly added filter must be discoverable via find_filter_in" + ); + assert_eq!(found.unwrap().name, "my-new-tool"); + } +} + +pub fn registry() -> &'static TomlFilterRegistry { + ®ISTRY +} + +fn collect_test_outcomes( + content: &str, + filter_name_opt: Option<&str>, + outcomes: &mut Vec, + all_filter_names: &mut Vec, + tested_filter_names: &mut std::collections::HashSet, +) { + let file: TomlFilterFile = match toml::from_str(content) { + Ok(f) => f, + Err(e) => { + eprintln!("[rtk] warning: TOML parse error during verify: {}", e); + return; + } + }; + + // Compile all filters and track their names + let mut compiled_filters: BTreeMap = BTreeMap::new(); + for (name, def) in file.filters { + all_filter_names.push(name.clone()); + match compile_filter(name.clone(), def) { + Ok(f) => { + compiled_filters.insert(name, f); + } + Err(e) => eprintln!("[rtk] warning: filter '{}' compilation error: {}", name, e), + } + } + + // Run tests + for (filter_name, tests) in file.tests { + if let Some(name) = filter_name_opt + && filter_name != name + { + continue; + } + + tested_filter_names.insert(filter_name.clone()); + + let compiled = match compiled_filters.get(&filter_name) { + Some(f) => f, + None => { + eprintln!( + "[rtk] warning: [[tests.{}]] references unknown filter", + filter_name + ); + continue; + } + }; + + for test in tests { + let actual = apply_filter(compiled, &test.input); + // Trim trailing newlines: TOML multiline strings end with a newline + let actual_cmp = actual.trim_end_matches('\n').to_string(); + let expected_cmp = test.expected.trim_end_matches('\n').to_string(); + outcomes.push(TestOutcome { + filter_name: filter_name.clone(), + test_name: test.name, + passed: actual_cmp == expected_cmp, + actual: actual_cmp, + expected: expected_cmp, + }); + } + } +} diff --git a/crates/sparkplane-rtk/src/core/truncate.rs b/crates/sparkplane-rtk/src/core/truncate.rs new file mode 100644 index 0000000..58285f6 --- /dev/null +++ b/crates/sparkplane-rtk/src/core/truncate.rs @@ -0,0 +1,60 @@ +//! Global truncation caps shared by every filter. See `src/core/README.md` +//! ("Truncation Caps") for the cap classes, config policy, and deviation rules. + +/// Errors: most actionable, shown the most. +pub const CAP_ERRORS: usize = 20; +/// Warnings and test failures: lower signal density than errors. +pub const CAP_WARNINGS: usize = 10; +/// Flat lists (PRs, services, packages): one line per item. +pub const CAP_LIST: usize = 20; +/// Inventories (`pip list`, `docker images`): exhaustive lookups. +pub const CAP_INVENTORY: usize = 50; + +/// A cap reduced for a verbose data class. Falls back to `cap` when `by >= cap` +/// so a deviation can never empty the list; `0` stays `0`. `const fn`, underflow-safe. +pub const fn reduced(cap: usize, by: usize) -> usize { + if by < cap { cap - by } else { cap } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reduced_preserves_current_values() { + assert_eq!(reduced(CAP_WARNINGS, 5), 5); + assert_eq!(reduced(CAP_LIST, 5), 15); + } + + #[test] + fn reduced_falls_back_to_cap_when_offset_too_large() { + assert_eq!(reduced(4, 5), 4); + assert_eq!(reduced(5, 5), 5); + } + + #[test] + fn reduced_honors_zero_cap() { + assert_eq!(reduced(0, 5), 0); + } + + // Sweep every plausible (cap, by) a future config could produce and assert + // the invariants that make caps safe: the result never wraps past `cap`, and + // the offset never empties a non-zero cap. `usize::MAX` covers a wraparound bug. + #[test] + fn reduced_is_underflow_safe_across_all_inputs() { + for cap in 0..=64usize { + for by in [0usize, 1, 5, 10, 64, usize::MAX] { + let r = reduced(cap, by); + assert!(r <= cap, "reduced({cap}, {by}) = {r} exceeds cap (wrapped)"); + if cap == 0 { + assert_eq!(r, 0, "zero cap must stay zero"); + } else { + assert!(r >= 1, "reduced({cap}, {by}) = {r} emptied a non-zero cap"); + } + if by < cap { + assert_eq!(r, cap - by, "exact deviation must be preserved"); + } + } + } + } +} diff --git a/crates/sparkplane-rtk/src/core/utils.rs b/crates/sparkplane-rtk/src/core/utils.rs new file mode 100644 index 0000000..f2e89dd --- /dev/null +++ b/crates/sparkplane-rtk/src/core/utils.rs @@ -0,0 +1,26 @@ +use regex::Regex; +use std::sync::LazyLock; +pub fn truncate(s: &str, max_len: usize) -> String { + let char_count = s.chars().count(); + if char_count <= max_len { + s.to_string() + } else if max_len < 3 { + // If max_len is too small, just return "..." + "...".to_string() + } else { + format!("{}...", s.chars().take(max_len - 3).collect::()) + } +} +pub fn strip_ansi(text: &str) -> String { + static ANSI_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\x1b\[[0-9;]*[a-zA-Z]").unwrap()); + + ANSI_RE.replace_all(text, "").to_string() +} +pub fn join_with_overflow(items: &[String], total: usize, max: usize, label: &str) -> String { + let mut out = items.join("\n"); + if total > max { + out.push_str(&format!("\n… +{} more {}", total - max, label)); + } + out +} diff --git a/crates/sparkplane-rtk/src/git.rs b/crates/sparkplane-rtk/src/git.rs new file mode 100644 index 0000000..1ee8b9d --- /dev/null +++ b/crates/sparkplane-rtk/src/git.rs @@ -0,0 +1,35 @@ +pub fn filter_status_with_args(output: &str) -> String { + let mut result = Vec::new(); + + for line in output.lines() { + let trimmed = line.trim(); + + // Skip empty lines + if trimmed.is_empty() { + continue; + } + + // Skip git hints - can appear at start or within line + if trimmed.starts_with("(use \"git") + || trimmed.starts_with("(create/copy files") + || trimmed.contains("(use \"git add") + || trimmed.contains("(use \"git restore") + { + continue; + } + + // Special case: clean working tree + if trimmed.contains("nothing to commit") && trimmed.contains("working tree clean") { + result.push(trimmed.to_string()); + break; + } + + result.push(line.to_string()); + } + + if result.is_empty() { + "ok".to_string() + } else { + result.join("\n") + } +} diff --git a/crates/sparkplane-rtk/src/lib.rs b/crates/sparkplane-rtk/src/lib.rs new file mode 100644 index 0000000..66e42c5 --- /dev/null +++ b/crates/sparkplane-rtk/src/lib.rs @@ -0,0 +1,132 @@ +//! Embedded RTK output filters. Command execution and metrics belong to Sparkplane. +// Vendored modules retain upstream style and public helpers for future filter imports. +#[allow(dead_code, clippy::all)] +mod cargo; +#[allow(dead_code, clippy::all)] +mod git; +#[allow(dead_code, clippy::all)] +mod pytest; +#[allow(dead_code, clippy::all)] +mod core { + pub mod constants; + pub mod guard; + pub mod stream; + pub mod toml_filter; + pub mod truncate; + pub mod utils; + pub mod tracking { + pub fn estimate_tokens(value: &str) -> usize { + value.len().div_ceil(4) + } + } + pub mod tee { + pub fn force_tee_hint(_: &str, _: &str) -> Option { + None + } + pub fn force_tee_tail_hint(_: &str, _: &str, _: usize) -> Option { + None + } + } +} + +pub const REVISION: &str = "6d4b77eadee1c66dc1f68466ad77e96d1b6e4989"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Filter { + GitStatus, + Cargo(&'static str), + Pytest, + Declarative(String), +} + +/// Conservative shell-only profile. Explicit output formats must stay byte exact. +pub fn select(args: &[String]) -> Option { + if args.iter().any(|a| { + a.starts_with('/') + || a.contains('=') + || [ + "--json", + "--porcelain", + "--format", + "--output", + "--raw", + "--null", + "-z", + "--message-format", + "--xml", + "--csv", + "--help", + "--version", + ] + .iter() + .any(|flag| a == flag || a.starts_with(&format!("{flag}="))) + }) { + return None; + } + match (args.first()?.as_str(), args.get(1).map(String::as_str)) { + ("git", Some("status")) => Some(Filter::GitStatus), + ("cargo", Some("test")) => Some(Filter::Cargo("test")), + ("cargo", Some("build")) => Some(Filter::Cargo("build")), + ("cargo", Some("check")) => Some(Filter::Cargo("check")), + ("cargo", Some("clippy")) => Some(Filter::Cargo("clippy")), + ("pytest", _) => Some(Filter::Pytest), + ("jq", _) => None, + _ => { + let command = args.join(" "); + core::toml_filter::find_matching_filter(&command) + .map(|f| Filter::Declarative(f.name.clone())) + } + } +} + +/// No truncation of unsuccessful executions: retain every diagnostic. +pub fn filter(profile: &Filter, output: &[u8], exit_code: i32) -> Vec { + let Ok(raw) = std::str::from_utf8(output) else { + return output.to_vec(); + }; + if exit_code != 0 || output.contains(&0) { + return output.to_vec(); + } + let compact = match profile { + Filter::GitStatus => git::filter_status_with_args(raw), + Filter::Cargo("test") => cargo::filter_cargo_test(raw), + Filter::Cargo(label) => cargo::filter_cargo_build_labeled(raw, label, exit_code), + Filter::Pytest => pytest::filter_pytest_output(raw), + Filter::Declarative(name) => { + let registry = core::toml_filter::registry(); + let Some(filter) = registry.filters.iter().find(|f| &f.name == name) else { + return output.to_vec(); + }; + core::toml_filter::apply_filter(filter, raw) + } + }; + let compact = if compact.ends_with('\n') { + compact + } else { + format!("{compact}\n") + }; + if compact.len() >= output.len() { + output.to_vec() + } else { + compact.into_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn cargo_success_compresses_and_failure_is_exact() { + let raw = b"running 2 tests\ntest first ... ok\ntest second ... ok\ntest result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s\n"; + let compact = filter(&Filter::Cargo("test"), raw, 0); + assert!(compact.len() < raw.len()); + assert!(String::from_utf8(compact).unwrap().contains("2 passed")); + assert_eq!(filter(&Filter::Cargo("test"), raw, 1), raw); + } + #[test] + fn builtins_compile_and_binary_is_exact() { + let registry = core::toml_filter::registry(); + assert!(registry.filters.len() > 50); + assert_eq!(filter(&Filter::GitStatus, b"\xff\x00", 0), b"\xff\x00"); + } +} diff --git a/crates/sparkplane-rtk/src/pytest.rs b/crates/sparkplane-rtk/src/pytest.rs new file mode 100644 index 0000000..b7e0e0d --- /dev/null +++ b/crates/sparkplane-rtk/src/pytest.rs @@ -0,0 +1,290 @@ +const MAX_XFAIL: usize = CAP_WARNINGS; +const MAX_PYTEST_FAILURES: usize = CAP_WARNINGS; +use crate::core::truncate::CAP_WARNINGS; +use crate::core::utils::truncate; +const PYTEST_NO_TESTS: &str = "Pytest: No tests collected"; + +pub(crate) fn filter_pytest_output(output: &str) -> String { + let mut state = ParseState::Header; + let mut test_files: Vec = Vec::new(); + let mut failures: Vec = Vec::new(); + let mut current_failure: Vec = Vec::new(); + let mut xfail_lines: Vec = Vec::new(); + let mut summary_line = String::new(); + + for line in output.lines() { + let trimmed = line.trim(); + + // State transitions + if trimmed.starts_with("===") && trimmed.contains("test session starts") { + state = ParseState::Header; + continue; + } else if trimmed.starts_with("===") && trimmed.contains("FAILURES") { + state = ParseState::Failures; + continue; + } else if trimmed.starts_with("===") && trimmed.contains("short test summary") { + state = ParseState::Summary; + // Save current failure if any + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } + continue; + } else if trimmed.starts_with("===") + && (trimmed.contains("passed") + || trimmed.contains("failed") + || trimmed.contains("skipped")) + { + summary_line = trimmed.to_string(); + continue; + // quiet mode (-q): bare summary without === wrapper, e.g. "5 failed, 1698 passed, 2 skipped in 108.89s" + } else if summary_line.is_empty() + && !trimmed.starts_with("===") + && !trimmed.starts_with("FAILED") + && !trimmed.starts_with("ERROR") + && (trimmed.contains(" passed") + || trimmed.contains(" failed") + || trimmed.contains(" skipped")) + && trimmed.contains(" in ") + { + summary_line = trimmed.to_string(); + continue; + } + + // Process based on state + match state { + ParseState::Header => { + if trimmed.starts_with("collected") { + state = ParseState::TestProgress; + } + } + ParseState::TestProgress => { + // Lines like "tests/test_foo.py .... [ 40%]" + if !trimmed.is_empty() + && !trimmed.starts_with("===") + && (trimmed.contains(".py") || trimmed.contains("%]")) + { + test_files.push(trimmed.to_string()); + } + } + ParseState::Failures => { + // Collect failure details + if trimmed.starts_with("___") { + // New failure section + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } + current_failure.push(trimmed.to_string()); + } else if !trimmed.is_empty() && !trimmed.starts_with("===") { + current_failure.push(trimmed.to_string()); + } + } + ParseState::Summary => { + // FAILED test lines + if trimmed.starts_with("FAILED") || trimmed.starts_with("ERROR") { + failures.push(trimmed.to_string()); + } else if trimmed.starts_with("XFAIL") || trimmed.starts_with("XPASS") { + xfail_lines.push(trimmed.to_string()); + } + } + } + } + + // Save last failure if any + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + } + + // Build compact output + build_pytest_summary(&summary_line, &test_files, &failures, &xfail_lines) +} + +#[derive(Default)] +struct PytestCounts { + passed: usize, + failed: usize, + skipped: usize, + xfailed: usize, + xpassed: usize, + duration: Option, +} + +fn build_pytest_summary( + summary: &str, + _test_files: &[String], + failures: &[String], + xfail_lines: &[String], +) -> String { + let counts = parse_summary_line(summary); + let PytestCounts { + passed, + failed, + skipped, + xfailed, + xpassed, + duration, + } = counts; + + if passed == 0 && failed == 0 && skipped == 0 && xfailed == 0 && xpassed == 0 { + return PYTEST_NO_TESTS.to_string(); + } + + let extras_present = skipped > 0 || xfailed > 0 || xpassed > 0 || !xfail_lines.is_empty(); + let duration_suffix = duration + .as_deref() + .map(|value| format!(" in {value}")) + .unwrap_or_default(); + + if failed == 0 && passed > 0 && !extras_present { + return format!("Pytest: {passed} passed{duration_suffix}"); + } + + let mut result = String::new(); + result.push_str(&format!("Pytest: {} passed, {} failed", passed, failed)); + if skipped > 0 { + result.push_str(&format!(", {} skipped", skipped)); + } + if xfailed > 0 { + result.push_str(&format!(", {} xfailed", xfailed)); + } + if xpassed > 0 { + result.push_str(&format!(", {} xpassed", xpassed)); + } + result.push_str(&duration_suffix); + result.push('\n'); + + // Surface xfail/xpass entries (with their reasons) — XPASS in particular + // signals that something expected-to-fail now passes. + if !xfail_lines.is_empty() { + result.push_str("\nExpected-failure outcomes:\n"); + for line in xfail_lines.iter().take(MAX_XFAIL) { + result.push_str(&format!(" {}\n", truncate(line, 120))); + } + if xfail_lines.len() > MAX_XFAIL { + result.push_str(&format!(" … +{} more\n", xfail_lines.len() - MAX_XFAIL)); + let all_xfail = xfail_lines.join("\n"); + if let Some(hint) = + crate::core::tee::force_tee_tail_hint(&all_xfail, "pytest-xfail", MAX_XFAIL + 1) + { + result.push_str(&format!(" {}\n", hint)); + } + } + } + + if failures.is_empty() { + return result.trim().to_string(); + } + + // Show failures (limit to key information) + result.push_str("\nFailures:\n"); + + for (i, failure) in failures.iter().take(MAX_PYTEST_FAILURES).enumerate() { + // Extract test name and key error info + let lines: Vec<&str> = failure.lines().collect(); + + // First line is usually test name (after ___) + if let Some(first_line) = lines.first() { + if first_line.starts_with("___") { + // Extract test name between ___ + let test_name = first_line.trim_matches('_').trim(); + result.push_str(&format!("{}. [FAIL] {}\n", i + 1, test_name)); + } else if first_line.starts_with("FAILED") { + // Summary format: "FAILED tests/test_foo.py::test_bar - AssertionError" + let parts: Vec<&str> = first_line.split(" - ").collect(); + if let Some(test_path) = parts.first() { + let test_name = test_path.trim_start_matches("FAILED "); + result.push_str(&format!("{}. [FAIL] {}\n", i + 1, test_name)); + } + if parts.len() > 1 { + result.push_str(&format!(" {}\n", truncate(parts[1], 100))); + } + continue; + } + } + + // Show relevant error lines (assertions, errors, file locations) + let mut relevant_lines = 0; + for line in &lines[1..] { + let line_lower = line.to_lowercase(); + let is_relevant = line.trim().starts_with('>') + || line.trim().starts_with('E') + || line_lower.contains("assert") + || line_lower.contains("error") + || line.contains(".py:"); + + if is_relevant && relevant_lines < 3 { + result.push_str(&format!(" {}\n", truncate(line, 100))); + relevant_lines += 1; + } + } + + if i < failures.len() - 1 { + result.push('\n'); + } + } + + if failures.len() > MAX_PYTEST_FAILURES { + result.push_str(&format!( + "\n… +{} more failures\n", + failures.len() - MAX_PYTEST_FAILURES + )); + let all_failures = failures.join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_failures, "pytest-failures") { + result.push_str(&format!(" {}\n", hint)); + } + } + + result.trim().to_string() +} + +fn parse_summary_line(summary: &str) -> PytestCounts { + let mut counts = PytestCounts { + duration: parse_duration(summary), + ..PytestCounts::default() + }; + + // Parse lines like "=== 4 passed, 1 failed, 2 xfailed, 1 xpassed in 0.50s ===" + for part in summary.split(',') { + let words: Vec<&str> = part.split_whitespace().collect(); + for (i, word) in words.iter().enumerate() { + if i == 0 { + continue; + } + let Ok(n) = words[i - 1].parse::() else { + continue; + }; + // Order matters: "xpassed"/"xfailed" contain "passed"/"failed". + if word.contains("xpassed") { + counts.xpassed = n; + } else if word.contains("xfailed") { + counts.xfailed = n; + } else if word.contains("passed") { + counts.passed = n; + } else if word.contains("failed") { + counts.failed = n; + } else if word.contains("skipped") { + counts.skipped = n; + } + } + } + + counts +} + +fn parse_duration(summary: &str) -> Option { + let unwrapped = summary.trim().trim_matches('=').trim(); + let (_, duration_tail) = unwrapped.rsplit_once(" in ")?; + let duration = duration_tail.split_whitespace().next()?; + let seconds = duration.strip_suffix('s')?; + seconds.parse::().ok()?; + Some(duration.to_string()) +} + +#[derive(Clone, Copy)] +enum ParseState { + Header, + TestProgress, + Failures, + Summary, +} diff --git a/crates/sparkplane-rtk/upstream/Cargo.toml b/crates/sparkplane-rtk/upstream/Cargo.toml new file mode 100644 index 0000000..17dd7d8 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/Cargo.toml @@ -0,0 +1,89 @@ +[package] +name = "rtk" +version = "0.49.0" +edition = "2024" +rust-version = "1.91" +authors = ["Patrick Szymkowiak"] +description = "Rust Token Killer - High-performance CLI proxy to minimize LLM token consumption" +license = "Apache-2.0" +homepage = "https://www.rtk-ai.app" +repository = "https://github.com/rtk-ai/rtk" +readme = "README.md" +keywords = ["cli", "llm", "token", "filter", "productivity"] +categories = ["command-line-utilities", "development-tools"] + +[dependencies] +clap = { version = "4", features = ["derive"] } +anyhow = "1.0" +# Console code page decoding for child-process output. Kept off the +# cfg(windows) target so the mapping and the incremental UTF-8 walk stay +# compiled — and unit-tested — on every platform; only the code page *lookup* +# is Windows-specific. codepage covers the ANSI/DBCS pages encoding_rs +# implements, oem_cp the legacy OEM/DOS pages (437, 850, …) it does not. +encoding_rs = "0.8" +codepage = "0.1" +oem_cp = "2" +ignore = "0.4" +walkdir = "2" +regex = "1" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["preserve_order"] } +colored = "3" +dirs = "5" +rusqlite = { version = "0.31", features = ["bundled"] } +toml = "0.8" +toml_edit = "0.22" +chrono = "0.4" +tempfile = "3" +sha2 = "0.10" +ureq = "2" +getrandom = "0.4" +flate2 = "1.0" +quick-xml = "0.37" +which = "8" + +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.59", features = [ + "Win32_System_Console", + "Win32_Globalization", +] } + +[build-dependencies] +toml = "0.8" + +[dev-dependencies] +# Safe, closure-scoped env mutation for tests. std::env::set_var is unsafe +# in edition 2024 and has no safe std replacement; temp-env restores the +# previous value on completion or panic and serializes callers internally. +temp-env = "0.3" + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 +panic = "abort" +strip = true + +# cargo-deb configuration +[package.metadata.deb] +maintainer = "Patrick Szymkowiak" +copyright = "2024 Patrick Szymkowiak" +license-file = ["LICENSE", "0"] +extended-description = "rtk filters and compresses command outputs before they reach your LLM context, saving 60-90% of tokens." +section = "utility" +priority = "optional" +assets = [ + ["target/release/rtk", "usr/bin/", "755"], +] +# cargo-generate-rpm configuration +[package.metadata.generate-rpm] +assets = [ + { source = "target/release/rtk", dest = "/usr/bin/rtk", mode = "755" }, +] + +[lints.rust] +unsafe_code = "deny" +warnings = "deny" diff --git a/crates/sparkplane-rtk/upstream/LICENSE b/crates/sparkplane-rtk/upstream/LICENSE new file mode 100644 index 0000000..0afaf4b --- /dev/null +++ b/crates/sparkplane-rtk/upstream/LICENSE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 rtk-ai and rtk-ai Labs + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/sparkplane-rtk/upstream/src/cmds/git/git_cmd.rs b/crates/sparkplane-rtk/upstream/src/cmds/git/git_cmd.rs new file mode 100644 index 0000000..88b61e1 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/cmds/git/git_cmd.rs @@ -0,0 +1,6833 @@ +//! Filters git output — log, status, diff, and more — keeping just the essential info. + +use crate::core::arg_tokenizer::{ + self, Attachment, Dialect, Token, TokenKind, ValueSpec, is_digit_run, +}; +use crate::core::args_utils; +use crate::core::guard::never_worse; +use crate::core::runner::{self, RunOptions}; +use crate::core::stream::{ + self, CaptureResult, FilterMode, LineHandler, LineStreamFilter, StdinMode, exec_capture, + exec_capture_stdin, +}; +use crate::core::tracking; +use crate::core::truncate::{CAP_LIST, CAP_WARNINGS}; +use crate::core::utils::{exit_code_from_status, join_with_overflow, resolved_command, strip_ansi}; +use anyhow::{Context, Result}; +use std::ffi::OsString; +use std::process::Command; + +#[derive(Debug, Clone)] +pub enum GitCommand { + Diff, + Log, + Status, + Show, + Add, + Commit, + Checkout, + Push, + Pull, + Branch, + Fetch, + Stash { subcommand: Option }, + Worktree, +} + +/// Create a git Command with global options (e.g. -C, -c, --git-dir, --work-tree) +/// prepended before any subcommand arguments. +fn git_cmd(global_args: &[String]) -> Command { + let mut cmd = resolved_command("git"); + for arg in global_args { + cmd.arg(arg); + } + cmd +} + +/// Create a git Command for internal parsing that must be locale-stable. +/// +/// We only use this for non-user-facing parses where RTK depends on git's +/// English status phrases. User-visible passthrough output keeps the user's +/// locale. +fn git_cmd_c_locale(global_args: &[String]) -> Command { + let mut cmd = git_cmd(global_args); + cmd.env("LC_ALL", "C"); + cmd +} + +fn uses_compact_status_path(args: &[String]) -> bool { + let tokens = arg_tokenizer::tokenize(args); + + if tokens.is_empty() { + return true; + } + + let mut saw_branch = false; + let mut saw_flag = false; + for token in &tokens { + match (token.kind, token.text) { + // A `--` with no pathspec after it selects nothing, so `git status -sb --` is + // `git status -sb`, and a lone `--` is plain `git status`. + (TokenKind::DashDash, _) => {} + (TokenKind::Short, "b") | (TokenKind::Long, "branch") => { + saw_branch = true; + saw_flag = true; + } + (TokenKind::Short, "s") | (TokenKind::Long, "short") => saw_flag = true, + _ => return false, + } + } + + saw_branch || !saw_flag +} + +fn build_status_command(args: &[String], global_args: &[String]) -> Command { + let mut cmd = git_cmd(global_args); + cmd.arg("status"); + if uses_compact_status_path(args) { + cmd.args(["--porcelain", "-b"]); + } else { + cmd.args(args); + } + cmd +} + +pub fn run( + cmd: GitCommand, + args: &[String], + max_lines: Option, + verbose: u8, + global_args: &[String], +) -> Result { + // Centralized here, once, rather than ad hoc per handler. `stash` needs the region put + // back together first: clap carves its subcommand positional out of the same trailing args, + // and restore_double_dash requires the whole region (fed only the remainder it slices one + // token short, which turned `git stash -- -p` into an interactive `stash push -p`). + let (cmd, args) = match cmd { + GitCommand::Stash { subcommand } => { + let mut region: Vec = subcommand.into_iter().collect(); + region.extend_from_slice(args); + let region = args_utils::restore_double_dash(®ion); + let (subcommand, rest) = split_stash_region(®ion); + (GitCommand::Stash { subcommand }, rest) + } + other => (other, args_utils::restore_double_dash(args)), + }; + let args = &args; + match cmd { + GitCommand::Diff => run_diff(args, max_lines, verbose, global_args), + GitCommand::Log => run_log(args, max_lines, verbose, global_args), + GitCommand::Status => run_status(args, verbose, global_args), + GitCommand::Show => run_show(args, max_lines, verbose, global_args), + GitCommand::Add => run_add(args, verbose, global_args), + GitCommand::Commit => run_commit(args, verbose, global_args), + GitCommand::Checkout => run_checkout(args, verbose, global_args), + GitCommand::Push => run_push(args, verbose, global_args), + GitCommand::Pull => run_pull(args, verbose, global_args), + GitCommand::Branch => run_branch(args, verbose, global_args), + GitCommand::Fetch => run_fetch(args, verbose, global_args), + GitCommand::Stash { subcommand } => { + run_stash(subcommand.as_deref(), args, verbose, global_args) + } + GitCommand::Worktree => run_worktree(args, verbose, global_args), + } +} + +/// Splits a restored `stash` region back into subcommand and remainder, the way clap did +/// before the `--` came back: a leading token that is neither a flag nor the boundary. +fn split_stash_region(region: &[String]) -> (Option, Vec) { + let tokens = arg_tokenizer::tokenize(region); + match tokens.first() { + Some(token) if token.kind == TokenKind::Positional => { + (Some(region[0].clone()), region[1..].to_vec()) + } + _ => (None, region.to_vec()), + } +} + +/// `-s`/`--no-patch` ask `git diff` for no body at all, so there is nothing to compact and +/// RTK's own `--stat` header would answer a question the user did not ask. On `git show` the +/// same flags ask for the commit summary, which is exactly what the compact form prints, so +/// this is deliberately not part of [`diff_wants_raw_shape`]. +fn suppresses_diff_body(token: &Token<'_>) -> bool { + matches!( + (token.kind, token.text), + (TokenKind::Long, "no-patch" | "quiet") | (TokenKind::Short, "s") + ) +} + +/// True for a token asking git for patch output: `-p`/`-u`/`--patch`, and the context-width +/// flags that imply a patch (`-U3`, `--unified=3`, `-W`/`--function-context`). +/// Whether the body ends up suppressed once git's own arbitration is applied: `-s`, +/// `--no-patch` and `--quiet` lose to a *later* `-p`/`--patch`/`-U` and win over an earlier +/// one (`git show -s -p` prints the diff, `git show -p -s` does not -- git 2.53). Taking the +/// suppressors order-independently swallowed a patch the user had asked for last. +fn body_is_suppressed(tokens: &[Token<'_>]) -> bool { + // `-s`/`--no-patch` and a patch request resolve by last flag wins: `git show -s -p` prints + // the diff, `git show -p -s` does not (git 2.53). + let mut suppressed = false; + for token in tokens { + if hard_suppresses_diff_body(token) { + suppressed = true; + } else if requests_patch_output(token) { + suppressed = false; + } + } + // `--quiet` does not play that game. In `show` it loses to a patch request from either + // side -- `--quiet -p` and `-p --quiet` both print the diff -- and only suppresses when + // nothing else asked for output. Treating it as a third spelling of `-s` dropped a patch + // the user had asked for. + suppressed || (tokens.iter().any(is_quiet_flag) && !tokens.iter().any(requests_patch_output)) +} + +/// `--quiet`, which is `--exit-code`'s companion rather than a shape flag. Only `run_diff` +/// treats it as one, and there it takes the raw route before any of this is consulted. +fn is_quiet_flag(token: &Token<'_>) -> bool { + (token.kind, token.text) == (TokenKind::Long, "quiet") +} + +/// The suppressors that really do replace the body: `-s` and its long spelling. +fn hard_suppresses_diff_body(token: &Token<'_>) -> bool { + matches!( + (token.kind, token.text), + (TokenKind::Long, "no-patch") | (TokenKind::Short, "s") + ) +} + +fn requests_patch_output(token: &Token<'_>) -> bool { + match token.kind { + TokenKind::Long => matches!( + token.text, + "patch" | "patch-with-stat" | "patch-with-raw" | "unified" | "function-context" + ), + TokenKind::Short => matches!(token.text, "p" | "u" | "U" | "W"), + _ => false, + } +} + +/// `args` with the patch-shape flags removed, so a stat-only header cannot be outranked by +/// them wherever git would have read them. Rebuilt per token rather than per arg: every short +/// flag in a `-xyz` cluster shares one `source_index`, so dropping the whole arg would take +/// its siblings with it -- `-pl 100` lost the `-l` and left `100` behind as a bogus revision. +/// `args` with any `--oneline` removed, by the token's own `source_index` so a pathspec of that +/// name past `--` is left alone. +fn args_without_oneline(args: &[String], tokens: &[Token<'_>]) -> Vec { + let dropped: Vec = tokens + .iter() + .filter(|t| t.kind == TokenKind::Long && t.text == "oneline") + .map(|t| t.source_index) + .collect(); + if dropped.is_empty() { + return args.to_vec(); + } + args.iter() + .enumerate() + .filter(|(index, _)| !dropped.contains(index)) + .map(|(_, arg)| arg.clone()) + .collect() +} + +fn args_without_patch_shape(args: &[String], tokens: &[Token<'_>]) -> Vec { + let mut out: Vec = Vec::with_capacity(args.len()); + for (index, arg) in args.iter().enumerate() { + let owned: Vec<&Token<'_>> = tokens + .iter() + .filter(|t| t.source_index == index && t.kind != TokenKind::Positional) + .collect(); + if owned.is_empty() || !owned.iter().any(|t| requests_patch_output(t)) { + out.push(arg.clone()); + continue; + } + // A cluster keeps whatever letters were not shape flags, with their attached value. + let kept: Vec<&&Token<'_>> = owned + .iter() + .filter(|t| t.kind == TokenKind::Short && !requests_patch_output(t)) + .collect(); + if kept.is_empty() { + continue; + } + let mut rebuilt = String::from("-"); + for token in &kept { + rebuilt.push_str(token.text); + } + if let Some(attached) = kept.last().and_then(|t| t.attached) { + rebuilt.push_str(attached); + } + out.push(rebuilt); + } + out +} + +fn run_diff( + args: &[String], + max_lines: Option, + verbose: u8, + global_args: &[String], +) -> Result { + let timer = tracking::TimedExecution::start(); + + let tokens = tokenize_git_diff_args(args); + let wants_stat = tokens + .iter() + .any(|t| diff_wants_raw_shape(t, &tokens) || suppresses_diff_body(t)); + + // Compact diff is the default RTK behavior; --no-compact is RTK's own pseudo-flag. The + // strip below removes the arg this token came from, so detection and removal can't + // disagree -- re-matching the string dropped a pathspec of the same name past `--`. + // Any spelling counts, attached value or not: `--no-compact` is RTK's own, so git would + // only answer `error: invalid option` if a value form leaked through. + let no_compact: Vec = tokens + .iter() + .filter(|t| t.kind == TokenKind::Long && t.text == "no-compact") + .map(|t| t.source_index) + .collect(); + let wants_compact = no_compact.is_empty() && !emits_word_diff(&tokens); + + if wants_stat || !wants_compact { + // User wants stat or explicitly no compacting - pass through directly + let mut cmd = git_cmd(global_args); + cmd.arg("diff"); + for (index, arg) in args.iter().enumerate() { + if no_compact.contains(&index) { + continue; // RTK flag, not a git flag + } + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git diff")?; + + // A non-zero exit does not mean there was nothing to say: `git diff --check` reports + // every whitespace error on stdout and *then* exits 2. Printed verbatim, since the + // report's payload *is* trailing whitespace and `--stat` has a leading column space. + print!("{}", result.stdout); + + if !result.success() { + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr.trim()); + } + timer.track( + &format!("git diff {}", args.join(" ")), + &format!("rtk git diff {} (passthrough)", args.join(" ")), + &result.stdout, + &result.stdout, + ); + return Ok(result.exit_code); + } + + timer.track( + &format!("git diff {}", args.join(" ")), + &format!("rtk git diff {} (passthrough)", args.join(" ")), + &result.stdout, + &result.stdout, + ); + + return Ok(0); + } + + // Default RTK behavior: stat first, then compacted diff. `--no-patch --stat` forces the + // header to be stat-only whatever the user asked for -- `git diff --stat -p` (or -U3, -W, + // ...) emits the patch too, and RTK then printed it again, compacted, for 2.4x the raw + // output. The flags go before the user's own `--`, where git still reads them as options. + // The user's own command runs first, because only it can give git's verdict on what they + // typed. The stat header below runs with the patch-shape flags stripped, so it answers a + // *different* command: `git diff -Uabc nonexistent-ref` is `error: --unified expects a + // numerical value` (129) to git, and the stripped probe reported `ambiguous argument` (128) + // instead. A probe is decoration; it must never be the thing that reports failure. + let mut diff_cmd = git_cmd(global_args); + diff_cmd.arg("diff"); + for arg in args { + diff_cmd.arg(arg); + } + + let diff_result = exec_capture(&mut diff_cmd).context("Failed to run git diff")?; + + if !diff_result.success() { + if !diff_result.stderr.trim().is_empty() { + eprint!("{}", diff_result.stderr); + } + timer.track( + &format!("git diff {}", args.join(" ")), + &format!("rtk git diff {}", args.join(" ")), + &diff_result.stdout, + &diff_result.stdout, + ); + return Ok(diff_result.exit_code); + } + + // `--no-patch --stat` forces the header to be stat-only whatever the user asked for -- + // `git diff --stat -p` (or -U3, -W, ...) emits the patch too, and RTK then printed it + // again, compacted, for 2.4x the raw output. The flags go before the user's own `--`, + // where git still reads them as options. A failure here costs the header, not the command. + let mut cmd = git_cmd(global_args); + cmd.args(["diff", "--no-patch", "--stat"]); + for arg in args_without_patch_shape(args, &tokens) { + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git diff")?; + + if verbose > 0 { + eprintln!("Git diff summary:"); + } + + let printed = if !diff_result.stdout.is_empty() { + let compacted = compact_diff(&strip_ansi(&diff_result.stdout), max_lines.unwrap_or(500)); + format!("{}\n\nChanges:\n{}", result.stdout.trim(), compacted) + } else { + result.stdout.trim().to_string() + }; + + let raw = format!("{}\n{}", result.stdout, diff_result.stdout); + let shown = never_worse(&raw, &printed); + println!("{}", shown); + + timer.track( + &format!("git diff {}", args.join(" ")), + &format!("rtk git diff {}", args.join(" ")), + &raw, + shown, + ); + + Ok(0) +} + +/// `git show` with RTK's own shape flags in front. `drop_patch_shape` removes the user's +/// patch-enabling flags, which the summary and stat steps need (git takes the last one, so a +/// `-p` anywhere would bring the patch back) but the diff step must not -- `-U5` sets the +/// context width of the patch RTK is about to compact. +/// git takes the last output-format flag, so a `-p` left anywhere in the args -- including +/// after a revision, where RTK's flags cannot be placed -- would re-enable the patch in the +/// summary and stat steps and the result stopped being smaller than raw. +/// True when the user asked `git show` for a commit format of their own, which RTK cannot +/// compact around -- its own `--pretty=format:` would be overridden by theirs. +/// +/// `--oneline` is deliberately absent, unlike `run_log`'s equivalent check: it does not replace +/// the summary with something unparseable, so routing the whole command raw over it cost 3.7x +/// the output on a real commit (116 KB against 31 KB), on the one metric this tool exists for. +/// It does outrank RTK's own `--pretty=format:`, which is passed before the user's args, so the +/// stat step drops it from what it forwards (see `args_without_oneline`); otherwise the commit +/// header that step suppresses comes back and the summary prints twice. +fn show_wants_format(tokens: &[Token<'_>]) -> bool { + tokens + .iter() + .any(|t| t.kind == TokenKind::Long && matches!(t.text, "format" | "pretty")) +} + +fn show_cmd( + global_args: &[String], + args: &[String], + tokens: &[Token<'_>], + rtk_flags: &[&str], + drop_patch_shape: bool, +) -> Command { + let mut cmd = git_cmd(global_args); + cmd.arg("show"); + cmd.args(rtk_flags); + let forwarded = if drop_patch_shape { + args_without_patch_shape(args, tokens) + } else { + args.to_vec() + }; + for arg in forwarded { + cmd.arg(arg); + } + cmd +} + +fn run_show( + args: &[String], + max_lines: Option, + verbose: u8, + global_args: &[String], +) -> Result { + let timer = tracking::TimedExecution::start(); + + // Re-insert `--` when clap's trailing_var_arg consumed it (issue #1215), same as + // run_diff/run_checkout. Without this the pathspec separator never reaches + // `show_positionals`, so `git show -- ` would misread the + // colon'd pathspec as a `:` blob and dump it instead of a commit-diff. + let args = &args_utils::restore_double_dash(args); + + // Pick one of three handlers for `git show`. `show_route` decides the blob case + // FIRST (see its docs): the two branches below are mutually exclusive on `route`, + // so their source order does not affect which one runs. + let positionals = show_positionals(args); + // Authoritative blob decision (belt-and-suspenders). `show_route` is the cheap pure + // pre-filter: does ANY positional look like `rev:path`? Only then do we probe with + // `git cat-file -t` — and we probe EVERY colon positional, not just the first. The + // cluster-aware `show_positionals` already drops option-value operands (`-S 'url:1'`, + // the `-G` value in `-wG a:b HEAD:blob`, …), but probing all candidates means that + // even if the walker ever missed some exotic short-flag cluster, a non-object like + // `a:b` is still rejected by `cat-file` and the REAL blob elsewhere on the line is + // rescued instead of being silently dropped or misrouted through lossy decoding. + // * exactly one blob → window it (the byte-safe path below), + // * more than one → git concatenates the objects with no separator, so the + // hint can't reconstruct them → raw passthrough, + // * zero → ordinary commit-diff / --stat classification. + // Any presence of a blob object takes the byte-safe path: its output carries raw + // blob bytes that the commit-diff path's lossy UTF-8 decode would corrupt (a Latin-1 + // `0xF1` becomes the `U+FFFD` replacement char — verified). + let (route, blob_objects) = match show_route(args) { + ShowRoute::Blob => { + let blobs: Vec<&String> = blob_candidates(args) + .into_iter() + .filter(|c| probe_is_blob(global_args, c)) + .collect(); + if blobs.is_empty() { + (commit_or_stat_route(args), blobs) + } else { + (ShowRoute::Blob, blobs) + } + } + other => (other, Vec::new()), + }; + + if route == ShowRoute::StatOrFormat { + let mut cmd = git_cmd(global_args); + cmd.arg("show"); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git show")?; + // Verbatim, and before the exit check: `git show --check` reports on stdout and then + // exits 2, so returning early on failure threw the whole report away. + print!("{}", result.stdout); + if !result.success() { + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr.trim()); + } + return Ok(result.exit_code); + } + + timer.track( + &format!("git show {}", args.join(" ")), + &format!("rtk git show {} (passthrough)", args.join(" ")), + &result.stdout, + &result.stdout, + ); + + return Ok(0); + } + + if route == ShowRoute::Blob { + let mut cmd = git_cmd(global_args); + cmd.arg("show"); + for arg in args { + cmd.arg(arg); + } + // Capture raw bytes: `git show` of an ISO-8859/Latin-1 file (e.g. Oracle + // PL/SQL `.pck`) is not UTF-8, and we must preserve git's exact bytes for the + // passthrough path below — decoding into a `String` first would lose them. + let result = + crate::core::stream::exec_capture_bytes(&mut cmd).context("Failed to run git show")?; + let label = format!("git show {}", args.join(" ")); + let rtk_label = format!("rtk git show {}", args.join(" ")); + if !result.success() { + eprint!( + "{}", + crate::core::utils::decode_process_output(&result.stderr) + ); + return Ok(result.exit_code); + } + // git can warn on stderr (e.g. CRLF / autocrlf notices) while still exiting 0; + // surface it instead of swallowing it just because the command succeeded. + if !result.stderr.is_empty() { + eprint!( + "{}", + crate::core::utils::decode_process_output(&result.stderr) + ); + } + // Fidelity invariant: "byte-identical unless we successfully windowed." + // + // Windowing shows a head and points at the rest with `git show 'rev:path' | + // tail -n +N`. That recovery reconstructs the file EXACTLY only if the head we + // printed is a BYTE-EXACT prefix of what git wrote — which holds only when the + // content is valid UTF-8 as-is. Any transcode (Latin-1 → UTF-8), BOM rewrite, + // UTF-16 (whose `tail`-sliced bytes are UTF-16 that won't concatenate with a + // UTF-8 head), U+FFFD from lossy decoding, or true binary would make the head + // diverge from git's bytes and silently break recovery — and can even inflate + // the output past what git emitted. + // + // So we window ONLY content that is valid UTF-8 recoverable byte-for-byte AND + // whose flags/pathspec don't perturb the dump (see the `can_window` gate below: + // no `--textconv`/`--filters`/`--ext-diff`, no trailing `-- `, a single + // sole blob object); everything else goes through `emit_raw_bytes_passthrough`, + // byte-identical to a plain `git show`. This makes the recovery hint reconstruct + // exactly whenever we DO window, and scopes windowing to the common text-lockfile + // case the filter exists for. + // + // Larger blobs window regardless of whether stdout is a pipe or a TTY: the whole + // point of the filter is to shrink what the agent reads, and the agent reads + // through a pipe. That mirrors the diff/log filters (which also compact in a + // pipe); a consumer that needs the full content follows the `| tail -n +N` + // recovery hint — the same tradeoff `git log | grep` already makes. + // (`is_terminal()` is not a reliable "human is watching" signal anyway: the hook + // always pipes rtk's stdout — verified `isatty: False` — and CI agents hand rtk + // a pseudo-TTY. Had this branch gated on it, it would have been the first gate to + // drop CONTENT; every other `stdout().is_terminal()` gate in the tree governs + // only PRESENTATION — color, line-buffering, curl formatting.) + let text = match std::str::from_utf8(&result.stdout) { + // Valid UTF-8: the string's bytes ARE git's bytes, so a head prefix + tail + // recovery is byte-exact. Below the budget it passes through unchanged; over + // it, `compact_blob_show` windows it (or declines and returns it whole). + Ok(s) => s, + // Latin-1, UTF-16/BOM, or binary: recovery would not be byte-exact, so pass + // the raw bytes through verbatim, tracked as a passthrough. + Err(_) => { + return emit_raw_bytes_passthrough( + &result.stdout, + &label, + &rtk_label, + &timer, + result.exit_code, + ); + } + }; + // A blob dump is unfiltered file content: cap large text blobs to a byte + // budget with a tail-recovery pointer, mirroring how the commit-diff path + // below caps at `max_lines`. `--max-lines` does not apply here — blob output + // is bounded by bytes, not lines. + // + // Window ONLY when git emits exactly this one blob and nothing that makes the + // `git show : | tail` recovery hint diverge from git's bytes: + // * a single positional that is the sole blob object — git concatenates + // multiple objects with no separator, so cutting the first would silently + // drop the rest, and that concatenation is not the single `rev:path` the + // hint reconstructs; + // * no content-transforming flag (`--textconv`/`--filters`/`--ext-diff` + // rewrite the dump, and the hint omits them — see `has_content_transform_flag`); + // * no trailing `-- ` (also omitted from the hint — see + // `has_trailing_pathspec`). + // Everything else passes the raw bytes through, byte-identical to plain git show. + // Below the budget, `compact_blob_show` returns the text unchanged, so a windowed + // single-object print stays byte-identical to git there too. + let can_window = positionals.len() == 1 + && blob_objects.len() == 1 + && !has_content_transform_flag(args) + && !has_trailing_pathspec(args); + if can_window { + let shown = compact_blob_show(text, blob_objects[0], global_args); + print!("{}", shown); + // Track savings against the bytes git actually wrote (`result.stdout`). + timer.track_bytes(&label, &rtk_label, result.stdout.len(), &shown); + return Ok(0); + } + // Not windowable (multiple concatenated objects, a content-transforming flag, or + // a trailing pathspec): pass through byte-identical. + return emit_raw_bytes_passthrough( + &result.stdout, + &label, + &rtk_label, + &timer, + result.exit_code, + ); + } + + // The compacted commit-diff path from here down. `show_route` above already decided the + // blob and summary cases, so what is left is the one shape RTK renders itself. + let tokens = tokenize_git_diff_args(args); + + // Get raw output for tracking + let mut raw_cmd = git_cmd(global_args); + raw_cmd.arg("show"); + for arg in args { + raw_cmd.arg(arg); + } + let raw_result = exec_capture(&mut raw_cmd).context("Failed to run git show")?; + // git's verdict on the command as the user typed it, before the steps below run it again + // with RTK's own flags -- the stat step strips the patch-shape flags, so it succeeds where + // git refused (`git show -pq HEAD` exits 128 raw) and the compaction looked like success. + if !raw_result.success() { + if !raw_result.stderr.trim().is_empty() { + eprint!("{}", raw_result.stderr); + } + timer.track( + &format!("git show {}", args.join(" ")), + &format!("rtk git show {}", args.join(" ")), + &raw_result.stdout, + &raw_result.stdout, + ); + return Ok(raw_result.exit_code); + } + let raw_output = raw_result.stdout; + + // Step 1: one-line commit summary + let mut summary_cmd = show_cmd( + global_args, + args, + &tokens, + &["--no-patch", "--pretty=format:%h %s (%ar) <%an>"], + true, + ); + let summary_result = exec_capture(&mut summary_cmd).context("Failed to run git show")?; + if !summary_result.success() { + eprintln!("{}", summary_result.stderr); + return Ok(summary_result.exit_code); + } + let mut printed = summary_result.stdout.trim().to_string(); + + if body_is_suppressed(&tokens) { + // `git show -s` is the commit summary and nothing else, so the stat and diff steps + // below would print what the user explicitly suppressed. + let shown = never_worse(&raw_output, &printed); + println!("{}", shown); + timer.track( + &format!("git show {}", args.join(" ")), + &format!("rtk git show {}", args.join(" ")), + &raw_output, + shown, + ); + return Ok(0); + } + + // Step 2: --stat summary + // `--oneline` is dropped, not outranked: RTK's `--pretty=format:` suppresses the commit + // header this step must not repeat, and git resolves the two by last flag wins, so a user + // `--oneline` re-enabled it and the summary printed twice. Appending RTK's flag after the + // user's instead would put it past their `--`, where git reads it as a pathspec. + // Re-tokenized, because dropping `--oneline` shifts every later argument down one and + // `tokens` still addresses the unshifted `args`. Stale indices make the patch-shape strip + // below drop the wrong argument: `--oneline -p ` loses the `` and keeps a bare + // `-p`, which outranks RTK's `--no-patch` -- HEAD's stat and patch under another + // commit's header. + let stat_args = args_without_oneline(args, &tokens); + let stat_tokens = tokenize_git_diff_args(&stat_args); + let mut stat_cmd = show_cmd( + global_args, + &stat_args, + &stat_tokens, + &["--no-patch", "--stat", "--pretty=format:"], + true, + ); + let stat_result = exec_capture(&mut stat_cmd).context("Failed to run git show --stat")?; + let stat_text = stat_result.stdout.trim(); + if !stat_text.is_empty() { + printed.push('\n'); + printed.push_str(stat_text); + } + + // Step 3: compacted diff + // No `--patch` here: a patch is `show`'s default, and forcing it would override a user + // `-s`/`--no-patch`, whose whole point is that there is no body to print. + let mut diff_cmd = show_cmd(global_args, args, &tokens, &["--pretty=format:"], false); + let diff_result = exec_capture(&mut diff_cmd).context("Failed to run git show (diff)")?; + let diff_text = diff_result.stdout.trim(); + + if !diff_text.is_empty() { + if verbose > 0 { + printed.push_str("\n\nChanges:"); + } + let compacted = compact_diff(&strip_ansi(diff_text), max_lines.unwrap_or(500)); + printed.push('\n'); + printed.push_str(&compacted); + } + + let shown = never_worse(&raw_output, &printed); + println!("{}", shown); + + timer.track( + &format!("git show {}", args.join(" ")), + &format!("rtk git show {}", args.join(" ")), + &raw_output, + shown, + ); + + Ok(0) +} + +/// Whether these args make git emit a word diff rather than a line diff. +/// +/// `compact_diff` reads a unified or combined diff: a body line's first column +/// (or columns) is a marker and the rest is content. A word diff drops the +/// marker entirely and puts `[-removed-]` / `{+added+}` inline, so its body +/// lines are arbitrary content in the marker position. A line starting with `+` +/// then counts as an addition, one starting with `\` is dropped as a +/// no-newline annotation, and one whose content happens to start `diff --` +/// opens a new file section. There is nothing to compact faithfully, so these +/// modes pass through. +/// +/// `--word-diff=none` is the mode that turns a word diff back off, leaving an +/// ordinary unified diff to compact. Modes are last-one-wins, which is what +/// that mode is for: overriding an alias or an earlier flag on the same line. +/// +/// Takes tokens rather than raw args: a `--word-diff` consumed as another option's value +/// (`--author --word-diff`) or sitting past `--` as a pathspec is not a word diff request, and +/// real git agrees on both. +fn emits_word_diff(tokens: &[Token]) -> bool { + let mut word_diff = false; + for token in tokens { + if token.kind != TokenKind::Long { + continue; + } + match token.text { + "word-diff" => word_diff = token.attached != Some("none"), + // `--color-words[=]` takes a regex rather than a mode, so there is no `none` + // to honour on that spelling. + "color-words" | "word-diff-regex" => word_diff = true, + _ => {} + } + } + word_diff +} + +/// `rev:path` names a blob. The caller filters to free positionals first, so a flag's own +/// value (`--pretty=format:...`) never reaches here. +fn is_blob_show_arg(arg: &str) -> bool { + // Detect `rev:path` style arguments while ignoring flags like `--pretty=format:...`. + // `:/text` is a commit-message search, not a blob, so it is excluded. `:path` and + // `:N:path` (index / merge-stage blobs) start with `:` but ARE blobs, so only the + // `:/` prefix is filtered out there. + // + // Magic pathspecs (`:(exclude)…`, `:(top)…`, `:!…`, and `:^…` — an exact synonym + // of `:!…` for exclude magic) also start with `:` but are NOT blobs: they only + // ever appear as pathspecs, so exclude them too. An option value with a colon + // (`git show -S 'a:b' HEAD`) is handled by `show_positionals`, which skips a + // flag's operand via git's argument grammar, so it never reaches here as a blob. + !arg.starts_with('-') + && !arg.starts_with(":/") + && !arg.starts_with(":(") + && !arg.starts_with(":!") + && !arg.starts_with(":^") + && arg.contains(':') + // A colons-only token (`:`, `::`) is never a blob object: route it to git + // rather than the blob window, which would only surface git's own error. + && arg.chars().any(|c| c != ':') +} + +/// The positional (non-option) arguments of a `git show` — its objects. Options and +/// the value tokens they consume (`-S 'url:1'`, `-L 1,2:file`) are dropped via git's +/// own flag/value grammar ([`flag_token_consumes_next`]), so an option operand that +/// happens to contain a colon is never mistaken for a blob and truncated. This handles +/// short-flag CLUSTERS too (`-wG a:b`, `-pS a:b`), whose value-flag tail git re-parses. +/// Args after a `--` are pathspecs, never objects, so a colon in a filename there +/// (`-- weird:name`) is excluded by scanning only the args before the first `--`; a +/// trailing `-- ` beside a real object arg is thus ignored (git still dumps the +/// blob) rather than emptying the list. +fn show_positionals(args: &[String]) -> Vec<&String> { + let rev_args = match args.iter().position(|a| a == "--") { + Some(sep) => &args[..sep], + None => args, + }; + let mut positionals = Vec::new(); + let mut iter = rev_args.iter(); + while let Some(arg) = iter.next() { + if arg.starts_with('-') { + if flag_token_consumes_next(arg) { + iter.next(); // skip this flag's value token + } + continue; + } + positionals.push(arg); + } + positionals +} + +/// Whether `arg` is a flag that consumes the following argument as its value. +/// +/// Delegates to [`log_takes_value`] so git's flag/value grammar lives in one table rather than +/// two that can drift: the same predicate the tokenizer folds with. `AttachedOnly` flags are +/// excluded because they never take a separate token (`-M50` attaches, `-M 50` does not), and +/// the cluster-position rule for solo-only flags is applied by the caller below. +fn consumes_next_token_as_value(arg: &str) -> bool { + let (kind, name) = match arg.strip_prefix("--") { + Some(rest) => (TokenKind::Long, rest), + None => match arg.strip_prefix('-') { + Some(rest) => (TokenKind::Short, rest), + None => return false, + }, + }; + log_takes_value(kind, name).is_some_and(|spec| spec.attachment != Attachment::AttachedOnly) +} + +/// Whether a flag token consumes the NEXT arg as its value (so `show_positionals` must +/// skip it). Handles long flags (`--grep foo`) via [`consumes_next_token_as_value`] and +/// short-flag CLUSTERS (`-wG foo`, `-pS bar`), which git re-parses char by char. +/// +/// Inside a cluster, the first value-taking short flag (`-S -G -I -L -O -l -n`) takes +/// the REST of the cluster as an INLINE value when more chars follow it (`-Sfoo` == `-S +/// foo`, so it does NOT consume the next arg), or the NEXT arg when it is the cluster's +/// last char (`-wG` == `-w -G`, consuming the next arg). Any earlier char is a boolean +/// flag we skip over. This reuses the single flag/value table rather than re-tokenizing +/// git's whole grammar, so `git show -wG x:y HEAD:big` correctly treats `x:y` as `-G`'s +/// value and `HEAD:big` as the object. +// +// TODO(after #3681): replace this short-cluster walk with the ValueSpec factorization; +// the per-char logic here is exactly what a ValueSpec table subsumes. +fn flag_token_consumes_next(arg: &str) -> bool { + // A short cluster is a single leading `-` followed by non-empty flag chars (not the + // `--long` form and not the bare `-` stdin sentinel). Everything else (`--foo`, `-`) + // uses the exact-match table directly. + match arg.strip_prefix('-') { + Some(cluster) if !cluster.is_empty() && !cluster.starts_with('-') => { + for (i, c) in cluster.char_indices() { + if is_short_value_flag(c) { + // Consumes the next arg only if no inline value follows in-cluster. + return i + c.len_utf8() == cluster.len(); + } + } + false + } + _ => consumes_next_token_as_value(arg), + } +} + +/// Whether a single-letter short flag takes a value (`-S`, `-G`, `-L`, …). Derived from +/// [`consumes_next_token_as_value`] so the flag/value table stays the single source of +/// truth and no parallel list can drift out of sync. +fn is_short_value_flag(c: char) -> bool { + c.is_ascii() && consumes_next_token_as_value(format!("-{c}").as_str()) +} + +/// The `git show` positionals that look like `:` blob objects — the +/// windowing candidates. ALL of them are returned (not just the first) so `run_show` +/// can `cat-file`-probe every one: a value operand a missed exotic cluster might leave +/// behind is rejected by the probe, while the real blob elsewhere on the line is found. +fn blob_candidates(args: &[String]) -> Vec<&String> { + show_positionals(args) + .into_iter() + .filter(|a| is_blob_show_arg(a)) + .collect() +} + +/// Whether a `git show` invocation carries any content-transforming flag +/// (`--textconv`/`--filters`/`--ext-diff`) that rewrites a blob's bytes. The `git show +/// : | tail` recovery hint omits these flags, so its output would not match +/// what git printed; their presence forces byte-identical raw passthrough instead of +/// windowing. The `--no-*` spellings restore the default (no rewrite) and are safe, so +/// only the enabling spellings count. Flags precede `--`, so scan up to it. +fn has_content_transform_flag(args: &[String]) -> bool { + args.iter() + .take_while(|a| *a != "--") + .any(|a| matches!(a.as_str(), "--textconv" | "--filters" | "--ext-diff")) +} + +/// Whether a `-- ` with at least one path after it is present. Empirically +/// git ignores a trailing pathspec for a `rev:path` blob dump (verified: output is +/// byte-identical with and without it), but the recovery hint omits it, so rather than +/// bank on that holding for every git version and pathspec form we force byte-identical +/// raw passthrough whenever one accompanies a blob. +fn has_trailing_pathspec(args: &[String]) -> bool { + matches!(args.iter().position(|a| a == "--"), Some(sep) if args.len() > sep + 1) +} + +/// Which `git show` handler an invocation routes to. +#[derive(Debug, PartialEq, Eq)] +enum ShowRoute { + /// A `:` blob dump → byte-safe decode/window path. + Blob, + /// `--stat`/`--numstat`/`--shortstat`/`--pretty`/`--format` with NO blob target + /// → summary-only passthrough. + StatOrFormat, + /// An ordinary commit → compacted commit-diff. + CommitDiff, +} + +/// Classify a `git show` invocation. Blob detection wins over --stat/--format +/// because git accepts and silently ignores those flags when the argument resolves +/// to a blob (verified against git 2.39 — output byte-identical to a plain blob +/// dump), still emitting the full file. Routing such a call to the --stat passthrough +/// would send it through lossy UTF-8 decoding and reintroduce the blob corruption the +/// byte-safe path fixes, so the blob case is checked first. +/// +/// This is a cheap, pure PRE-FILTER: a `Blob` result here only means SOME positional +/// *looks like* `rev:path`. The authoritative blob decision is a `git cat-file -t` +/// probe run by `run_show` (see `probe_is_blob`) on those candidates — the pre-filter +/// keeps the probe off every other invocation. +fn show_route(args: &[String]) -> ShowRoute { + if !blob_candidates(args).is_empty() { + return ShowRoute::Blob; + } + commit_or_stat_route(args) +} + +/// Classify a `git show` that is NOT a blob dump: `--stat`/`--format`/word-diff +/// summary passthrough vs. a compacted commit-diff. Split out from `show_route` so +/// `run_show` can fall back to it when the `cat-file` probe rejects a `rev:path` +/// candidate (a flag value like `-S 'url:1'`, or a tree/commit/bogus object). +fn commit_or_stat_route(args: &[String]) -> ShowRoute { + // Tokenized, not string-matched: a pathspec named `--stat` past the boundary is not the + // flag, `--prettyish` is not `--pretty`, and the shapes RTK cannot compact are more than + // the three stat spellings -- `--raw`, `--name-only`, `--check` and the rest route here + // too. A word/color-word diff has no unified-diff markers for `compact_diff` to read, so + // it passes through like a summary; it only ever applies to a commit-diff, so it is + // checked after the blob case above. + let tokens = tokenize_git_diff_args(args); + if tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)) + || show_wants_format(&tokens) + || emits_word_diff(&tokens) + { + return ShowRoute::StatOrFormat; + } + ShowRoute::CommitDiff +} + +/// Authoritatively decide whether a `git show` argument is a blob, by asking git +/// instead of mirroring its flag grammar. `git cat-file -t ` prints the object +/// type; only an exact `blob` is a windowing target. A tree/commit/tag, or any +/// non-zero exit (a flag value that isn't an object, e.g. `-S 'url:1'`, or a bogus +/// `rev:path`), is not a blob and routes to the normal commit-diff / passthrough path. +/// +/// Run with the SAME global args as the command (`git_cmd(global_args)`) so `-C`, +/// `-c`, `--git-dir`, `--work-tree` resolve the object in the right repo — the same +/// requirement the recovery hint has. Only called on the path where the arg already +/// looks like `rev:path` (the `show_route` pre-filter), so the ~1 ms subprocess never +/// runs on an ordinary commit show. +/// +// TODO(after #3681): once ValueSpec factorization lands, a flag pre-filter can avoid +// the cat-file probe on the common path. +fn probe_is_blob(global_args: &[String], arg: &str) -> bool { + let mut cmd = git_cmd(global_args); + cmd.args(["cat-file", "-t", arg]); + match exec_capture(&mut cmd) { + Ok(result) if result.success() => result.stdout.trim() == "blob", + _ => false, + } +} + +/// Byte budget for a blob preview before truncation kicks in (~2k tokens). +const MAX_BLOB_BYTES: Budget = Budget(8192); + +/// Byte budget after which a blob preview is truncated. A newtype rather than a bare +/// `usize` so a call site can't silently pass some other length in its place. +#[derive(Clone, Copy)] +struct Budget(usize); + +/// Decide how to window a `git show :` blob. Returns +/// `Some((head, remaining_lines, tail_offset))` when the blob should be truncated, +/// or `None` to pass it through unchanged: a small blob, a binary blob, a tree +/// listing, or a single line too long to cut at a line boundary. +/// +/// Pure and side-effect free so it can be unit-tested against real blob fixtures. +fn blob_truncation(raw: &str, budget: Budget) -> Option<(&str, usize, usize)> { + let Budget(budget) = budget; + // No content-sniffing for object type here: the caller only reaches this function + // for an arg the `git cat-file -t` probe has already confirmed is a blob, so a tree + // listing never arrives (and the old `raw.starts_with("tree ")` heuristic — a false + // positive for Newick/NEXUS blobs that genuinely begin "tree " — is gone with it). + // A defensive binary guard is likewise unnecessary now: `run_show` only calls this + // on valid UTF-8 (`\0`/`U+FFFD` content routes to byte-exact passthrough upstream), + // but keep the cheap check so the pure function stays safe when unit-tested directly. + if raw.contains('\0') || raw.contains('\u{FFFD}') { + return None; + } + if raw.len() <= budget { + return None; + } + // Slicing a `&str` at a byte index that is not a UTF-8 char boundary panics, so + // walk the budget back to the nearest boundary at or below it before cutting. + let mut end = budget; + while end > 0 && !raw.is_char_boundary(end) { + end -= 1; + } + // Cut at the last line boundary inside the window. No newline means a single line + // longer than the budget, which tail-based recovery cannot re-window: pass through. + let cut = raw[..end].rfind('\n')? + 1; + let head = &raw[..cut]; + let head_lines = head.lines().count(); + let total_lines = raw.lines().count(); + let remaining = total_lines.checked_sub(head_lines).filter(|&r| r > 0)?; + // `tail -n +offset` recovers everything from the first un-shown line onward. + Some((head, remaining, head_lines + 1)) +} + +/// POSIX single-quote a blob arg so a hint with a space or shell metachar in the path +/// stays copy-paste safe. +fn shell_single_quote(s: &str) -> String { + format!("'{}'", s.replace('\'', r"'\''")) +} + +/// Window a single blob dump: cap a large text blob and point at the rest with a hint +/// re-derived from the blob arg itself — `git show : | tail -n +N`. +/// +/// The hint deliberately does not lean on a tee file. The tee is capped at +/// `max_file_size` (1 MB by default), so it would refuse to store exactly the giant +/// lockfiles this filter most wants to trim, leaving the biggest blobs un-windowed; +/// re-running `git show` reproduces the tail at any size, with no cap and no I/O. +fn compact_blob_show(raw: &str, blob_arg: &str, global_args: &[String]) -> String { + let Some((head, remaining, offset)) = blob_truncation(raw, MAX_BLOB_BYTES) else { + return raw.to_string(); + }; + // Carry the command's global args (`-C `, `-c k=v`, `--git-dir`, `--work-tree`) + // into the hint so it is runnable from anywhere, not just the repo root: without + // them `rtk git -C /repo show HEAD:big` would emit `git show 'HEAD:big' | tail …`, + // which fails outside /repo. Each is shell-quoted for copy-paste safety. + let mut prefix = String::new(); + for arg in global_args { + prefix.push_str(&shell_single_quote(arg)); + prefix.push(' '); + } + // Through `rtk proxy`, like the `diff` hints: a bare `git show` is what RTK's own hook + // rewrites back into `rtk git show`, which windows the dump a second time and leaves + // `tail` nothing but this hint to print -- one line where the reader asked for the rest + // of the file. `proxy` is the documented way to reach the unfiltered command. + let hint = format!( + "[see remaining: rtk proxy git {}show {} | tail -n +{}]", + prefix, + shell_single_quote(blob_arg), + offset + ); + let out = format!("{}... (+{} lines) {}\n", head, remaining, hint); + never_worse(raw, &out).to_string() +} + +/// Write raw bytes straight to stdout, tracked as a passthrough. Used when the blob +/// must reach the caller byte-for-byte — binary or ambiguously-encoded content that +/// decoding would corrupt, or a passthrough where any rewrite (BOM strip, transcode) +/// would diverge from what plain `git show` emits — so it goes through the locked +/// stdout handle as bytes rather than a `String`. +fn emit_raw_bytes_passthrough( + bytes: &[u8], + label: &str, + rtk_label: &str, + timer: &tracking::TimedExecution, + exit_code: i32, +) -> Result { + use std::io::Write; + let stdout = std::io::stdout(); + let mut handle = stdout.lock(); + handle + .write_all(bytes) + .context("Failed to write blob to stdout")?; + timer.track_passthrough(label, rtk_label); + Ok(exit_code) +} + +/// Path named by a diff section header. +/// +/// `diff --git a/p b/p` carries the path twice; `diff --cc p` and +/// `diff --combined p` carry it once, as the whole remainder of the line. Only +/// the two-path form can be split at its midpoint, so the header kind decides +/// which shape to read: `diff --cc dup dup` names one file called `dup dup`, +/// not the file `dup` twice. +/// +/// Under the default `core.quotepath`, git wraps a path in `"` and escapes any +/// non-ASCII byte, control character, quote or backslash inside it — but not a +/// space. The quoting is undone here, so the header carries the path as it is +/// on disk and a `grep` over the output finds it by name. +fn diff_header_path(line: &str) -> String { + let Some(rest) = line.splitn(3, ' ').nth(2) else { + return "unknown".to_string(); + }; + if !line.starts_with("diff --git ") { + return unquote_path(rest); + } + if let Some(path) = same_path_twice(rest) { + return path; + } + // A rename names two different paths, and the destination is the second. + if let Some(quoted) = rest + .split(" \"b/") + .nth(1) + .and_then(|dst| dst.strip_suffix('"')) + { + return unescape_path(quoted); + } + match rest.split(" b/").nth(1) { + Some(path) => path.to_string(), + None => unquote_path(rest), + } +} + +/// The path a `diff --git` header names twice, split at the midpoint. +/// +/// Anything but a rename names the same path on both sides, so the two halves +/// are the same length and the separating space sits dead centre. Splitting +/// there instead of on the first ` b/` keeps a path that contains that +/// substring — a file under a directory named `x b`. Prefixes are then dropped +/// by matching the halves against each other rather than by name, so +/// `--no-prefix` and any custom `--src-prefix` / `--dst-prefix` read alike. +/// +/// `None` for a rename, whose halves differ past their first component, and for +/// anything else the two halves disagree on; both fall through to the ` b/` +/// split. A `--no-prefix` rename between two directories is the one shape this +/// cannot tell from a prefix pair — space-separated paths with no prefix are +/// ambiguous by construction — and it reads as the shared trailing path. +fn same_path_twice(rest: &str) -> Option { + if rest.len().is_multiple_of(2) { + return None; + } + let mid = rest.len() / 2; + // A space at the midpoint is a char boundary, so both halves are valid. + if rest.as_bytes().get(mid) != Some(&b' ') { + return None; + } + let (left, right) = (unquote_path(&rest[..mid]), unquote_path(&rest[mid + 1..])); + if left == right { + return Some(left); + } + let (_, left_path) = left.split_once('/')?; + let (_, right_path) = right.split_once('/')?; + (left_path == right_path).then(|| right_path.to_string()) +} + +/// Undo git's `core.quotepath` quoting: `"a/\303\251.txt"` becomes `a/é.txt`. +/// +/// A path git did not quote is returned as-is, so either form can be passed. +fn unquote_path(raw: &str) -> String { + match raw.strip_prefix('"').and_then(|r| r.strip_suffix('"')) { + Some(quoted) => unescape_path(quoted), + None => raw.to_string(), + } +} + +/// Decode the C escapes inside a quoted path. +/// +/// The octal escapes spell out the path's bytes one at a time, so a multi-byte +/// character arrives as several of them; they are collected as bytes and +/// decoded once at the end rather than per escape. A path whose bytes are not +/// UTF-8 keeps replacement characters, which is as close as a `String` gets. +fn unescape_path(quoted: &str) -> String { + let bytes = quoted.as_bytes(); + let mut out: Vec = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] != b'\\' || i + 1 == bytes.len() { + out.push(bytes[i]); + i += 1; + continue; + } + let escape = bytes[i + 1]; + if escape.is_ascii_digit() { + let end = (i + 4).min(bytes.len()); + let octal = std::str::from_utf8(&bytes[i + 1..end]) + .ok() + .and_then(|digits| u8::from_str_radix(digits, 8).ok()); + match octal { + Some(byte) => { + out.push(byte); + i = end; + } + // Not an octal escape after all: keep the backslash verbatim. + None => { + out.push(bytes[i]); + i += 1; + } + } + continue; + } + out.push(match escape { + b'a' => 0x07, + b'b' => 0x08, + b't' => b'\t', + b'n' => b'\n', + b'v' => 0x0b, + b'f' => 0x0c, + b'r' => b'\r', + // `\"` and `\\` stand for themselves. + other => other, + }); + i += 2; + } + String::from_utf8_lossy(&out).into_owned() +} + +/// Line budget a hunk header declares, and how wide its body prefix is. +struct HunkHeader { + /// Lines the hunk spans in each parent, in marker-column order. One entry + /// for a unified `@@`, one per parent for a combined `@@@`. + parents: Vec, + /// Lines the hunk spans in the result file. + new: usize, + /// Marker columns: 1 for `@@`, and one per parent for a combined `@@@`. + prefix_width: usize, +} + +impl HunkHeader { + /// Whether every declared line has been accounted for, which is where the + /// hunk body ends. A combined hunk is not done until *every* parent's + /// budget is spent: a line removed from only the second parent spends that + /// parent's budget and neither the first's nor the result's. + fn exhausted(&self) -> bool { + self.new == 0 && self.parents.iter().all(|&remaining| remaining == 0) + } + + /// Charge a body line against the budgets it occupies. + /// + /// Column `i` is the line's marker against parent `i + 1`. `-` there means + /// the line is in that parent and is being removed; a space on a line that + /// is not a removal means the line is in that parent unchanged. Both spend + /// one of that parent's lines. `+` there, or a space on a removal line, + /// means the line is not in that parent at all. + /// + /// Collapsing the columns into one add/delete pair, as an aggregate over + /// the whole prefix does, loses that distinction and leaves a combined + /// hunk's budget unable to converge. + fn consume(&mut self, markers: &[u8]) { + let is_add = markers.contains(&b'+'); + let is_del = markers.contains(&b'-'); + for (i, remaining) in self.parents.iter_mut().enumerate() { + let column = markers.get(i).copied(); + let present = if is_del { + column == Some(b'-') + } else { + // A line shorter than the prefix reads as context, which is + // what a bare blank line in a unified diff body is. + column != Some(b'+') + }; + if present { + *remaining = remaining.saturating_sub(1); + } + } + // In the result file unless the line is a pure deletion. + if is_add || !is_del { + self.new = self.new.saturating_sub(1); + } + } +} + +/// Parse `@@ -a,b +c,d @@` and the combined `@@@ -a,b -c,d +e,f @@@`. +/// +/// The counts bound the hunk body, which is what lets the body end where the +/// hunk ends rather than running on until the next header. Anything after it — +/// an mbox envelope, a `--` signature, trailing prose — is then outside every +/// hunk and cannot be read as diff content. A count is 1 when the header omits +/// it (`@@ -1 +1 @@`). +fn parse_hunk_header(line: &str) -> Option { + let at_run = line.len() - line.trim_start_matches('@').len(); + if at_run < 2 { + return None; + } + let body = line[at_run..].split('@').next()?; + + let mut parents: Vec = Vec::new(); + let mut new = None; + for group in body.split_whitespace() { + let Some(rest) = group.strip_prefix(['-', '+']) else { + continue; + }; + let count = match rest.split_once(',') { + Some((_, c)) => c.parse::().ok()?, + None => 1, + }; + if group.starts_with('-') { + // A combined header lists one range per parent, in the same order + // as the marker columns. + parents.push(count); + } else { + new = Some(count); + } + } + + // `@@` has one marker column, `@@@` two, and so on for more parents. + let prefix_width = at_run - 1; + // A well-formed header lists exactly one range per marker column. When it + // does not, only the columns can be charged, so trust them: an untracked + // parent would otherwise sit at its declared count forever and the hunk + // would never close, while a parent with no column of its own would be + // charged against nothing. A missing range gets `usize::MAX`, which keeps + // the hunk open to the next header rather than dropping its body. + if parents.len() != prefix_width { + parents.resize(prefix_width, usize::MAX); + } + + Some(HunkHeader { + parents, + new: new.unwrap_or(0), + prefix_width, + }) +} + +/// Render the note for change lines dropped past `max_hunk_lines`, split by +/// sign so an anchored `^-` / `^+` audit can tell what it did not see. +fn hunk_truncation_note(deletions: usize, additions: usize) -> Option { + fn count(n: usize, noun: &str) -> String { + if n == 1 { + format!("{} {}", n, noun) + } else { + format!("{} {}s", n, noun) + } + } + match (deletions, additions) { + (0, 0) => None, + (0, a) => Some(format!(" ... ({} truncated)", count(a, "addition"))), + (d, 0) => Some(format!(" ... ({} truncated)", count(d, "deletion"))), + (d, a) => Some(format!( + " ... ({}, {} truncated)", + count(d, "deletion"), + count(a, "addition") + )), + } +} + +/// Emit the buffered leading context, charged against the diff-wide budget. +/// +/// Keeps the lines closest to the change when the budget cannot take all of +/// them. Called wherever a hunk closes as well as at its first change line: +/// context buffered by a hunk that ends without one would otherwise be dropped, +/// leaving a bare hunk header with nothing under it. +fn flush_leading_context( + buffer: &mut Vec, + result: &mut Vec, + total: &mut usize, + cap: usize, +) { + let room = cap.saturating_sub(*total); + let keep = buffer.len().min(room); + let skip = buffer.len() - keep; + for ctx in buffer.drain(..).skip(skip) { + result.push(ctx); + } + *total += keep; +} + +pub(crate) fn compact_diff(diff: &str, max_lines: usize) -> String { + let mut result = Vec::new(); + let mut current_file = String::new(); + let mut added = 0; + let mut removed = 0; + let mut hunk: Option = None; + let mut hunk_shown = 0; + let mut skipped_add = 0usize; + let mut skipped_del = 0usize; + let mut leading_context: Vec = Vec::new(); + let mut leading_context_total = 0usize; + let max_hunk_lines = 100; + // Context before a hunk's first change, up to three lines per hunk and + // `max_lines / 10` across the diff. It does not count against `max_lines`, + // so it cannot displace change lines, and the diff-wide cap is what bounds + // the overrun that exemption would otherwise allow: a diff of many small + // hunks would otherwise spend three exempt lines on every one of them. + let max_leading_context = 3; + let leading_context_cap = max_lines / 10; + let mut was_truncated = false; + + for line in diff.lines() { + // Every diff section header (`--git`, `--cc`, `--combined`) opens a new + // file and closes any open hunk, so the `---` / `+++` headers that + // follow it are never read as hunk content. + if line.starts_with("diff --") { + flush_leading_context( + &mut leading_context, + &mut result, + &mut leading_context_total, + leading_context_cap, + ); + if let Some(note) = hunk_truncation_note(skipped_del, skipped_add) { + result.push(note); + was_truncated = true; + skipped_del = 0; + skipped_add = 0; + } + if !current_file.is_empty() && (added > 0 || removed > 0) { + result.push(format!(" +{} -{}", added, removed)); + } + current_file = diff_header_path(line); + result.push(format!("\n{}", current_file)); + added = 0; + removed = 0; + hunk = None; + hunk_shown = 0; + } else if let Some(header) = parse_hunk_header(line) { + flush_leading_context( + &mut leading_context, + &mut result, + &mut leading_context_total, + leading_context_cap, + ); + if let Some(note) = hunk_truncation_note(skipped_del, skipped_add) { + result.push(note); + was_truncated = true; + skipped_del = 0; + skipped_add = 0; + } + hunk = Some(header); + hunk_shown = 0; + // Preserve the full unified diff hunk header, including trailing + // function / symbol context after the second @@ marker. + result.push(line.to_string()); + } else if let Some(header) = hunk.as_mut() { + if header.exhausted() { + hunk = None; + continue; + } + if line.starts_with('\\') { + // "\ No newline at end of file" annotates the line above and + // occupies no line in either file. + continue; + } + + // Slice the marker columns as bytes. `prefix_width` counts columns, + // and the markers are ASCII by construction, but the body content + // right after them is not: `--word-diff` emits body lines with no + // marker column at all, so a `char`-unaware `&line[..width]` splits + // a leading multi-byte character and panics. + let width = header.prefix_width.min(line.len()); + let markers = &line.as_bytes()[..width]; + let is_add = markers.contains(&b'+'); + let is_del = markers.contains(&b'-'); + header.consume(markers); + + // Hunk bodies emit at column 0 in git's own unified shape, so + // `^+` / `^-` anchor. rtk's own annotations stay indented so those + // same anchors never match them. Inside a hunk every `+`/`-` line + // is content: the `---` / `+++` file headers only ever appear + // before the first hunk header. + if is_add || is_del { + if is_add { + added += 1; + } + if is_del { + removed += 1; + } + if hunk_shown < max_hunk_lines { + // The context immediately preceding the change, so the body + // reads as contiguous with it. The diff-wide budget is + // charged on emit rather than on buffering, so a line the + // ring evicted never costs anything. + flush_leading_context( + &mut leading_context, + &mut result, + &mut leading_context_total, + leading_context_cap, + ); + result.push(line.to_string()); + hunk_shown += 1; + } else if is_del { + skipped_del += 1; + } else { + skipped_add += 1; + } + leading_context.clear(); + } else if hunk_shown > 0 { + if hunk_shown < max_hunk_lines { + result.push(line.to_string()); + hunk_shown += 1; + } + } else if leading_context_total < leading_context_cap { + // Keep the last `max_leading_context` lines rather than the + // first: with `-U10` or `--function-context` the first ones sit + // ten lines above the change and would imply an adjacency the + // file does not have. + if leading_context.len() == max_leading_context { + leading_context.remove(0); + } + leading_context.push(line.to_string()); + } + + if header.exhausted() { + hunk = None; + flush_leading_context( + &mut leading_context, + &mut result, + &mut leading_context_total, + leading_context_cap, + ); + } + } + + if result.len().saturating_sub(leading_context_total) >= max_lines { + result.push("\n... (more changes truncated)".to_string()); + was_truncated = true; + break; + } + } + + // Flush last hunk + flush_leading_context( + &mut leading_context, + &mut result, + &mut leading_context_total, + leading_context_cap, + ); + if let Some(note) = hunk_truncation_note(skipped_del, skipped_add) { + result.push(note); + was_truncated = true; + } + + if !current_file.is_empty() && (added > 0 || removed > 0) { + result.push(format!(" +{} -{}", added, removed)); + } + + if was_truncated { + result.push("[full diff: rtk git diff --no-compact]".to_string()); + } + + result.join("\n") +} + +/// RTK's default `git log` limit, applied whenever the user names none. +const DEFAULT_LOG_LIMIT: usize = 10; +const DEFAULT_LOG_LIMIT_ARG: &str = "-10"; + +/// `git log ` for the raw passthrough, carrying RTK's default limit unless the user named +/// one. [`run_passthrough`] streams straight to the terminal, so the limit has to be in the args +/// or it never applies: a patch request became the whole history, 411k lines against 50 for +/// plain `rtk git log`. +/// +/// The limit goes first, ahead of every user argument. `injection_point` is the wrong tool here: +/// it only guarantees "before the user's `--`", while git requires options to precede *all* +/// positionals ("fatal: -10 option must come before non-option arguments") and a pathspec needs +/// no boundary to be one. Only the limit is injected -- `--no-merges` would gut `--cc`/`-c`, +/// whose entire purpose is the merge diff. +fn raw_log_passthrough_args(args: &[String], capped: bool) -> Vec { + let mut out = vec![OsString::from("log")]; + if capped { + out.push(OsString::from(DEFAULT_LOG_LIMIT_ARG)); + } + out.extend(args.iter().map(OsString::from)); + out +} + +/// True when RTK's default limit applies: the user named no limit of their own and did not +/// bound the walk with a revision range. +fn raw_log_is_capped(tokens: &[Token<'_>]) -> bool { + !has_limit_flag(tokens) && !bounds_the_walk(tokens) +} + +/// Whether the walk holds more commits than RTK's default limit shows -- the question the +/// truncation notice answers. +/// +/// Asks git for the one commit just past the limit and reads only whether anything came back. +/// Nothing about the printed shape can disfigure that answer, which is what counting commits +/// out of the emitted output could never manage: `--oneline` leaves no commit header, +/// `log.decorate` and `--graph` disfigure the one there is, `-z` runs the walk onto a single +/// line, `--line-prefix` puts something in front of it, colour puts an escape there, and a +/// SHA-256 object name is not 40 characters. Each of those silenced the notice on a cap that +/// had taken most of the history away. +/// +/// RTK's flags go in front of every user argument: git refuses an option that follows a +/// positional, so a bare pathspec would otherwise make the probe fail outright. That leaves +/// the user's flags last, where git's last-flag-wins hands the format back to them -- harmless +/// except for `--pretty`/`--format`/`--oneline`, since an empty `--pretty=format:` prints a +/// commit as no bytes at all and would read as "nothing left". Those three are dropped from +/// what is forwarded, along with `--skip`, which RTK's own has to absorb, `--exit-code`, which +/// would make the probe report failure on a walk git was perfectly happy with, `--output`, +/// which is a redirect that would truncate the file the command just wrote, and the +/// patch-shape flags, which are work with no answer in them. +/// +/// `None` when git refuses the command: RTK then says nothing rather than guessing. +fn walk_exceeds_limit( + global_args: &[String], + args: &[String], + tokens: &[Token<'_>], + limit: usize, +) -> Option { + let mut cmd = git_cmd(global_args); + cmd.arg("log"); + // No `--no-patch`: git refuses it beside `--name-only`/`--name-status`, and the strip in + // `log_probe_args` has already taken the patch-shape flags out of what is forwarded. + cmd.args(["--pretty=format:%H"]); + // `--skip` is counted where the walk starts; a diff-based filter is applied after it. So + // for those the skip lands on commits the filter would have dropped and the probe reports + // on a commit the user was never going to see -- `git log -p -S needle` matching twice + // claimed a cap of ten. `--max-count` is applied last, after every filter, so there the + // question becomes how many came back rather than whether one did. + let counting = selects_by_diff(tokens); + if counting { + cmd.arg(format!("--max-count={}", limit.saturating_add(1))); + } else { + cmd.args(["--max-count=1".to_string()]); + cmd.arg(format!( + "--skip={}", + limit.saturating_add(user_skip(tokens)) + )); + } + cmd.args(log_probe_args(args, tokens)); + + let result = exec_capture(&mut cmd).ok()?; + if !result.success() { + return None; + } + if !counting { + return Some(!result.stdout.trim().is_empty()); + } + // One line per commit is RTK's own `--pretty` talking, but the user's `--stat`, + // `--name-only` or `--graph` still add lines of their own, so commits are counted by + // shape. ANSI first: `color.ui=always` puts an escape in front of the rail. + let shown = strip_ansi(&result.stdout) + .lines() + .filter(|line| is_commit_name(line)) + .count(); + Some(shown > limit) +} + +/// True for a `--pretty=format:%H` line, ignoring any `--graph` rail in front of it. No upper +/// bound on the length: a SHA-256 object name is 64 characters, and rejecting it counted every +/// commit in such a repository as none. +fn is_commit_name(line: &str) -> bool { + let name = + line.trim_matches(|c: char| c.is_whitespace() || matches!(c, '*' | '|' | '/' | '\\')); + name.len() >= 7 && name.chars().all(|c| c.is_ascii_hexdigit()) +} + +/// True when the walk is narrowed by what the commits changed rather than by where they are. +/// git applies these after `--skip` and before `--max-count`, which is what makes the two +/// probes above answer different questions. +fn selects_by_diff(tokens: &[Token<'_>]) -> bool { + arg_tokenizer::before_dashdash(tokens).iter().any(|t| { + match t.kind { + TokenKind::Long => matches!( + t.text, + "find-object" | "diff-filter" | "pickaxe-regex" | "pickaxe-all" + ), + // `-S` and `-G`. + TokenKind::Short => matches!(t.text, "S" | "G"), + _ => false, + } + }) +} + +/// The user arguments [`walk_exceeds_limit`] forwards: everything that bounds the walk, and +/// nothing that decides what git does with it. +/// +/// Three classes come out. `--pretty`/`--format`/`--oneline`, because RTK's own `--pretty` is +/// written first and would otherwise lose the last-flag-wins arbitration -- an empty +/// `--pretty=format:` then prints a commit as no bytes at all, which reads as "nothing left" +/// (`--summary` and `--dirstat` print nothing of their own for a plain modification, so the +/// probe would have only that format to go on). `--skip`, because RTK's has to absorb it for +/// the same reason. And `--output=`, which is not a format at all but a redirect: left +/// in, the probe reruns it and truncates the file the command it follows has just written. +/// +/// The patch-shape flags go too, through the same strip the `show` header uses. The probe +/// reads only whether git printed anything, so a patch is work with no answer in it -- and +/// with `diff.external` or a `textconv` driver configured, work that runs the user's own +/// program one more time than they asked for. +fn log_probe_args(args: &[String], tokens: &[Token<'_>]) -> Vec { + let mut dropped: Vec = Vec::new(); + for token in tokens { + if token.kind != TokenKind::Long + || !matches!( + token.text, + "pretty" | "format" | "oneline" | "skip" | "output" | "exit-code" + ) + { + continue; + } + dropped.push(token.source_index); + // `--skip 5` spends a second argument on its value; `--skip=5` does not. + if let Some(index) = token.linked + && let Some(value) = tokens.get(index) + { + dropped.push(value.source_index); + } + } + let kept: Vec = args + .iter() + .enumerate() + .filter(|(index, _)| !dropped.contains(index)) + .map(|(_, arg)| arg.clone()) + .collect(); + // Re-tokenized, because the drop above shifted every index `tokens` holds. + let kept_tokens = tokenize_git_log_args(&kept); + args_without_patch_shape(&kept, &kept_tokens) +} + +/// The user's own `--skip`, which RTK's has to absorb: git takes the last one, and RTK's is +/// written first. Last one wins here too, matching git. Unparseable means git will reject the +/// command anyway, so 0 is as good an answer as any. +fn user_skip(tokens: &[Token<'_>]) -> usize { + tokens + .iter() + .rfind(|t| t.kind == TokenKind::Long && t.text == "skip") + .and_then(|t| t.value(tokens)) + .and_then(|value| value.parse().ok()) + .unwrap_or(0) +} + +/// True when the arguments already bound the walk, so RTK's default limit would only take away +/// what the user explicitly asked for -- `git log -p HEAD~15..HEAD` means those 15 commits. +/// +/// A revision range is the one bound that is exact; `--since` and friends narrow the walk but +/// not to any particular size, so those still get the limit (and now the notice with it). A +/// relative path is excluded because it is a pathspec, not a range. +fn bounds_the_walk(tokens: &[Token<'_>]) -> bool { + arg_tokenizer::before_dashdash(tokens).iter().any(|t| { + t.is_free_positional() + && t.text.contains("..") + && !t.text.starts_with("./") + && !t.text.starts_with("../") + }) +} + +fn run_log( + args: &[String], + _max_lines: Option, + verbose: u8, + global_args: &[String], +) -> Result { + let tokens = tokenize_git_log_args(args); + + if tokens.iter().any(|t| log_wants_raw_shape(t, &tokens)) { + let capped = raw_log_is_capped(&tokens); + let passthrough_args = raw_log_passthrough_args(args, capped); + if !capped { + return run_passthrough(&passthrough_args, global_args, verbose); + } + // Capped, so the output is bounded and worth capturing: it is the only way to say + // whether the limit actually took anything away. Announcing it up-front announced a + // truncation that had not happened, on a two-commit repo, and announced it ahead of + // commands git then rejected outright. + let timer = tracking::TimedExecution::start(); + let mut cmd = git_cmd(global_args); + cmd.args(&passthrough_args); + let result = exec_capture(&mut cmd).context("Failed to run git log")?; + print!("{}", result.stdout); + if !result.stderr.trim().is_empty() { + eprint!("{}", result.stderr); + } + timer.track( + &format!("git log {}", args.join(" ")), + &format!("rtk git log {} (passthrough)", args.join(" ")), + &result.stdout, + &result.stdout, + ); + // Before the exit check, not after: `--exit-code` makes a perfectly successful + // `git log` exit 1, and returning first left that cap unannounced. A command git + // really did refuse silences the notice anyway -- the probe fails on it too. + // + // Asked unconditionally, never skipped on empty output: `--output=` sends the + // whole run to a file and `--summary --pretty=format:` prints nothing for a plain + // modification, and both of those are capped walks with a notice owed. The probe + // walks the history a second time when a filter like `-S` matches little, which is + // the price of an answer that does not depend on what the run happened to print. + if walk_exceeds_limit(global_args, args, &tokens, DEFAULT_LOG_LIMIT) == Some(true) { + eprintln!( + "[rtk] capped at {} commits; pass -n for more", + DEFAULT_LOG_LIMIT + ); + } + if !result.success() { + return Ok(result.exit_code); + } + return Ok(0); + } + + let timer = tracking::TimedExecution::start(); + + let mut cmd = git_cmd(global_args); + cmd.arg("log"); + + // Check if user provided format flags + let has_format_flag = tokens + .iter() + .any(|t| t.kind == TokenKind::Long && matches!(t.text, "format" | "oneline" | "pretty")); + + // Check if user provided limit flag (-N, -n N, --max-count=N, --max-count N) + let has_limit_flag = has_limit_flag(&tokens); + + // Apply RTK defaults only if user didn't specify them + // Use %b (body) to preserve first line of commit body for agent context + // (BREAKING CHANGE, Closes #xxx, design notes) + if !has_format_flag { + cmd.args(["--pretty=format:%h %s (%ar) <%an>%n%b%n---END---"]); + } + + // Determine limit: respect user's explicit -N flag, use sensible defaults otherwise + let (limit, user_set_limit) = if has_limit_flag { + // User explicitly passed -N / -n N / --max-count=N → respect their choice + let n = parse_limit_from_tokens(&tokens).unwrap_or(10); + (n, true) + } else if has_format_flag { + // --oneline / --pretty without -N: user wants compact output, allow more + cmd.arg("-50"); + (50, false) + } else { + // No flags at all: default to 10 + cmd.arg(DEFAULT_LOG_LIMIT_ARG); + (10, false) + }; + + // Only add --no-merges if user didn't explicitly request merge commits. Any + // `--min-parents=N` with N >= 2 asks for merges; pinning it to 2 let `--min-parents=3` + // collect RTK's `--no-merges` as well, and the two constraints select nothing at all. + let wants_merges = tokens.iter().any(|t| { + t.kind == TokenKind::Long + && (t.text == "merges" + || t.text == "no-merges" + || (t.text == "min-parents" + && t.attached + .is_some_and(|v| v.parse::().is_ok_and(|n| n >= 2)))) + }); + // Don't add --no-merges if user explicitly requested merges or an exact count (-n N / --max-count) + if !wants_merges && !has_limit_flag { + cmd.arg("--no-merges"); + } + + // Pass all user arguments + for arg in args { + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git log")?; + + if !result.success() { + eprintln!("{}", result.stderr); + return Ok(result.exit_code); + } + + if verbose > 0 { + eprintln!("Git log output:"); + } + + // Post-process: truncate long messages, cap lines only if RTK set the default + let filtered = filter_log_output(&result.stdout, limit, user_set_limit, has_format_flag); + let filtered = never_worse(&result.stdout, &filtered).to_string(); + println!("{}", filtered); + + timer.track( + &format!("git log {}", args.join(" ")), + &format!("rtk git log {}", args.join(" ")), + &result.stdout, + &filtered, + ); + + Ok(0) +} + +/// The long flags `git log`, `diff` and `show` all take a value for. Shared deliberately: these +/// are revision-walk options all three parse the same way. The *short* grammar is where they +/// differ, so each subcommand states its own below. +/// +/// E.g. `--grep -p` searches messages for the literal string "-p"; it does not request patch +/// output. +fn shared_long_takes_value(name: &str) -> bool { + matches!( + name, + "after" + | "anchored" + | "author" + | "before" + | "color-moved-ws" + | "committer" + | "date" + | "decorate-refs" + | "decorate-refs-exclude" + | "diff-algorithm" + | "diff-filter" + | "diff-merges" + | "dst-prefix" + | "encoding" + | "exclude" + | "find-object" + | "glob" + | "grep" + | "grep-reflog" + | "ignore-matching-lines" + | "inter-hunk-context" + | "line-prefix" + | "max-age" + | "max-count" + | "max-depth" + | "min-age" + | "output" + | "output-indicator-context" + | "output-indicator-new" + | "output-indicator-old" + | "rotate-to" + | "since" + | "since-as-filter" + | "skip" + | "skip-to" + | "src-prefix" + | "stat-count" + | "stat-graph-width" + | "stat-name-width" + | "stat-width" + | "until" + | "word-diff-regex" + | "ws-error-highlight" + ) +} + +/// `git log`'s grammar. `-M`/`-U`/`-C`/`-B` take an optional attached number and never a +/// separate token; `-n` and `-l` take one only when solo (`git log -pn 2` fails against git +/// 2.53, and `-l` is kept solo-only out of caution -- only run_log uses this, where a stray +/// positional is inert). +fn log_takes_value(kind: TokenKind, name: &str) -> Option { + match kind { + TokenKind::Long => shared_long_takes_value(name).then(ValueSpec::value), + TokenKind::Short => match name { + "B" | "C" | "M" | "U" => Some(ValueSpec::attached_only()), + "G" | "I" | "L" | "O" | "S" => Some(ValueSpec::value()), + "l" | "n" => Some(ValueSpec::solo_only()), + _ => None, + }, + _ => None, + } +} + +/// `diff`/`show`'s grammar. Same long list, but `-l` is the rename limit here and *does* +/// cluster: `git diff -wl 100` works where `git log -cl 2` does not. Sharing log's short +/// grammar made RTK read the 100 as a pathspec and splice its own flags in front of it. +fn diff_takes_value(kind: TokenKind, name: &str) -> Option { + match kind { + TokenKind::Long => shared_long_takes_value(name).then(ValueSpec::value), + TokenKind::Short => match name { + "B" | "C" | "M" | "U" => Some(ValueSpec::attached_only()), + "G" | "I" | "L" | "O" | "S" | "l" => Some(ValueSpec::value()), + "n" => Some(ValueSpec::solo_only()), + _ => None, + }, + _ => None, + } +} + +fn tokenize_git_diff_args(args: &[String]) -> Vec> { + arg_tokenizer::tokenize_grammar(args, &diff_takes_value, Dialect::Posix) +} + +fn tokenize_git_log_args(args: &[String]) -> Vec> { + arg_tokenizer::tokenize_grammar(args, &log_takes_value, Dialect::Posix) +} + +#[cfg(test)] +fn real_flag_args(args: &[String]) -> Vec<&str> { + tokenize_git_log_args(args) + .iter() + .filter(|t| matches!(t.kind, TokenKind::Long | TokenKind::Short)) + .map(|t| t.text) + .collect() +} + +/// True for git log flags that change the *shape* of git's raw output (patch text, diffstat, +/// name lists) in a way incompatible with RTK's injected `--pretty=format` markers, requiring +/// the raw passthrough path instead (see [`requests_raw_log_output`]). `diff`/`show` use the +/// narrower [`diff_wants_raw_shape`]/[`show_wants_raw_shape`] instead. +fn log_wants_raw_shape(token: &Token<'_>, tokens: &[Token<'_>]) -> bool { + // Every `--diff-merges` format but `off`/`none` emits a patch (git 2.53), and log's + // one-line-per-commit compaction cannot represent one -- the same reason `-p` is listed + // below. git takes the value attached or as the next token, so both spellings are read. + if token.kind == TokenKind::Long && token.text == "diff-merges" { + return !matches!(token.value(tokens), None | Some("none" | "off")); + } + match token.kind { + TokenKind::Long => matches!( + token.text, + // A binary patch is meant to be fed back to `git apply`; compacting it destroys + // that, so it takes the raw route rather than merely being kept out of the header. + "binary" + // `--cc`/`--remerge-diff` imply `-p` on merge commits (8 diff lines against + // git 2.53 where plain log has none), and the log compaction drops the patch + // with no tee to recover it from. + | "cc" + | "remerge-diff" + | "compact-summary" + | "dirstat" + // Prefixes every output line, including the `diff --git`/`@@` markers the + // compaction keys on, so nothing parses and the body came back empty. + | "line-prefix" + | "name-only" + | "name-status" + | "numstat" + | "patch" + | "patch-with-raw" + | "patch-with-stat" + | "raw" + | "shortstat" + | "stat" + | "summary" + | "unified" + ), + // `-U` makes `git log` emit a patch the one-line-per-commit compaction cannot + // represent, and `-c` is the combined-diff form of `--cc`. `-W`/`--function-context` + // is *not* here: against git 2.53 it leaves `git log` byte-identical to plain log. + TokenKind::Short => matches!(token.text, "U" | "c" | "p" | "u"), + _ => false, + } +} + +/// `diff`'s raw-output grammar: `show`'s, plus `--quiet`. A strict superset, so it composes +/// rather than repeating the list -- `git diff --quiet` prints nothing and exits 1 on a +/// difference (git 2.53), so there is nothing to compact and the exit code has to survive. +fn diff_wants_raw_shape(token: &Token<'_>, tokens: &[Token<'_>]) -> bool { + matches!((token.kind, token.text), (TokenKind::Long, "quiet")) + || show_wants_raw_shape(token, tokens) +} + +/// `show`'s raw-output grammar. `--check` reports whitespace errors and exits 2; `--exit-code` +/// prints the whole patch and exits 1. `--quiet` is deliberately absent: in `show` it is a +/// synonym of `-s` (exit 0, body suppressed, git 2.53), which [`suppresses_diff_body`] renders +/// as the compact summary -- claiming it here made `git show --quiet` raw-pass the header its +/// own synonyms compact. +/// +/// Excludes `--patch`/`-p`/`-u` and `--unified`/`-U`: unlike `log`, `diff`/`show`'s default +/// output already *is* patch text, so those are redundant with the default rather than a shape +/// RTK cannot produce. Delegates the rest to [`log_wants_raw_shape`] so the two can't drift. +fn show_wants_raw_shape(token: &Token<'_>, tokens: &[Token<'_>]) -> bool { + if matches!( + (token.kind, token.text), + (TokenKind::Long, "check" | "exit-code") + ) { + return true; + } + // Only `--diff-merges`' combined formats produce the two marker columns `compact_diff` + // misreads -- the same reason `-c`/`--cc` are raw here. The rest are ordinary + // single-column patches, which is already `show`'s default output. + if token.kind == TokenKind::Long && token.text == "diff-merges" { + return matches!( + token.value(tokens), + Some("c" | "cc" | "combined" | "dense-combined") + ); + } + // `-c` is the exception: it is the combined-diff form of `--cc`, not a patch request, and + // `compact_diff` reads a combined diff's two marker columns as one -- `git show -c` on a + // merge reported `+54 -8` where git's own stat says 156 insertions and 0 deletions. + if (token.kind == TokenKind::Short && token.text != "c") + || (token.kind == TokenKind::Long && matches!(token.text, "patch" | "unified")) + { + return false; + } + log_wants_raw_shape(token, tokens) +} + +/// Test-only convenience wrapper. +#[cfg(test)] +fn requests_raw_log_output(args: &[String]) -> bool { + let tokens = tokenize_git_log_args(args); + tokens.iter().any(|t| log_wants_raw_shape(t, &tokens)) +} + +/// Parse the user-specified limit from git log args. +/// Handles: -20, -n 20, --max-count=20, --max-count 20 +/// `run_log` shares a single tokenization via [`parse_limit_from_tokens`] +/// instead; this convenience wrapper exists for tests. +#[cfg(test)] +fn parse_user_limit(args: &[String]) -> Option { + parse_limit_from_tokens(&tokenize_git_log_args(args)) +} + +/// True if the user explicitly requested a commit-count limit (-N, -n N, --max-count=N, +/// --max-count N). +fn has_limit_flag(tokens: &[Token<'_>]) -> bool { + tokens.iter().any(|t| match t.kind { + TokenKind::Long => t.text == "max-count", + // "n" only counts if it actually captured a value -- e.g. clustered with another short + // flag (log_takes_value's solo_only spec refuses to link a value there), it's + // just an inert letter, not a real limit request. + TokenKind::Short => (t.text == "n" && t.value(tokens).is_some()) || is_digit_run(t.text), + _ => false, + }) +} + +fn parse_limit_from_tokens(tokens: &[Token<'_>]) -> Option { + for token in tokens { + let value = match token.kind { + // --max-count=20 (attached) or --max-count 20 (two-token form). + TokenKind::Long if token.text == "max-count" => token.value(tokens), + // -20 (combined digit form): the token itself is the count. + TokenKind::Short if is_digit_run(token.text) => Some(token.text), + // -n 20 (two-token form) or -n's value if ever attached. + TokenKind::Short if token.text == "n" => token.value(tokens), + _ => None, + }; + if let Some(n) = value.and_then(|v| v.parse::().ok()) { + return Some(n); + } + } + None +} + +/// When `user_set_limit` is true, the user explicitly passed `-N` to git log, +/// so we skip line capping (git already returns exactly N commits) and use a +/// wider truncation threshold (120 chars) to preserve commit context that LLMs +/// need for rebase/squash operations. +pub(crate) fn filter_log_output( + output: &str, + limit: usize, + user_set_limit: bool, + user_format: bool, +) -> String { + let truncate_width = if user_set_limit { 120 } else { 80 }; + + // When user specified their own format (--oneline, --pretty, --format), + // RTK did not inject ---END--- markers. Use simple line-based truncation. + if user_format { + let lines: Vec<&str> = output.lines().collect(); + let max_lines = if user_set_limit { lines.len() } else { limit }; + return lines + .iter() + .take(max_lines) + .map(|l| truncate_line(l, truncate_width)) + .collect::>() + .join("\n"); + } + + // RTK injected format: split output into commit blocks separated by ---END--- + let commits: Vec<&str> = output.split("---END---").collect(); + let max_commits = if user_set_limit { commits.len() } else { limit }; + + let mut result = Vec::new(); + for block in commits.iter().take(max_commits) { + let block = block.trim(); + if block.is_empty() { + continue; + } + let mut lines = block.lines(); + // First line is the header: hash subject (date) + let header = match lines.next() { + Some(h) => truncate_line(h.trim(), truncate_width), + None => continue, + }; + // Remaining lines are the body — keep up to 3 non-empty, non-trailer lines + let all_body_lines: Vec<&str> = lines + .map(|l| l.trim()) + .filter(|l| { + !l.is_empty() + && !l.starts_with("Signed-off-by:") + && !l.starts_with("Co-authored-by:") + }) + .collect(); + let body_omitted = all_body_lines.len().saturating_sub(3); + let body_lines = &all_body_lines[..all_body_lines.len().min(3)]; + + if body_lines.is_empty() { + result.push(header); + } else { + let mut entry = header; + for body in body_lines { + entry.push_str(&format!("\n {}", truncate_line(body, truncate_width))); + } + if body_omitted > 0 { + entry.push_str(&format!("\n [+{} lines omitted]", body_omitted)); + } + result.push(entry); + } + } + + result.join("\n").trim().to_string() +} + +/// Truncate a single line to `width` characters, appending "..." if needed +fn truncate_line(line: &str, width: usize) -> String { + if line.chars().count() > width { + let truncated: String = line.chars().take(width - 3).collect(); + format!("{}...", truncated) + } else { + line.to_string() + } +} + +pub(crate) fn format_status_output(porcelain: &str) -> String { + format_status_inner(porcelain, None) +} + +pub(crate) fn format_status_output_detached(porcelain: &str, detached_ref: &str) -> String { + format_status_inner(porcelain, Some(detached_ref)) +} + +fn format_status_inner(porcelain: &str, detached: Option<&str>) -> String { + let lines: Vec<&str> = porcelain + .lines() + .filter(|line| !line.trim().is_empty()) + .collect(); + + if lines.is_empty() { + return "Clean working tree".to_string(); + } + + let mut output = Vec::new(); + + if let Some(branch_line) = lines.first() { + if branch_line.starts_with("##") { + let branch = branch_line.trim_start_matches("## "); + let display = detached.unwrap_or(branch); + output.push(format!("* {}", display)); + } else { + output.push((*branch_line).to_string()); + } + } + + for line in lines.iter().skip(1) { + output.push((*line).to_string()); + } + + if lines.len() == 1 && lines[0].starts_with("##") { + output.push("clean — nothing to commit".to_string()); + } + + output.join("\n") +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GitStatusState { + Rebase, + MergeConflicts, + MergeReadyToCommit, + CherryPick, + Revert, + Bisect, + Am, + SparseCheckout, +} + +impl GitStatusState { + fn summary(self) -> &'static str { + match self { + Self::Rebase => "rebase in progress", + Self::MergeConflicts => "merge in progress. unresolved conflicts", + Self::MergeReadyToCommit => "merge in progress. no conflicts", + Self::CherryPick => "cherry-pick in progress", + Self::Revert => "revert in progress", + Self::Bisect => "bisect in progress", + Self::Am => "am session in progress", + Self::SparseCheckout => "sparse checkout enabled", + } + } +} + +const REBASE_INDICATORS: &[&str] = &[ + "rebase in progress", + "You are currently rebasing", + "You are currently editing", + "You are currently splitting", + "Last command done", + "Next command to do", + "No commands remaining", +]; + +fn detect_status_state(line: &str) -> Option { + if line.contains("All conflicts fixed but you are still merging") { + Some(GitStatusState::MergeReadyToCommit) + } else if line.contains("You have unmerged paths") { + Some(GitStatusState::MergeConflicts) + } else if line.contains("You are currently cherry-picking") { + Some(GitStatusState::CherryPick) + } else if line.contains("You are currently reverting") { + Some(GitStatusState::Revert) + } else if line.contains("You are currently bisecting") { + Some(GitStatusState::Bisect) + } else if line.contains("You are in the middle of an am session") { + Some(GitStatusState::Am) + } else if line.contains("You are in a sparse checkout") { + Some(GitStatusState::SparseCheckout) + } else if REBASE_INDICATORS.iter().any(|i| line.contains(i)) { + Some(GitStatusState::Rebase) + } else { + None + } +} + +/// `git status --porcelain -b` (compact mode) omits the state header for rebase/merge/ +/// cherry-pick/etc, so an in-progress rebase can look like a clean status. Extracts a compact +/// summary of that state from plain `git status` output instead. `None` if none is in progress. +fn extract_state_header(raw: &str) -> Option { + // Headers of the file-change blocks — everything relevant to state appears + // above these in git's output, so they double as a terminator. + const STOPPERS: &[&str] = &[ + "Changes to be committed:", + "Changes not staged for commit:", + "Untracked files:", + "Unmerged paths:", + "no changes added to commit", + "nothing to commit", + "nothing added to commit", + ]; + + for line in raw.lines() { + let stripped = line.trim(); + + if STOPPERS.iter().any(|s| stripped.starts_with(s)) { + break; + } + + if let Some(state) = detect_status_state(stripped) { + return Some(state.summary().to_string()); + } + } + + None +} + +/// Porcelain `-b` collapses a detached HEAD to the opaque `## HEAD (no branch)`, which can be +/// misread as a branch literally named `HEAD`. Extracts the explicit "HEAD detached at/from +/// " line from plain `git status` output instead. `None` if HEAD is on a branch. +fn extract_detached_head(raw: &str) -> Option { + raw.lines() + .map(str::trim) + .find(|l| l.starts_with("HEAD detached ")) + .map(str::to_string) +} + +/// Minimal filtering for git status with user-provided args +fn filter_status_with_args(output: &str) -> String { + let mut result = Vec::new(); + + for line in output.lines() { + let trimmed = line.trim(); + + // Skip empty lines + if trimmed.is_empty() { + continue; + } + + // Skip git hints - can appear at start or within line + if trimmed.starts_with("(use \"git") + || trimmed.starts_with("(create/copy files") + || trimmed.contains("(use \"git add") + || trimmed.contains("(use \"git restore") + { + continue; + } + + // Special case: clean working tree + if trimmed.contains("nothing to commit") && trimmed.contains("working tree clean") { + result.push(trimmed.to_string()); + break; + } + + result.push(line.to_string()); + } + + if result.is_empty() { + "ok".to_string() + } else { + result.join("\n") + } +} + +fn run_status(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + // Keep a narrow compact path for no-arg status and branch/short-only flags. + // More complex explicit args still use the existing minimal-filter path. + if !uses_compact_status_path(args) { + let mut cmd = build_status_command(args, global_args); + let result = exec_capture(&mut cmd).context("Failed to run git status")?; + + if !result.success() { + if !result.stderr.trim().is_empty() { + eprint!("{}", result.stderr); + } + timer.track( + &format!("git status {}", args.join(" ")), + &format!("rtk git status {}", args.join(" ")), + &result.stdout, + &result.stdout, + ); + return Ok(result.exit_code); + } + + if verbose > 0 || !result.stderr.is_empty() { + eprint!("{}", result.stderr); + } + + // Apply minimal filtering: strip ANSI, remove hints, empty lines + let filtered = filter_status_with_args(&result.stdout); + let filtered = never_worse(&result.stdout, &filtered).to_string(); + print!("{}", filtered); + + timer.track( + &format!("git status {}", args.join(" ")), + &format!("rtk git status {}", args.join(" ")), + &result.stdout, + &filtered, + ); + + return Ok(0); + } + + let mut raw_cmd = git_cmd_c_locale(global_args); + raw_cmd.arg("status"); + raw_cmd.args(args); + let raw_output = exec_capture(&mut raw_cmd) + .map(|r| r.stdout) + .unwrap_or_default(); + + let mut cmd = build_status_command(args, global_args); + let result = exec_capture(&mut cmd).context("Failed to run git status")?; + + if !result.success() { + let message = if result.stderr.contains("not a git repository") { + "Not a git repository".to_string() + } else { + result.stderr.trim().to_string() + }; + if !message.is_empty() { + eprintln!("{}", message); + } + let original_cmd = if args.is_empty() { + "git status".to_string() + } else { + format!("git status {}", args.join(" ")) + }; + let rtk_cmd = if args.is_empty() { + "rtk git status".to_string() + } else { + format!("rtk git status {}", args.join(" ")) + }; + let shown = never_worse(&raw_output, &message); + timer.track(&original_cmd, &rtk_cmd, &raw_output, shown); + return Ok(result.exit_code); + } + + let formatted = match extract_detached_head(&raw_output) { + Some(detached_ref) => format_status_output_detached(&result.stdout, &detached_ref), + None => format_status_output(&result.stdout), + }; + + // Surface in-progress state (rebase/merge/cherry-pick/bisect/am) from the + // plain-status output we already captured for tracking. Porcelain omits it + // and hiding it misleads the user about the true repo state. + let final_output = match extract_state_header(&raw_output) { + Some(state) => format!("{}\n{}", state, formatted), + None => formatted, + }; + + let shown = never_worse(&raw_output, &final_output); + println!("{}", shown); + + let original_cmd = if args.is_empty() { + "git status".to_string() + } else { + format!("git status {}", args.join(" ")) + }; + let rtk_cmd = if args.is_empty() { + "rtk git status".to_string() + } else { + format!("rtk git status {}", args.join(" ")) + }; + + timer.track(&original_cmd, &rtk_cmd, &raw_output, shown); + + Ok(0) +} + +fn run_add(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + let mut cmd = git_cmd(global_args); + cmd.arg("add"); + + // Pass all arguments directly to git (flags like -A, -p, --all, etc.) + if args.is_empty() { + cmd.arg("."); + } else { + for arg in args { + cmd.arg(arg); + } + } + + let result = exec_capture(&mut cmd).context("Failed to run git add")?; + + if verbose > 0 { + eprintln!("git add executed"); + } + + let raw_output = format!("{}\n{}", result.stdout, result.stderr); + + if result.success() { + // Count what was added + let mut stat_cmd = git_cmd(global_args); + stat_cmd.args(["diff", "--cached", "--stat", "--shortstat"]); + let stat_result = exec_capture(&mut stat_cmd).context("Failed to check staged files")?; + + // Mirror git's own behaviour: a no-op `git add` is silent. Emitting a + // generic "ok" here is misleading — an agent can't tell "staged N files" + // from "staged nothing" when both print "ok". + let compact = if stat_result.stdout.trim().is_empty() { + String::new() + } else { + // Parse "1 file changed, 5 insertions(+)" format + let short = stat_result.stdout.lines().last().unwrap_or("").trim(); + if short.is_empty() { + "ok".to_string() + } else { + format!("ok {}", short) + } + }; + + if !compact.is_empty() { + println!("{}", compact); + } else if !result.stderr.trim().is_empty() { + // Nothing staged, but git had something to say about why (`git add --` answers + // "Nothing specified, nothing added" with a hint). Printing neither the count nor + // git's own explanation leaves the agent unable to tell that from a crash. + eprintln!("{}", result.stderr.trim()); + } + + timer.track( + &format!("git add {}", args.join(" ")), + &format!("rtk git add {}", args.join(" ")), + &raw_output, + &compact, + ); + } else { + eprintln!("FAILED: git add"); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + if !result.stdout.trim().is_empty() { + eprintln!("{}", result.stdout); + } + return Ok(result.exit_code); + } + + Ok(0) +} + +fn build_commit_command(args: &[String], global_args: &[String]) -> Command { + let mut cmd = git_cmd(global_args); + cmd.arg("commit"); + for arg in args { + cmd.arg(arg); + } + cmd +} + +/// Parse the first line of `git commit` success output and return a compact token. +/// Handles: `[main abc1234def] message`, `[main (root-commit) abc1234def] msg`, +/// localized variants, and multibyte branch names. +fn parse_commit_output(line: &str) -> String { + // Locate the brackets rather than assume the line starts with '[': git prints hook output + // first, and slicing from byte 1 would panic on a multi-byte leading character. + let (Some(open), Some(bracket_end)) = (line.find('['), line.find(']')) else { + return "ok".to_string(); + }; + if open >= bracket_end { + return "ok".to_string(); + } + + let bracket_content = &line[open + 1..bracket_end]; + let hash = bracket_content.split_whitespace().next_back().unwrap_or(""); + if hash.chars().count() >= 7 { + let short_hash: String = hash.chars().take(7).collect(); + format!("ok {}", short_hash) + } else { + "ok".to_string() + } +} + +fn run_commit(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + let original_cmd = format!("git commit {}", args.join(" ")); + + if verbose > 0 { + eprintln!("{}", original_cmd); + } + + // stdin is inherited so an interactive editor, GPG passphrase prompt or + // credential helper still reaches the terminal. + let CaptureResult { + stdout, + stderr, + exit_code, + } = exec_capture_stdin(&mut build_commit_command(args, global_args)) + .context("Failed to run git commit")?; + let raw_output = format!("{}\n{}", stdout, stderr); + + match classify_commit_outcome(exit_code == 0, &stdout, exit_code) { + CommitOutcome::Ok(compact) => { + println!("{}", compact); + timer.track(&original_cmd, "rtk git commit", &raw_output, &compact); + Ok(0) + } + CommitOutcome::Failed(code) => { + if !stderr.trim().is_empty() { + eprint!("{}", stderr); + } + if !stdout.trim().is_empty() { + eprint!("{}", stdout); + } + timer.track(&original_cmd, "rtk git commit", &raw_output, &raw_output); + Ok(code) + } + } +} + +/// Outcome of a `git commit`: a non-success status propagates the exit code +/// rather than being reported as "ok" (#2494). +enum CommitOutcome { + Ok(String), + Failed(i32), +} + +/// Classify a `git commit` result. +fn classify_commit_outcome(success: bool, stdout: &str, exit_code: i32) -> CommitOutcome { + if success { + // Extract commit hash from output + let compact = stdout + .lines() + .next() + .map(parse_commit_output) + .unwrap_or_else(|| "ok".to_string()); + CommitOutcome::Ok(compact) + } else { + CommitOutcome::Failed(exit_code) + } +} + +fn run_checkout(args: &[String], verbose: u8, global_args: &[String]) -> Result { + if verbose > 0 { + eprintln!("git checkout"); + } + + // The user's locale, per git_cmd_c_locale's own contract: this child's stderr is shown + // verbatim on failure. When the English "Switched to branch ..." scan misses, the + // args-based fallback below still names the branch. + let mut cmd = git_cmd(global_args); + cmd.arg("checkout"); + for arg in args { + cmd.arg(arg); + } + + let args_display = args.join(" "); + let args_for_filter = args.to_vec(); + runner::run_filtered_with_exit( + cmd, + "git checkout", + &args_display, + move |raw, exit_code| format_checkout_output(&args_for_filter, raw, exit_code), + RunOptions::with_tee("git_checkout"), + ) +} + +fn format_checkout_output(args: &[String], raw: &str, exit_code: i32) -> String { + if exit_code == 0 { + format_checkout_success(args, raw) + } else { + filter_checkout_failure(raw) + } +} + +fn format_checkout_success(args: &[String], raw: &str) -> String { + let tokens = arg_tokenizer::tokenize_grammar(args, &checkout_takes_value, Dialect::Posix); + + if let Some(restored) = checkout_restored_count(&tokens) { + return format!( + "ok {} {}", + restored, + pluralize(restored, "file restored", "files restored") + ); + } + for line in raw.lines().map(str::trim) { + if let Some(branch) = quoted_suffix(line, "Switched to a new branch ") { + return format!("ok {} (new)", branch); + } + if let Some(branch) = quoted_suffix(line, "Switched to branch ") { + return format!("ok {}", branch); + } + if let Some(branch) = quoted_suffix(line, "Already on ") { + return format!("ok {}", branch); + } + if let Some(rest) = line.strip_prefix("HEAD is now at ") { + let hash = rest.split_whitespace().next().unwrap_or("HEAD"); + return format!("ok HEAD {}", hash); + } + if line.starts_with("Updated ") && line.contains(" path") { + return format!("ok {}", line.to_ascii_lowercase()); + } + } + + // Both of these are the fallback for when the scan above misses, never a short-circuit + // past it: `-B` creates *or* resets, and only git knows which happened, so claiming the + // name early cost `git checkout -Bfoo` its `(new)` marker. + // + // The scan is English-only and this child deliberately keeps the user's locale (see + // run_checkout), so under another locale `-B` reaches `ok ` here even when it + // created the branch. That is a weaker answer, never a wrong one -- the marker is omitted + // rather than claimed falsely -- but it does mean `(new)` is not guaranteed. + if let Some(branch) = checkout_new_branch_arg(&tokens) { + return format!("ok {} (new)", branch); + } + if let Some(branch) = checkout_reset_branch_arg(&tokens) { + return format!("ok {}", branch); + } + if let Some(branch) = checkout_branch_arg(&tokens) { + return format!("ok {}", branch); + } + + "ok".to_string() +} + +/// The options git consumes a separate token for: `--orphan`/`-b`/`-B` take a branch name, +/// `--conflict` a style, `--pathspec-from-file` a file (all confirmed against git 2.53, which +/// answers "requires a value"). `-t`/`--track`/`--detach` and any other `-`-prefixed token are +/// booleans. Shared by every `checkout_*_arg` helper below via one +/// [`arg_tokenizer::tokenize`] call instead of each hand-rolling its own scan over `args`. +fn checkout_takes_value(kind: TokenKind, name: &str) -> Option { + match kind { + TokenKind::Long => { + matches!(name, "conflict" | "orphan" | "pathspec-from-file").then(ValueSpec::value) + } + TokenKind::Short => matches!(name, "B" | "b").then(ValueSpec::value), + _ => None, + } +} + +fn checkout_restored_count(tokens: &[Token<'_>]) -> Option { + let separator = arg_tokenizer::dashdash_index(tokens)?; + let count = tokens[separator + 1..] + .iter() + .filter(|t| !t.text.is_empty()) + .count(); + (count > 0).then_some(count) +} + +fn checkout_new_branch_arg<'a>(tokens: &[Token<'a>]) -> Option<&'a str> { + tokens.iter().find_map(|t| match t.kind { + TokenKind::Long if t.text == "orphan" => t.value(tokens), + TokenKind::Short if t.text == "b" => t.value(tokens), + _ => None, + }) +} + +fn checkout_reset_branch_arg<'a>(tokens: &[Token<'a>]) -> Option<&'a str> { + tokens + .iter() + .find(|t| t.kind == TokenKind::Short && t.text == "B") + .and_then(|t| t.value(tokens)) +} + +fn checkout_branch_arg<'a>(tokens: &[Token<'a>]) -> Option<&'a str> { + if arg_tokenizer::has_dashdash(tokens) { + return None; + } + tokens + .iter() + // A bare `-` is git's "the branch I was on before", not a branch name to echo back. + .find(|t| t.is_free_positional() && t.text != "-") + .map(|t| t.text) +} + +fn quoted_suffix<'a>(line: &'a str, prefix: &str) -> Option<&'a str> { + line.strip_prefix(prefix) + .and_then(|rest| rest.strip_prefix('\'')) + .and_then(|rest| rest.strip_suffix('\'')) +} + +fn pluralize<'a>(count: usize, singular: &'a str, plural: &'a str) -> &'a str { + if count == 1 { singular } else { plural } +} + +fn filter_checkout_failure(raw: &str) -> String { + let mut important = Vec::new(); + let mut in_file_list = false; + + for line in raw.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + + let is_header = trimmed.starts_with("error:") + || trimmed.starts_with("fatal:") + || trimmed.starts_with("CONFLICT"); + + if is_header { + in_file_list = trimmed.contains("following") + && trimmed.contains("files") + && trimmed.ends_with(':'); + important.push(trimmed.to_string()); + continue; + } + + if in_file_list { + if trimmed.starts_with("Please ") || trimmed.starts_with("Aborting") { + in_file_list = false; + } else if line.starts_with(char::is_whitespace) { + important.push(line.to_string()); + continue; + } + } + + if trimmed.starts_with("Aborting") { + important.push(trimmed.to_string()); + } + } + + if important.is_empty() { + raw.trim().to_string() + } else { + important.join("\n") + } +} + +// Git push progress prefixes (stderr) — dropped from the stream. +const GIT_PUSH_NOISE_PREFIXES: &[&str] = &[ + "Enumerating objects:", + "Counting objects:", + "Compressing objects:", + "Writing objects:", + "Delta compression using", + "Total ", +]; + +#[derive(Default)] +struct GitPushLineHandler { + up_to_date: bool, + pushed_ref: Option, +} + +impl LineHandler for GitPushLineHandler { + fn should_skip(&mut self, line: &str) -> bool { + if line.is_empty() { + return true; + } + let trimmed = line.trim_start(); + GIT_PUSH_NOISE_PREFIXES + .iter() + .any(|p| trimmed.starts_with(p)) + } + + fn observe_line(&mut self, line: &str) { + if line.contains("Everything up-to-date") { + self.up_to_date = true; + } + if self.pushed_ref.is_none() + && let Some(idx) = line.find(" -> ") + { + let after = &line[idx + 4..]; + if let Some(dest) = after.split_whitespace().next() { + self.pushed_ref = Some(dest.to_string()); + } + } + } + + fn format_summary(&self, exit_code: i32, _raw: &str) -> Option { + if exit_code != 0 { + return None; + } + let summary = if self.up_to_date { + "ok (up-to-date)".to_string() + } else if let Some(dest) = &self.pushed_ref { + format!("ok {}", dest) + } else { + "ok".to_string() + }; + Some(format!("{}\n", summary)) + } +} + +fn run_push(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git push"); + } + + let mut cmd = git_cmd(global_args); + cmd.arg("push"); + for arg in args { + cmd.arg(arg); + } + + let cmd_label = format!("git push {}", args.join(" ")); + let filter = LineStreamFilter::new(GitPushLineHandler::default()); + let result = stream::run_streaming( + &mut cmd, + StdinMode::Inherit, + FilterMode::Streaming(Box::new(filter)), + ) + .context("Failed to run git push")?; + + timer.track( + &cmd_label, + &format!("rtk {}", cmd_label), + &result.raw, + &result.filtered, + ); + + Ok(result.exit_code) +} + +fn run_pull(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git pull"); + } + + let mut cmd = git_cmd(global_args); + cmd.arg("pull"); + for arg in args { + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git pull")?; + + let raw_output = format!("{}\n{}", result.stdout, result.stderr); + + if result.success() { + let compact = if result.stdout.contains("Already up to date") + || result.stdout.contains("Already up-to-date") + { + "ok (up-to-date)".to_string() + } else { + // Count files changed + let mut files = 0; + let mut insertions = 0; + let mut deletions = 0; + + for line in result.stdout.lines() { + if line.contains("file") && line.contains("changed") { + // Parse "3 files changed, 10 insertions(+), 2 deletions(-)" + for part in line.split(',') { + let part = part.trim(); + if part.contains("file") { + files = part + .split_whitespace() + .next() + .and_then(|n| n.parse().ok()) + .unwrap_or(0); + } else if part.contains("insertion") { + insertions = part + .split_whitespace() + .next() + .and_then(|n| n.parse().ok()) + .unwrap_or(0); + } else if part.contains("deletion") { + deletions = part + .split_whitespace() + .next() + .and_then(|n| n.parse().ok()) + .unwrap_or(0); + } + } + } + } + + if files > 0 { + format!("ok {} files +{} -{}", files, insertions, deletions) + } else { + "ok".to_string() + } + }; + + println!("{}", compact); + + timer.track( + &format!("git pull {}", args.join(" ")), + &format!("rtk git pull {}", args.join(" ")), + &raw_output, + &compact, + ); + } else { + eprintln!("FAILED: git pull"); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + if !result.stdout.trim().is_empty() { + eprintln!("{}", result.stdout); + } + return Ok(result.exit_code); + } + + Ok(0) +} + +fn branch_takes_value(kind: TokenKind, name: &str) -> Option { + // -c/-C/-m/-M/-d/-D are followed by positional branch names, not a "flag value" in the + // attached/separate-value sense, so they're excluded here. -u IS a genuine value-taking flag + // (its short form takes the same single upstream-ref value as --set-upstream-to) and must be + // included alongside its long form, or `git branch -u origin/main` leaves "origin/main" as + // an unlinked Positional token instead of -u's linked value. + match kind { + TokenKind::Long => matches!( + name, + "contains" + | "format" + | "merged" + | "no-contains" + | "no-merged" + | "points-at" + | "set-upstream-to" + | "sort" + ) + .then(ValueSpec::value), + TokenKind::Short => (name == "u").then(ValueSpec::value), + _ => None, + } +} + +fn run_branch(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git branch"); + } + + let tokens = arg_tokenizer::tokenize_grammar(args, &branch_takes_value, Dialect::Posix); + + // Detect write operations: delete, rename, copy, upstream tracking + let has_action_flag = tokens.iter().any(|t| match t.kind { + TokenKind::Short => matches!(t.text, "C" | "D" | "M" | "c" | "d" | "m" | "u"), + TokenKind::Long => matches!( + t.text, + "edit-description" | "set-upstream-to" | "unset-upstream" + ), + _ => false, + }); + + // Detect flags that produce specific output (not a branch list) + let has_show_flag = tokens + .iter() + .any(|t| t.kind == TokenKind::Long && t.text == "show-current"); + + // Detect list-mode flags + let has_list_flag = tokens.iter().any(|t| match t.kind { + TokenKind::Short => matches!(t.text, "a" | "r"), + TokenKind::Long => matches!( + t.text, + "all" + | "contains" + | "format" + | "list" + | "merged" + | "no-contains" + | "no-merged" + | "points-at" + | "remotes" + | "sort" + ), + _ => false, + }); + + // Detect positional arguments (not flags) — indicates branch creation. A value consumed by + // a preceding flag (e.g. -u/--set-upstream-to's upstream ref) is that flag's value, not an + // independent positional branch name, so linked tokens are excluded. + let has_positional_arg = tokens.iter().any(|t| t.is_free_positional()); + + // --show-current: passthrough with raw stdout (not "ok") + if has_show_flag { + let mut cmd = git_cmd(global_args); + cmd.arg("branch"); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git branch")?; + let combined = result.combined(); + + let trimmed = result.stdout.trim(); + timer.track( + &format!("git branch {}", args.join(" ")), + &format!("rtk git branch {}", args.join(" ")), + &combined, + trimmed, + ); + + if result.success() { + println!("{}", trimmed); + } else { + eprintln!("FAILED: git branch {}", args.join(" ")); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + return Ok(result.exit_code); + } + return Ok(0); + } + + // Write operation: action flags, or positional args without list flags (= branch creation) + if has_action_flag || (has_positional_arg && !has_list_flag) { + let mut cmd = git_cmd(global_args); + cmd.arg("branch"); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git branch")?; + let combined = result.combined(); + + let msg = if result.success() { "ok" } else { &combined }; + + timer.track( + &format!("git branch {}", args.join(" ")), + &format!("rtk git branch {}", args.join(" ")), + &combined, + msg, + ); + + if result.success() { + println!("ok"); + } else { + eprintln!("FAILED: git branch {}", args.join(" ")); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + if !result.stdout.trim().is_empty() { + eprintln!("{}", result.stdout); + } + return Ok(result.exit_code); + } + return Ok(0); + } + + // List mode: show compact branch list + let mut cmd = git_cmd(global_args); + cmd.arg("branch"); + if !has_list_flag { + cmd.arg("-a"); + } + cmd.arg("--no-color"); + for arg in args { + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git branch")?; + + if !result.success() { + if !result.stderr.trim().is_empty() { + eprint!("{}", result.stderr); + } + timer.track( + &format!("git branch {}", args.join(" ")), + &format!("rtk git branch {}", args.join(" ")), + &result.stdout, + &result.stdout, + ); + return Ok(result.exit_code); + } + + let filtered = filter_branch_output(&result.stdout); + let filtered = never_worse(&result.stdout, &filtered).to_string(); + println!("{}", filtered); + + timer.track( + &format!("git branch {}", args.join(" ")), + &format!("rtk git branch {}", args.join(" ")), + &result.stdout, + &filtered, + ); + + Ok(0) +} + +fn filter_branch_output(output: &str) -> String { + let mut current = String::new(); + let mut local: Vec = Vec::new(); + let mut remote: Vec = Vec::new(); + let mut seen_remote: std::collections::HashSet = std::collections::HashSet::new(); + + for line in output.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + + if let Some(branch) = line.strip_prefix("* ") { + current = branch.to_string(); + } else if let Some(rest) = line.strip_prefix("remotes/") { + if let Some(slash_pos) = rest.find('/') { + let branch = &rest[slash_pos + 1..]; + if branch.starts_with("HEAD ") { + continue; + } + if seen_remote.insert(branch.to_string()) { + remote.push(branch.to_string()); + } + } + } else { + local.push(line.to_string()); + } + } + + let mut result = Vec::new(); + result.push(format!("* {}", current)); + + if !local.is_empty() { + for b in &local { + result.push(format!(" {}", b)); + } + } + + if !remote.is_empty() { + let remote_only: Vec<&String> = remote + .iter() + .filter(|r| *r != ¤t && !local.contains(r)) + .collect(); + if !remote_only.is_empty() { + const MAX_REMOTE_BRANCHES: usize = CAP_WARNINGS; + result.push(format!(" remote-only ({}):", remote_only.len())); + for b in remote_only.iter().take(MAX_REMOTE_BRANCHES) { + result.push(format!(" {}", b)); + } + if remote_only.len() > MAX_REMOTE_BRANCHES { + result.push(format!( + " ... +{} more", + remote_only.len() - MAX_REMOTE_BRANCHES + )); + } + } + } + + result.join("\n") +} + +fn run_fetch(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git fetch"); + } + + let mut cmd = git_cmd(global_args); + cmd.arg("fetch"); + for arg in args { + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git fetch")?; + let raw = result.combined(); + + if !result.success() { + eprintln!("FAILED: git fetch"); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + return Ok(result.exit_code); + } + + // Count new refs from stderr (git fetch outputs to stderr) + let new_refs: usize = result + .stderr + .lines() + .filter(|l| l.contains("->") || l.contains("[new")) + .count(); + + let msg = if new_refs > 0 { + format!("ok fetched ({} new refs)", new_refs) + } else { + "ok fetched".to_string() + }; + + println!("{}", msg); + timer.track("git fetch", "rtk git fetch", &raw, &msg); + + Ok(0) +} + +/// Format status message for stash operations. +/// - For create operations (push/save): checks for "No local changes" +/// - For other operations: uses "ok stash " format +fn format_stash_message(subcommand: Option<&str>, result: &CaptureResult) -> String { + match subcommand { + None | Some("push") | Some("save") => { + // A successful stash collapses to "ok stashed" (the WIP ref/sha git + // prints isn't needed to `git stash pop`). But a no-op must NOT look + // like success — pass git's "No local changes to save" through so the + // agent can tell nothing was stashed. + if result.combined().contains("No local changes") { + "No local changes to save".to_string() + } else { + "ok stashed".to_string() + } + } + Some(sub) => format!("ok stash {}", sub), + } +} + +/// True if `-p`/`--patch` was requested. Note: `-u` means `--include-untracked` here, not `-p`. +/// +/// The "nothing takes a value" predicate *is* `stash show`'s grammar for this question, not a +/// placeholder: git parses `-p`/`-u` itself before handing the rest to the revision machinery, +/// so no flag it sees here consumes a following token, and treating one as if it did swallowed +/// the `-p` after it (confirmed against git 2.53 with `git stash show --author -p`). +fn stash_show_wants_patch(args: &[String]) -> bool { + let tokens = arg_tokenizer::tokenize(args); + tokens.iter().any(|t| match t.kind { + TokenKind::Long => t.text == "patch", + TokenKind::Short => t.text == "p", + _ => false, + }) +} + +fn run_stash( + subcommand: Option<&str>, + args: &[String], + verbose: u8, + global_args: &[String], +) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git stash {:?}", subcommand); + } + + match subcommand { + Some("list") => { + let mut cmd = git_cmd(global_args); + cmd.args(["stash", "list"]); + let result = exec_capture(&mut cmd).context("Failed to run git stash list")?; + + if result.stdout.trim().is_empty() { + if !result.success() && !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr.trim()); + } + timer.track("git stash list", "rtk git stash list", &result.stdout, ""); + return Ok(result.exit_code); + } + + let filtered = filter_stash_list(&result.stdout); + let filtered = never_worse(&result.stdout, &filtered).to_string(); + println!("{}", filtered); + timer.track( + "git stash list", + "rtk git stash list", + &result.stdout, + &filtered, + ); + } + Some("show") => { + let asked_for_patch = stash_show_wants_patch(args); + + let mut cmd = git_cmd(global_args); + cmd.args(["stash", "show"]); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git stash show")?; + + if result.stdout.trim().is_empty() { + if !result.success() && !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr.trim()); + } + timer.track("git stash show", "rtk git stash show", &result.stdout, ""); + return Ok(result.exit_code); + } + + // What git actually produced settles it, not what the flags predicted: `git stash + // show --` (or any other extra argument) flips git to diff output, and running the + // stat filter over patch text yields nothing at all. + let patch_mode = asked_for_patch + || result + .stdout + .lines() + .any(|line| line.starts_with("diff --git ") || line.starts_with("diff --cc ")); + + // Log's grammar, unlike `stash_show_wants_patch`'s: `stash show` parses `-p`/`-u` + // itself, but hands the rest to the revision machinery, which does consume a + // following `--word-diff` as `--author`'s value. + let filtered = if patch_mode && !emits_word_diff(&tokenize_git_log_args(args)) { + compact_diff(&result.stdout, 100) + } else if patch_mode { + result.stdout.clone() + } else { + compact_stash_stat(&result.stdout) + }; + let shown = crate::core::runner::emit_guarded(&filtered, None, &result.stdout); + timer.track( + "git stash show", + "rtk git stash show", + &result.stdout, + &shown, + ); + } + Some("apply") | Some("branch") | Some("clear") | Some("create") | Some("drop") + | Some("export") | Some("import") | Some("pop") | Some("store") => { + let sub = subcommand.unwrap(); + let mut cmd = git_cmd(global_args); + cmd.args(["stash", sub]); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git stash")?; + let combined = result.combined(); + + let msg = if result.success() { + let msg = format_stash_message(subcommand, &result); + println!("{}", msg); + msg + } else { + eprintln!("FAILED: git stash {}", sub); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + combined.clone() + }; + + timer.track( + &format!("git stash {}", sub), + &format!("rtk git stash {}", sub), + &combined, + &msg, + ); + + if !result.success() { + return Ok(result.exit_code); + } + } + // Default: "git stash [push] [--] [...]" or "git stash save []" + Some(_) | None => { + let (sub, arg) = match subcommand { + Some("save") => ("save", None), + Some("push") => ("push", None), + Some(s) => ("push", Some(s)), + None => ("push", None), + }; + let mut cmd = git_cmd(global_args); + cmd.args(["stash", sub]); + if let Some(arg) = arg { + cmd.arg(arg); + } + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git stash")?; + let combined = result.combined(); + + let msg = if result.success() { + let msg = format_stash_message(subcommand, &result); + println!("{}", msg); + msg + } else { + eprintln!("FAILED: git stash {}", sub); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + combined.clone() + }; + + timer.track( + &format!("git stash {}", sub), + &format!("rtk git stash {}", sub), + &combined, + &msg, + ); + + if !result.success() { + return Ok(result.exit_code); + } + } + } + + Ok(0) +} + +fn filter_stash_list(output: &str) -> String { + // Format: "stash@{0}: WIP on main: abc1234 commit message" + let mut result = Vec::new(); + for line in output.lines() { + if let Some(colon_pos) = line.find(": ") { + let index = &line[..colon_pos]; + let rest = &line[colon_pos + 2..]; + // Compact: strip "WIP on branch:" prefix if present + let message = if let Some(second_colon) = rest.find(": ") { + rest[second_colon + 2..].trim() + } else { + rest.trim() + }; + result.push(format!("{}: {}", index, message)); + } else { + result.push(line.to_string()); + } + } + result.join("\n") +} + +fn compact_stash_stat(raw: &str) -> String { + let (files, summary) = parse_stash_stat(raw); + if files.is_empty() { + return raw.trim_end().to_string(); + } + let total = files.len(); + let mut out = join_with_overflow(&files[..total.min(CAP_LIST)], total, CAP_LIST, "files"); + if total > CAP_LIST + && let Some(hint) = + crate::core::tee::force_tee_tail_hint(&files.join("\n"), "git-stash-show", CAP_LIST + 1) + { + out.push(' '); + out.push_str(&hint); + } + if !summary.is_empty() { + out.push('\n'); + out.push_str(&compress_stat_summary(&summary)); + } + out +} + +fn compress_stat_summary(summary: &str) -> String { + summary + .replace("insertions(+)", "+") + .replace("insertion(+)", "+") + .replace("deletions(-)", "-") + .replace("deletion(-)", "-") + .replace("files changed", "changed") + .replace("file changed", "changed") + .replace(",", "") +} + +fn parse_stash_stat(stat: &str) -> (Vec, String) { + let stat = strip_ansi(stat); + let mut files = Vec::new(); + let mut summary = String::new(); + + for line in stat.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + match diffstat_row(line) { + Some(row) => files.push(row), + None => summary = line.to_string(), + } + } + + (files, summary) +} + +fn diffstat_row(line: &str) -> Option { + let bar = line.rfind('|')?; + let path = line[..bar].trim(); + let rhs = line[bar + 1..].trim(); + let is_diffstat_row = rhs.starts_with("Bin") || rhs.starts_with(|c: char| c.is_ascii_digit()); + if path.is_empty() || !is_diffstat_row { + return None; + } + if rhs.starts_with("Bin") { + return Some(format!("{} (binary)", path)); + } + let count = rhs.split_whitespace().next().unwrap_or(""); + let sign = match (rhs.contains('+'), rhs.contains('-')) { + (true, true) => " +-", + (true, false) => " +", + (false, true) => " -", + (false, false) => "", + }; + Some(format!("{} {}{}", path, count, sign)) +} + +/// True when a `git worktree` write action was asked to report what it did: `prune --dry-run` +/// names every worktree it would remove and that list is the whole point of the command, while +/// `git worktree add`'s progress lines are exactly what RTK's "ok" replaces. +fn worktree_asked_for_report(tokens: &[Token<'_>]) -> bool { + arg_tokenizer::before_dashdash(tokens) + .iter() + .any(|t| match t.kind { + TokenKind::Long => matches!(t.text, "dry-run" | "verbose"), + TokenKind::Short => matches!(t.text, "n" | "v"), + _ => false, + }) +} + +fn run_worktree(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git worktree list"); + } + + // The subcommand is the first positional, not any arg that happens to spell one (a + // worktree path named "move", say). + let tokens = arg_tokenizer::tokenize(args); + let subcommand = arg_tokenizer::before_dashdash(&tokens) + .iter() + .find(|t| t.is_free_positional()) + .map(|t| t.text); + + // Only a bare listing is RTK's to compact. A write action gets the terse "ok"; everything + // else -- `list` with flags of its own like `--porcelain`, a subcommand git grows later, + // or a `--` with no subcommand, which real git rejects -- passes through verbatim, since + // RTK cannot know what its output means. + let compact_list = tokens.is_empty() || (subcommand == Some("list") && tokens.len() == 1); + let write_action = matches!( + subcommand, + Some("add" | "remove" | "prune" | "lock" | "unlock" | "move" | "repair") + ); + let asked_for_report = worktree_asked_for_report(&tokens); + + if !compact_list && !write_action { + let mut cmd = git_cmd(global_args); + cmd.arg("worktree"); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git worktree")?; + print!("{}", result.stdout); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr.trim()); + } + timer.track( + &format!("git worktree {}", args.join(" ")), + &format!("rtk git worktree {} (passthrough)", args.join(" ")), + &result.stdout, + &result.stdout, + ); + return Ok(result.exit_code); + } + + if write_action { + let mut cmd = git_cmd(global_args); + cmd.arg("worktree"); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git worktree")?; + let combined = result.combined(); + + let said = if asked_for_report { + combined.trim() + } else { + "" + }; + // Track what RTK prints, not what git said: on success that is the report or "ok". + let msg = if !result.success() { + combined.as_str() + } else if said.is_empty() { + "ok" + } else { + said + }; + + timer.track( + &format!("git worktree {}", args.join(" ")), + &format!("rtk git worktree {}", args.join(" ")), + &combined, + msg, + ); + + if result.success() { + if said.is_empty() { + println!("ok"); + } else { + println!("{said}"); + } + } else { + eprintln!("FAILED: git worktree {}", args.join(" ")); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + return Ok(result.exit_code); + } + return Ok(0); + } + + // Default: list mode + let mut cmd = git_cmd(global_args); + cmd.args(["worktree", "list"]); + let result = exec_capture(&mut cmd).context("Failed to run git worktree list")?; + + if !result.success() { + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + timer.track( + "git worktree list", + "rtk git worktree", + &result.stdout, + &result.stderr, + ); + return Ok(result.exit_code); + } + + let filtered = filter_worktree_list(&result.stdout); + let filtered = never_worse(&result.stdout, &filtered).to_string(); + println!("{}", filtered); + timer.track( + "git worktree list", + "rtk git worktree", + &result.stdout, + &filtered, + ); + + Ok(0) +} + +fn filter_worktree_list(output: &str) -> String { + let home = dirs::home_dir() + .map(|h| h.to_string_lossy().to_string()) + .unwrap_or_default(); + + let mut result = Vec::new(); + for line in output.lines() { + if line.trim().is_empty() { + continue; + } + // Format: "/path/to/worktree abc1234 [branch]" + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() >= 3 { + let mut path = parts[0].to_string(); + if !home.is_empty() && path.starts_with(&home) { + path = format!("~{}", &path[home.len()..]); + } + let hash = parts[1]; + let branch = parts[2..].join(" "); + result.push(format!("{} {} {}", path, hash, branch)); + } else { + result.push(line.to_string()); + } + } + result.join("\n") +} + +/// Runs an unsupported git subcommand by passing it through directly +pub fn run_passthrough(args: &[OsString], global_args: &[String], verbose: u8) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git passthrough: {:?}", args); + } + let status = git_cmd(global_args) + .args(args) + .status() + .context("Failed to run git")?; + + let args_str = tracking::args_display(args); + timer.track_passthrough( + &format!("git {}", args_str), + &format!("rtk git {} (passthrough)", args_str), + ); + + if !status.success() { + return Ok(exit_code_from_status(&status, "git")); + } + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_branch_dash_u_links_its_upstream_value_not_a_free_positional() { + // -u's short form must link its value like its long form --set-upstream-to does. + let args = vec!["-u".to_string(), "origin/main".to_string()]; + let tokens = arg_tokenizer::tokenize_grammar(&args, &branch_takes_value, Dialect::Posix); + assert_eq!(tokens[0].kind, TokenKind::Short); + assert_eq!(tokens[0].text, "u"); + assert_eq!(tokens[0].value(&tokens), Some("origin/main")); + assert_eq!(tokens[1].kind, TokenKind::Positional); + assert!( + tokens[1].linked.is_some(), + "origin/main must be linked as -u's value, not a free-standing positional" + ); + } + + #[test] + fn test_checkout_new_branch_arg_accepts_glued_short_flag() { + // `-bmy-branch` (glued) and `-b my-branch` (separate) must both work. + let args = vec!["-bmy-branch".to_string()]; + let tokens = arg_tokenizer::tokenize_grammar(&args, &checkout_takes_value, Dialect::Posix); + assert_eq!(checkout_new_branch_arg(&tokens), Some("my-branch")); + + let args = vec!["-b".to_string(), "my-branch".to_string()]; + let tokens = arg_tokenizer::tokenize_grammar(&args, &checkout_takes_value, Dialect::Posix); + assert_eq!(checkout_new_branch_arg(&tokens), Some("my-branch")); + } + + #[test] + fn test_checkout_reset_branch_arg_accepts_glued_short_flag() { + // Same glued-form guarantee as -b, for -B (force-create/reset). + let args = vec!["-Bmy-branch".to_string()]; + let tokens = arg_tokenizer::tokenize_grammar(&args, &checkout_takes_value, Dialect::Posix); + assert_eq!(checkout_reset_branch_arg(&tokens), Some("my-branch")); + } + + #[test] + fn test_git_cmd_no_global_args() { + let cmd = git_cmd(&[]); + let program = cmd.get_program().to_string_lossy().to_string(); + // On Windows, resolved_command returns full path (e.g. "C:\Program Files\Git\bin\git.exe") + let basename = std::path::Path::new(&program) + .file_stem() + .unwrap() + .to_string_lossy() + .to_string(); + assert_eq!(basename, "git"); + let args: Vec<_> = cmd.get_args().collect(); + assert!(args.is_empty()); + } + + #[test] + fn test_git_cmd_with_directory() { + let global_args = vec!["-C".to_string(), "/tmp".to_string()]; + let cmd = git_cmd(&global_args); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!(args, vec!["-C", "/tmp"]); + } + + #[test] + fn test_git_cmd_with_multiple_global_args() { + let global_args = vec![ + "-C".to_string(), + "/tmp".to_string(), + "-c".to_string(), + "user.name=test".to_string(), + "--git-dir".to_string(), + "/foo/.git".to_string(), + ]; + let cmd = git_cmd(&global_args); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!( + args, + vec![ + "-C", + "/tmp", + "-c", + "user.name=test", + "--git-dir", + "/foo/.git" + ] + ); + } + + #[test] + fn test_git_cmd_with_boolean_flags() { + let global_args = vec!["--no-pager".to_string(), "--bare".to_string()]; + let cmd = git_cmd(&global_args); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!(args, vec!["--no-pager", "--bare"]); + } + + #[test] + fn test_git_cmd_c_locale_sets_stable_env() { + let cmd = git_cmd_c_locale(&[]); + let envs: Vec<_> = cmd + .get_envs() + .map(|(key, value)| { + ( + key.to_string_lossy().to_string(), + value.expect("env value").to_string_lossy().to_string(), + ) + }) + .collect(); + assert!(envs.contains(&("LC_ALL".to_string(), "C".to_string()))); + } + + #[test] + fn test_build_status_command_default_compact() { + let cmd = build_status_command(&[], &[]); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!(args, vec!["status", "--porcelain", "-b"]); + } + + #[test] + fn test_uses_compact_status_path_for_branch_and_short_flags() { + assert!(uses_compact_status_path(&["-b".to_string()])); + assert!(uses_compact_status_path(&["--branch".to_string()])); + assert!(uses_compact_status_path(&["-sb".to_string()])); + assert!(uses_compact_status_path(&[ + "-s".to_string(), + "-b".to_string() + ])); + assert!(uses_compact_status_path(&[ + "--short".to_string(), + "--branch".to_string() + ])); + assert!(!uses_compact_status_path(&["-s".to_string()])); + assert!(!uses_compact_status_path(&["--short".to_string()])); + assert!(!uses_compact_status_path(&["--porcelain".to_string()])); + assert!(!uses_compact_status_path(&["-uno".to_string()])); + } + + #[test] + fn test_build_status_command_with_user_args_passthrough() { + let args = vec!["--short".to_string(), "--branch".to_string()]; + let cmd = build_status_command(&args, &[]); + let cmd_args: Vec<_> = cmd.get_args().collect(); + assert_eq!(cmd_args, vec!["status", "--porcelain", "-b"]); + } + + #[test] + fn test_build_status_command_with_incompatible_user_args_passthrough() { + let args = vec!["--porcelain".to_string(), "-uno".to_string()]; + let cmd = build_status_command(&args, &[]); + let cmd_args: Vec<_> = cmd.get_args().collect(); + assert_eq!(cmd_args, vec!["status", "--porcelain", "-uno"]); + } + + #[test] + fn test_run_status_compact_propagates_non_repo_failure() { + // #2497: a `git status` failure other than "not a git repository" + // (here: a corrupt index) must propagate a non-zero exit, not be + // flattened into "Clean working tree" + exit 0. + let dir = tempfile::tempdir().expect("tempdir"); + let p = dir.path().to_string_lossy().into_owned(); + assert!( + Command::new("git") + .args(["-C", &p, "init", "-q"]) + .status() + .expect("git init") + .success(), + "git init should succeed" + ); + std::fs::write(dir.path().join(".git/index"), "corrupt-index").expect("corrupt index"); + let global = vec!["-C".to_string(), p]; + let code = run_status(&[], 0, &global).expect("run_status"); + assert_ne!( + code, 0, + "corrupt-index git status must not be reported as success" + ); + } + + #[test] + fn test_compact_diff() { + let diff = r#"diff --git a/foo.rs b/foo.rs +--- a/foo.rs ++++ b/foo.rs +@@ -1,3 +1,4 @@ + fn main() { ++ println!("hello"); + } +"#; + let result = compact_diff(diff, 100); + assert!(result.contains("foo.rs")); + assert!(result.contains("+")); + } + + #[test] + fn test_compact_diff_hunk_lines_are_grep_anchorable() { + let diff = "diff --git a/f.txt b/f.txt\n\ + --- a/f.txt\n\ + +++ b/f.txt\n\ + @@ -1,5 +1,4 @@\n\ + \x20keep1\n\ + -DELETED_A\n\ + \x20keep2\n\ + -DELETED_B\n\ + +ADDED\n"; + let result = compact_diff(diff, 100); + + let removed: Vec<&str> = result.lines().filter(|l| l.starts_with('-')).collect(); + let added: Vec<&str> = result.lines().filter(|l| l.starts_with('+')).collect(); + + assert_eq!( + removed, + vec!["-DELETED_A", "-DELETED_B"], + "`^-` must anchor" + ); + assert_eq!(added, vec!["+ADDED"], "`^+` must anchor"); + + // rtk's own tally stays indented so these same greps never count it as + // a diff line. Without this, `^+` would pick up the "+1 -2" summary. + assert!(result.contains(" +1 -2"), "tally must stay indented"); + + // Context lines keep git's leading space, so they are not `^-`/`^+`. + // Both are emitted: the one before the first change as well as the one + // between changes. + assert!( + result.lines().any(|l| l == " keep1"), + "leading context must survive, got:\n{}", + result + ); + assert!(result.lines().any(|l| l == " keep2")); + } + + #[test] + fn test_compact_diff_keeps_content_starting_with_plus_or_minus() { + // `---` / `+++` are file headers only before the first `@@`. Inside a + // hunk, `++i;` and `-- sql comment` are content and must be neither + // dropped from the body nor missing from the tally. + let diff = "diff --git a/f.sql b/f.sql\n\ + --- a/f.sql\n\ + +++ b/f.sql\n\ + @@ -1,2 +1,2 @@\n\ + --- sql comment\n\ + +++i;\n"; + let result = compact_diff(diff, 100); + + assert!( + result.lines().any(|l| l == "--- sql comment"), + "deleted SQL comment must survive, got:\n{}", + result + ); + assert!( + result.lines().any(|l| l == "+++i;"), + "added `++i;` must survive, got:\n{}", + result + ); + assert!( + result.contains(" +1 -1"), + "tally must count both, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_leading_context_has_its_own_budget() { + // Leading context must not consume the 100-line change budget: a hunk + // opening with more context than the budget still shows every change. + let mut diff = String::from("diff --git a/f.rs b/f.rs\n@@ -1,120 +1,120 @@\n"); + for i in 0..20 { + diff.push_str(&format!(" ctx{}\n", i)); + } + for i in 0..100 { + diff.push_str(&format!("-del{}\n", i)); + } + let result = compact_diff(&diff, 1000); + + let ctx = result.lines().filter(|l| l.starts_with(" ctx")).count(); + let dels = result.lines().filter(|l| l.starts_with("-del")).count(); + assert_eq!(ctx, 3, "leading context is capped, got:\n{}", result); + assert_eq!( + dels, 100, + "every change must still be shown, got:\n{}", + result + ); + assert!( + !result.contains("truncated"), + "no change was dropped, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_combined_diff_headers_are_not_hunk_content() { + // `diff --cc` sections do not match `diff --git`, so the reset fires on + // the `diff --` prefix and the `---` / `+++` headers of every section + // stay outside the hunk body. + let diff = "diff --cc a.txt\n\ + index ba2906d,e45c9c2..0000000\n\ + --- a/a.txt\n\ + +++ b/a.txt\n\ + @@@ -1,1 -1,1 +1,5 @@@\n\ + ++<<<<<<< HEAD\n\ + \x20+main\n\ + ++=======\n\ + + side\n\ + ++>>>>>>> side\n\ + diff --cc z.txt\n\ + index ba2906d,e45c9c2..0000000\n\ + --- a/z.txt\n\ + +++ b/z.txt\n\ + @@@ -1,1 -1,1 +1,5 @@@\n\ + ++<<<<<<< HEAD\n\ + \x20+main\n\ + ++=======\n\ + + side\n\ + ++>>>>>>> side\n"; + let result = compact_diff(diff, 500); + + assert!( + !result.lines().any(|l| l.starts_with("+++ b/")), + "file headers must not reach the hunk body, got:\n{}", + result + ); + assert!(result.contains("z.txt"), "got:\n{}", result); + // A combined diff carries one marker column per parent, so ` +main` is + // an addition against the second parent. The tally counts all five + // added lines per file; an anchored `^+` sees only the four whose + // marker sits in column 1. That gap is documented in FEATURES.md. + assert_eq!( + result.matches(" +5 -0").count(), + 2, + "column-2 markers must be counted, got:\n{}", + result + ); + let anchored = result.lines().filter(|l| l.starts_with('+')).count(); + assert_eq!(anchored, 8, "four per file anchor, got:\n{}", result); + } + + #[test] + fn test_compact_diff_mbox_signature_is_not_a_deletion() { + // `gh pr diff --patch` yields an mbox: a bare `---` before the diffstat + // and a `-- ` signature after each patch, both at column 0. The hunk + // ends where its declared line counts run out, so neither is read as + // hunk content. + let diff = "From abc Mon Sep 17 00:00:00 2001\n\ + Subject: [PATCH 1/2] one\n\ + \n\ + ---\n\ + \x20f.txt | 2 +-\n\ + \n\ + diff --git a/f.txt b/f.txt\n\ + --- a/f.txt\n\ + +++ b/f.txt\n\ + @@ -1,2 +1,2 @@\n\ + -old1\n\ + +new1\n\ + \x20tail1\n\ + -- \n\ + 2.40.0\n\ + \n\ + From def Mon Sep 17 00:00:00 2001\n\ + Subject: [PATCH 2/2] two\n\ + \n\ + ---\n\ + diff --git a/g.txt b/g.txt\n\ + --- a/g.txt\n\ + +++ b/g.txt\n\ + @@ -1,2 +1,2 @@\n\ + -old2\n\ + +new2\n\ + \x20tail2\n\ + -- \n\ + 2.40.0\n"; + let result = compact_diff(diff, 500); + + let removed: Vec<&str> = result.lines().filter(|l| l.starts_with('-')).collect(); + assert_eq!( + removed, + vec!["-old1", "-old2"], + "only real deletions anchor, got:\n{}", + result + ); + assert!( + !result.contains("Subject:"), + "mbox envelope must stay out of the body, got:\n{}", + result + ); + assert!( + result.contains(" +1 -1"), + "tally counts real changes only, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_leading_context_is_adjacent_to_the_change() { + // With `-U10` the first context lines sit ten lines above the change. + // Emitting those would tell the reader that ctx3 precedes the deletion + // when ctx10 does. + let mut diff = String::from("diff --git a/f.rs b/f.rs\n@@ -1,11 +1,11 @@\n"); + for i in 1..=10 { + diff.push_str(&format!(" ctx{}\n", i)); + } + diff.push_str("-old\n+new\n"); + let result = compact_diff(&diff, 500); + + let ctx: Vec<&str> = result.lines().filter(|l| l.starts_with(" ctx")).collect(); + assert_eq!( + ctx, + vec![" ctx8", " ctx9", " ctx10"], + "the last context lines, not the first, got:\n{}", + result + ); + } + + #[test] + fn test_diff_header_path_keeps_spaces() { + assert_eq!( + diff_header_path("diff --git a/my file.txt b/my file.txt"), + "my file.txt" + ); + assert_eq!(diff_header_path("diff --cc my file.txt"), "my file.txt"); + assert_eq!( + diff_header_path("diff --combined my file.txt"), + "my file.txt" + ); + } + + #[test] + fn test_diff_header_path_handles_gits_quoted_paths() { + // Under the default `core.quotepath`, git escapes a non-ASCII path and + // wraps it in quotes, which removes the ` b/` separator the plain form + // is split on. Without the quoted form handled, the fallback returned + // the whole remainder — both paths — as the section header. + assert_eq!( + diff_header_path(r#"diff --git "a/été.txt" "b/été.txt""#), + r"été.txt" + ); + assert_eq!(diff_header_path(r#"diff --cc "été.txt""#), r"été.txt"); + // A rename quotes each side on its own. + assert_eq!( + diff_header_path(r#"diff --git a/plain.txt "b/ét.txt""#), + r"ét.txt" + ); + assert_eq!( + diff_header_path(r#"diff --git "a/ét.txt" b/plain.txt"#), + "plain.txt" + ); + } + + #[test] + fn test_diff_header_path_unescapes_gits_default_quoting() { + // What git actually emits under the default `core.quotepath`: one octal + // escape per byte, so the header has to be decoded rather than merely + // unwrapped, or `rtk git diff | grep été` finds nothing. + assert_eq!( + diff_header_path(r#"diff --git "a/\303\251t\303\251.txt" "b/\303\251t\303\251.txt""#), + "été.txt" + ); + assert_eq!( + diff_header_path(r#"diff --cc "\303\251t\303\251.txt""#), + "été.txt" + ); + assert_eq!( + diff_header_path(r#"diff --git a/plain.txt "b/\303\251t.txt""#), + "ét.txt" + ); + // The single-character escapes, and a backslash standing for itself. + assert_eq!( + diff_header_path(r#"diff --cc "tab\there.txt""#), + "tab\there.txt" + ); + assert_eq!( + diff_header_path(r#"diff --cc "quote\"here.txt""#), + "quote\"here.txt" + ); + assert_eq!( + diff_header_path(r#"diff --cc "back\\slash.txt""#), + r"back\slash.txt" + ); + } + + fn word_diff_from(args: &[&str]) -> bool { + let args: Vec = args.iter().map(|a| a.to_string()).collect(); + emits_word_diff(&tokenize_git_log_args(&args)) + } + + #[test] + fn test_emits_word_diff_detects_every_form() { + for flag in [ + "--word-diff", + "--word-diff=plain", + "--word-diff=porcelain", + "--word-diff-regex=.", + "--color-words", + "--color-words=.", + ] { + assert!(word_diff_from(&[flag]), "{} must pass through", flag); + } + assert!(!word_diff_from(&["--stat"])); + assert!(!word_diff_from(&["-U10"])); + assert!(!word_diff_from(&[])); + } + + #[test] + fn test_emits_word_diff_honours_the_none_mode() { + // `--word-diff=none` leaves an ordinary unified diff, which compacts + // like any other. Treating it as a word diff passed the whole raw diff + // through, so a defensive `--word-diff=none` lost every saving. + assert!(!word_diff_from(&["--word-diff=none"])); + // Modes are last-one-wins, which is what `none` exists to do. + assert!(!word_diff_from(&["--word-diff", "--word-diff=none"])); + assert!(word_diff_from(&["--word-diff=none", "--word-diff"])); + // `--color-words` takes a regex, so `none` there is a pattern. + assert!(word_diff_from(&["--color-words=none"])); + } + + #[test] + fn test_emits_word_diff_ignores_a_consumed_or_pathspec_word_diff() { + // Real git 2.53.0: `git diff --author --word-diff` emits an ordinary line diff (the + // flag is `--author`'s value), and `git diff -- --word-diff` treats it as a pathspec. + assert!(!word_diff_from(&["--author", "--word-diff"])); + // `-M` takes an attached value only, so it consumes nothing and git does word-diff: + // reading this as `-M`'s value would hand compact_diff a word diff to mangle. + assert!(word_diff_from(&["-M", "--word-diff"])); + assert!(!word_diff_from(&["--", "--word-diff"])); + // `--word-diff-regex` still requests one when its own value is flag-shaped. + assert!(word_diff_from(&["--word-diff-regex", "--stat"])); + } + + #[test] + fn test_parse_hunk_header_reconciles_ranges_with_marker_columns() { + // A well-formed header lists one range per marker column. When it does + // not, only the columns can be charged. A missing range must not leave + // an untracked parent holding the hunk open forever, and an extra one + // must not sit at its declared count with no column to spend it. + let h = parse_hunk_header("@@@ -1 +1 @@@").expect("two columns, one range"); + assert_eq!(h.prefix_width, 2); + assert_eq!(h.parents, vec![1, usize::MAX]); + + let h = parse_hunk_header("@@@ -1 -1 -1 +0,0 @@@").expect("two columns, three ranges"); + assert_eq!(h.prefix_width, 2); + assert_eq!(h.parents, vec![1, 1]); + } + + #[test] + fn test_compact_diff_extra_range_does_not_strand_a_hunk() { + // With the third range untracked, `--x` left it at 1 forever, so the + // hunk never closed and the mbox signature became its content. + let out = compact_diff( + "diff --cc f\n@@@ -1 -1 -1 +0,0 @@@\n--x\n-- \n2.40.0\n", + 100, + ); + assert!(out.contains("--x"), "got:\n{}", out); + assert!(!out.contains("2.40.0"), "got:\n{}", out); + assert!(!out.contains("-- "), "got:\n{}", out); + // One line, removed from both parents, is one deletion. + assert!(out.contains("+0 -1"), "got:\n{}", out); + } + + #[test] + fn test_compact_diff_missing_range_keeps_the_body() { + // One range for two columns: the untracked parent gets `usize::MAX`, so + // the hunk stays open to the next header rather than closing early and + // dropping ` -lost`. + let out = compact_diff("diff --cc f\n@@@ -1 +1 @@@\n +kept\n -lost\n", 100); + assert!(out.contains(" +kept"), "got:\n{}", out); + assert!(out.contains(" -lost"), "got:\n{}", out); + assert!(out.contains("+1 -1"), "got:\n{}", out); + } + + #[test] + fn test_diff_header_path_splits_the_pair_at_its_midpoint() { + // A file under a directory named `x b` puts the ` b/` separator inside + // the path, so the first match is the wrong one. + assert_eq!(diff_header_path("diff --git a/x b/y b/x b/y"), "x b/y"); + // `--no-prefix` and custom prefixes leave no ` b/` at all. + assert_eq!(diff_header_path("diff --git x x"), "x"); + assert_eq!( + diff_header_path("diff --git src/main.rs src/main.rs"), + "src/main.rs" + ); + // Prefixes are matched against each other, not by name, so a custom + // `--dst-prefix` reads like any other pair. + assert_eq!(diff_header_path("diff --git a/f.txt w/f.txt"), "f.txt"); + assert_eq!(diff_header_path("diff --git i/f.txt w/f.txt"), "f.txt"); + // A rename's halves disagree past their first component, so the ` b/` + // split still names the destination. + assert_eq!( + diff_header_path("diff --git a/old.txt b/new.txt"), + "new.txt" + ); + } + + #[test] + fn test_diff_header_path_does_not_split_single_path_headers() { + // `diff --cc` names one path. Splitting its remainder at the midpoint + // would read a file called `dup dup` as the file `dup` named twice. + assert_eq!(diff_header_path("diff --cc dup dup"), "dup dup"); + assert_eq!(diff_header_path("diff --combined dup dup"), "dup dup"); + assert_eq!(diff_header_path("diff --cc a/x b/x"), "a/x b/x"); + } + + #[test] + fn test_parse_hunk_header_counts() { + let h = parse_hunk_header("@@ -10,3 +10,4 @@ fn ctx() {").expect("unified header"); + assert_eq!( + (h.parents.as_slice(), h.new, h.prefix_width), + (&[3][..], 4, 1) + ); + + // Omitted counts mean one line. + let h = parse_hunk_header("@@ -1 +1 @@").expect("single-line header"); + assert_eq!((h.parents.as_slice(), h.new), (&[1][..], 1)); + + // A combined header lists one range per parent, in marker-column order. + // Every one of them bounds the hunk body. + let h = parse_hunk_header("@@@ -1,1 -1,4 +1,5 @@@").expect("combined header"); + assert_eq!( + (h.parents.as_slice(), h.new, h.prefix_width), + (&[1, 4][..], 5, 2) + ); + + assert!(parse_hunk_header("@ -1,1 +1,1 @").is_none()); + assert!(parse_hunk_header("-- ").is_none()); + assert!(parse_hunk_header("---").is_none()); + } + + #[test] + fn test_compact_diff_non_ascii_body_line_without_a_marker_does_not_panic() { + // `--word-diff` / `--color-words` emit body lines with no marker column, + // so content lands where the markers are sliced. Slicing by byte index + // split a leading multi-byte character and aborted the process. + let out = compact_diff( + "diff --git a/f.txt b/f.txt\n@@ -1,3 +1,3 @@\n-old\n+new\nécole ancienne ligne\n", + 100, + ); + assert!(out.contains("-old"), "got:\n{}", out); + assert!(out.contains("+new"), "got:\n{}", out); + assert!(out.contains("école ancienne ligne"), "got:\n{}", out); + } + + #[test] + fn test_compact_diff_combined_hunk_ends_at_its_declared_length() { + // Every parent's declared range bounds the body. Charging only the + // first parent left `old` unable to converge on real conflict output, + // so the hunk never closed by count and the mbox / signature / prose + // guard did not apply to combined sections at all. + let conflict = "diff --cc f.txt\n@@@ -1,1 -1,1 +1,5 @@@\n++<<<<<<<\n +main\n++=======\n+ side\n++>>>>>>>\n-- \ntrailing signature\n"; + let out = compact_diff(conflict, 100); + assert!(!out.contains("trailing signature"), "got:\n{}", out); + assert!(!out.contains("-- "), "got:\n{}", out); + assert!(out.contains("+5 -0"), "got:\n{}", out); + } + + #[test] + fn test_compact_diff_combined_hunk_keeps_second_parent_removals() { + // `-1,2 -1,4 +1,2`: two removals spend only the second parent's budget. + // Closing on the first parent and the result alone dropped them with no + // tally and no truncation note — a silent loss. + let out = compact_diff( + "diff --cc f.txt\n@@@ -1,2 -1,4 +1,2 @@@\n a\n b\n -x\n -y\n", + 100, + ); + assert!(out.contains(" -x"), "got:\n{}", out); + assert!(out.contains(" -y"), "got:\n{}", out); + assert!(out.contains("+0 -2"), "got:\n{}", out); + } + + #[test] + fn test_compact_diff_flushes_context_from_a_hunk_with_no_change_line() { + // The buffer drained only on the first change line, so a hunk that ends + // without one rendered as a bare header with nothing under it. + let out = compact_diff( + "diff --git a/g.txt b/g.txt\n@@ -1,3 +1,3 @@\n ctx1\n ctx2\n ctx3\n", + 100, + ); + assert!(out.contains(" ctx1"), "got:\n{}", out); + assert!(out.contains(" ctx3"), "got:\n{}", out); + } + + #[test] + fn test_compact_diff_leading_context_does_not_displace_change_lines() { + // Leading context is exempt from `max_lines`, so the same number of + // change lines survives whether or not the hunks open with context. + let build = |with_context: bool| { + let mut diff = String::new(); + for f in 0..30 { + diff.push_str(&format!("diff --git a/f{}.rs b/f{}.rs\n", f, f)); + diff.push_str("@@ -1,20 +1,20 @@\n"); + if with_context { + for c in 0..3 { + diff.push_str(&format!(" ctx{}_{}\n", f, c)); + } + } + for i in 0..12 { + diff.push_str(&format!("-del{}_{}\n", f, i)); + } + } + diff + }; + let count_changes = |out: &str| out.lines().filter(|l| l.starts_with("-del")).count(); + + let without = compact_diff(&build(false), 500); + let with = compact_diff(&build(true), 500); + assert_eq!( + count_changes(&with), + count_changes(&without), + "leading context displaced change lines:\n{}", + with + ); + } + + #[test] + fn test_compact_diff_leading_context_is_capped_across_the_diff() { + // The diff-wide cap is what bounds the exemption: without it, a diff of + // many small hunks would spend three exempt lines on each one. + let mut diff = String::new(); + for f in 0..200 { + diff.push_str(&format!("diff --git a/f{}.rs b/f{}.rs\n", f, f)); + diff.push_str("@@ -1,4 +1,4 @@\n"); + for c in 0..3 { + diff.push_str(&format!(" ctx{}_{}\n", f, c)); + } + diff.push_str(&format!("-del{}\n", f)); + } + let result = compact_diff(&diff, 500); + + let ctx = result.lines().filter(|l| l.starts_with(" ctx")).count(); + assert!( + ctx <= 50, + "leading context must stay within max_lines / 10, got {} in:\n{}", + ctx, + result + ); + } + + #[test] + fn test_hunk_truncation_note_counts_one_as_singular() { + assert_eq!( + hunk_truncation_note(1, 0).as_deref(), + Some(" ... (1 deletion truncated)") + ); + assert_eq!( + hunk_truncation_note(0, 1).as_deref(), + Some(" ... (1 addition truncated)") + ); + assert_eq!( + hunk_truncation_note(1, 2).as_deref(), + Some(" ... (1 deletion, 2 additions truncated)") + ); + assert_eq!(hunk_truncation_note(0, 0), None); + } + + #[test] + fn test_compact_diff_truncation_note_splits_by_sign() { + // An anchored `^-` audit needs to know how many deletions it did not + // see, which a merged "N lines truncated" cannot tell it. + let mut diff = String::from("diff --git a/f.rs b/f.rs\n@@ -1,160 +1,160 @@\n"); + for i in 0..80 { + diff.push_str(&format!("-del{}\n", i)); + diff.push_str(&format!("+add{}\n", i)); + } + let result = compact_diff(&diff, 1000); + + assert!( + result.contains(" ... (30 deletions, 30 additions truncated)"), + "expected per-sign truncation note, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_preserves_full_hunk_header_context() { + let diff = r#"diff --git a/foo.rs b/foo.rs +--- a/foo.rs ++++ b/foo.rs +@@ -10,3 +10,4 @@ fn important_context() { + fn main() { ++ println!("hello"); + } +"#; + let result = compact_diff(diff, 100); + assert!( + result.contains("@@ -10,3 +10,4 @@ fn important_context() {"), + "Expected full hunk header with trailing context, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_increased_hunk_limit() { + // Build a hunk with 25 changed lines — should NOT be truncated with limit 30 + let mut diff = + "diff --git a/big.rs b/big.rs\n--- a/big.rs\n+++ b/big.rs\n@@ -1,25 +1,25 @@\n" + .to_string(); + for i in 1..=25 { + diff.push_str(&format!("+line{}\n", i)); + } + let result = compact_diff(&diff, 500); + assert!( + !result.contains("... (truncated)"), + "25 lines should not be truncated with max_hunk_lines=30" + ); + assert!(result.contains("+line25")); + } + + #[test] + fn test_compact_diff_increased_total_limit() { + // Build a diff with 150 output result lines across multiple files — should NOT be cut at 100 + let mut diff = String::new(); + for f in 1..=5 { + diff.push_str(&format!("diff --git a/file{f}.rs b/file{f}.rs\n--- a/file{f}.rs\n+++ b/file{f}.rs\n@@ -1,20 +1,20 @@\n")); + for i in 1..=20 { + diff.push_str(&format!("+line{f}_{i}\n")); + } + } + let result = compact_diff(&diff, 500); + assert!( + !result.contains("more changes truncated"), + "5 files × 20 lines should not exceed max_lines=500" + ); + } + + #[test] + fn test_is_blob_show_arg() { + assert!(is_blob_show_arg("develop:modules/pairs_backtest.py")); + assert!(is_blob_show_arg("HEAD:src/main.rs")); + // Index / merge-stage blobs start with `:` but are still blobs. + assert!(is_blob_show_arg(":Cargo.toml")); + assert!(is_blob_show_arg(":2:conflict.rs")); + assert!(!is_blob_show_arg("--pretty=format:%h")); + assert!(!is_blob_show_arg("--format=short")); + assert!(!is_blob_show_arg("HEAD")); + // `:/text` is a commit-message search, not a blob. + assert!(!is_blob_show_arg(":/fix the bug")); + // Magic pathspecs are pathspecs, never blobs. + assert!(!is_blob_show_arg(":(exclude)b.txt")); + assert!(!is_blob_show_arg(":(top,glob)*.rs")); + assert!(!is_blob_show_arg(":!b.txt")); + // `:^` is an exact synonym of `:!` (exclude magic), also a pathspec. + assert!(!is_blob_show_arg(":^b.txt")); + // A colons-only token is not a blob object. + assert!(!is_blob_show_arg(":")); + assert!(!is_blob_show_arg("::")); + } + + /// Helper: build an args slice from string literals. + fn show_args(v: &[&str]) -> Vec { + v.iter().map(|x| x.to_string()).collect() + } + + #[test] + fn test_show_route_blob_wins_over_stat_and_format() { + // Blocking bug #1: git accepts and silently ignores --stat/--numstat/--pretty/ + // --format for a blob target and still dumps the file, so these must NOT shadow + // the blob path (which would send the blob through lossy UTF-8 decoding). + assert_eq!( + show_route(&show_args(&["--stat", "HEAD:legacy.pck"])), + ShowRoute::Blob + ); + assert_eq!( + show_route(&show_args(&["--numstat", "HEAD:legacy.pck"])), + ShowRoute::Blob + ); + assert_eq!( + show_route(&show_args(&["--format=medium", "HEAD:legacy.pck"])), + ShowRoute::Blob + ); + assert_eq!( + show_route(&show_args(&["--pretty=oneline", "HEAD:legacy.pck"])), + ShowRoute::Blob + ); + } + + #[test] + fn test_show_route_trailing_pathspec_stays_a_blob() { + // Blocking bug #2: a trailing `-- ` beside a blob arg is ignored by git + // (still a blob dump), so it must not disable blob handling. + let args = show_args(&["HEAD:Cargo.toml", "--", "Cargo.toml"]); + assert_eq!(show_route(&args), ShowRoute::Blob); + assert_eq!(blob_candidates(&args), vec![&"HEAD:Cargo.toml".to_string()]); + } + + #[test] + fn test_blob_candidates_skip_flag_operand_with_colon() { + // Blocking bug #2: `-S 'url:1'`'s operand contains a colon but is the value of + // a pickaxe flag, not an object — it must not be read as a blob and truncated. + // The commit is the real (colon-free) object, so this is a commit-diff. + let args = show_args(&["-S", "url:1", "HEAD"]); + assert!(blob_candidates(&args).is_empty()); + assert_eq!(show_route(&args), ShowRoute::CommitDiff); + // `-L :` operand likewise carries a colon. + let args = show_args(&["-L", "1,2:file.rs", "HEAD"]); + assert!(blob_candidates(&args).is_empty()); + // A real blob still routes as a blob even with a preceding value flag. + let args = show_args(&["-S", "needle", "HEAD:src/main.rs"]); + assert_eq!( + blob_candidates(&args), + vec![&"HEAD:src/main.rs".to_string()] + ); + // A value operand that IS a valid-looking blob (`-S 'HEAD:real'`) is the pickaxe + // value, not an object: the walker excludes it, so only the trailing commit + // remains and nothing is offered as a windowing candidate. + let args = show_args(&["-S", "HEAD:real", "HEAD"]); + assert!(blob_candidates(&args).is_empty()); + } + + #[test] + fn test_blob_candidates_short_flag_clusters() { + // BLOCKER: a short-flag cluster whose value-taking tail consumes the NEXT token + // (`-wG x:y` == `-w -G x:y`) must skip `x:y` and expose the real blob object, not + // mistake `x:y` for the blob. Covers the clusters git re-parses. + for cluster in ["-wG", "-pS", "-pI", "-pwG", "-wpG"] { + let args = show_args(&[cluster, "x:y", "HEAD:big.txt"]); + assert_eq!( + blob_candidates(&args), + vec![&"HEAD:big.txt".to_string()], + "cluster {cluster}: x:y is the value flag's operand, HEAD:big.txt the object", + ); + } + // An INLINE cluster value (`-Sfoo` == `-S foo`) does NOT consume the next token, + // so the following object is still exposed. + let args = show_args(&["-Sneedle", "HEAD:big.txt"]); + assert_eq!(blob_candidates(&args), vec![&"HEAD:big.txt".to_string()]); + // `-Gx:y` (value flag NOT last, inline value `x:y`) consumes no next token. + let args = show_args(&["-Gx:y", "HEAD:big.txt"]); + assert_eq!(blob_candidates(&args), vec![&"HEAD:big.txt".to_string()]); + // A boolean-only cluster (`-wp`) consumes nothing: the object stays a candidate. + let args = show_args(&["-wp", "HEAD:big.txt"]); + assert_eq!(blob_candidates(&args), vec![&"HEAD:big.txt".to_string()]); + } + + #[test] + fn test_show_route_colon_pathspec_after_dashdash_is_not_a_blob() { + // A filename containing a colon AFTER `--` is a pathspec, not a blob target. + // This `--` is representative of the real CLI path: clap strips the separator, + // but `run_show` restores it via `restore_double_dash` before calling + // `show_route`/`show_positionals` (verified live: `git show HEAD -- a:b.txt` + // renders a commit-diff, not a blob dump). + let args = show_args(&["HEAD", "--", "weird:name.txt"]); + assert_eq!(show_route(&args), ShowRoute::CommitDiff); + assert!(blob_candidates(&args).is_empty()); + } + + #[test] + fn test_show_route_plain_cases() { + assert_eq!( + show_route(&show_args(&["--stat", "HEAD"])), + ShowRoute::StatOrFormat + ); + assert_eq!(show_route(&show_args(&["HEAD"])), ShowRoute::CommitDiff); + assert_eq!( + show_route(&show_args(&["HEAD:src/main.rs"])), + ShowRoute::Blob + ); + } + + #[test] + fn test_blob_windowing_token_savings() { + // A filter must verify its savings claim with a real fixture. The enforced floor + // is 20% (CONTRIBUTING.md), and `blob_large.txt` (a real `git show + // HEAD:src/main.rs | head -350`, ~10.7 KB) has always cleared it: windowing to + // the 8 KiB head plus a few-token recovery hint is the savings floor. (An earlier + // revision introduced a synthetic ~107 KB fixture to chase a stale 60% figure + // that only `.claude/rules/cli-testing.md` still cites; that churn is reverted.) + fn count_tokens(text: &str) -> usize { + text.split_whitespace().count() + } + let raw = include_str!("../../../tests/fixtures/git/blob_large.txt"); + assert!(raw.len() > MAX_BLOB_BYTES.0); + let (head, _remaining, _offset) = + blob_truncation(raw, MAX_BLOB_BYTES).expect("large blob should window"); + let savings = 100.0 - (count_tokens(head) as f64 / count_tokens(raw) as f64 * 100.0); + assert!( + savings >= 20.0, + "expected ≥20% token savings, got {:.1}%", + savings + ); + } + + #[test] + fn test_blob_truncation_small_passthrough() { + // Real committed file, ~1.7KB < budget: passed through unchanged. + let small = include_str!("../../../Cargo.toml"); + assert!(blob_truncation(small, MAX_BLOB_BYTES).is_none()); + } + + #[test] + fn test_blob_truncation_large_windowed() { + // Real blob fixture (`git show HEAD:src/main.rs | head -350`), ~10.7KB. + let large = include_str!("../../../tests/fixtures/git/blob_large.txt"); + let (head, remaining, offset) = + blob_truncation(large, MAX_BLOB_BYTES).expect("large blob should window"); + assert!(head.len() <= MAX_BLOB_BYTES.0); + assert!(head.ends_with('\n'), "head must end at a line boundary"); + let head_lines = head.lines().count(); + // N formula: remaining == total - head_lines, offset == head_lines + 1. + assert_eq!(offset, head_lines + 1); + assert_eq!(remaining, large.lines().count() - head_lines); + assert!(remaining > 0); + } + + #[test] + fn test_blob_truncation_tree_passthrough() { + // Real tree listing (`git show HEAD:src`): `tree :\n\n...`. In the live + // path the `git cat-file -t` probe classifies a tree as non-blob before this + // function is ever reached, so the old `starts_with("tree ")` content-sniff was + // removed; this small (87 B) listing still declines here via the size check. + let tree = include_str!("../../../tests/fixtures/git/tree_listing.txt"); + assert!(tree.starts_with("tree ")); + assert!(blob_truncation(tree, MAX_BLOB_BYTES).is_none()); + } + + #[test] + fn test_blob_truncation_binary_passthrough() { + let mut binary = "x".repeat(9000); + binary.push('\0'); + assert!(blob_truncation(&binary, MAX_BLOB_BYTES).is_none()); + // Replacement char from lossy UTF-8 decoding of a binary blob. + let lossy = format!("{}\u{FFFD}", "y".repeat(9000)); + assert!(blob_truncation(&lossy, MAX_BLOB_BYTES).is_none()); + } + + #[test] + fn test_blob_truncation_giant_single_line_passthrough() { + // A single line longer than the budget has no line boundary to cut at. + let giant = "a".repeat(20_000); + assert!(blob_truncation(&giant, MAX_BLOB_BYTES).is_none()); + } + + #[test] + fn test_blob_truncation_utf8_boundary_no_panic() { + // Multibyte content so the byte budget can land mid-codepoint: must not panic. + let mut s = String::new(); + while s.len() < 9000 { + s.push_str("áéíóú-ñ-日本語"); + s.push('\n'); + } + // Exercise budgets straddling a multibyte char around the window edge. + let _ = blob_truncation(&s, MAX_BLOB_BYTES); + let _ = blob_truncation(&s, Budget(8191)); + let _ = blob_truncation(&s, Budget(8193)); + // Should still window (multi-line, over budget) without crashing. + assert!(blob_truncation(&s, MAX_BLOB_BYTES).is_some()); + } + + #[test] + fn test_blob_truncation_no_trailing_newline() { + let mut s = String::new(); + for i in 0..500 { + s.push_str(&format!("line number {i} with a bit of content here\n")); + } + s.pop(); // drop the final newline + let (head, remaining, offset) = blob_truncation(&s, MAX_BLOB_BYTES).expect("should window"); + assert_eq!(offset, head.lines().count() + 1); + assert_eq!(remaining, s.lines().count() - head.lines().count()); + } + + #[test] + fn test_compact_blob_show_hint_is_tee_independent() { + // The recovery hint re-derives the tail from the blob arg itself — no tee file, + // so it works at any size (N2) and quotes a path with a space for safe paste. + let mut s = String::new(); + for i in 0..2000 { + s.push_str(&format!( + "line {i} with enough content to exceed the byte budget\n" + )); + } + let offset = blob_truncation(&s, MAX_BLOB_BYTES) + .expect("should window") + .2; + let out = compact_blob_show(&s, "HEAD:my dir/big.lock", &[]); + assert!(out.len() < s.len(), "windowing must shrink the output"); + let expected = format!( + "[see remaining: rtk proxy git show 'HEAD:my dir/big.lock' | tail -n +{offset}]" + ); + assert!(out.contains(&expected), "hint missing/unquoted: {out:?}"); + assert!(!out.contains("tail -n +0")); + + // A path containing a single quote must be escaped `'\''` so the hint stays + // copy-paste safe (the dangerous branch of `shell_single_quote`). + let out_q = compact_blob_show(&s, "HEAD:it's/a.lock", &[]); + let expected_q = format!( + "[see remaining: rtk proxy git show 'HEAD:it'\\''s/a.lock' | tail -n +{offset}]" + ); + assert!( + out_q.contains(&expected_q), + "single-quote path unescaped: {out_q:?}" + ); + } + + #[test] + fn test_compact_blob_show_hint_survives_rtks_own_hook() { + // The hook rewrites a bare `git ...` into `rtk git ...`, which windows the dump a + // second time -- `tail` then has only the hint left to print. The hint must name a + // command the hook leaves alone. + let mut s = String::new(); + for i in 0..2000 { + s.push_str(&format!( + "line {i} with enough content to exceed the byte budget\n" + )); + } + let out = compact_blob_show(&s, "HEAD:big.lock", &[]); + let hint = out + .lines() + .next_back() + .and_then(|l| l.split_once("[see remaining: ")) + .map(|(_, rest)| rest.trim_end_matches(']')) + .expect("hint"); + assert!( + hint.starts_with("rtk proxy git "), + "hint must go through proxy: {hint:?}" + ); + assert_eq!( + crate::discover::registry::rewrite_command(hint, &[], &[]), + None, + "the hook must leave the hint alone: {hint:?}" + ); + } + + #[test] + fn test_compact_blob_show_small_passes_through() { + // Below the byte budget: returned unchanged, no hint. + let small = "a few\nshort\nlines\n"; + assert_eq!(compact_blob_show(small, "HEAD:x.txt", &[]), small); + } + + #[test] + fn test_compact_blob_show_hint_carries_global_args() { + // `rtk git -C /repo -c core.x=y show HEAD:big` must produce a hint that is + // runnable from OUTSIDE the repo: the global args go between `git` and `show`, + // each shell-quoted, so the recovery command targets the same repo. + let mut s = String::new(); + for i in 0..2000 { + s.push_str(&format!( + "line {i} with enough content to exceed the byte budget\n" + )); + } + let offset = blob_truncation(&s, MAX_BLOB_BYTES) + .expect("should window") + .2; + let globals = vec![ + "-C".to_string(), + "/tmp/my repo".to_string(), + "-c".to_string(), + "core.autocrlf=false".to_string(), + ]; + let out = compact_blob_show(&s, "HEAD:big.lock", &globals); + // Every token is shell-quoted (same policy as the blob arg) — quoting a flag + // like `-C` is a harmless no-op and keeps the hint copy-paste safe. + let expected = format!( + "[see remaining: rtk proxy git '-C' '/tmp/my repo' '-c' 'core.autocrlf=false' show 'HEAD:big.lock' | tail -n +{offset}]" + ); + assert!(out.contains(&expected), "global-args hint wrong: {out:?}"); + } + + #[test] + // `from_utf8` on a `include_bytes!` literal is exactly the point here (asserting the + // fixture is NOT valid UTF-8, matching `run_show`'s decision predicate), so silence + // clippy's "literal always errors" lint rather than obscure the intent. + #[allow(invalid_from_utf8)] + fn test_blob_latin1_fixture_passes_through() { + // Real ISO-8859-1 slice of an Oracle PL/SQL `.pck` (14 KB, contains "MÉTODO", + // NOT valid UTF-8). Earlier revisions transcoded it to UTF-8 and windowed it, + // but the head shown was then no longer a byte-exact prefix of git's bytes, so + // `git show 'rev:path' | tail -n +N` could not reconstruct the original. Under + // the fidelity invariant, `run_show` windows ONLY content that is valid UTF-8 + // byte-for-byte; anything that would need transcoding passes through unchanged. + // + // The decision predicate is `std::str::from_utf8(git_bytes).is_ok()`, so assert + // the fixture is NOT valid UTF-8 — which is exactly why it takes the byte-exact + // passthrough path rather than being windowed. + let bytes = include_bytes!("../../../tests/fixtures/git/latin1_blob.pck"); + assert!(bytes.len() > MAX_BLOB_BYTES.0); + assert!( + std::str::from_utf8(bytes).is_err(), + "a Latin-1 .pck is not valid UTF-8, so it must pass through byte-identically \ + (exact recovery is impossible once transcoded)" + ); + } + + #[test] + fn test_filter_branch_output() { + let output = "* main\n feature/auth\n fix/bug-123\n remotes/origin/HEAD -> origin/main\n remotes/origin/main\n remotes/origin/feature/auth\n remotes/origin/release/v2\n"; + let result = filter_branch_output(output); + assert!(result.contains("* main")); + assert!(result.contains("feature/auth")); + assert!(result.contains("fix/bug-123")); + // remote-only should show release/v2 but not main or feature/auth (already local) + assert!(result.contains("remote-only")); + assert!(result.contains("release/v2")); + } + + #[test] + fn test_filter_branch_no_remotes() { + let output = "* main\n develop\n"; + let result = filter_branch_output(output); + assert!(result.contains("* main")); + assert!(result.contains("develop")); + assert!(!result.contains("remote-only")); + } + + #[test] + fn test_filter_branch_multi_remote() { + let output = "* main\n develop\n remotes/origin/HEAD -> origin/main\n remotes/origin/main\n remotes/origin/feature-x\n remotes/upstream/main\n remotes/upstream/release-v3\n remotes/fork/main\n remotes/fork/experiment\n"; + let result = filter_branch_output(output); + assert!(result.contains("* main")); + assert!(result.contains("develop")); + assert!( + result.contains("feature-x"), + "origin branch shown: {}", + result + ); + assert!( + result.contains("release-v3"), + "upstream branch shown: {}", + result + ); + assert!( + result.contains("experiment"), + "fork branch shown: {}", + result + ); + assert!( + !result.contains("remotes/"), + "remote prefix stripped: {}", + result + ); + let main_count = result.matches("main").count(); + assert!( + main_count <= 2, + "main deduplicated across remotes (found {} occurrences): {}", + main_count, + result + ); + } + + #[test] + fn test_filter_stash_list() { + let output = + "stash@{0}: WIP on main: abc1234 fix login\nstash@{1}: On feature: def5678 wip\n"; + let result = filter_stash_list(output); + assert!(result.contains("stash@{0}: abc1234 fix login")); + assert!(result.contains("stash@{1}: def5678 wip")); + } + + #[test] + fn test_parse_stash_stat_strips_decorations() { + let raw = " del.md | 2 --\n keep.md | 5 ++++-\n logo.bin | Bin 0 -> 1024 bytes\n \ + new.rs | 40 ++++++++\n 4 files changed, 44 insertions(+), 3 deletions(-)\n"; + let (files, summary) = parse_stash_stat(raw); + assert_eq!( + files, + vec![ + "del.md 2 -", + "keep.md 5 +-", + "logo.bin (binary)", + "new.rs 40 +" + ] + ); + assert_eq!(summary, "4 files changed, 44 insertions(+), 3 deletions(-)"); + } + + #[test] + fn test_parse_stash_stat_collapsed_bar() { + let (files, _) = parse_stash_stat(" .claude/CLAUDE.md | 234 +-\n"); + assert_eq!(files, vec![".claude/CLAUDE.md 234 +-"]); + } + + #[test] + fn test_compact_stash_stat_passthrough_numstat() { + let raw = "0\t1\tdel.md\n3\t2\tkeep.md\n1\t0\tn1.rs\n"; + assert_eq!( + compact_stash_stat(raw), + "0\t1\tdel.md\n3\t2\tkeep.md\n1\t0\tn1.rs" + ); + } + + #[test] + fn test_compact_stash_stat_passthrough_name_only() { + let raw = "del.md\nkeep.md\nn1.rs\n"; + assert_eq!(compact_stash_stat(raw), "del.md\nkeep.md\nn1.rs"); + } + + #[test] + fn test_compress_stat_summary_variants() { + assert_eq!( + compress_stat_summary("4 files changed, 60 insertions(+), 313 deletions(-)"), + "4 changed 60 + 313 -" + ); + assert_eq!( + compress_stat_summary("1 file changed, 1 insertion(+)"), + "1 changed 1 +" + ); + assert_eq!( + compress_stat_summary("1 file changed, 1 deletion(-)"), + "1 changed 1 -" + ); + assert_eq!( + compress_stat_summary("2 files changed, 4 insertions(+), 1 deletion(-)"), + "2 changed 4 + 1 -" + ); + } + + #[test] + fn test_compact_stash_stat_compresses_summary() { + let raw = " a.txt | 2 ++\n 1 file changed, 2 insertions(+)\n"; + assert_eq!(compact_stash_stat(raw), "a.txt 2 +\n1 changed 2 +"); + } + + #[test] + fn test_parse_stash_stat_last_pipe_is_separator() { + let (files, _) = parse_stash_stat(" weird|name.txt | 3 +++\n"); + assert_eq!(files, vec!["weird|name.txt 3 +"]); + } + + #[test] + fn test_parse_stash_stat_strips_ansi() { + let (files, _) = parse_stash_stat(" a.txt | 2 \x1b[32m++\x1b[m\n"); + assert_eq!(files, vec!["a.txt 2 +"]); + } + + #[test] + fn test_parse_stash_stat_empty() { + let (files, summary) = parse_stash_stat(""); + assert!(files.is_empty()); + assert!(summary.is_empty()); + } + + #[test] + fn test_parse_stash_stat_unicode_and_malformed_never_panic() { + let _ = parse_stash_stat("not a diffstat at all"); + let _ = parse_stash_stat("| | |"); + let (files, _) = parse_stash_stat(" 日本語.md | 5 +++--\n"); + assert_eq!(files, vec!["日本語.md 5 +-"]); + } + + #[test] + fn test_parse_stash_stat_savings() { + use crate::core::tracking::estimate_tokens; + let raw = " CONTRIBUTING.md | 305 \ + ----------------------------------------------------------\n \ + README.md | 28 ++++--\n logo.bin | Bin 0 -> 2048 bytes\n \ + newfeature.rs | 40 ++++++++\n \ + 4 files changed, 60 insertions(+), 313 deletions(-)\n"; + let (files, summary) = parse_stash_stat(raw); + let compact = format!("{}\n{}", files.join("\n"), summary); + let savings = + 100.0 - (estimate_tokens(&compact) as f64 / estimate_tokens(raw) as f64 * 100.0); + assert!( + savings >= 40.0, + "expected >=40% savings, got {:.1}%", + savings + ); + } + + #[test] + fn test_run_stash_list_propagates_failure() { + let dir = tempfile::tempdir().expect("tempdir"); + let global = vec!["-C".to_string(), dir.path().to_string_lossy().into_owned()]; + let code = run_stash(Some("list"), &[], 0, &global).expect("run_stash list"); + assert_ne!(code, 0, "git stash list failure must propagate"); + } + + #[test] + fn test_run_stash_show_propagates_failure() { + let dir = tempfile::tempdir().expect("tempdir"); + let global = vec!["-C".to_string(), dir.path().to_string_lossy().into_owned()]; + let code = run_stash(Some("show"), &[], 0, &global).expect("run_stash show"); + assert_ne!(code, 0, "git stash show failure must propagate"); + } + + #[test] + fn test_filter_worktree_list() { + let output = + "/home/user/project abc1234 [main]\n/home/user/worktrees/feat def5678 [feature]\n"; + let result = filter_worktree_list(output); + assert!(result.contains("abc1234")); + assert!(result.contains("[main]")); + assert!(result.contains("[feature]")); + } + + #[test] + fn test_run_worktree_list_propagates_failure() { + // #2497: `git worktree list` outside a repo exits non-zero; rtk must not + // report success (empty output + exit 0). + let dir = tempfile::tempdir().expect("tempdir"); + let global = vec!["-C".to_string(), dir.path().to_string_lossy().into_owned()]; + let code = run_worktree(&[], 0, &global).expect("run_worktree"); + assert_ne!(code, 0, "git worktree list failure must propagate"); + } + + #[test] + fn test_worktree_asked_for_report() { + let report = |args: &[&str]| { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + worktree_asked_for_report(&arg_tokenizer::tokenize(&owned)) + }; + // `add` writes two progress lines that "ok" exists to replace, and has no --dry-run or + // --verbose of its own. + assert!(!report(&["add", "/tmp/w", "-b", "topic"])); + assert!(!report(&["prune"])); + assert!(!report(&["remove", "/tmp/w"])); + for spelling in [ + &["prune", "-n"][..], + &["prune", "--dry-run"][..], + &["prune", "-v"][..], + ] { + assert!(report(spelling), "{spelling:?} asks for a report"); + } + // A worktree path spelled like the flag is a path, not a request. + assert!(!report(&["remove", "--", "-n"])); + } + + #[test] + fn test_format_status_output_clean() { + let porcelain = "## main...origin/main\n"; + let result = format_status_output(porcelain); + assert_eq!(result, "* main...origin/main\nclean — nothing to commit"); + } + + #[test] + fn test_extract_state_header_clean_returns_none() { + let raw = "On branch main\nYour branch is up to date with 'origin/main'.\n\nnothing to commit, working tree clean\n"; + assert_eq!(extract_state_header(raw), None); + } + + #[test] + fn test_extract_state_header_no_state_with_changes_returns_none() { + let raw = "On branch main\nChanges not staged for commit:\n (use \"git add ...\" to update what will be committed)\n\tmodified: src/main.rs\n\nno changes added to commit\n"; + assert_eq!(extract_state_header(raw), None); + } + + #[test] + fn test_extract_state_header_editing_while_rebasing() { + let raw = "On branch feature\n\ninteractive rebase in progress; onto abc1234\nLast command done (1 command done):\n edit abc123 some message\nNo commands remaining.\nYou are currently editing a commit while rebasing branch 'feature' on 'abc1234'.\n (use \"git commit --amend\" to amend the current commit)\n (use \"git rebase --continue\" once you are satisfied with your changes)\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "rebase in progress"); + } + + #[test] + fn test_extract_state_header_merge_unresolved() { + let raw = "On branch main\nYou have unmerged paths.\n (fix conflicts and run \"git commit\")\n (use \"git merge --abort\" to abort the merge)\n\nUnmerged paths:\n\tboth modified: src/main.rs\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "merge in progress. unresolved conflicts"); + } + + #[test] + fn test_extract_state_header_cherry_pick() { + let raw = "On branch main\n\nYou are currently cherry-picking commit abc1234.\n (fix conflicts and run \"git cherry-pick --continue\")\n (use \"git cherry-pick --abort\" to cancel the cherry-pick operation)\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "cherry-pick in progress"); + } + + #[test] + fn test_extract_state_header_bisect() { + let raw = "On branch main\n\nYou are currently bisecting, started from branch 'main'.\n (use \"git bisect reset\" to get back to the original branch)\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "bisect in progress"); + } + + #[test] + fn test_extract_state_header_revert() { + let raw = "On branch main\n\nYou are currently reverting commit abc1234.\n (fix conflicts and run \"git revert --continue\")\n (use \"git revert --abort\" to cancel the revert operation)\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "revert in progress"); + } + + #[test] + fn test_extract_state_header_merge_in_middle() { + let raw = "On branch main\n\nAll conflicts fixed but you are still merging.\n (use \"git commit\" to conclude merge)\n\nChanges to be committed:\n\tmodified: src/main.rs\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "merge in progress. no conflicts"); + } + + #[test] + fn test_extract_state_header_am_session() { + let raw = "On branch main\n\nYou are in the middle of an am session.\n (use \"git am --continue\" to continue)\n (use \"git am --abort\" to restore the original branch)\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "am session in progress"); + } + + #[test] + fn test_extract_state_header_sparse_checkout() { + let raw = "On branch main\n\nYou are in a sparse checkout with 17% of tracked files present.\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "sparse checkout enabled"); + } + + #[test] + fn test_format_status_output_preserves_nested_untracked_paths() { + let porcelain = "## main\n?? tmp/c.txt\n?? tmp/nested/d.txt\n"; + let result = format_status_output(porcelain); + assert!(result.contains("* main")); + assert!(result.contains("?? tmp/c.txt")); + assert!(result.contains("?? tmp/nested/d.txt")); + assert!( + result.lines().all(|line| line != "?? tmp/"), + "Nested untracked files must not collapse back to a directory marker:\n{}", + result + ); + } + + #[test] + fn test_format_status_output_mixed_changes() { + let porcelain = r#"## main +M staged.rs + M modified.rs +A added.rs +?? untracked.txt +"#; + let result = format_status_output(porcelain); + assert!(result.contains("* main")); + assert!(result.contains("M staged.rs")); + assert!(result.contains(" M modified.rs")); + assert!(result.contains("A added.rs")); + assert!(result.contains("?? untracked.txt")); + assert!(!result.contains("Staged")); + assert!(!result.contains("Modified")); + assert!(!result.contains("Untracked")); + } + + #[test] + fn test_format_status_output_preserves_rename_and_conflict_lines() { + let porcelain = "## main\nR old.rs -> new.rs\nUU conflict.rs\nMM mixed.rs\n"; + let result = format_status_output(porcelain); + assert!(result.contains("* main")); + assert!(result.contains("R old.rs -> new.rs")); + assert!(result.contains("UU conflict.rs")); + assert!(result.contains("MM mixed.rs")); + assert!(!result.contains("conflicts:")); + } + + #[test] + fn test_run_passthrough_accepts_args() { + // Test that run_passthrough compiles and has correct signature + let _args: Vec = vec![OsString::from("tag"), OsString::from("--list")]; + // Compile-time verification that the function exists with correct signature + } + + #[test] + fn test_filter_log_output() { + let output = "abc1234 This is a commit message (2 days ago) \n\n---END---\ndef5678 Another commit (1 week ago) \n\n---END---\n"; + let result = filter_log_output(output, 10, false, false); + assert!(result.contains("abc1234")); + assert!(result.contains("def5678")); + assert_eq!(result.lines().count(), 2); + } + + #[test] + fn test_filter_log_output_with_body() { + // Commit with body: first non-trailer body line should appear indented + let output = "abc1234 feat: add feature (2 days ago) \nBREAKING CHANGE: removed old API\nSigned-off-by: Author \n---END---\ndef5678 fix: typo (1 day ago) \n\n---END---\n"; + let result = filter_log_output(output, 10, false, false); + assert!(result.contains("abc1234")); + assert!(result.contains("BREAKING CHANGE: removed old API")); + assert!(!result.contains("Signed-off-by:")); + // def5678 has no body — just header + assert!(result.contains("def5678")); + // 3 lines: header1, body1 indented, header2 + assert_eq!(result.lines().count(), 3); + } + + #[test] + fn test_filter_log_output_skips_trailers() { + // Body with only trailers should not produce a body line + let output = "abc1234 chore: bump (1 day ago) \nSigned-off-by: Bot \nCo-authored-by: Human \n---END---\n"; + let result = filter_log_output(output, 10, false, false); + assert!(result.contains("abc1234")); + assert!(!result.contains("Signed-off-by:")); + assert!(!result.contains("Co-authored-by:")); + assert_eq!(result.lines().count(), 1); + } + + #[test] + fn test_filter_log_output_truncate_long() { + let long_line = "abc1234 ".to_string() + &"x".repeat(100) + " (2 days ago) "; + let result = filter_log_output(&long_line, 10, false, false); + assert!(result.chars().count() < long_line.chars().count()); + assert!(result.contains("...")); + assert!(result.chars().count() <= 80); + } + + #[test] + fn test_filter_log_output_cap_lines() { + let output = (0..20) + .map(|i| format!("hash{} message {} (1 day ago) \n\n---END---", i, i)) + .collect::>() + .join("\n"); + let result = filter_log_output(&output, 5, false, false); + assert_eq!(result.lines().count(), 5); + } + + #[test] + fn test_filter_log_output_user_limit_no_cap() { + // When user explicitly passes -N, all N lines should be returned (no re-truncation) + let output = (0..20) + .map(|i| format!("hash{} message {} (1 day ago) \n\n---END---", i, i)) + .collect::>() + .join("\n"); + let result = filter_log_output(&output, 20, true, false); + assert_eq!( + result.lines().count(), + 20, + "User's -20 should return all 20 lines" + ); + } + + #[test] + fn test_filter_log_output_user_limit_wider_truncation() { + // When user explicitly passes -N, lines up to 120 chars should NOT be truncated + let line_90_chars = format!("abc1234 {} (2 days ago) ", "x".repeat(60)); + assert!(line_90_chars.chars().count() > 80); + assert!(line_90_chars.chars().count() < 120); + + let result_default = filter_log_output(&line_90_chars, 10, false, false); + let result_user = filter_log_output(&line_90_chars, 10, true, false); + + // Default truncates at 80 chars + assert!( + result_default.contains("..."), + "Default should truncate at 80 chars" + ); + // User-set limit uses wider threshold (120 chars) + assert!( + !result_user.contains("..."), + "User limit should not truncate 90-char line" + ); + } + + #[test] + fn test_parse_user_limit_combined() { + let args: Vec = vec!["-20".into()]; + assert_eq!(parse_user_limit(&args), Some(20)); + } + + #[test] + fn test_parse_user_limit_n_space() { + let args: Vec = vec!["-n".into(), "15".into()]; + assert_eq!(parse_user_limit(&args), Some(15)); + } + + #[test] + fn test_parse_user_limit_max_count_eq() { + let args: Vec = vec!["--max-count=30".into()]; + assert_eq!(parse_user_limit(&args), Some(30)); + } + + #[test] + fn test_parse_user_limit_max_count_space() { + let args: Vec = vec!["--max-count".into(), "25".into()]; + assert_eq!(parse_user_limit(&args), Some(25)); + } + + #[test] + fn test_parse_user_limit_none() { + let args: Vec = vec!["--oneline".into()]; + assert_eq!(parse_user_limit(&args), None); + } + + #[test] + fn test_parse_user_limit_malformed_combined_digit_run() { + // "-5x" isn't a valid git log limit (real git rejects it outright), but the digit-run + // rule only looks at the leading run and parses 5 anyway -- harmless since run_log bails + // out on git's own failure before ever using this value. + let args: Vec = vec!["-5x".into()]; + assert_eq!(parse_user_limit(&args), Some(5)); + } + + #[test] + fn test_patch_log_flags_request_raw_output() { + for flag in [ + "-p", + "-u", + "--patch", + "--patch-with-raw", + "--patch-with-stat", + ] { + let args = vec![flag.to_string()]; + assert!(requests_raw_log_output(&args), "{flag} should pass through"); + } + } + + #[test] + fn test_patch_flag_after_pathspec_separator_is_ignored() { + // `git log -- -p` means "show history for a path literally named -p", + // not "show patches" — the flag lookalike appears after `--`. + let args = vec!["--".to_string(), "-p".to_string()]; + assert!( + !requests_raw_log_output(&args), + "-p after -- is a pathspec, not a patch flag, and should stay on the filtered path" + ); + } + + #[test] + fn test_non_patch_log_flags_remain_filtered() { + for flag in ["--no-patch", "--oneline", "--format=%H"] { + let args = vec![flag.to_string()]; + assert!( + !requests_raw_log_output(&args), + "{flag} should remain on the filtered log path" + ); + } + } + + #[test] + fn test_diff_shape_flags_request_raw_output() { + // These change the shape of git's raw output (diffstat, name lists) + // the same way -p does — RTK's injected --pretty=format markers + // can't coexist with them, so they must stay on the raw path too. + for flag in [ + "--dirstat", + "--dirstat=files", + "--name-only", + "--name-status", + "--numstat", + "--raw", + "--shortstat", + "--stat", + "--stat=80", + "--summary", + ] { + let args = vec![flag.to_string()]; + assert!( + requests_raw_log_output(&args), + "{flag} changes output shape and should request raw output" + ); + } + } + + #[test] + fn test_diff_show_raw_shape_excludes_patch_flags() { + for flag in ["--patch", "-p", "-u"] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_log_args(&args); + assert!( + !tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)) + && !tokens.iter().any(|t| diff_wants_raw_shape(t, &tokens)), + "{flag} must stay on diff/show's compact path, not the raw passthrough path" + ); + } + } + + #[test] + fn test_diff_show_raw_shape_still_includes_other_shape_flags() { + for flag in [ + "--dirstat", + "--name-only", + "--name-status", + "--numstat", + "--patch-with-raw", + "--patch-with-stat", + "--raw", + "--shortstat", + "--stat", + "--summary", + ] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_log_args(&args); + assert!( + tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)) + && tokens.iter().any(|t| diff_wants_raw_shape(t, &tokens)), + "{flag} changes output shape and should still request the raw passthrough path" + ); + } + } + + #[test] + fn test_diff_shape_flag_as_value_of_grep_is_not_misdetected() { + // `git log --grep --stat` searches for the literal string + // "--stat"; git consumes it as --grep's value, not the --stat flag. + let args = vec!["--grep".to_string(), "--stat".to_string()]; + assert!( + !requests_raw_log_output(&args), + "--stat as the value of --grep should stay on the filtered path" + ); + } + + #[test] + fn test_patch_flag_as_value_of_grep_is_not_misdetected() { + // `git log --grep -p` searches commit messages for the literal + // string "-p"; git does not treat it as the patch flag. + for opt in [ + "--author", + "--committer", + "--diff-algorithm", + "--diff-filter", + "--grep", + "-G", + "-S", + ] { + let args = vec![opt.to_string(), "-p".to_string()]; + assert!( + !requests_raw_log_output(&args), + "-p as the value of {opt} should stay on the filtered path" + ); + } + } + + #[test] + fn test_patch_flag_still_detected_after_value_taking_option() { + // The value-taking option consumes only its own value token; + // a genuine -p later in the args still triggers the raw path. + let args = vec!["--grep".to_string(), "fix".to_string(), "-p".to_string()]; + assert!( + requests_raw_log_output(&args), + "a real -p after --grep's value should still request raw output" + ); + } + + #[test] + fn test_optional_value_options_do_not_consume_next_token() { + // These options only take an attached value (-U3, --unified=3, --expand-tabs=4, + // --max-parents=2); a bare separate token after them is not their value. + for opt in [ + "-U", + "--unified", + "--expand-tabs", + "--max-parents", + "--min-parents", + ] { + let args = vec![opt.to_string(), "-p".to_string()]; + assert!( + requests_raw_log_output(&args), + "a real -p after {opt} should still request raw output" + ); + } + } + + #[test] + fn test_glued_diff_shape_short_flag_is_not_misread_as_a_limit() { + // -M/-C/-B/-U take only an attached optional numeric value ("-M50"); the "50" must not + // decompose into a stray digit-run token misread as a "-5" limit flag. + for glued in ["-M50", "-C5", "-B10", "-U3"] { + let args = vec![glued.to_string()]; + assert_eq!( + parse_user_limit(&args), + None, + "{glued} must not be misread as a commit-count limit" + ); + } + } + + #[test] + fn test_clustered_short_flag_does_not_consume_separate_value() { + // `-n 2` consumes the separate "2", but clustered with another short flag (`-cn 2`), + // -n's value is only the (empty) remainder of the same arg, never the next token. + let args = vec!["-cn".to_string(), "2".to_string()]; + assert_eq!( + parse_user_limit(&args), + None, + "-n clustered with -c must not consume \"2\" as its value" + ); + + // The bare, standalone form must still work as documented. + let args = vec!["-n".to_string(), "2".to_string()]; + assert_eq!(parse_user_limit(&args), Some(2)); + } + + #[test] + fn test_diff_grammar_differs_from_logs_where_git_does() { + // `git diff -wl 100` clusters (rename limit); `git log -cl 2` does not. Sharing log's + // predicate made RTK read the 100 as a pathspec and splice its own flags before it. + let args = vec!["-wl".to_string(), "100".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert_eq!(tokens[1].text, "l"); + assert_eq!(tokens[1].value(&tokens), Some("100")); + + // Options git's own completion helper lists as value-taking that RTK had missed: their + // values were read as free positionals, which is what the project-path and pathspec + // logic keys on. + for opt in [ + "ignore-matching-lines", + "stat-graph-width", + "max-age", + "min-age", + ] { + let args = vec![format!("--{opt}"), "x".to_string(), "HEAD".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert_eq!(tokens[0].value(&tokens), Some("x"), "--{opt}"); + assert!( + !tokens[1].is_free_positional(), + "--{opt}'s value must not read as a positional" + ); + } + } + + #[test] + fn test_patch_shape_removal_keeps_the_rest_of_a_cluster() { + // Every short flag in `-pl` shares one source_index, so dropping the whole arg took + // `-l` with it and left its `100` behind as a bogus revision. + let rebuild = |args: &[&str]| -> Vec { + let args: Vec = args.iter().map(|a| a.to_string()).collect(); + let tokens = tokenize_git_diff_args(&args); + args_without_patch_shape(&args, &tokens) + }; + + assert_eq!(rebuild(&["-pl", "100"]), vec!["-l", "100"]); + assert_eq!(rebuild(&["-pw"]), vec!["-w"]); + assert_eq!(rebuild(&["-wU2"]), vec!["-w"]); + assert_eq!(rebuild(&["-p"]), Vec::::new()); + // Nothing to strip: flags and positionals pass through untouched. + assert_eq!( + rebuild(&["-w", "HEAD~1", "f.txt"]), + vec!["-w", "HEAD~1", "f.txt"] + ); + assert_eq!(rebuild(&["--author", "-p"]), vec!["--author", "-p"]); + } + + #[test] + fn test_stat_step_retokenizes_after_dropping_oneline() { + // The stat step drops `--oneline` and then strips patch-shape flags. Both steps read + // token indices, and the first shifts them, so the second must re-tokenize or it + // deletes the argument that inherited the patch flag's index -- here the revision. + let args: Vec = ["--oneline", "-p", "deadbee"] + .iter() + .map(|a| a.to_string()) + .collect(); + let tokens = tokenize_git_diff_args(&args); + let stat_args = args_without_oneline(&args, &tokens); + assert_eq!(stat_args, vec!["-p", "deadbee"]); + + let stat_tokens = tokenize_git_diff_args(&stat_args); + assert_eq!( + args_without_patch_shape(&stat_args, &stat_tokens), + vec!["deadbee"], + "the revision must survive; only the patch flag is dropped" + ); + assert_eq!( + args_without_patch_shape(&stat_args, &tokens), + vec!["-p"], + "stale indices invert the result: revision dropped, patch flag kept" + ); + } + + #[test] + fn test_diff_header_drops_the_users_patch_flags() { + // git takes the last shape flag and accepts options after a revision, so a `-p` + // written there outranks RTK's header flags wherever they are placed -- the header + // then carried the whole patch and RTK printed it again, compacted. + let args = vec!["HEAD~1".to_string(), "-p".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert_eq!(args_without_patch_shape(&args, &tokens), vec!["HEAD~1"]); + + for flag in ["-p", "-u", "--patch", "-U5", "-W", "--function-context"] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!( + args_without_patch_shape(&args, &tokens).is_empty(), + "{flag}" + ); + } + // A flag that only tunes the diff must survive into the header. + let args = vec!["-w".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert_eq!(args_without_patch_shape(&args, &tokens), vec!["-w"]); + } + + #[test] + fn test_suppression_flags_are_raw_for_diff_but_compact_for_show() { + // `-s` asks diff for no body (nothing to compact) but asks show for the commit + // summary, which is exactly what the compact form prints. + for flag in ["-s", "--no-patch"] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!(tokens.iter().any(suppresses_diff_body), "{flag}"); + assert!( + !tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)), + "{flag} must stay on show's compact path" + ); + } + // --check replaces the body with a whitespace report in both. + let args = vec!["--check".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!(tokens.iter().any(|t| show_wants_raw_shape(t, &tokens))); + assert!(tokens.iter().any(|t| diff_wants_raw_shape(t, &tokens))); + } + + #[test] + fn test_quiet_is_raw_for_diff_but_compact_for_show() { + // Verified against git 2.53: `git diff --quiet` prints nothing and exits 1 on a + // difference, while `git show --quiet` exits 0 and prints the header its synonyms + // `-s`/`--no-patch` compact. Claiming it for show raw-passed that header. + let args = vec!["--quiet".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!( + tokens.iter().any(|t| diff_wants_raw_shape(t, &tokens)), + "diff needs the exit code" + ); + assert!( + !tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)), + "show's --quiet is -s, which suppresses_diff_body renders as the summary" + ); + assert!(tokens.iter().any(suppresses_diff_body)); + } + + #[test] + fn test_raw_log_passthrough_keeps_rtks_default_limit() { + let built = |args: &[&str]| { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + let tokens = tokenize_git_log_args(&owned); + assert!( + tokens.iter().any(|t| log_wants_raw_shape(t, &tokens)), + "{args:?} must route raw" + ); + raw_log_passthrough_args(&owned, raw_log_is_capped(&tokens)) + .iter() + .map(|a| a.to_string_lossy().into_owned()) + .collect::>() + }; + + // The limit precedes every user argument. git rejects an option that follows a + // positional -- "fatal: -10 option must come before non-option arguments" -- and a + // pathspec needs no `--` to be a positional, so anchoring on the boundary is not enough. + assert_eq!(built(&["-p"]), ["log", "-10", "-p"]); + assert_eq!( + built(&["-p", "src/main.rs"]), + ["log", "-10", "-p", "src/main.rs"], + "a bare pathspec still has to come after the limit" + ); + assert_eq!( + built(&["--stat", "--", "src/main.rs"]), + ["log", "-10", "--stat", "--", "src/main.rs"] + ); + + // A limit the user set is left alone, in every spelling has_limit_flag knows. + for limit in [&["-5"][..], &["-n", "5"][..], &["--max-count=5"][..]] { + let args: Vec<&str> = std::iter::once("-p").chain(limit.iter().copied()).collect(); + let got = built(&args); + assert!( + !got.contains(&DEFAULT_LOG_LIMIT_ARG.to_string()), + "{args:?} already names a limit, got {got:?}" + ); + } + } + + #[test] + fn test_show_keeps_compacting_under_oneline() { + let gate = |args: &[&str]| { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + show_wants_format(&tokenize_git_diff_args(&owned)) + }; + // The user's own format displaces RTK's summary entirely: nothing to compact around. + assert!(gate(&["--pretty=format:%H"])); + assert!(gate(&["--format=%H"])); + // `--oneline` only outranks RTK's own one-line summary. Compaction stays on, or one + // display flag turns the whole command into a raw passthrough. + assert!(!gate(&["--oneline"])); + assert!(!gate(&[])); + } + + #[test] + fn test_body_suppression_follows_gits_last_flag_wins() { + // git 2.53: `git show -s -p` prints the diff, `git show -p -s` does not. Taking the + // suppressors order-independently swallowed a patch the user asked for last. + // `--quiet` is deliberately not in the last-wins group: git show ignores it for shape + // whenever a patch is requested, from either side, and honours it only when nothing + // else asked for output. Every row measured against git 2.53. + let cases: [(&[&str], bool); 13] = [ + (&["-s"], true), + (&["--no-patch"], true), + (&["--quiet"], true), + (&["-s", "-p"], false), + (&["-p", "-s"], true), + (&["--no-patch", "--patch"], false), + (&["--patch", "--no-patch"], true), + (&["--quiet", "-p"], false), + (&["-p", "--quiet"], false), + (&["--quiet", "-U0"], false), + (&["-U0", "--quiet"], false), + (&["--quiet", "-s", "-p"], false), + (&["-p", "-s", "--quiet"], true), + ]; + for (args, expected) in cases { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + let tokens = tokenize_git_diff_args(&owned); + assert_eq!(body_is_suppressed(&tokens), expected, "{args:?}"); + } + } + + #[test] + fn test_raw_log_limit_leaves_an_explicitly_bounded_walk_alone() { + let built = |args: &[&str]| { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + let tokens = tokenize_git_log_args(&owned); + raw_log_passthrough_args(&owned, raw_log_is_capped(&tokens)) + .iter() + .any(|a| a == DEFAULT_LOG_LIMIT_ARG) + }; + // A revision range is an exact bound the user chose; capping it takes away what they + // asked for -- `git log -p HEAD~15..HEAD` came back with 10 of the 15. + assert!(!built(&["-p", "HEAD~15..HEAD"])); + assert!(!built(&["-p", "HEAD~3...HEAD"])); + assert!(!built(&["-p", "origin/main..HEAD"])); + // Unbounded still gets the limit, or a patch request prints the whole history. + assert!(built(&["-p"])); + assert!(built(&["-p", "HEAD"])); + // A relative pathspec is not a range. + assert!(built(&["-p", "../sibling"])); + assert!(built(&["-p", "./a..b"])); + // And an explicit limit still wins over both. + assert!(!built(&["-p", "-5"])); + } + + #[test] + fn test_line_prefix_takes_the_raw_route() { + // It prefixes every output line, `diff --git` and `@@` markers included, so the + // compaction parses nothing and printed a diffstat over an empty `Changes:` section. + for args in [ + &["--line-prefix=XX"][..], + &["--line-prefix", "XX"][..], + // git takes the next token as the value, so a following flag is the prefix text. + &["--line-prefix", "--stat"][..], + ] { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + let log = tokenize_git_log_args(&owned); + assert!( + log.iter().any(|t| log_wants_raw_shape(t, &log)), + "{args:?} on log" + ); + let diff = tokenize_git_diff_args(&owned); + assert!( + diff.iter().any(|t| diff_wants_raw_shape(t, &diff)), + "{args:?} on diff" + ); + } + } + + #[test] + fn test_diff_merges_routes_by_the_format_it_names() { + // Verified against git 2.53 on a conflict-resolved merge: every format but off/none + // emits a patch, and only the combined ones use the two `@@@` marker columns. + let route = |args: &[&str]| { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + let log = tokenize_git_log_args(&owned); + let show = tokenize_git_diff_args(&owned); + ( + log.iter().any(|t| log_wants_raw_shape(t, &log)), + show.iter().any(|t| show_wants_raw_shape(t, &show)), + ) + }; + + for fmt in ["c", "cc", "combined", "dense-combined"] { + // Combined: raw for both -- compact_diff reads the two marker columns as one. + assert_eq!( + route(&[&format!("--diff-merges={fmt}")]), + (true, true), + "={fmt}" + ); + // git takes the value as the next token too, and that spelling must route alike. + assert_eq!( + route(&["--diff-merges", fmt]), + (true, true), + "separate {fmt}" + ); + } + + for fmt in ["1", "first-parent", "m", "on", "r", "remerge", "separate"] { + // A single-column patch: log still cannot represent one, show's default already is + // one, so the two subcommands disagree here on purpose. + assert_eq!( + route(&[&format!("--diff-merges={fmt}")]), + (true, false), + "={fmt}" + ); + assert_eq!( + route(&["--diff-merges", fmt]), + (true, false), + "separate {fmt}" + ); + } + + for fmt in ["none", "off"] { + // No patch at all, so nothing to escape the compact path for. + assert_eq!( + route(&[&format!("--diff-merges={fmt}")]), + (false, false), + "={fmt}" + ); + assert_eq!( + route(&["--diff-merges", fmt]), + (false, false), + "separate {fmt}" + ); + } + + // No value is a git error either way; RTK must not read it as a patch request. + assert_eq!(route(&["--diff-merges"]), (false, false)); + } + + #[test] + fn test_combined_diff_short_flag_is_raw_for_show_like_its_long_form() { + // `-c` is `--cc`'s combined-diff form, not a redundant patch request: compact_diff + // reads a combined diff's two marker columns as one, so `git show -c ` came + // back as `+54 -8` against git's own 156 insertions / 0 deletions. + for flag in ["-c", "--cc"] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!( + tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)), + "{flag} must take show's raw route" + ); + assert!( + tokens.iter().any(|t| diff_wants_raw_shape(t, &tokens)), + "{flag} for diff too" + ); + } + // The other short flags stay on the compact path: they only restate the default. + for flag in ["-p", "-u", "-U3"] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!( + !tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)), + "{flag}" + ); + } + } + + #[test] + fn test_log_output_flags_are_not_a_patch_request() { + // Neither changes `git log`'s output at all (byte-identical to plain `git log` + // against git 2.53), so routing them raw skipped RTK's own -10 and printed the + // entire history. + for flag in ["--quiet", "--exit-code"] { + let args = vec![flag.to_string()]; + assert!( + !requests_raw_log_output(&args), + "{flag} leaves git log's output unchanged, so it has no shape to escape to" + ); + } + } + + #[test] + fn test_stat_header_flags_land_before_the_users_pathspec_boundary() { + // `--no-patch --stat` after the user's `--` would be pathspecs, not options. + let args = vec!["--".to_string(), "src/".to_string()]; + let tokens = tokenize_git_log_args(&args); + assert_eq!(arg_tokenizer::injection_point(&tokens, args.len()), 0); + + let args = vec!["-p".to_string()]; + let tokens = tokenize_git_log_args(&args); + assert_eq!(arg_tokenizer::injection_point(&tokens, args.len()), 1); + } + + #[test] + fn test_blob_show_detection_ignores_flag_values_and_pathspecs() { + // Only a free positional before `--` can name a blob: `--author 'a:b'` is that flag's + // value and `-- a:b` is a pathspec, and neither should force raw passthrough. + let blob = |args: &[&str]| -> bool { + let args: Vec = args.iter().map(|a| a.to_string()).collect(); + // diff's grammar, the one run_show actually uses: log's disagrees on clustered + // `-l`, so the test would assert a classification that never ships. + let tokens = tokenize_git_diff_args(&args); + arg_tokenizer::before_dashdash(&tokens) + .iter() + .any(|t| t.is_free_positional() && is_blob_show_arg(t.text)) + }; + + assert!(blob(&["HEAD:src/main.rs"])); + assert!(!blob(&["--author", "a:b", "HEAD"])); + assert!(!blob(&["--", "a:b"])); + assert!(!blob(&["--pretty=format:%h"])); + } + + #[test] + fn test_status_compact_path_survives_a_bare_double_dash() { + // `git status --` selects no pathspec, so it is `git status` -- and must keep the + // compact porcelain path rather than falling through to git's prose output. The + // separator has to be ignored wherever it sits, not only when it is the only token. + assert!(uses_compact_status_path(&[])); + assert!(uses_compact_status_path(&["--".to_string()])); + assert!(uses_compact_status_path(&[ + "-sb".to_string(), + "--".to_string() + ])); + assert!(uses_compact_status_path(&[ + "-s".to_string(), + "-b".to_string(), + "--".to_string() + ])); + assert!(uses_compact_status_path(&["-sb".to_string()])); + assert!(uses_compact_status_path(&["-b".to_string()])); + // A real pathspec still passes through. + assert!(!uses_compact_status_path(&[ + "--".to_string(), + "f.txt".to_string() + ])); + } + + #[test] + fn test_split_stash_region_keeps_the_boundary_out_of_the_subcommand() { + // `git stash -- -p` is a pathspec, not interactive patch mode: the restored region + // starts at the boundary, so nothing is taken as the subcommand. + let owned = |args: &[&str]| -> Vec { args.iter().map(|a| a.to_string()).collect() }; + + let (subcommand, rest) = split_stash_region(&owned(&["--", "-p"])); + assert_eq!(subcommand, None); + assert_eq!(rest, owned(&["--", "-p"])); + + let (subcommand, rest) = split_stash_region(&owned(&["show", "-p"])); + assert_eq!(subcommand.as_deref(), Some("show")); + assert_eq!(rest, owned(&["-p"])); + + let (subcommand, rest) = split_stash_region(&owned(&["push", "--", "file.txt"])); + assert_eq!(subcommand.as_deref(), Some("push")); + assert_eq!(rest, owned(&["--", "file.txt"])); + + let (subcommand, rest) = split_stash_region(&owned(&["-p"])); + assert_eq!(subcommand, None); + assert_eq!(rest, owned(&["-p"])); + } + + #[test] + fn test_stash_show_wants_patch_ignores_dash_p_pathspec_after_double_dash() { + // A pathspec literally named "-p" after `--` must not be mistaken for the flag. + let args = vec!["--".to_string(), "-p".to_string()]; + assert!(!stash_show_wants_patch(&args)); + + let args = vec!["-p".to_string()]; + assert!(stash_show_wants_patch(&args)); + let args = vec!["--patch".to_string()]; + assert!(stash_show_wants_patch(&args)); + } + + #[test] + fn test_stash_show_wants_patch_does_not_swallow_dash_p_as_log_only_option_value() { + // git log's grammar (where --author takes a separate value) must not apply here. + let args = vec!["--author".to_string(), "-p".to_string()]; + assert!(stash_show_wants_patch(&args)); + + let args = vec!["--grep".to_string(), "-p".to_string()]; + assert!(stash_show_wants_patch(&args)); + } + + #[test] + fn test_stash_show_wants_patch_does_not_treat_dash_u_as_patch() { + // -u means --include-untracked for stash show, not -p (unlike git log, where -u is a + // -p synonym) -- conflating them routed -u's stat-only output through compact_diff, + // which only renders patch content, producing silently empty output. + let args = vec!["-u".to_string()]; + assert!(!stash_show_wants_patch(&args)); + + let args = vec!["-u".to_string(), "-p".to_string()]; + assert!(stash_show_wants_patch(&args)); + } + + #[test] + fn test_has_limit_flag_ignores_a_clustered_n_with_no_captured_value() { + // A clustered "n" with no captured value (e.g. "-cn") must not count as a limit. + let args = vec!["-cn".to_string(), "2".to_string()]; + let tokens = tokenize_git_log_args(&args); + assert!(!has_limit_flag(&tokens)); + + // The bare, standalone form still counts. + let args = vec!["-n".to_string(), "2".to_string()]; + let tokens = tokenize_git_log_args(&args); + assert!(has_limit_flag(&tokens)); + } + + #[test] + fn test_real_flag_args_drops_value_taking_option_values() { + // `--grep`'s value is not itself a flag and must not appear in the + // filtered set, even when it looks like -N, --pretty, or --merges. + let args = vec!["--grep".to_string(), "-5".to_string()]; + assert_eq!(real_flag_args(&args), vec!["grep"]); + } + + #[test] + fn test_real_flag_args_keeps_limit_flag_drops_its_value() { + let args = vec!["-n".to_string(), "15".to_string()]; + assert_eq!(real_flag_args(&args), vec!["n"]); + + let args = vec!["--max-count".to_string(), "25".to_string()]; + assert_eq!(real_flag_args(&args), vec!["max-count"]); + } + + #[test] + fn test_real_flag_args_keeps_genuine_flags() { + let args = vec![ + "--grep".to_string(), + "fix".to_string(), + "--oneline".to_string(), + ]; + assert_eq!(real_flag_args(&args), vec!["grep", "oneline"]); + } + + #[test] + fn test_grep_value_looking_like_limit_flag_is_not_misdetected() { + // `git log --grep -5` searches commit messages for the literal + // string "-5"; it is not a request to limit output to 5 commits. + let args = vec!["--grep".to_string(), "-5".to_string()]; + assert!( + !real_flag_args(&args).iter().any(|arg| is_digit_run(arg)), + "-5 as the value of --grep should not be seen as a limit flag" + ); + assert_eq!( + parse_user_limit(&args), + None, + "-5 as the value of --grep should not be parsed as a limit" + ); + } + + #[test] + fn test_grep_value_looking_like_format_flag_is_not_misdetected() { + // `git log --grep --pretty` searches for the literal string + // "--pretty"; git consumes it as --grep's value, not a format flag. + let args = vec!["--grep".to_string(), "--pretty".to_string()]; + assert!( + !real_flag_args(&args).contains(&"pretty"), + "--pretty as the value of --grep should not be seen as a format flag" + ); + } + + #[test] + fn test_grep_value_looking_like_merges_flag_is_not_misdetected() { + // `git log --grep --merges` searches for the literal string + // "--merges"; git consumes it as --grep's value, not --merges. + let args = vec!["--grep".to_string(), "--merges".to_string()]; + assert!( + !real_flag_args(&args).contains(&"merges"), + "--merges as the value of --grep should not be seen as the --merges flag" + ); + } + + #[test] + fn test_parse_user_limit_skips_foreign_option_values() { + // A real limit later in the args is still found after a + // value-taking option's value is skipped. + let args = vec!["--grep".to_string(), "-5".to_string(), "-20".to_string()]; + assert_eq!(parse_user_limit(&args), Some(20)); + } + + #[test] + fn test_log_arg_tokens_stop_at_pathspec_separator() { + // `git log -- -5` means "history for the path literally named -5", + // not a limit flag — tokens after `--` must be ignored entirely. + let args = vec!["--".to_string(), "-5".to_string()]; + assert!( + real_flag_args(&args).is_empty(), + "-5 after -- is a pathspec, not a flag" + ); + assert_eq!( + parse_user_limit(&args), + None, + "-5 after -- should not be parsed as a limit" + ); + } + + #[test] + fn test_filter_log_output_token_savings() { + fn count_tokens(text: &str) -> usize { + text.split_whitespace().count() + } + // Simulate verbose git log output (default format with full metadata) + let input = (0..20) + .map(|i| { + format!( + "commit abc123{:02x}\nAuthor: User Name \nDate: Mon Mar 10 10:00:00 2026 +0000\n\n fix: commit message number {}\n\n Extended body with details about the change.\n", + i, i + ) + }) + .collect::>() + .join("\n"); + let output = filter_log_output(&input, 10, false, false); + let savings = 100.0 - (count_tokens(&output) as f64 / count_tokens(&input) as f64 * 100.0); + assert!( + savings >= 60.0, + "Expected ≥60% token savings, got {:.1}%", + savings + ); + } + + #[test] + fn test_filter_status_with_args() { + let output = r#"On branch main +Your branch is up to date with 'origin/main'. + +Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + modified: src/main.rs + +no changes added to commit (use "git add" and/or "git commit -a") +"#; + let result = filter_status_with_args(output); + eprintln!("Result:\n{}", result); + assert!(result.contains("On branch main")); + assert!(result.contains("modified: src/main.rs")); + assert!( + !result.contains("(use \"git"), + "Result should not contain git hints" + ); + } + + #[test] + fn test_filter_status_with_args_clean() { + let output = "nothing to commit, working tree clean\n"; + let result = filter_status_with_args(output); + assert!(result.contains("nothing to commit")); + } + + #[test] + fn test_filter_log_output_multibyte() { + // Thai characters: each is 3 bytes. A line with >80 bytes but few chars + let thai_msg = format!("abc1234 {} (2 days ago) ", "ก".repeat(30)); + let result = filter_log_output(&thai_msg, 10, false, false); + // Should not panic + assert!(result.contains("abc1234")); + // The line has 30 Thai chars + other text, so > 80 chars total + // truncate_line now counts chars, not bytes + // 30 Thai + ~33 other = 63 chars < 80 threshold, so no truncation + assert!(result.contains("abc1234")); + } + + #[test] + fn test_filter_log_output_emoji() { + let emoji_msg = "abc1234 🎉🎊🎈🎁🎂🎄🎃🎆🎇✨🎉🎊🎈🎁🎂🎄🎃🎆🎇✨ (1 day ago) "; + let result = filter_log_output(emoji_msg, 10, false, false); + // Should not panic + // 20 emoji + ~30 other chars = ~50 chars < 80, no truncation needed + assert!(result.contains("abc1234")); + } + + #[test] + fn test_format_status_output_thai_filename() { + let porcelain = "## main\n M สวัสดี.txt\n?? ทดสอบ.rs\n"; + let result = format_status_output(porcelain); + // Should not panic + assert!(result.contains("* main")); + assert!(result.contains("สวัสดี.txt")); + assert!(result.contains("ทดสอบ.rs")); + } + + #[test] + fn test_format_status_output_emoji_filename() { + let porcelain = "## main\nA 🎉-party.txt\n M 日本語ファイル.rs\n"; + let result = format_status_output(porcelain); + assert!(result.contains("* main")); + } + + // --- commit output parsing --- + + #[test] + fn test_parse_commit_output_normal() { + let line = "[main abc1234def] add feature"; + assert_eq!(parse_commit_output(line), "ok abc1234"); + } + + #[test] + fn test_parse_commit_output_root_commit() { + let line = "[main (root-commit) abc1234def] initial commit"; + assert_eq!(parse_commit_output(line), "ok abc1234"); + } + + /// Regression test: multibyte branch name must not panic (was byte-slicing before fix) + #[test] + fn test_parse_commit_output_multibyte_branch() { + let line = "[分支名 abc1234def] 提交消息"; + assert_eq!(parse_commit_output(line), "ok abc1234"); + } + + /// Regression test: Thai branch name (3 bytes per char) + #[test] + fn test_parse_commit_output_thai_branch() { + let line = "[สาขา abc1234def] commit message"; + assert_eq!(parse_commit_output(line), "ok abc1234"); + } + + /// Regression: git prints hook output before its own summary. A first line + /// that opens with a multi-byte character and contains ']' used to panic on + /// `line[1..]` ("byte index 1 is not a char boundary"). + #[test] + fn test_parse_commit_output_multibyte_prefix_does_not_panic() { + assert_eq!(parse_commit_output("✅ lint passed]"), "ok"); + assert_eq!(parse_commit_output("→ hook] done"), "ok"); + } + + /// The same shape as above, but with a real summary after the hook text — + /// the hash must still be found via the bracket pair. + #[test] + fn test_parse_commit_output_after_multibyte_hook_prefix() { + assert_eq!( + parse_commit_output("✅ [main abc1234def] add feature"), + "ok abc1234" + ); + } + + /// A U+FFFD from lossily decoded output is itself multi-byte. + #[test] + fn test_parse_commit_output_replacement_char_prefix() { + assert_eq!(parse_commit_output("\u{FFFD}oops]"), "ok"); + } + + /// A closing bracket before any opening one must not slice backwards. + #[test] + fn test_parse_commit_output_close_before_open() { + assert_eq!(parse_commit_output("] stray [main abc1234def]"), "ok"); + } + + #[test] + fn test_parse_commit_output_no_bracket() { + let line = "some other output"; + assert_eq!(parse_commit_output(line), "ok"); + } + + #[test] + fn test_parse_commit_output_short_hash() { + // Hash shorter than 7 chars — treat as "ok" (no hash shown) + let line = "[main abc12] message"; + assert_eq!(parse_commit_output(line), "ok"); + } + + #[test] + fn test_parse_commit_output_empty() { + assert_eq!(parse_commit_output(""), "ok"); + } + + // --- commit outcome classification (issue #2494) --- + + #[test] + fn test_classify_commit_success_extracts_hash() { + match classify_commit_outcome(true, "[main abc1234def] add feature", 0) { + CommitOutcome::Ok(s) => assert_eq!(s, "ok abc1234"), + CommitOutcome::Failed(_) => panic!("successful commit must be Ok"), + } + } + + #[test] + fn test_classify_commit_success_empty_stdout() { + match classify_commit_outcome(true, "", 0) { + CommitOutcome::Ok(s) => assert_eq!(s, "ok"), + CommitOutcome::Failed(_) => panic!("successful commit must be Ok"), + } + } + + #[test] + fn test_classify_commit_nothing_to_commit_is_failure() { + match classify_commit_outcome( + false, + "On branch main\nnothing to commit, working tree clean", + 1, + ) { + CommitOutcome::Failed(code) => assert_eq!(code, 1), + CommitOutcome::Ok(s) => panic!("nothing-to-commit must not be ok: {}", s), + } + } + + #[test] + fn test_classify_commit_hook_abort_propagates_exit_code() { + match classify_commit_outcome(false, "pre-commit hook failed", 2) { + CommitOutcome::Failed(code) => assert_eq!(code, 2), + CommitOutcome::Ok(_) => panic!("hook abort must be a failure"), + } + } + + /// Regression test: --oneline and other user format flags must preserve all commits. + /// Before fix, filter_log_output split on ---END--- which doesn't exist when + /// the user specifies their own format, resulting in only 2 commits surviving. + #[test] + fn test_filter_log_output_user_format_oneline() { + let oneline_output = "abc1234 feat: add feature\n\ + def5678 fix: typo\n\ + ghi9012 chore: bump deps\n\ + jkl3456 docs: update readme\n\ + mno7890 test: add tests\n"; + + let result = filter_log_output(oneline_output, 10, false, true); + // All 5 lines must survive — no ---END--- splitting + assert_eq!(result.lines().count(), 5); + assert!(result.contains("abc1234")); + assert!(result.contains("mno7890")); + } + + #[test] + fn test_filter_log_output_user_format_with_limit() { + let oneline_output = "abc1234 feat: add feature\n\ + def5678 fix: typo\n\ + ghi9012 chore: bump deps\n\ + jkl3456 docs: update readme\n\ + mno7890 test: add tests\n"; + + // user_set_limit=true means respect all lines (no cap) + let result = filter_log_output(oneline_output, 3, true, true); + assert_eq!(result.lines().count(), 5); + + // user_set_limit=false means cap at limit + let result = filter_log_output(oneline_output, 3, false, true); + assert_eq!(result.lines().count(), 3); + } + + /// Regression test: `git branch ` must create, not list. + /// Before fix, positional args fell into list mode which added `-a`, + /// turning creation into a pattern-filtered listing (silent no-op). + #[test] + #[ignore] // Integration test: requires git repo + fn test_branch_creation_not_swallowed() { + let branch = "test-rtk-create-branch-regression"; + // Create branch via run_branch + run_branch(&[branch.to_string()], 0, &[]).expect("run_branch should succeed"); + // Verify it exists + let output = Command::new("git") + .args(["branch", "--list", branch]) + .output() + .expect("git branch --list should work"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(branch), + "Branch '{}' was not created. run_branch silently swallowed the creation.", + branch + ); + // Cleanup + let _ = Command::new("git").args(["branch", "-d", branch]).output(); + } + + /// Regression test: `git branch ` must create from commit. + #[test] + #[ignore] // Integration test: requires git repo + fn test_branch_creation_from_commit() { + let branch = "test-rtk-create-from-commit"; + run_branch(&[branch.to_string(), "HEAD".to_string()], 0, &[]) + .expect("run_branch with start-point should succeed"); + let output = Command::new("git") + .args(["branch", "--list", branch]) + .output() + .expect("git branch --list should work"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(branch), + "Branch '{}' was not created from commit.", + branch + ); + let _ = Command::new("git").args(["branch", "-d", branch]).output(); + } + + #[test] + fn test_commit_single_message() { + let args = vec!["-m".to_string(), "fix: typo".to_string()]; + let cmd = build_commit_command(&args, &[]); + let cmd_args: Vec<_> = cmd + .get_args() + .map(|a| a.to_string_lossy().to_string()) + .collect(); + assert_eq!(cmd_args, vec!["commit", "-m", "fix: typo"]); + } + + #[test] + fn test_commit_multiple_messages() { + let args = vec![ + "-m".to_string(), + "feat: add multi-paragraph support".to_string(), + "-m".to_string(), + "This allows git commit -m \"title\" -m \"body\".".to_string(), + ]; + let cmd = build_commit_command(&args, &[]); + let cmd_args: Vec<_> = cmd + .get_args() + .map(|a| a.to_string_lossy().to_string()) + .collect(); + assert_eq!( + cmd_args, + vec![ + "commit", + "-m", + "feat: add multi-paragraph support", + "-m", + "This allows git commit -m \"title\" -m \"body\"." + ] + ); + } + + // #327: git commit -am "msg" must pass -am through to git + #[test] + fn test_commit_am_flag() { + let args = vec!["-am".to_string(), "quick fix".to_string()]; + let cmd = build_commit_command(&args, &[]); + let cmd_args: Vec<_> = cmd + .get_args() + .map(|a| a.to_string_lossy().to_string()) + .collect(); + assert_eq!(cmd_args, vec!["commit", "-am", "quick fix"]); + } + + #[test] + fn test_commit_amend() { + let args = vec![ + "--amend".to_string(), + "-m".to_string(), + "new msg".to_string(), + ]; + let cmd = build_commit_command(&args, &[]); + let cmd_args: Vec<_> = cmd + .get_args() + .map(|a| a.to_string_lossy().to_string()) + .collect(); + assert_eq!(cmd_args, vec!["commit", "--amend", "-m", "new msg"]); + } + + #[test] + #[ignore] // Requires `cargo build` first — run with `cargo test --ignored` + fn test_git_status_not_a_repo_exits_nonzero() { + // Run rtk git status in a directory that is not a git repo + let tmp = std::env::temp_dir().join("rtk_test_not_a_repo"); + let _ = std::fs::create_dir_all(&tmp); + + // Build the path to the test binary + let bin_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join("debug") + .join("rtk"); + assert!( + bin_path.exists(), + "Debug binary not found at {:?} — run `cargo build` first", + bin_path + ); + let output = std::process::Command::new(&bin_path) + .args(["git", "status"]) + .current_dir(&tmp) + .output() + .expect("Failed to run rtk"); + + // Should exit with non-zero (128 from git) + assert!( + !output.status.success(), + "Expected non-zero exit code for git status outside a repo, got {:?}", + output.status.code() + ); + + // Message should be on stderr, not stdout + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stderr.to_lowercase().contains("not a git repository"), + "Expected 'not a git repository' on stderr, got stderr={:?}, stdout={:?}", + stderr, + stdout + ); + + let _ = std::fs::remove_dir_all(&tmp); + } + + // --- truncation accuracy --- + + #[test] + fn test_format_status_output_shows_every_file_when_many_are_dirty() { + let mut porcelain = String::from("## main...origin/main\n"); + for i in 0..25 { + porcelain.push_str(&format!("M staged_file_{}.rs\n", i)); + } + let result = format_status_output(&porcelain); + assert!( + result.contains("staged_file_24.rs"), + "Expected the last staged file to remain visible, got:\n{}", + result + ); + assert!( + result.lines().count() == 26, + "Expected branch + all 25 staged files, got:\n{}", + result + ); + assert!( + !result.contains("... +"), + "Status output must not hide dirty paths behind overflow markers:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_recovery_hint_present() { + // A hunk with 110 lines exceeds max_hunk_lines (100), triggers truncation + // The recovery hint must appear so LLMs can re-fetch the full diff + let mut diff = String::new(); + diff.push_str("diff --git a/large.rs b/large.rs\n"); + diff.push_str("--- a/large.rs\n"); + diff.push_str("+++ b/large.rs\n"); + diff.push_str("@@ -1,150 +1,150 @@\n"); + for i in 0..110 { + diff.push_str(&format!("+added line {}\n", i)); + } + let result = compact_diff(&diff, 500); + assert!( + result.contains("[full diff: rtk git diff --no-compact]"), + "Expected recovery hint when hunk is truncated, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_hunk_truncation_count_accurate() { + // 150 change lines in one hunk: 100 shown, 50 silently dropped + // Must report the exact count, not just "(truncated)" + let mut diff = String::from( + "diff --git a/large.rs b/large.rs\n--- a/large.rs\n+++ b/large.rs\n@@ -1,150 +1,150 @@\n", + ); + for i in 0..150 { + diff.push_str(&format!("+line {}\n", i)); + } + let result = compact_diff(&diff, 500); + assert!( + result.contains("50 additions truncated"), + "Expected '50 additions truncated' (150 - 100 = 50), got:\n{}", + result + ); + } + + #[test] + fn test_extract_detached_head_returns_line() { + let raw = "HEAD detached at abc1234\nnothing to commit, working tree clean\n"; + assert_eq!( + extract_detached_head(raw), + Some("HEAD detached at abc1234".to_string()) + ); + } + + #[test] + fn test_extract_detached_head_on_branch_is_none() { + let raw = "On branch main\nnothing to commit, working tree clean\n"; + assert!(extract_detached_head(raw).is_none()); + } + + #[test] + fn test_format_status_output_detached_head() { + let porcelain = "## HEAD (no branch)\n M src/main.rs\n"; + let result = format_status_output_detached(porcelain, "HEAD detached at abc1234"); + assert!( + result.contains("HEAD detached at abc1234"), + "should use explicit detached ref, got: {result}" + ); + assert!( + !result.contains("HEAD (no branch)"), + "should not show opaque porcelain string, got: {result}" + ); + } + + #[test] + fn test_filter_log_output_body_omission_indicator() { + // Commit with 6 meaningful body lines: only 3 shown, must signal "+3 lines omitted" + let body_lines = (1..=6) + .map(|i| format!("body line {}", i)) + .collect::>() + .join("\n"); + let output = format!( + "abc1234 feat: big change (1 day ago) \n{}\n---END---\n", + body_lines + ); + let result = filter_log_output(&output, 10, false, false); + assert!( + result.contains("+3 lines omitted"), + "Expected '+3 lines omitted' when 6 body lines truncated to 3, got:\n{}", + result + ); + } + + fn run_push_filter(input: &str, exit_code: i32) -> String { + use crate::core::stream::StreamFilter; + let mut f = LineStreamFilter::new(GitPushLineHandler::default()); + let mut out = String::new(); + for line in input.lines() { + if let Some(s) = f.feed_line(line) { + out.push_str(&s); + } + } + out.push_str(&f.flush()); + if let Some(s) = f.on_exit(exit_code, input) { + out.push_str(&s); + } + out + } + + #[test] + fn test_push_filter_drops_progress_phases() { + let input = "\ +Enumerating objects: 5, done. +Counting objects: 100% (5/5), done. +Delta compression using up to 8 threads +Compressing objects: 100% (3/3), done. +Writing objects: 100% (3/3), 312 bytes | 312.00 KiB/s, done. +Total 3 (delta 2), reused 0 (delta 0) +To https://github.com/foo/bar.git + abc1234..def5678 master -> master +"; + let result = run_push_filter(input, 0); + for prefix in GIT_PUSH_NOISE_PREFIXES { + assert!( + !result.contains(prefix), + "noise prefix '{}' leaked through, got: {}", + prefix, + result + ); + } + assert!(result.contains("To https://github.com/foo/bar.git")); + assert!(result.contains("master -> master")); + assert!(result.ends_with("ok master\n"), "got: {}", result); + } + + #[test] + fn test_push_filter_up_to_date_summary() { + let input = "Everything up-to-date\n"; + let result = run_push_filter(input, 0); + assert!(result.contains("Everything up-to-date")); + assert!(result.ends_with("ok (up-to-date)\n"), "got: {}", result); + } + + #[test] + fn test_push_filter_passes_remote_messages_through() { + let input = "\ +remote: Resolving deltas: 100% (2/2), completed with 2 local objects. +remote: GitHub found 1 vulnerability on foo/bar's default branch (1 moderate). +To https://github.com/foo/bar.git + abc1234..def5678 feature -> feature +"; + let result = run_push_filter(input, 0); + assert!(result.contains("remote: Resolving deltas")); + assert!(result.contains("remote: GitHub found 1 vulnerability")); + assert!(result.ends_with("ok feature\n"), "got: {}", result); + } + + #[test] + fn test_push_filter_no_summary_on_failure() { + let input = "\ +To https://github.com/foo/bar.git + ! [rejected] master -> master (non-fast-forward) +error: failed to push some refs to 'https://github.com/foo/bar.git' +"; + let result = run_push_filter(input, 1); + assert!(result.contains("[rejected]")); + assert!(result.contains("error: failed to push")); + assert!( + !result.contains("ok "), + "summary leaked on failure, got: {}", + result + ); + } + + #[test] + fn test_push_filter_first_ref_wins_for_summary() { + let input = "\ +To https://github.com/foo/bar.git + abc1234..def5678 feat/a -> feat/a + 1111111..2222222 feat/b -> feat/b +"; + let result = run_push_filter(input, 0); + assert!(result.ends_with("ok feat/a\n"), "got: {}", result); + } + + #[test] + fn test_push_filter_token_savings_on_verbose_output() { + let input = "\ +Enumerating objects: 142, done. +Counting objects: 100% (142/142), done. +Delta compression using up to 8 threads +Compressing objects: 100% (88/88), done. +Writing objects: 100% (104/104), 28.50 KiB | 14.25 MiB/s, done. +Total 104 (delta 64), reused 0 (delta 0), pack-reused 0 +remote: Resolving deltas: 100% (64/64), completed with 24 local objects. +To https://github.com/foo/bar.git + abc1234..def5678 master -> master +"; + let result = run_push_filter(input, 0); + let count_tokens = |s: &str| s.split_whitespace().count(); + let input_tokens = count_tokens(input); + let output_tokens = count_tokens(&result); + let savings = 100.0 - (output_tokens as f64 / input_tokens as f64 * 100.0); + assert!( + savings >= 60.0, + "expected >=60% savings, got {:.1}% (in={}, out={})", + savings, + input_tokens, + output_tokens + ); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/cmds/python/pytest_cmd.rs b/crates/sparkplane-rtk/upstream/src/cmds/python/pytest_cmd.rs new file mode 100644 index 0000000..f32ac7d --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/cmds/python/pytest_cmd.rs @@ -0,0 +1,604 @@ +//! Filters pytest output to show only failures and the summary line. + +use crate::core::config; +use crate::core::runner; +use crate::core::truncate::CAP_WARNINGS; +use crate::core::utils::{resolved_command, strip_ansi, tool_exists, truncate}; +use anyhow::Result; + +const MAX_XFAIL: usize = CAP_WARNINGS; +const MAX_PYTEST_FAILURES: usize = CAP_WARNINGS; + +#[derive(Debug, PartialEq)] +enum ParseState { + Header, + TestProgress, + Failures, + Summary, +} + +pub fn run(args: &[String], verbose: u8) -> Result { + let mut cmd = if tool_exists("pytest") { + resolved_command("pytest") + } else { + let mut c = resolved_command("python"); + c.arg("-m").arg("pytest"); + c + }; + + let has_tb_flag = args.iter().any(|a| a.starts_with("--tb")); + let has_quiet_flag = args.iter().any(|a| a == "-q" || a == "--quiet"); + // Only treat a short `-r…` as pytest's report flag (not `--randomly-seed` etc.) + let has_report_flag = args + .iter() + .any(|a| a.starts_with("-r") && !a.starts_with("--")); + + if !has_tb_flag { + cmd.arg("--tb=short"); + } + if !has_quiet_flag { + cmd.arg("-q"); + } + // Surface xfailed/xpassed (and their reasons) in the short summary section + // so the compact output can report expected failures and — crucially — + // unexpected passes (XPASS), which signal a behavior change. + if !has_report_flag { + cmd.arg("-rxX"); + } + + for arg in args { + cmd.arg(arg); + } + + if verbose > 0 { + eprintln!("Running: pytest --tb=short -q {}", args.join(" ")); + } + + runner::run_filtered_with_exit( + cmd, + "pytest", + &args.join(" "), + |raw, exit_code| { + let clean = strip_ansi(raw); + let filtered = filter_pytest_output(&clean); + // Any other failure parsed as empty means the run broke before reporting. + if exit_code != 0 && exit_code != PYTEST_EXIT_NO_TESTS && filtered == PYTEST_NO_TESTS { + return truncate(clean.trim(), config::limits().passthrough_max_chars); + } + filtered + }, + runner::RunOptions::stdout_only().tee("pytest"), + ) +} + +const PYTEST_NO_TESTS: &str = "Pytest: No tests collected"; +const PYTEST_EXIT_NO_TESTS: i32 = 5; + +pub(crate) fn filter_pytest_output(output: &str) -> String { + let mut state = ParseState::Header; + let mut test_files: Vec = Vec::new(); + let mut failures: Vec = Vec::new(); + let mut current_failure: Vec = Vec::new(); + let mut xfail_lines: Vec = Vec::new(); + let mut summary_line = String::new(); + + for line in output.lines() { + let trimmed = line.trim(); + + // State transitions + if trimmed.starts_with("===") && trimmed.contains("test session starts") { + state = ParseState::Header; + continue; + } else if trimmed.starts_with("===") && trimmed.contains("FAILURES") { + state = ParseState::Failures; + continue; + } else if trimmed.starts_with("===") && trimmed.contains("short test summary") { + state = ParseState::Summary; + // Save current failure if any + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } + continue; + } else if trimmed.starts_with("===") + && (trimmed.contains("passed") + || trimmed.contains("failed") + || trimmed.contains("skipped")) + { + summary_line = trimmed.to_string(); + continue; + // quiet mode (-q): bare summary without === wrapper, e.g. "5 failed, 1698 passed, 2 skipped in 108.89s" + } else if summary_line.is_empty() + && !trimmed.starts_with("===") + && !trimmed.starts_with("FAILED") + && !trimmed.starts_with("ERROR") + && (trimmed.contains(" passed") + || trimmed.contains(" failed") + || trimmed.contains(" skipped")) + && trimmed.contains(" in ") + { + summary_line = trimmed.to_string(); + continue; + } + + // Process based on state + match state { + ParseState::Header => { + if trimmed.starts_with("collected") { + state = ParseState::TestProgress; + } + } + ParseState::TestProgress => { + // Lines like "tests/test_foo.py .... [ 40%]" + if !trimmed.is_empty() + && !trimmed.starts_with("===") + && (trimmed.contains(".py") || trimmed.contains("%]")) + { + test_files.push(trimmed.to_string()); + } + } + ParseState::Failures => { + // Collect failure details + if trimmed.starts_with("___") { + // New failure section + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } + current_failure.push(trimmed.to_string()); + } else if !trimmed.is_empty() && !trimmed.starts_with("===") { + current_failure.push(trimmed.to_string()); + } + } + ParseState::Summary => { + // FAILED test lines + if trimmed.starts_with("FAILED") || trimmed.starts_with("ERROR") { + failures.push(trimmed.to_string()); + } else if trimmed.starts_with("XFAIL") || trimmed.starts_with("XPASS") { + xfail_lines.push(trimmed.to_string()); + } + } + } + } + + // Save last failure if any + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + } + + // Build compact output + build_pytest_summary(&summary_line, &test_files, &failures, &xfail_lines) +} + +#[derive(Default)] +struct PytestCounts { + passed: usize, + failed: usize, + skipped: usize, + xfailed: usize, + xpassed: usize, + duration: Option, +} + +fn build_pytest_summary( + summary: &str, + _test_files: &[String], + failures: &[String], + xfail_lines: &[String], +) -> String { + let counts = parse_summary_line(summary); + let PytestCounts { + passed, + failed, + skipped, + xfailed, + xpassed, + duration, + } = counts; + + if passed == 0 && failed == 0 && skipped == 0 && xfailed == 0 && xpassed == 0 { + return PYTEST_NO_TESTS.to_string(); + } + + let extras_present = skipped > 0 || xfailed > 0 || xpassed > 0 || !xfail_lines.is_empty(); + let duration_suffix = duration + .as_deref() + .map(|value| format!(" in {value}")) + .unwrap_or_default(); + + if failed == 0 && passed > 0 && !extras_present { + return format!("Pytest: {passed} passed{duration_suffix}"); + } + + let mut result = String::new(); + result.push_str(&format!("Pytest: {} passed, {} failed", passed, failed)); + if skipped > 0 { + result.push_str(&format!(", {} skipped", skipped)); + } + if xfailed > 0 { + result.push_str(&format!(", {} xfailed", xfailed)); + } + if xpassed > 0 { + result.push_str(&format!(", {} xpassed", xpassed)); + } + result.push_str(&duration_suffix); + result.push('\n'); + + // Surface xfail/xpass entries (with their reasons) — XPASS in particular + // signals that something expected-to-fail now passes. + if !xfail_lines.is_empty() { + result.push_str("\nExpected-failure outcomes:\n"); + for line in xfail_lines.iter().take(MAX_XFAIL) { + result.push_str(&format!(" {}\n", truncate(line, 120))); + } + if xfail_lines.len() > MAX_XFAIL { + result.push_str(&format!(" … +{} more\n", xfail_lines.len() - MAX_XFAIL)); + let all_xfail = xfail_lines.join("\n"); + if let Some(hint) = + crate::core::tee::force_tee_tail_hint(&all_xfail, "pytest-xfail", MAX_XFAIL + 1) + { + result.push_str(&format!(" {}\n", hint)); + } + } + } + + if failures.is_empty() { + return result.trim().to_string(); + } + + // Show failures (limit to key information) + result.push_str("\nFailures:\n"); + + for (i, failure) in failures.iter().take(MAX_PYTEST_FAILURES).enumerate() { + // Extract test name and key error info + let lines: Vec<&str> = failure.lines().collect(); + + // First line is usually test name (after ___) + if let Some(first_line) = lines.first() { + if first_line.starts_with("___") { + // Extract test name between ___ + let test_name = first_line.trim_matches('_').trim(); + result.push_str(&format!("{}. [FAIL] {}\n", i + 1, test_name)); + } else if first_line.starts_with("FAILED") { + // Summary format: "FAILED tests/test_foo.py::test_bar - AssertionError" + let parts: Vec<&str> = first_line.split(" - ").collect(); + if let Some(test_path) = parts.first() { + let test_name = test_path.trim_start_matches("FAILED "); + result.push_str(&format!("{}. [FAIL] {}\n", i + 1, test_name)); + } + if parts.len() > 1 { + result.push_str(&format!(" {}\n", truncate(parts[1], 100))); + } + continue; + } + } + + // Show relevant error lines (assertions, errors, file locations) + let mut relevant_lines = 0; + for line in &lines[1..] { + let line_lower = line.to_lowercase(); + let is_relevant = line.trim().starts_with('>') + || line.trim().starts_with('E') + || line_lower.contains("assert") + || line_lower.contains("error") + || line.contains(".py:"); + + if is_relevant && relevant_lines < 3 { + result.push_str(&format!(" {}\n", truncate(line, 100))); + relevant_lines += 1; + } + } + + if i < failures.len() - 1 { + result.push('\n'); + } + } + + if failures.len() > MAX_PYTEST_FAILURES { + result.push_str(&format!( + "\n… +{} more failures\n", + failures.len() - MAX_PYTEST_FAILURES + )); + let all_failures = failures.join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_failures, "pytest-failures") { + result.push_str(&format!(" {}\n", hint)); + } + } + + result.trim().to_string() +} + +fn parse_summary_line(summary: &str) -> PytestCounts { + let mut counts = PytestCounts { + duration: parse_duration(summary), + ..PytestCounts::default() + }; + + // Parse lines like "=== 4 passed, 1 failed, 2 xfailed, 1 xpassed in 0.50s ===" + for part in summary.split(',') { + let words: Vec<&str> = part.split_whitespace().collect(); + for (i, word) in words.iter().enumerate() { + if i == 0 { + continue; + } + let Ok(n) = words[i - 1].parse::() else { + continue; + }; + // Order matters: "xpassed"/"xfailed" contain "passed"/"failed". + if word.contains("xpassed") { + counts.xpassed = n; + } else if word.contains("xfailed") { + counts.xfailed = n; + } else if word.contains("passed") { + counts.passed = n; + } else if word.contains("failed") { + counts.failed = n; + } else if word.contains("skipped") { + counts.skipped = n; + } + } + } + + counts +} + +fn parse_duration(summary: &str) -> Option { + let unwrapped = summary.trim().trim_matches('=').trim(); + let (_, duration_tail) = unwrapped.rsplit_once(" in ")?; + let duration = duration_tail.split_whitespace().next()?; + let seconds = duration.strip_suffix('s')?; + seconds.parse::().ok()?; + Some(duration.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_filter_pytest_all_pass() { + let output = r#"=== test session starts === +platform darwin -- Python 3.11.0 +collected 5 items + +tests/test_foo.py ..... [100%] + +=== 5 passed in 0.50s ==="#; + + let result = filter_pytest_output(output); + assert!(result.contains("Pytest")); + assert!(result.contains("5 passed")); + assert_eq!(result, "Pytest: 5 passed in 0.50s"); + } + + #[test] + fn test_filter_pytest_with_failures() { + let output = r#"=== test session starts === +collected 5 items + +tests/test_foo.py ..F.. [100%] + +=== FAILURES === +___ test_something ___ + + def test_something(): +> assert False +E assert False + +tests/test_foo.py:10: AssertionError + +=== short test summary info === +FAILED tests/test_foo.py::test_something - assert False +=== 4 passed, 1 failed in 0.50s ==="#; + + let result = filter_pytest_output(output); + assert!(result.contains("4 passed, 1 failed")); + assert!(result.starts_with("Pytest: 4 passed, 1 failed in 0.50s")); + assert!(result.contains("test_something")); + assert!(result.contains("assert False")); + } + + #[test] + fn test_filter_pytest_multiple_failures() { + let output = r#"=== test session starts === +collected 3 items + +tests/test_foo.py FFF [100%] + +=== FAILURES === +___ test_one ___ +E AssertionError: expected 5 + +___ test_two ___ +E ValueError: invalid value + +=== short test summary info === +FAILED tests/test_foo.py::test_one - AssertionError: expected 5 +FAILED tests/test_foo.py::test_two - ValueError: invalid value +FAILED tests/test_foo.py::test_three - KeyError +=== 3 failed in 0.20s ==="#; + + let result = filter_pytest_output(output); + assert!(result.contains("3 failed")); + assert!(result.contains("test_one")); + assert!(result.contains("test_two")); + assert!(result.contains("expected 5")); + } + + #[test] + fn test_filter_pytest_no_tests() { + let output = r#"=== test session starts === +collected 0 items + +=== no tests ran in 0.00s ==="#; + + let result = filter_pytest_output(output); + assert!(result.contains("No tests collected")); + } + + #[test] + fn test_parse_summary_line() { + let c = parse_summary_line("=== 5 passed in 0.50s ==="); + assert_eq!((c.passed, c.failed, c.skipped), (5, 0, 0)); + assert_eq!(c.duration.as_deref(), Some("0.50s")); + + let c = parse_summary_line("=== 4 passed, 1 failed in 0.50s ==="); + assert_eq!((c.passed, c.failed, c.skipped), (4, 1, 0)); + assert_eq!(c.duration.as_deref(), Some("0.50s")); + + let c = parse_summary_line("=== 3 passed, 1 failed, 2 skipped in 1.0s ==="); + assert_eq!((c.passed, c.failed, c.skipped), (3, 1, 2)); + assert_eq!(c.duration.as_deref(), Some("1.0s")); + + let c = parse_summary_line("=== 2 passed, 1 failed, 2 xfailed, 1 xpassed in 1.0s ==="); + assert_eq!((c.passed, c.failed, c.xfailed, c.xpassed), (2, 1, 2, 1)); + assert_eq!(c.duration.as_deref(), Some("1.0s")); + } + + #[test] + fn test_filter_pytest_quiet_mode_preserves_duration() { + let output = "2 passed in 3.04s"; + + assert_eq!(filter_pytest_output(output), "Pytest: 2 passed in 3.04s"); + } + + #[test] + fn test_filter_pytest_long_run_preserves_seconds_duration() { + let output = "2 passed in 65.12s (0:01:05)"; + + assert_eq!(filter_pytest_output(output), "Pytest: 2 passed in 65.12s"); + } + + #[test] + fn test_parse_duration_rejects_tails_that_are_not_seconds() { + assert_eq!( + parse_duration("=== 2 passed in 0.50s ===").as_deref(), + Some("0.50s") + ); + assert_eq!( + parse_duration("2 passed in 65.12s (0:01:05)").as_deref(), + Some("65.12s") + ); + assert_eq!(parse_duration("=== 2 passed ==="), None); + assert_eq!(parse_duration("2 passed in seconds"), None); + assert_eq!(parse_duration("2 passed in 0.50"), None); + } + + #[test] + fn test_filter_pytest_duration_on_every_summary_shape() { + // Wrapped long-run form with a warnings part before the duration. + let output = "=== 5 passed, 2 warnings in 65.12s (0:01:05) ==="; + assert_eq!(filter_pytest_output(output), "Pytest: 5 passed in 65.12s"); + + // Only-skipped run takes the multi-line emit path. + let output = "=== 3 skipped in 0.10s ==="; + assert_eq!( + filter_pytest_output(output), + "Pytest: 0 passed, 0 failed, 3 skipped in 0.10s" + ); + + // Quiet-mode failures line keeps its duration on the multi-line path too. + let output = "5 failed, 1698 passed, 2 skipped in 108.89s"; + assert_eq!( + filter_pytest_output(output), + "Pytest: 1698 passed, 5 failed, 2 skipped in 108.89s" + ); + } + + #[test] + fn test_filter_pytest_xfail_caps_and_tee_hint() { + let mut lines = String::from("=== test session starts ===\ncollected 30 items\n\n"); + lines.push_str("test_x.py "); + for _ in 0..15 { + lines.push('x'); + } + lines.push_str("\n\n=== short test summary info ===\n"); + for i in 0..15 { + lines.push_str(&format!( + "XFAIL test_x.py::test_case_{i} - known issue #{i}\n" + )); + } + lines.push_str("=== 0 passed, 15 xfailed in 0.05s ===\n"); + + let result = filter_pytest_output(&lines); + let xfail_in_section = result + .split("Expected-failure outcomes:") + .nth(1) + .unwrap_or(""); + let listed = xfail_in_section + .lines() + .filter(|l| l.trim().starts_with("XFAIL")) + .count(); + assert!(listed <= 10, "MAX_XFAIL cap not enforced: listed {listed}"); + assert!(result.contains("… +5 more"), "missing '+N more': {result}"); + } + + #[test] + fn test_filter_pytest_xfail_xpass() { + let output = r#"=== test session starts === +collected 5 items + +test_math.py ..xxX [100%] + +=== short test summary info === +XFAIL test_math.py::test_division_by_zero - known bug in division +XFAIL test_math.py::test_float_precision - float precision issue — bug #42 +XPASS test_math.py::test_unexpected_pass - this should fail but currently passes +=== 2 passed, 2 xfailed, 1 xpassed in 0.05s ==="#; + + let result = filter_pytest_output(output); + assert!(result.contains("xfailed"), "got: {result}"); + assert!(result.contains("xpassed"), "got: {result}"); + assert!(result.contains("XPASS"), "got: {result}"); + assert!(result.contains("float precision"), "got: {result}"); + assert!(result.contains("test_division_by_zero"), "got: {result}"); + } + + #[test] + fn test_filter_pytest_quiet_mode_failures() { + // In -q mode, the final summary line has NO === wrapper + // This was causing "No tests collected" to be reported incorrectly + let output = r#"=== test session starts === +platform linux -- Python 3.12.11, pytest-8.1.0 +collected 1705 items + +.......F....... + +=== FAILURES === +___ test_something ___ + +E AssertionError: expected True + +=== short test summary info === +FAILED tests/test_foo.py::test_something - AssertionError +5 failed, 1698 passed, 2 skipped in 108.89s"#; + + let result = filter_pytest_output(output); + assert!( + !result.contains("No tests collected"), + "Should not report 'No tests collected' when tests ran. Got: {}", + result + ); + assert!( + result.contains("1698") || result.contains("5 failed"), + "Should show actual test counts. Got: {}", + result + ); + } + + #[test] + fn test_filter_pytest_only_skipped() { + // If only skipped tests, should NOT say "No tests collected" + let output = r#"=== test session starts === +collected 3 items + +=== 3 skipped in 0.10s ==="#; + + let result = filter_pytest_output(output); + assert!( + !result.contains("No tests collected"), + "Should not say 'No tests collected' when tests were skipped. Got: {}", + result + ); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/cmds/rust/cargo_cmd.rs b/crates/sparkplane-rtk/upstream/src/cmds/rust/cargo_cmd.rs new file mode 100644 index 0000000..3960418 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/cmds/rust/cargo_cmd.rs @@ -0,0 +1,2899 @@ +//! Filters cargo output — build errors, test results, clippy warnings. + +use crate::core::args_utils; +use crate::core::runner; +use crate::core::stream::{BlockHandler, BlockStreamFilter, StreamFilter}; +use crate::core::truncate::{CAP_ERRORS, CAP_LIST, CAP_WARNINGS}; +use crate::core::utils::{join_with_overflow, resolved_command, truncate}; +use anyhow::Result; +use serde::Deserialize; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::ffi::OsString; +use std::sync::LazyLock; + +#[derive(Debug, Clone)] +pub enum CargoCommand { + Build, + Test, + Clippy, + Check, + Install, + Nextest, +} + +pub fn run(cmd: CargoCommand, args: &[String], verbose: u8) -> Result { + match cmd { + CargoCommand::Build => run_build(args, verbose), + CargoCommand::Test => run_test(args, verbose), + CargoCommand::Clippy => run_clippy(args, verbose), + CargoCommand::Check => run_check(args, verbose), + CargoCommand::Install => run_install(args, verbose), + CargoCommand::Nextest => run_nextest(args, verbose), + } +} + +// --- Stream handlers --- + +struct CargoBuildHandler { + compiled: usize, + warnings: usize, + error_count: usize, + finished_line: Option, + label: &'static str, +} + +impl CargoBuildHandler { + fn with_label(label: &'static str) -> Self { + Self { + compiled: 0, + warnings: 0, + error_count: 0, + finished_line: None, + label, + } + } +} + +impl BlockHandler for CargoBuildHandler { + fn should_skip(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if trimmed.starts_with("Compiling") || trimmed.starts_with("Checking") { + self.compiled += 1; + return true; + } + if trimmed.starts_with("Downloading") || trimmed.starts_with("Downloaded") { + return true; + } + if trimmed.starts_with("Finished") { + self.finished_line = Some(trimmed.to_string()); + return true; + } + if line.starts_with("warning:") && line.contains("generated") && line.contains("warning") { + return true; + } + if (line.starts_with("error:") || line.starts_with("error[")) + && (line.contains("aborting due to") || line.contains("could not compile")) + { + return true; + } + false + } + + fn is_block_start(&mut self, line: &str) -> bool { + if line.starts_with("error[") || line.starts_with("error:") { + self.error_count += 1; + return true; + } + if line.starts_with("warning:") || line.starts_with("warning[") { + self.warnings += 1; + return true; + } + false + } + + fn is_block_continuation(&mut self, line: &str, block: &[String]) -> bool { + !(line.trim().is_empty() && block.len() > 3) + } + + fn format_summary(&self, exit_code: i32, raw: &str) -> Option { + if self.error_count == 0 && self.warnings == 0 && exit_code == 0 { + let summary = + cargo_build_success_line(self.compiled, self.finished_line.as_deref(), self.label); + return Some(crate::core::guard::never_worse(raw, &summary).to_string()); + } + // The streamed path only runs for non-json build/check; error blocks are + // emitted live, so the summary carries no rendered diagnostics. + let empty = JsonDiagnostics { + errors: Vec::new(), + warnings: Vec::new(), + }; + Some(cargo_build_failure_summary( + self.compiled, + self.error_count, + self.warnings, + &empty, + self.label, + exit_code, + )) + } +} + +struct CargoTestHandler { + in_failure_section: bool, + in_failure_names: bool, + summary_lines: Vec, + has_compile_errors: bool, +} + +impl CargoTestHandler { + fn new() -> Self { + Self { + in_failure_section: false, + in_failure_names: false, + summary_lines: Vec::new(), + has_compile_errors: false, + } + } + + /// Compacted `cargo test` summary, before the never-worse guard. + fn compute_test_summary(&self, raw: &str) -> Option { + if self.summary_lines.is_empty() { + let json = extract_json_diagnostics(raw); + if self.has_compile_errors || !json.errors.is_empty() { + // Content-based (exit 0): a real compile error yields "cargo test: N + // errors"; a bare "could not compile" leaves the raw tail fallback. + let build_filtered = filter_cargo_build_labeled(raw, "test", 0); + if build_filtered.contains("cargo test:") { + return Some(format!("{}\n", build_filtered)); + } + // Fallback: last 5 meaningful lines + let meaningful: Vec<&str> = raw + .lines() + .filter(|l| !l.trim().is_empty() && !l.trim_start().starts_with("Compiling")) + .collect(); + let last5: Vec<&str> = meaningful.iter().rev().take(5).rev().copied().collect(); + return Some(format!("{}\n", last5.join("\n"))); + } + } + + // No failures emitted — aggregate pass results + let mut aggregated: Option = None; + let mut all_parsed = true; + + for line in &self.summary_lines { + if let Some(parsed) = AggregatedTestResult::parse_line(line) { + if let Some(ref mut agg) = aggregated { + agg.merge(&parsed); + } else { + aggregated = Some(parsed); + } + } else { + all_parsed = false; + break; + } + } + + if all_parsed + && let Some(agg) = aggregated + && agg.suites > 0 + { + return Some(format!("{}\n", agg.format_compact())); + } + + // Fallback: show raw summary lines + if !self.summary_lines.is_empty() { + let mut s = String::new(); + for line in &self.summary_lines { + s.push_str(line); + s.push('\n'); + } + return Some(s); + } + + None + } +} + +impl BlockHandler for CargoTestHandler { + fn should_skip(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if trimmed.starts_with("Compiling") + || trimmed.starts_with("Downloading") + || trimmed.starts_with("Downloaded") + || trimmed.starts_with("Finished") + { + return true; + } + if line.starts_with("running ") { + return true; + } + if line.starts_with("test ") && line.ends_with("... ok") { + return true; + } + // Track compile errors for fallback + if trimmed.starts_with("error[") || trimmed.starts_with("error:") { + self.has_compile_errors = true; + } + // "failures:" toggles section state + if line == "failures:" { + if self.in_failure_section { + // Second "failures:" = list of failure names — skip them + self.in_failure_names = true; + } + self.in_failure_section = true; + return true; + } + // Skip the failure name listing section + if self.in_failure_names { + if line.starts_with("test result:") { + self.in_failure_names = false; + self.in_failure_section = false; + self.summary_lines.push(line.to_string()); + return true; + } + return true; + } + if line.starts_with("test result:") { + self.summary_lines.push(line.to_string()); + self.in_failure_section = false; + return true; + } + false + } + + fn is_block_start(&mut self, line: &str) -> bool { + self.in_failure_section && line.starts_with("---- ") + } + + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + self.in_failure_section && !line.starts_with("---- ") + } + + fn format_summary(&self, _exit_code: i32, raw: &str) -> Option { + // Same never-worse guard as CargoBuildHandler (#3430 review): if the + // compacted summary ends up larger than the raw output (e.g. a tiny + // `cargo test` run), keep the raw output instead of "compacting" it + // into something bigger. + let summary = self.compute_test_summary(raw)?; + Some(crate::core::guard::never_worse(raw, &summary).to_string()) + } +} + +/// Generic cargo command runner with filtering. +/// Builds the Command with restored `--` separator, then delegates to shared runner. +fn run_cargo_filtered( + subcommand: &str, + args: &[String], + verbose: u8, + filter_fn: F, +) -> Result +where + F: Fn(&str) -> String, +{ + let mut cmd = resolved_command("cargo"); + cmd.arg(subcommand); + + let restored_args = args_utils::restore_double_dash(args); + for arg in &restored_args { + cmd.arg(arg); + } + + if verbose > 0 { + eprintln!("Running: cargo {} {}", subcommand, restored_args.join(" ")); + } + + runner::run_filtered( + cmd, + &format!("cargo {}", subcommand), + &restored_args.join(" "), + filter_fn, + runner::RunOptions::with_tee(&format!("cargo_{}", subcommand)), + ) +} + +/// Same as `run_cargo_filtered` but the filter also receives the child exit code, +/// so it can tell a genuine failure from a clean run when no diagnostics parse. +fn run_cargo_filtered_with_exit( + subcommand: &str, + args: &[String], + verbose: u8, + filter_fn: F, +) -> Result +where + F: Fn(&str, i32) -> String, +{ + let mut cmd = resolved_command("cargo"); + cmd.arg(subcommand); + + let restored_args = args_utils::restore_double_dash(args); + for arg in &restored_args { + cmd.arg(arg); + } + + if verbose > 0 { + eprintln!("Running: cargo {} {}", subcommand, restored_args.join(" ")); + } + + runner::run_filtered_with_exit( + cmd, + &format!("cargo {}", subcommand), + &restored_args.join(" "), + filter_fn, + runner::RunOptions::with_tee(&format!("cargo_{}", subcommand)), + ) +} + +fn run_cargo_streamed( + subcommand: &str, + args: &[String], + verbose: u8, + filter: Box, +) -> Result { + let mut cmd = resolved_command("cargo"); + cmd.arg(subcommand); + + let restored_args = args_utils::restore_double_dash(args); + for arg in &restored_args { + cmd.arg(arg); + } + + if verbose > 0 { + eprintln!("Running: cargo {} {}", subcommand, restored_args.join(" ")); + } + + runner::run_streamed( + cmd, + &format!("cargo {}", subcommand), + &restored_args.join(" "), + filter, + runner::RunOptions::with_tee(&format!("cargo_{}", subcommand)), + ) +} + +fn has_json_message_format(args: &[String]) -> bool { + let mut json = false; + let mut iter = args.iter(); + while let Some(arg) = iter.next() { + if let Some(val) = arg.strip_prefix("--message-format=") { + json = val.contains("json"); + } else if arg == "--message-format" + && let Some(val) = iter.next() + { + json = val.contains("json"); + } + } + json +} + +fn run_build(args: &[String], verbose: u8) -> Result { + if has_json_message_format(args) { + return run_cargo_filtered_with_exit("build", args, verbose, |o, exit| { + filter_cargo_build_labeled(o, "build", exit) + }); + } + run_cargo_streamed( + "build", + args, + verbose, + Box::new(BlockStreamFilter::new(CargoBuildHandler::with_label( + "build", + ))), + ) +} + +fn run_test(args: &[String], verbose: u8) -> Result { + // No json branch here on purpose: --message-format=json only reformats the + // build phase, the test harness output stays human-readable. CargoTestHandler + // reads both — it aggregates the `test result:` lines and, on a compile error, + // parses the json diagnostics in format_summary. + run_cargo_streamed( + "test", + args, + verbose, + Box::new(BlockStreamFilter::new(CargoTestHandler::new())), + ) +} + +fn run_clippy(args: &[String], verbose: u8) -> Result { + if has_json_message_format(args) { + return run_cargo_filtered_with_exit("clippy", args, verbose, filter_cargo_clippy_json); + } + run_cargo_filtered("clippy", args, verbose, filter_cargo_clippy) +} + +fn run_check(args: &[String], verbose: u8) -> Result { + if has_json_message_format(args) { + return run_cargo_filtered_with_exit("check", args, verbose, |o, exit| { + filter_cargo_build_labeled(o, "check", exit) + }); + } + run_cargo_streamed( + "check", + args, + verbose, + Box::new(BlockStreamFilter::new(CargoBuildHandler::with_label( + "check", + ))), + ) +} + +fn run_install(args: &[String], verbose: u8) -> Result { + run_cargo_filtered("install", args, verbose, filter_cargo_install) +} + +fn run_nextest(args: &[String], verbose: u8) -> Result { + run_cargo_filtered("nextest", args, verbose, filter_cargo_nextest) +} + +/// Format crate name + version into a display string +fn format_crate_info(name: &str, version: &str, fallback: &str) -> String { + if name.is_empty() { + fallback.to_string() + } else if version.is_empty() { + name.to_string() + } else { + format!("{} {}", name, version) + } +} + +/// Filter cargo install output - strip dep compilation, keep installed/replaced/errors +fn filter_cargo_install(output: &str) -> String { + let mut errors: Vec = Vec::new(); + let mut error_count = 0; + let mut compiled = 0; + let mut in_error = false; + let mut current_error = Vec::new(); + let mut installed_crate = String::new(); + let mut installed_version = String::new(); + let mut replaced_lines: Vec = Vec::new(); + let mut already_installed = false; + let mut ignored_line = String::new(); + + for line in output.lines() { + let trimmed = line.trim_start(); + + // Strip noise: dep compilation, downloading, locking, etc. + if trimmed.starts_with("Compiling") { + compiled += 1; + continue; + } + if trimmed.starts_with("Downloading") + || trimmed.starts_with("Downloaded") + || trimmed.starts_with("Locking") + || trimmed.starts_with("Updating") + || trimmed.starts_with("Adding") + || trimmed.starts_with("Finished") + || trimmed.starts_with("Blocking waiting for file lock") + { + continue; + } + + // Keep: Installing line (extract crate name + version) + if trimmed.starts_with("Installing") { + let rest = trimmed.strip_prefix("Installing").unwrap_or("").trim(); + if !rest.is_empty() && !rest.starts_with('/') { + if let Some((name, version)) = rest.split_once(' ') { + installed_crate = name.to_string(); + installed_version = version.to_string(); + } else { + installed_crate = rest.to_string(); + } + } + continue; + } + + // Keep: Installed line (extract crate + version if not already set) + if trimmed.starts_with("Installed") { + let rest = trimmed.strip_prefix("Installed").unwrap_or("").trim(); + if !rest.is_empty() && installed_crate.is_empty() { + let mut parts = rest.split_whitespace(); + if let (Some(name), Some(version)) = (parts.next(), parts.next()) { + installed_crate = name.to_string(); + installed_version = version.to_string(); + } + } + continue; + } + + // Keep: Replacing/Replaced lines + if trimmed.starts_with("Replacing") || trimmed.starts_with("Replaced") { + replaced_lines.push(trimmed.to_string()); + continue; + } + + // Keep: "Ignored package" (already up to date) + if trimmed.starts_with("Ignored package") { + already_installed = true; + ignored_line = trimmed.to_string(); + continue; + } + + // Keep: actionable warnings (e.g., "be sure to add `/path` to your PATH") + // Skip summary lines like "warning: `crate` generated N warnings" + if line.starts_with("warning:") { + if !(line.contains("generated") && line.contains("warning")) { + replaced_lines.push(line.to_string()); + } + continue; + } + + // Detect error blocks + if line.starts_with("error[") || line.starts_with("error:") { + if line.contains("aborting due to") || line.contains("could not compile") { + continue; + } + if in_error && !current_error.is_empty() { + errors.push(current_error.join("\n")); + current_error.clear(); + } + error_count += 1; + in_error = true; + current_error.push(line.to_string()); + } else if in_error { + if line.trim().is_empty() && current_error.len() > 3 { + errors.push(current_error.join("\n")); + current_error.clear(); + in_error = false; + } else { + current_error.push(line.to_string()); + } + } + } + + if !current_error.is_empty() { + errors.push(current_error.join("\n")); + } + + // Already installed / up to date + if already_installed { + let info = ignored_line.split('`').nth(1).unwrap_or(&ignored_line); + return format!("cargo install: {} already installed", info); + } + + // Errors + if error_count > 0 { + let crate_info = format_crate_info(&installed_crate, &installed_version, ""); + let deps_info = if compiled > 0 { + format!(", {} deps compiled", compiled) + } else { + String::new() + }; + + let mut result = String::new(); + if crate_info.is_empty() { + result.push_str(&format!( + "cargo install: {} error{}{}\n", + error_count, + if error_count > 1 { "s" } else { "" }, + deps_info + )); + } else { + result.push_str(&format!( + "cargo install: {} error{} ({}{})\n", + error_count, + if error_count > 1 { "s" } else { "" }, + crate_info, + deps_info + )); + } + + const MAX_INSTALL_ERRORS: usize = CAP_ERRORS; + for (i, err) in errors.iter().enumerate().take(MAX_INSTALL_ERRORS) { + result.push_str(err); + result.push('\n'); + if i < errors.len() - 1 { + result.push('\n'); + } + } + + if errors.len() > MAX_INSTALL_ERRORS { + result.push_str(&format!( + "\n… +{} more issues\n", + errors.len() - MAX_INSTALL_ERRORS + )); + let all_errors = errors.join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_errors, "cargo-build-errors") + { + result.push_str(&format!(" {}\n", hint)); + } + } + + return result.trim().to_string(); + } + + // Success + let crate_info = format_crate_info(&installed_crate, &installed_version, "package"); + + let mut result = format!("cargo install ({}, {} deps compiled)", crate_info, compiled); + + for line in &replaced_lines { + result.push_str(&format!("\n {}", line)); + } + + result +} + +/// Push a completed failure block (header + body) into the failures list, then clear the buffers. +fn flush_failure_block(header: &mut String, body: &mut Vec, failures: &mut Vec) { + if header.is_empty() { + return; + } + let mut block = header.clone(); + if !body.is_empty() { + block.push('\n'); + block.push_str(&body.join("\n")); + } + failures.push(block); + header.clear(); + body.clear(); +} + +/// Filter cargo nextest output - show failures + compact summary +fn filter_cargo_nextest(output: &str) -> String { + let summary_re = regex::Regex::new( + r"Summary \[\s*([\d.]+)s\]\s+(\d+) tests? run:\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) skipped)?" + ).expect("invalid nextest summary regex"); + + let starting_re = regex::Regex::new(r"Starting \d+ tests? across (\d+) binar(?:y|ies)") + .expect("invalid nextest starting regex"); + + let mut failures: Vec = Vec::new(); + let mut in_failure_block = false; + let mut past_summary = false; + let mut current_failure_header = String::new(); + let mut current_failure_body = Vec::new(); + let mut summary_line = String::new(); + let mut binaries: u32 = 0; + let mut has_cancel_line = false; + + for line in output.lines() { + let trimmed = line.trim(); + + // Strip compilation noise + if trimmed.starts_with("Compiling") + || trimmed.starts_with("Downloading") + || trimmed.starts_with("Downloaded") + || trimmed.starts_with("Finished") + || trimmed.starts_with("Locking") + || trimmed.starts_with("Updating") + { + continue; + } + + // Strip separator lines (────) + if trimmed.starts_with("────") { + continue; + } + + // Skip post-summary recap lines (FAIL duplicates + "error: test run failed") + if past_summary { + continue; + } + + // Parse binary count from Starting line + if trimmed.starts_with("Starting") { + if let Some(caps) = starting_re.captures(trimmed) + && let Some(m) = caps.get(1) + { + binaries = m.as_str().parse().unwrap_or(0); + } + continue; + } + + // Strip PASS lines + if trimmed.starts_with("PASS") { + if in_failure_block { + flush_failure_block( + &mut current_failure_header, + &mut current_failure_body, + &mut failures, + ); + in_failure_block = false; + } + continue; + } + + // Detect FAIL lines + if trimmed.starts_with("FAIL") { + // Close previous failure block if any + if in_failure_block { + flush_failure_block( + &mut current_failure_header, + &mut current_failure_body, + &mut failures, + ); + } + current_failure_header = trimmed.to_string(); + in_failure_block = true; + continue; + } + + // Cancellation notice + if trimmed.starts_with("Cancelling") || trimmed.starts_with("Canceling") { + has_cancel_line = true; + continue; + } + + // Nextest run ID line + if trimmed.starts_with("Nextest run ID") { + continue; + } + + // Parse summary + if trimmed.starts_with("Summary") { + summary_line = trimmed.to_string(); + if in_failure_block { + flush_failure_block( + &mut current_failure_header, + &mut current_failure_body, + &mut failures, + ); + in_failure_block = false; + } + past_summary = true; + continue; + } + + // Collect failure body lines (stdout/stderr sections) + if in_failure_block { + current_failure_body.push(line.to_string()); + } + } + + // Close last failure block + if in_failure_block { + flush_failure_block( + &mut current_failure_header, + &mut current_failure_body, + &mut failures, + ); + } + + // Parse summary with regex + if let Some(caps) = summary_re.captures(&summary_line) { + let duration = caps.get(1).map_or("?", |m| m.as_str()); + let passed: u32 = caps + .get(3) + .and_then(|m| m.as_str().parse().ok()) + .unwrap_or(0); + let failed: u32 = caps + .get(4) + .and_then(|m| m.as_str().parse().ok()) + .unwrap_or(0); + let skipped: u32 = caps + .get(5) + .and_then(|m| m.as_str().parse().ok()) + .unwrap_or(0); + + let binary_text = match binaries.cmp(&1) { + Ordering::Greater => format!("{} binaries", binaries), + Ordering::Equal => "1 binary".to_string(), + Ordering::Less => String::new(), + }; + + if failed == 0 { + // All pass - compact single line + let mut parts = vec![format!("{} passed", passed)]; + if skipped > 0 { + parts.push(format!("{} skipped", skipped)); + } + let meta = if binary_text.is_empty() { + format!("{}s", duration) + } else { + format!("{}, {}s", binary_text, duration) + }; + return format!("cargo nextest: {} ({})", parts.join(", "), meta); + } + + // With failures - show failure details then summary + let mut result = String::new(); + + for failure in &failures { + result.push_str(failure); + result.push('\n'); + } + + if has_cancel_line { + result.push_str("Cancelling due to test failure\n"); + } + + let mut summary_parts = vec![format!("{} passed", passed)]; + if failed > 0 { + summary_parts.push(format!("{} failed", failed)); + } + if skipped > 0 { + summary_parts.push(format!("{} skipped", skipped)); + } + let meta = if binary_text.is_empty() { + format!("{}s", duration) + } else { + format!("{}, {}s", binary_text, duration) + }; + result.push_str(&format!( + "cargo nextest: {} ({})", + summary_parts.join(", "), + meta + )); + + return result.trim().to_string(); + } + + // Fallback: if summary regex didn't match, show what we have + if !failures.is_empty() { + let mut result = String::new(); + for failure in &failures { + result.push_str(failure); + result.push('\n'); + } + if !summary_line.is_empty() { + result.push_str(&summary_line); + } + return result.trim().to_string(); + } + + if !summary_line.is_empty() { + return summary_line; + } + + // Empty or unrecognized + String::new() +} + +struct JsonDiagnostics { + errors: Vec, + warnings: Vec, +} + +#[derive(Deserialize)] +struct CargoJsonLine { + reason: String, + message: Option, +} + +#[derive(Deserialize)] +struct CargoDiagnostic { + level: String, + #[serde(default)] + message: String, + rendered: Option, +} + +fn extract_json_diagnostics(raw: &str) -> JsonDiagnostics { + let mut errors = Vec::new(); + let mut warnings = Vec::new(); + for line in raw.lines() { + let line = line.trim_start(); + if !line.starts_with('{') { + continue; + } + let Ok(entry) = serde_json::from_str::(line) else { + continue; + }; + if entry.reason != "compiler-message" { + continue; + } + let Some(msg) = entry.message else { + continue; + }; + let bucket = match msg.level.as_str() { + "error" | "error: internal compiler error" => &mut errors, + "warning" => &mut warnings, + _ => continue, + }; + if msg.message.starts_with("aborting due to") + || msg.message.starts_with("could not compile") + || (msg.message.contains("warning") && msg.message.contains("generated")) + { + continue; + } + if let Some(rendered) = msg.rendered { + bucket.push( + crate::core::utils::strip_ansi(&rendered) + .trim_end() + .to_string(), + ); + } else if !msg.message.is_empty() { + bucket.push(msg.message); + } + } + JsonDiagnostics { errors, warnings } +} + +fn merge_diag_counts( + error_count: usize, + warnings: usize, + json: &JsonDiagnostics, +) -> (usize, usize) { + ( + error_count.max(json.errors.len()), + warnings.max(json.warnings.len()), + ) +} + +fn cargo_build_success_line(compiled: usize, finished: Option<&str>, label: &str) -> String { + match finished { + Some(f) => format!("cargo {} ({} crates compiled)\n{}\n", label, compiled, f), + None => format!("cargo {} ({} crates compiled)\n", label, compiled), + } +} + +fn cargo_build_failure_summary( + compiled: usize, + errors: usize, + warnings: usize, + json: &JsonDiagnostics, + label: &str, + exit_code: i32, +) -> String { + let mut out = if errors == 0 && warnings == 0 { + format!("cargo {}: failed (exit {})\n", label, exit_code) + } else { + format!( + "cargo {}: {} errors, {} warnings ({} crates)\n", + label, errors, warnings, compiled + ) + }; + if !json.errors.is_empty() { + let shown: Vec = json.errors.iter().take(CAP_ERRORS).cloned().collect(); + out.push_str(&join_with_overflow( + &shown, + json.errors.len(), + CAP_ERRORS, + "errors", + )); + out.push('\n'); + } + if !json.warnings.is_empty() { + let shown: Vec = json.warnings.iter().take(CAP_WARNINGS).cloned().collect(); + out.push_str(&join_with_overflow( + &shown, + json.warnings.len(), + CAP_WARNINGS, + "warnings", + )); + out.push('\n'); + } + if json.errors.len() > CAP_ERRORS || json.warnings.len() > CAP_WARNINGS { + let full = json + .errors + .iter() + .chain(json.warnings.iter()) + .map(String::as_str) + .collect::>() + .join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&full, "cargo-json-issues") { + out.push_str(&format!(" {}\n", hint)); + } + } + out +} + +#[cfg(test)] +fn filter_cargo_build(output: &str) -> String { + filter_cargo_build_labeled(output, "build", 0) +} + +fn filter_cargo_build_labeled(output: &str, label: &'static str, exit_code: i32) -> String { + let mut handler = CargoBuildHandler::with_label(label); + let mut blocks: Vec> = Vec::new(); + let mut current_block: Vec = Vec::new(); + let mut in_block = false; + + for line in output.lines() { + if handler.should_skip(line) { + continue; + } + if handler.is_block_start(line) { + if in_block && !current_block.is_empty() { + blocks.push(std::mem::take(&mut current_block)); + } + in_block = true; + current_block.push(line.to_string()); + } else if in_block { + if handler.is_block_continuation(line, ¤t_block) { + current_block.push(line.to_string()); + } else { + blocks.push(std::mem::take(&mut current_block)); + in_block = false; + } + } + } + if !current_block.is_empty() { + blocks.push(current_block); + } + + let json = extract_json_diagnostics(output); + let (errors, warnings) = merge_diag_counts(handler.error_count, handler.warnings, &json); + + if errors == 0 && warnings == 0 && exit_code == 0 { + let summary = + cargo_build_success_line(handler.compiled, handler.finished_line.as_deref(), label); + return crate::core::guard::never_worse(output, &summary).to_string(); + } + + let mut result = + cargo_build_failure_summary(handler.compiled, errors, warnings, &json, label, exit_code); + const MAX_CHECK_BLOCKS: usize = CAP_ERRORS; + for (i, blk) in blocks.iter().enumerate().take(MAX_CHECK_BLOCKS) { + result.push_str(&blk.join("\n")); + result.push('\n'); + if i < blocks.len() - 1 { + result.push('\n'); + } + } + if blocks.len() > MAX_CHECK_BLOCKS { + result.push_str(&format!( + "\n… +{} more issues\n", + blocks.len() - MAX_CHECK_BLOCKS + )); + let all_blocks: String = blocks + .iter() + .map(|b| b.join("\n")) + .collect::>() + .join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_blocks, "cargo-check-issues") { + result.push_str(&format!(" {}\n", hint)); + } + } + result.trim().to_string() +} + +/// Aggregated test results for compact display +#[derive(Debug, Default, Clone)] +struct AggregatedTestResult { + passed: usize, + failed: usize, + ignored: usize, + measured: usize, + filtered_out: usize, + suites: usize, + duration_secs: f64, + has_duration: bool, +} + +impl AggregatedTestResult { + /// Parse a test result summary line + /// Format: "test result: ok. 15 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s" + fn parse_line(line: &str) -> Option { + static RE: LazyLock = LazyLock::new(|| { + regex::Regex::new( + r"test result: (\w+)\.\s+(\d+) passed;\s+(\d+) failed;\s+(\d+) ignored;\s+(\d+) measured;\s+(\d+) filtered out(?:;\s+finished in ([\d.]+)s)?" + ).unwrap() + }); + + let caps = RE.captures(line)?; + let status = caps.get(1)?.as_str(); + + // Only aggregate if status is "ok" (all tests passed) + if status != "ok" { + return None; + } + + let passed = caps.get(2)?.as_str().parse().ok()?; + let failed = caps.get(3)?.as_str().parse().ok()?; + let ignored = caps.get(4)?.as_str().parse().ok()?; + let measured = caps.get(5)?.as_str().parse().ok()?; + let filtered_out = caps.get(6)?.as_str().parse().ok()?; + + let (duration_secs, has_duration) = if let Some(duration_match) = caps.get(7) { + (duration_match.as_str().parse().unwrap_or(0.0), true) + } else { + (0.0, false) + }; + + Some(Self { + passed, + failed, + ignored, + measured, + filtered_out, + suites: 1, + duration_secs, + has_duration, + }) + } + + /// Merge another test result into this one + fn merge(&mut self, other: &Self) { + self.passed += other.passed; + self.failed += other.failed; + self.ignored += other.ignored; + self.measured += other.measured; + self.filtered_out += other.filtered_out; + self.suites += other.suites; + self.duration_secs += other.duration_secs; + self.has_duration = self.has_duration && other.has_duration; + } + + /// Format as compact single line + fn format_compact(&self) -> String { + let mut parts = vec![format!("{} passed", self.passed)]; + + if self.ignored > 0 { + parts.push(format!("{} ignored", self.ignored)); + } + if self.filtered_out > 0 { + parts.push(format!("{} filtered out", self.filtered_out)); + } + + let counts = parts.join(", "); + + let suite_text = if self.suites == 1 { + "1 suite".to_string() + } else { + format!("{} suites", self.suites) + }; + + if self.has_duration { + format!( + "cargo test: {} ({}, {:.2}s)", + counts, suite_text, self.duration_secs + ) + } else { + format!("cargo test: {} ({})", counts, suite_text) + } + } +} + +pub(crate) fn filter_cargo_test(output: &str) -> String { + let mut failures: Vec = Vec::new(); + let mut summary_lines: Vec = Vec::new(); + let mut in_failure_section = false; + let mut current_failure = Vec::new(); + + for line in output.lines() { + // Skip compilation lines + if line.trim_start().starts_with("Compiling") + || line.trim_start().starts_with("Downloading") + || line.trim_start().starts_with("Downloaded") + || line.trim_start().starts_with("Finished") + { + continue; + } + + // Skip "running N tests" and individual "test ... ok" lines + if line.starts_with("running ") || (line.starts_with("test ") && line.ends_with("... ok")) { + continue; + } + + // Detect failures section + if line == "failures:" { + in_failure_section = true; + continue; + } + + if in_failure_section { + if line.starts_with("test result:") { + in_failure_section = false; + summary_lines.push(line.to_string()); + } else if line.starts_with(" ") || line.starts_with("---- ") { + current_failure.push(line.to_string()); + } else if line.trim().is_empty() && !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } else if !line.trim().is_empty() { + current_failure.push(line.to_string()); + } + } + + // Capture test result summary + if !in_failure_section && line.starts_with("test result:") { + summary_lines.push(line.to_string()); + } + } + + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + } + + let mut result = String::new(); + + if failures.is_empty() && !summary_lines.is_empty() { + // All passed - try to aggregate + let mut aggregated: Option = None; + let mut all_parsed = true; + + for line in &summary_lines { + if let Some(parsed) = AggregatedTestResult::parse_line(line) { + if let Some(ref mut agg) = aggregated { + agg.merge(&parsed); + } else { + aggregated = Some(parsed); + } + } else { + all_parsed = false; + break; + } + } + + // If all lines parsed successfully and we have at least one suite, return compact format + if all_parsed + && let Some(agg) = aggregated + && agg.suites > 0 + { + return agg.format_compact(); + } + + // Fallback: use original behavior if regex failed + for line in &summary_lines { + result.push_str(&format!("{}\n", line)); + } + return result.trim().to_string(); + } + + if !failures.is_empty() { + result.push_str(&format!("FAILURES ({}):\n", failures.len())); + const MAX_FAILURES: usize = CAP_WARNINGS; + for (i, failure) in failures.iter().enumerate().take(MAX_FAILURES) { + result.push_str(&format!("{}. {}\n", i + 1, truncate(failure, 200))); + } + if failures.len() > MAX_FAILURES { + result.push_str(&format!( + "\n… +{} more failures\n", + failures.len() - MAX_FAILURES + )); + let all_failures = failures.join("\n\n"); + if let Some(hint) = + crate::core::tee::force_tee_hint(&all_failures, "cargo-test-failures") + { + result.push_str(&format!(" {}\n", hint)); + } + } + result.push('\n'); + } + + for line in &summary_lines { + result.push_str(&format!("{}\n", line)); + } + + if result.trim().is_empty() { + let json = extract_json_diagnostics(output); + let has_compile_errors = !json.errors.is_empty() + || output.lines().any(|line| { + let trimmed = line.trim_start(); + trimmed.starts_with("error[") || trimmed.starts_with("error:") + }); + + if has_compile_errors { + let build_filtered = filter_cargo_build_labeled(output, "test", 0); + if build_filtered.contains("cargo test:") { + return build_filtered; + } + } + + // Fallback: show last meaningful lines + let meaningful: Vec<&str> = output + .lines() + .filter(|l| !l.trim().is_empty() && !l.trim_start().starts_with("Compiling")) + .collect(); + for line in meaningful.iter().rev().take(5).rev() { + result.push_str(&format!("{}\n", line)); + } + } + + result.trim().to_string() +} + +/// Filter cargo clippy output - show full error blocks, group warnings by lint rule +fn filter_cargo_clippy(output: &str) -> String { + let mut by_rule: HashMap> = HashMap::new(); + let mut error_count = 0; + let mut warning_count = 0; + // Each entry is a full multi-line error block (headline + location + code context) + let mut error_blocks: Vec> = Vec::new(); + + let mut current_rule = String::new(); + let mut in_error = false; + let mut current_block: Vec = Vec::new(); + + for line in output.lines() { + // Skip compilation progress lines + if line.trim_start().starts_with("Compiling") + || line.trim_start().starts_with("Checking") + || line.trim_start().starts_with("Downloading") + || line.trim_start().starts_with("Downloaded") + || line.trim_start().starts_with("Finished") + { + if in_error && !current_block.is_empty() { + error_blocks.push(current_block.clone()); + current_block.clear(); + in_error = false; + } + continue; + } + + // Skip noise: summary counts and abort lines + if (line.contains("generated") && line.contains("warning")) + || line.contains("aborting due to") + || line.contains("could not compile") + { + continue; + } + + let is_error_line = line.starts_with("error:") || line.starts_with("error["); + let is_warning_line = line.starts_with("warning:") || line.starts_with("warning["); + + if is_error_line || is_warning_line { + // Flush any in-progress error block before starting a new diagnostic + if in_error && !current_block.is_empty() { + error_blocks.push(current_block.clone()); + current_block.clear(); + } + in_error = false; + + if is_error_line { + error_count += 1; + in_error = true; + current_block.push(line.to_string()); + } else { + warning_count += 1; + } + + // Extract rule/error-code from brackets for warning grouping + current_rule = if let Some(bracket_start) = line.rfind('[') { + if let Some(bracket_end) = line.rfind(']') { + line[bracket_start + 1..bracket_end].to_string() + } else { + line.to_string() + } + } else { + let prefix = if is_error_line { + "error: " + } else { + "warning: " + }; + line.strip_prefix(prefix).unwrap_or(line).to_string() + }; + } else if line.trim_start().starts_with("--> ") { + let location = line.trim_start().trim_start_matches("--> ").to_string(); + if !current_rule.is_empty() { + by_rule + .entry(current_rule.clone()) + .or_default() + .push(location); + } + if in_error { + current_block.push(line.to_string()); + } + } else if in_error { + if line.trim().is_empty() { + // Blank line terminates the error block + if !current_block.is_empty() { + error_blocks.push(current_block.clone()); + current_block.clear(); + } + in_error = false; + } else if current_block.len() < 15 { + // Collect code-context lines (|, ^, = note:, help:, etc.) + current_block.push(line.to_string()); + } + } + } + + // Flush final error block + if in_error && !current_block.is_empty() { + error_blocks.push(current_block); + } + + if error_count == 0 && warning_count == 0 { + return "cargo clippy: No issues found".to_string(); + } + + let mut result = String::new(); + result.push_str(&format!( + "cargo clippy: {} errors, {} warnings\n", + error_count, warning_count + )); + + // Show full error blocks so developers can see what needs fixing + if !error_blocks.is_empty() { + const MAX_CLIPPY_ERRORS: usize = CAP_WARNINGS; + result.push_str("\nErrors:\n"); + for block in error_blocks.iter().take(MAX_CLIPPY_ERRORS) { + for block_line in block { + result.push_str(&format!(" {}\n", truncate(block_line, 160))); + } + result.push('\n'); + } + if error_blocks.len() > MAX_CLIPPY_ERRORS { + result.push_str(&format!( + " … +{} more errors\n", + error_blocks.len() - MAX_CLIPPY_ERRORS + )); + let all_blocks: String = error_blocks + .iter() + .map(|b| b.join("\n")) + .collect::>() + .join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_blocks, "cargo-clippy-errors") + { + result.push_str(&format!(" {}\n", hint)); + } + } + } + + // Sort warning rules by frequency + let mut rule_counts: Vec<_> = by_rule.iter().collect(); + rule_counts.sort_by_key(|b| std::cmp::Reverse(b.1.len())); + + const MAX_RULES: usize = CAP_LIST; + for (rule, locations) in rule_counts.iter().take(MAX_RULES) { + result.push_str(&format!(" {} ({}x)\n", rule, locations.len())); + for loc in locations.iter().take(3) { + result.push_str(&format!(" {}\n", loc)); + } + if locations.len() > 3 { + result.push_str(&format!(" … +{} more\n", locations.len() - 3)); + } + } + + if by_rule.len() > MAX_RULES { + result.push_str(&format!("\n… +{} more rules\n", by_rule.len() - MAX_RULES)); + let all_rules = rule_counts + .iter() + .map(|(rule, locs)| format!("{} ({}x)", rule, locs.len())) + .collect::>() + .join("\n"); + if let Some(hint) = + crate::core::tee::force_tee_tail_hint(&all_rules, "cargo-clippy-rules", MAX_RULES + 1) + { + result.push_str(&format!(" {}\n", hint)); + } + } + + result.trim().to_string() +} + +fn filter_cargo_clippy_json(output: &str, exit_code: i32) -> String { + let json = extract_json_diagnostics(output); + if json.errors.is_empty() && json.warnings.is_empty() && exit_code == 0 { + return "cargo clippy: No issues found".to_string(); + } + filter_cargo_build_labeled(output, "clippy", exit_code) +} + +pub fn run_passthrough(args: &[OsString], verbose: u8) -> Result { + crate::core::runner::run_passthrough("cargo", args, verbose) +} + +#[cfg(test)] +mod tests { + const TINY_BUILD_SUCCESS_OUTPUT: &str = + " Finished dev [unoptimized + debuginfo] target(s) in 0.01s\n"; + + use super::*; + use crate::core::args_utils::restore_double_dash_with_raw; + + #[test] + fn test_restore_double_dash_with_separator() { + // rtk cargo test -- --nocapture → clap gives ["--nocapture"] + let args: Vec = vec!["--nocapture".into()]; + let raw = vec![ + "rtk".into(), + "cargo".into(), + "test".into(), + "--".into(), + "--nocapture".into(), + ]; + let result = restore_double_dash_with_raw(&args, &raw); + assert_eq!(result, vec!["--", "--nocapture"]); + } + + #[test] + fn test_restore_double_dash_with_test_name() { + // rtk cargo test my_test -- --nocapture → clap gives ["my_test", "--nocapture"] + let args: Vec = vec!["my_test".into(), "--nocapture".into()]; + let raw = vec![ + "rtk".into(), + "cargo".into(), + "test".into(), + "my_test".into(), + "--".into(), + "--nocapture".into(), + ]; + let result = restore_double_dash_with_raw(&args, &raw); + assert_eq!(result, vec!["my_test", "--", "--nocapture"]); + } + + #[test] + fn test_restore_double_dash_without_separator() { + // rtk cargo test my_test → no --, args unchanged + let args: Vec = vec!["my_test".into()]; + let raw = vec![ + "rtk".into(), + "cargo".into(), + "test".into(), + "my_test".into(), + ]; + let result = restore_double_dash_with_raw(&args, &raw); + assert_eq!(result, vec!["my_test"]); + } + + #[test] + fn test_restore_double_dash_empty_args() { + let args: Vec = vec![]; + let raw = vec!["rtk".into(), "cargo".into(), "test".into()]; + let result = restore_double_dash_with_raw(&args, &raw); + assert!(result.is_empty()); + } + + #[test] + fn test_restore_double_dash_clippy() { + // rtk cargo clippy -- -D warnings → clap gives ["-D", "warnings"] + let args: Vec = vec!["-D".into(), "warnings".into()]; + let raw = vec![ + "rtk".into(), + "cargo".into(), + "clippy".into(), + "--".into(), + "-D".into(), + "warnings".into(), + ]; + let result = restore_double_dash_with_raw(&args, &raw); + assert_eq!(result, vec!["--", "-D", "warnings"]); + } + + #[test] + fn test_restore_double_dash_clippy_with_package_flags() { + // rtk cargo clippy -p my-service -p my-crate -- -D warnings + // Clap with trailing_var_arg preserves "--" when args precede it + // → clap gives ["-p", "my-service", "-p", "my-crate", "--", "-D", "warnings"] + let args: Vec = vec![ + "-p".into(), + "my-service".into(), + "-p".into(), + "my-crate".into(), + "--".into(), + "-D".into(), + "warnings".into(), + ]; + let raw = vec![ + "rtk".into(), + "cargo".into(), + "clippy".into(), + "-p".into(), + "my-service".into(), + "-p".into(), + "my-crate".into(), + "--".into(), + "-D".into(), + "warnings".into(), + ]; + let result = restore_double_dash_with_raw(&args, &raw); + // Should NOT double the "--" + assert_eq!( + result, + vec!["-p", "my-service", "-p", "my-crate", "--", "-D", "warnings"] + ); + // Verify only one "--" exists + assert_eq!(result.iter().filter(|a| *a == "--").count(), 1); + } + + #[test] + fn test_filter_cargo_build_success() { + let output = r#" Compiling libc v0.2.153 + Compiling cfg-if v1.0.0 + Compiling rtk v0.5.0 + Finished dev [unoptimized + debuginfo] target(s) in 15.23s +"#; + let result = filter_cargo_build(output); + assert!(result.contains("cargo build")); + assert!(result.contains("3 crates compiled")); + } + + #[test] + fn test_filter_cargo_build_success_uses_raw_when_summary_is_larger() { + let output = TINY_BUILD_SUCCESS_OUTPUT; + let result = filter_cargo_build(output); + assert_eq!(result, output); + } + + #[test] + fn test_cargo_test_summary_uses_raw_when_summary_is_larger() { + // A one-line compile failure is already minimal: prefixing it with the + // "cargo test: N errors, ..." header emits more tokens than the raw + // output, so the never-worse guard must keep the raw output. + const CARGO_COMPILE_FAILURE_EXIT_CODE: i32 = 101; + let raw = "error[E0433]: failed to resolve: use of undeclared type `Foo`\n"; + + let mut handler = CargoTestHandler::new(); + for line in raw.lines() { + handler.should_skip(line); + } + + let unguarded = handler + .compute_test_summary(raw) + .expect("compile failure yields a summary"); + assert!( + unguarded.len() > raw.len(), + "expected the compacted summary to be larger than the raw output, got {:?}", + unguarded + ); + assert_eq!( + handler + .format_summary(CARGO_COMPILE_FAILURE_EXIT_CODE, raw) + .as_deref(), + Some(raw) + ); + } + + #[test] + fn test_filter_cargo_build_errors() { + let output = r#" Compiling rtk v0.5.0 +error[E0308]: mismatched types + --> src/main.rs:10:5 + | +10| "hello" + | ^^^^^^^ expected `i32`, found `&str` + +error: aborting due to 1 previous error +"#; + let result = filter_cargo_build(output); + assert!(result.contains("1 errors")); + assert!(result.contains("E0308")); + assert!(result.contains("mismatched types")); + } + + #[test] + fn test_filter_cargo_test_all_pass() { + let output = r#" Compiling rtk v0.5.0 + Finished test [unoptimized + debuginfo] target(s) in 2.53s + Running target/debug/deps/rtk-abc123 + +running 15 tests +test utils::tests::test_truncate_short_string ... ok +test utils::tests::test_truncate_long_string ... ok +test utils::tests::test_strip_ansi_simple ... ok + +test result: ok. 15 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s +"#; + let result = filter_cargo_test(output); + assert!( + result.contains("cargo test: 15 passed (1 suite, 0.01s)"), + "Expected compact format, got: {}", + result + ); + assert!(!result.contains("Compiling")); + assert!(!result.contains("test utils")); + } + + #[test] + fn test_filter_cargo_test_failures() { + let output = r#"running 5 tests +test foo::test_a ... ok +test foo::test_b ... FAILED +test foo::test_c ... ok + +failures: + +---- foo::test_b stdout ---- +thread 'foo::test_b' panicked at 'assert_eq!(1, 2)' + +failures: + foo::test_b + +test result: FAILED. 4 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +"#; + let result = filter_cargo_test(output); + assert!(result.contains("FAILURES")); + assert!(result.contains("test_b")); + assert!(result.contains("test result:")); + } + + #[test] + fn test_filter_cargo_test_multi_suite_all_pass() { + let output = r#" Compiling rtk v0.5.0 + Finished test [unoptimized + debuginfo] target(s) in 2.53s + Running unittests src/lib.rs (target/debug/deps/rtk-abc123) + +running 50 tests +test result: ok. 50 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.45s + + Running unittests src/main.rs (target/debug/deps/rtk-def456) + +running 30 tests +test result: ok. 30 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.30s + + Running tests/integration.rs (target/debug/deps/integration-ghi789) + +running 25 tests +test result: ok. 25 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.25s + + Doc-tests rtk + +running 32 tests +test result: ok. 32 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.45s +"#; + let result = filter_cargo_test(output); + assert!( + result.contains("cargo test: 137 passed (4 suites, 1.45s)"), + "Expected aggregated format, got: {}", + result + ); + assert!(!result.contains("running")); + } + + #[test] + fn test_filter_cargo_test_multi_suite_with_failures() { + let output = r#" Running unittests src/lib.rs + +running 20 tests +test result: ok. 20 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.10s + + Running unittests src/main.rs + +running 15 tests +test foo::test_bad ... FAILED + +failures: + +---- foo::test_bad stdout ---- +thread panicked at 'assertion failed' + +test result: FAILED. 14 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s + + Running tests/integration.rs + +running 10 tests +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s +"#; + let result = filter_cargo_test(output); + // Should NOT aggregate when there are failures + assert!(result.contains("FAILURES"), "got: {}", result); + assert!(result.contains("test_bad"), "got: {}", result); + assert!(result.contains("test result:"), "got: {}", result); + // Should show individual summaries + assert!(result.contains("20 passed"), "got: {}", result); + assert!(result.contains("14 passed"), "got: {}", result); + assert!(result.contains("10 passed"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_test_all_suites_zero_tests() { + let output = r#" Running unittests src/empty1.rs + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running unittests src/empty2.rs + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/empty3.rs + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s +"#; + let result = filter_cargo_test(output); + assert!( + result.contains("cargo test: 0 passed (3 suites, 0.00s)"), + "Expected compact format for zero tests, got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_test_with_ignored_and_filtered() { + let output = r#" Running unittests src/lib.rs + +running 50 tests +test result: ok. 45 passed; 0 failed; 3 ignored; 0 measured; 2 filtered out; finished in 0.50s + + Running tests/integration.rs + +running 20 tests +test result: ok. 18 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.20s +"#; + let result = filter_cargo_test(output); + assert!( + result.contains("cargo test: 63 passed, 5 ignored, 2 filtered out (2 suites, 0.70s)"), + "Expected compact format with ignored and filtered, got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_test_single_suite_compact() { + let output = r#" Running unittests src/main.rs + +running 15 tests +test result: ok. 15 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s +"#; + let result = filter_cargo_test(output); + assert!( + result.contains("cargo test: 15 passed (1 suite, 0.01s)"), + "Expected singular 'suite', got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_test_regex_fallback() { + let output = r#" Running unittests src/main.rs + +running 15 tests +test result: MALFORMED LINE WITHOUT PROPER FORMAT +"#; + let result = filter_cargo_test(output); + // Should fallback to original behavior (show line without checkmark) + assert!( + result.contains("test result: MALFORMED"), + "Expected fallback format, got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_test_compile_error_preserves_error_header() { + let output = r#" Compiling rtk v0.31.0 (/workspace/projects/rtk) +error[E0425]: cannot find value `missing_symbol` in this scope + --> tests/repro_compile_fail.rs:3:13 + | +3 | let _ = missing_symbol; + | ^^^^^^^^^^^^^^ not found in this scope + +For more information about this error, try `rustc --explain E0425`. +error: could not compile `rtk` (test "repro_compile_fail") due to 1 previous error +"#; + let result = filter_cargo_test(output); + assert!(result.contains("cargo test: 1 errors, 0 warnings (1 crates)")); + assert!(result.contains("error[E0425]"), "got: {}", result); + assert!( + result.contains("--> tests/repro_compile_fail.rs:3:13"), + "got: {}", + result + ); + assert!(!result.starts_with('|'), "got: {}", result); + } + + #[test] + fn test_filter_cargo_clippy_clean() { + let output = r#" Checking rtk v0.5.0 + Finished dev [unoptimized + debuginfo] target(s) in 1.53s +"#; + let result = filter_cargo_clippy(output); + assert!(result.contains("cargo clippy: No issues found")); + } + + #[test] + fn test_filter_cargo_clippy_warnings() { + let output = r#" Checking rtk v0.5.0 +warning: unused variable: `x` [unused_variables] + --> src/main.rs:10:9 + | +10| let x = 5; + | ^ help: if this is intentional, prefix it with an underscore: `_x` + +warning: this function has too many arguments [clippy::too_many_arguments] + --> src/git.rs:16:1 + | +16| pub fn run(a: i32, b: i32, c: i32, d: i32, e: i32, f: i32, g: i32, h: i32) {} + | + +warning: `rtk` (bin) generated 2 warnings + Finished dev [unoptimized + debuginfo] target(s) in 1.53s +"#; + let result = filter_cargo_clippy(output); + assert!(result.contains("0 errors, 2 warnings")); + assert!(result.contains("unused_variables")); + assert!(result.contains("clippy::too_many_arguments")); + } + + #[test] + fn test_filter_cargo_clippy_includes_error_details() { + let output = r#" Checking rtk v0.5.0 +error: struct literals are not allowed here +warning: unused variable: `x` [unused_variables] + Finished dev [unoptimized + debuginfo] target(s) in 1.53s +"#; + let result = filter_cargo_clippy(output); + assert!(result.contains("cargo clippy: 1 errors, 1 warnings")); + assert!(result.contains("Errors:")); + assert!(result.contains("struct literals are not allowed here")); + } + + #[test] + fn test_filter_cargo_clippy_shows_full_error_block() { + // Full multi-line error block must be shown so the developer can debug + let output = r#" Checking rtk v0.5.0 +error[E0308]: mismatched types + --> src/main.rs:10:5 + | +9 | fn foo() -> i32 { + | --- expected `i32` because of return type +10| "hello" + | ^^^^^^^ expected `i32`, found `&str` + +error: aborting due to 1 previous error +"#; + let result = filter_cargo_clippy(output); + assert!( + result.contains("cargo clippy: 1 errors, 0 warnings"), + "got: {}", + result + ); + assert!( + result.contains("error[E0308]: mismatched types"), + "got: {}", + result + ); + assert!(result.contains("src/main.rs:10:5"), "got: {}", result); + assert!( + result.contains("expected `i32`, found `&str`"), + "got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_clippy_multiple_errors_show_all_blocks() { + let output = r#"error[E0308]: mismatched types + --> src/foo.rs:5:3 + +error[E0425]: cannot find value `x` + --> src/bar.rs:12:9 + +error: aborting due to 2 previous errors +"#; + let result = filter_cargo_clippy(output); + assert!(result.contains("2 errors"), "got: {}", result); + assert!(result.contains("src/foo.rs:5:3"), "got: {}", result); + assert!(result.contains("src/bar.rs:12:9"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_success() { + let output = r#" Installing rtk v0.11.0 + Downloading crates ... + Downloaded anyhow v1.0.80 + Downloaded clap v4.5.0 + Compiling libc v0.2.153 + Compiling cfg-if v1.0.0 + Compiling anyhow v1.0.80 + Compiling clap v4.5.0 + Compiling rtk v0.11.0 + Finished `release` profile [optimized] target(s) in 45.23s + Replacing /Users/user/.cargo/bin/rtk + Replaced package `rtk v0.9.4` with `rtk v0.11.0` (/Users/user/.cargo/bin/rtk) +"#; + let result = filter_cargo_install(output); + assert!(result.contains("cargo install"), "got: {}", result); + assert!(result.contains("rtk v0.11.0"), "got: {}", result); + assert!(result.contains("5 deps compiled"), "got: {}", result); + assert!(result.contains("Replaced"), "got: {}", result); + assert!(!result.contains("Compiling"), "got: {}", result); + assert!(!result.contains("Downloading"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_replace() { + let output = r#" Installing rtk v0.11.0 + Compiling rtk v0.11.0 + Finished `release` profile [optimized] target(s) in 10.0s + Replacing /Users/user/.cargo/bin/rtk + Replaced package `rtk v0.9.4` with `rtk v0.11.0` (/Users/user/.cargo/bin/rtk) +"#; + let result = filter_cargo_install(output); + assert!(result.contains("cargo install"), "got: {}", result); + assert!(result.contains("Replacing"), "got: {}", result); + assert!(result.contains("Replaced"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_error() { + let output = r#" Installing rtk v0.11.0 + Compiling rtk v0.11.0 +error[E0308]: mismatched types + --> src/main.rs:10:5 + | +10| "hello" + | ^^^^^^^ expected `i32`, found `&str` + +error: aborting due to 1 previous error +"#; + let result = filter_cargo_install(output); + assert!(result.contains("cargo install: 1 error"), "got: {}", result); + assert!(result.contains("E0308"), "got: {}", result); + assert!(result.contains("mismatched types"), "got: {}", result); + assert!(!result.contains("aborting"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_already_installed() { + let output = r#" Ignored package `rtk v0.11.0`, is already installed +"#; + let result = filter_cargo_install(output); + assert!(result.contains("already installed"), "got: {}", result); + assert!(result.contains("rtk v0.11.0"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_up_to_date() { + let output = r#" Ignored package `cargo-deb v2.1.0 (/Users/user/cargo-deb)`, is already installed +"#; + let result = filter_cargo_install(output); + assert!(result.contains("already installed"), "got: {}", result); + assert!(result.contains("cargo-deb v2.1.0"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_empty_output() { + let result = filter_cargo_install(""); + assert!(result.contains("cargo install"), "got: {}", result); + assert!(result.contains("0 deps compiled"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_path_warning() { + let output = r#" Installing rtk v0.11.0 + Compiling rtk v0.11.0 + Finished `release` profile [optimized] target(s) in 10.0s + Replacing /Users/user/.cargo/bin/rtk + Replaced package `rtk v0.9.4` with `rtk v0.11.0` (/Users/user/.cargo/bin/rtk) +warning: be sure to add `/Users/user/.cargo/bin` to your PATH +"#; + let result = filter_cargo_install(output); + assert!(result.contains("cargo install"), "got: {}", result); + assert!( + result.contains("be sure to add"), + "PATH warning should be kept: {}", + result + ); + assert!(result.contains("Replaced"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_multiple_errors() { + let output = r#" Installing rtk v0.11.0 + Compiling rtk v0.11.0 +error[E0308]: mismatched types + --> src/main.rs:10:5 + | +10| "hello" + | ^^^^^^^ expected `i32`, found `&str` + +error[E0425]: cannot find value `foo` + --> src/lib.rs:20:9 + | +20| foo + | ^^^ not found in this scope + +error: aborting due to 2 previous errors +"#; + let result = filter_cargo_install(output); + assert!( + result.contains("2 errors"), + "should show 2 errors: {}", + result + ); + assert!(result.contains("E0308"), "got: {}", result); + assert!(result.contains("E0425"), "got: {}", result); + assert!(!result.contains("aborting"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_locking_and_blocking() { + let output = r#" Locking 45 packages to latest compatible versions + Blocking waiting for file lock on package cache + Downloading crates ... + Downloaded serde v1.0.200 + Compiling serde v1.0.200 + Compiling rtk v0.11.0 + Finished `release` profile [optimized] target(s) in 30.0s + Installing rtk v0.11.0 +"#; + let result = filter_cargo_install(output); + assert!(result.contains("cargo install"), "got: {}", result); + assert!(!result.contains("Locking"), "got: {}", result); + assert!(!result.contains("Blocking"), "got: {}", result); + assert!(!result.contains("Downloading"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_from_path() { + let output = r#" Installing /Users/user/projects/rtk + Compiling rtk v0.11.0 + Finished `release` profile [optimized] target(s) in 10.0s +"#; + let result = filter_cargo_install(output); + // Path-based install: crate info not extracted from path + assert!(result.contains("cargo install"), "got: {}", result); + assert!(result.contains("1 deps compiled"), "got: {}", result); + } + + #[test] + fn test_format_crate_info() { + assert_eq!(format_crate_info("rtk", "v0.11.0", ""), "rtk v0.11.0"); + assert_eq!(format_crate_info("rtk", "", ""), "rtk"); + assert_eq!(format_crate_info("", "", "package"), "package"); + assert_eq!(format_crate_info("", "v0.1.0", "fallback"), "fallback"); + } + + #[test] + fn test_filter_cargo_nextest_all_pass() { + let output = r#" Compiling rtk v0.15.2 + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.04s +──────────────────────────── + Starting 301 tests across 1 binary + PASS [ 0.009s] (1/301) rtk::bin/rtk cargo_cmd::tests::test_one + PASS [ 0.008s] (2/301) rtk::bin/rtk cargo_cmd::tests::test_two + PASS [ 0.007s] (301/301) rtk::bin/rtk cargo_cmd::tests::test_last +──────────────────────────── + Summary [ 0.192s] 301 tests run: 301 passed, 0 skipped +"#; + let result = filter_cargo_nextest(output); + assert_eq!( + result, "cargo nextest: 301 passed (1 binary, 0.192s)", + "got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_with_failures() { + let output = r#" Starting 4 tests across 1 binary (1 test skipped) + PASS [ 0.006s] (1/4) test-proj tests::passing_test + FAIL [ 0.006s] (2/4) test-proj tests::failing_test + + stderr ─── + + thread 'tests::failing_test' panicked at src/lib.rs:15:9: + assertion `left == right` failed + left: 1 + right: 2 + + Cancelling due to test failure: 2 tests still running + PASS [ 0.007s] (3/4) test-proj tests::another_passing + FAIL [ 0.006s] (4/4) test-proj tests::another_failing + + stderr ─── + + thread 'tests::another_failing' panicked at src/lib.rs:20:9: + something went wrong + +──────────────────────────── + Summary [ 0.007s] 4 tests run: 2 passed, 2 failed, 1 skipped + FAIL [ 0.006s] (2/4) test-proj tests::failing_test + FAIL [ 0.006s] (4/4) test-proj tests::another_failing +error: test run failed +"#; + let result = filter_cargo_nextest(output); + assert!( + result.contains("tests::failing_test"), + "should contain first failure: {}", + result + ); + assert!( + result.contains("tests::another_failing"), + "should contain second failure: {}", + result + ); + assert!( + result.contains("panicked"), + "should contain stderr detail: {}", + result + ); + assert!( + result.contains("2 passed, 2 failed, 1 skipped"), + "should contain summary: {}", + result + ); + assert!( + !result.contains("PASS"), + "should not contain PASS lines: {}", + result + ); + // Post-summary FAIL recaps must not create duplicate FAIL header entries + // (test names may appear in both header and stderr body naturally) + assert_eq!( + result.matches("FAIL [").count(), + 2, + "should have exactly 2 FAIL headers (no post-summary duplicates): {}", + result + ); + assert!( + !result.contains("error: test run failed"), + "should not contain post-summary error line: {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_with_skipped() { + let output = r#" Starting 50 tests across 2 binaries (3 tests skipped) + PASS [ 0.010s] (1/50) rtk::bin/rtk test_one + PASS [ 0.010s] (50/50) rtk::bin/rtk test_last +──────────────────────────── + Summary [ 0.500s] 50 tests run: 50 passed, 3 skipped +"#; + let result = filter_cargo_nextest(output); + assert_eq!( + result, "cargo nextest: 50 passed, 3 skipped (2 binaries, 0.500s)", + "got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_single_failure_detail() { + let output = r#" Starting 2 tests across 1 binary + PASS [ 0.005s] (1/2) proj tests::good + FAIL [ 0.005s] (2/2) proj tests::bad + + stderr ─── + + thread 'tests::bad' panicked at src/lib.rs:5:9: + assertion failed: false + +──────────────────────────── + Summary [ 0.010s] 2 tests run: 1 passed, 1 failed + FAIL [ 0.005s] (2/2) proj tests::bad +error: test run failed +"#; + let result = filter_cargo_nextest(output); + assert!( + result.contains("assertion failed: false"), + "should show panic message: {}", + result + ); + assert!( + result.contains("1 passed, 1 failed"), + "should show summary: {}", + result + ); + // Post-summary recap must not duplicate FAIL headers + assert_eq!( + result.matches("FAIL [").count(), + 1, + "should have exactly 1 FAIL header (no post-summary duplicate): {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_multiple_binaries() { + let output = r#" Starting 100 tests across 5 binaries + PASS [ 0.010s] (100/100) test_last +──────────────────────────── + Summary [ 1.234s] 100 tests run: 100 passed, 0 skipped +"#; + let result = filter_cargo_nextest(output); + assert_eq!( + result, "cargo nextest: 100 passed (5 binaries, 1.234s)", + "got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_compilation_stripped() { + let output = r#" Compiling serde v1.0.200 + Compiling rtk v0.15.2 + Downloading crates ... + Finished `test` profile [unoptimized + debuginfo] target(s) in 5.00s +──────────────────────────── + Starting 10 tests across 1 binary + PASS [ 0.010s] (10/10) test_last +──────────────────────────── + Summary [ 0.050s] 10 tests run: 10 passed, 0 skipped +"#; + let result = filter_cargo_nextest(output); + assert!( + !result.contains("Compiling"), + "should strip Compiling: {}", + result + ); + assert!( + !result.contains("Downloading"), + "should strip Downloading: {}", + result + ); + assert!( + !result.contains("Finished"), + "should strip Finished: {}", + result + ); + assert!( + result.contains("cargo nextest: 10 passed"), + "got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_empty() { + let result = filter_cargo_nextest(""); + assert!(result.is_empty(), "got: {}", result); + } + + #[test] + fn test_filter_cargo_nextest_cancellation_notice() { + let output = r#" Starting 3 tests across 1 binary + FAIL [ 0.005s] (1/3) proj tests::bad + + stderr ─── + + thread panicked at 'oops' + + Cancelling due to test failure: 2 tests still running +──────────────────────────── + Summary [ 0.010s] 3 tests run: 2 passed, 1 failed + FAIL [ 0.005s] (1/3) proj tests::bad +error: test run failed +"#; + let result = filter_cargo_nextest(output); + assert!( + result.contains("Cancelling due to test failure"), + "should include cancel notice: {}", + result + ); + assert!( + result.contains("1 failed"), + "should show failure count: {}", + result + ); + // Post-summary recap must not duplicate FAIL headers + assert_eq!( + result.matches("FAIL [").count(), + 1, + "should have exactly 1 FAIL header (no post-summary duplicate): {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_summary_regex_fallback() { + let output = r#" Starting 5 tests across 1 binary + PASS [ 0.005s] (5/5) test_last +──────────────────────────── + Summary MALFORMED LINE +"#; + let result = filter_cargo_nextest(output); + assert!( + result.contains("Summary MALFORMED"), + "should fall back to raw summary: {}", + result + ); + } + + // --- Streaming handler tests --- + + use crate::core::stream::tests::run_block_filter; + + #[test] + fn test_cargo_build_stream_success() { + let input = " Compiling libc v0.2.153\n Compiling cfg-if v1.0.0\n Compiling rtk v0.5.0\n Finished dev [unoptimized + debuginfo] target(s) in 15.23s\n"; + let mut f = BlockStreamFilter::new(CargoBuildHandler::with_label("build")); + let result = run_block_filter(&mut f, input, 0); + assert!(result.contains("3 crates compiled"), "got: {}", result); + assert!(result.contains("Finished"), "got: {}", result); + assert!(!result.contains("Compiling"), "got: {}", result); + } + + #[test] + fn test_cargo_build_stream_success_uses_raw_when_summary_is_larger() { + let input = TINY_BUILD_SUCCESS_OUTPUT; + let mut f = BlockStreamFilter::new(CargoBuildHandler::with_label("build")); + let result = run_block_filter(&mut f, input, 0); + assert_eq!(result, input); + } + + #[test] + fn test_cargo_build_stream_json_success() { + let input = concat!( + " Compiling demo v0.1.0 (/tmp/demo)\n", + r#"{"reason":"compiler-artifact","package_id":"demo 0.1.0","target":{"name":"demo"},"executable":"/tmp/demo/target/debug/demo"}"#, + "\n", + " Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.39s\n", + r#"{"reason":"build-finished","success":true}"#, + "\n", + ); + let mut f = BlockStreamFilter::new(CargoBuildHandler::with_label("build")); + let result = run_block_filter(&mut f, input, 0); + assert!(result.contains("1 crates compiled"), "got: {}", result); + assert!(result.contains("Finished"), "got: {}", result); + assert!(!result.contains("error"), "got: {}", result); + assert!(!result.contains("Compiling"), "got: {}", result); + } + + #[test] + fn test_cargo_build_stream_errors() { + let input = r#" Compiling rtk v0.5.0 +error[E0308]: mismatched types + --> src/main.rs:10:5 + | +10| "hello" + | ^^^^^^^ expected `i32`, found `&str` + +error: aborting due to 1 previous error +"#; + let mut f = BlockStreamFilter::new(CargoBuildHandler::with_label("build")); + let result = run_block_filter(&mut f, input, 1); + assert!(result.contains("E0308"), "got: {}", result); + assert!(result.contains("mismatched types"), "got: {}", result); + assert!(result.contains("1 errors"), "got: {}", result); + assert!(!result.contains("aborting"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_build_json_warning_rendered() { + let input = concat!( + " Compiling v_cargo v0.1.0 (/tmp/v_cargo)\n", + r#"{"reason":"compiler-message","package_id":"v_cargo 0.1.0","message":{"level":"warning","message":"unused variable: `x`","rendered":"warning: unused variable: `x`\n --> src/main.rs:2:9"}}"#, + "\n", + r#"{"reason":"build-finished","success":true}"#, + "\n", + ); + let result = filter_cargo_build_labeled(input, "build", 0); + assert!(result.contains("unused variable"), "got: {}", result); + assert!(result.contains("1 warnings"), "got: {}", result); + assert!(!result.contains("crates compiled"), "got: {}", result); + } + + #[test] + fn test_cargo_check_stream_success_label() { + let input = " Checking demo v0.1.0 (/tmp/demo)\n Finished dev [unoptimized + debuginfo] target(s) in 0.42s\n"; + let mut f = BlockStreamFilter::new(CargoBuildHandler::with_label("check")); + let result = run_block_filter(&mut f, input, 0); + assert!(result.contains("cargo check"), "got: {}", result); + assert!(!result.contains("cargo build"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_build_labeled_clippy_json_not_clean() { + let input = concat!( + " Checking demo v0.1.0 (/tmp/demo)\n", + r#"{"reason":"compiler-message","message":{"code":{"code":"E0308"},"level":"error","message":"mismatched types","rendered":"error[E0308]: mismatched types\n --> src/main.rs:2:18"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let result = filter_cargo_build_labeled(input, "clippy", 1); + assert!(result.contains("cargo clippy:"), "got: {}", result); + assert!(result.contains("1 errors"), "got: {}", result); + assert!( + !result.contains("No issues found"), + "json clippy errors must not be swallowed: {}", + result + ); + } + + #[test] + fn test_filter_cargo_clippy_json_clean() { + let input = concat!( + " Checking demo v0.1.0 (/tmp/demo)\n", + r#"{"reason":"compiler-artifact","package_id":"demo 0.1.0","target":{"name":"demo"}}"#, + "\n", + r#"{"reason":"build-finished","success":true}"#, + "\n", + ); + assert_eq!( + filter_cargo_clippy_json(input, 0), + "cargo clippy: No issues found" + ); + } + + #[test] + fn test_filter_cargo_clippy_json_warnings() { + let input = concat!( + " Checking demo v0.1.0 (/tmp/demo)\n", + r#"{"reason":"compiler-message","message":{"level":"warning","message":"unused variable: `x`","rendered":"warning: unused variable: `x`\n --> src/main.rs:2:9"}}"#, + "\n", + r#"{"reason":"build-finished","success":true}"#, + "\n", + ); + let result = filter_cargo_clippy_json(input, 0); + assert!(result.contains("unused variable"), "got: {}", result); + assert!( + result.contains("cargo clippy: 0 errors, 1 warnings"), + "got: {}", + result + ); + } + + #[test] + fn test_batch_filter_nonzero_exit_without_diagnostics_is_not_compiled() { + // build-script failure: exit 101, no compiler-message diagnostics parsed. + let input = concat!( + " Compiling demo v0.1.0 (/tmp/demo)\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let result = filter_cargo_build_labeled(input, "build", 101); + assert!( + !result.contains("crates compiled"), + "a failed build must not be reported as compiled: {}", + result + ); + assert!( + result.contains("cargo build: failed (exit 101)"), + "got: {}", + result + ); + } + + #[test] + fn test_failure_summary_line_is_first() { + let json = JsonDiagnostics { + errors: vec!["error[E0308]: mismatched types".to_string()], + warnings: Vec::new(), + }; + let out = cargo_build_failure_summary(1, 1, 0, &json, "build", 101); + assert!( + out.starts_with("cargo build: 1 errors"), + "summary line must come first: {}", + out + ); + } + + #[test] + fn test_filter_cargo_build_labeled_check() { + let input = concat!( + r#"{"reason":"compiler-message","message":{"level":"error","message":"mismatched types","rendered":"error[E0308]: mismatched types\n --> src/main.rs:2:18"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let result = filter_cargo_build_labeled(input, "check", 1); + assert!(result.contains("cargo check:"), "got: {}", result); + assert!(!result.contains("cargo build:"), "got: {}", result); + } + + #[test] + fn test_extract_json_diagnostics_strips_ansi() { + let output = concat!( + r#"{"reason":"compiler-message","message":{"level":"error","message":"mismatched types","rendered":"\u001b[1m\u001b[31merror[E0308]\u001b[0m: mismatched types"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let json = extract_json_diagnostics(output); + assert_eq!(json.errors.len(), 1); + assert!( + !json.errors[0].contains('\u{1b}'), + "ansi escapes must be stripped: {:?}", + json.errors[0] + ); + assert!( + json.errors[0].contains("error[E0308]"), + "got: {:?}", + json.errors[0] + ); + } + + #[test] + fn test_extract_json_diagnostics_skips_generated_summary() { + let output = concat!( + r#"{"reason":"compiler-message","message":{"level":"warning","message":"unused variable: `x`","rendered":"warning: unused variable: `x`"}}"#, + "\n", + r#"{"reason":"compiler-message","message":{"level":"warning","message":"`demo` (bin \"demo\") generated 1 warning","rendered":"warning: `demo` (bin \"demo\") generated 1 warning"}}"#, + "\n", + r#"{"reason":"build-finished","success":true}"#, + "\n", + ); + let json = extract_json_diagnostics(output); + assert_eq!( + json.warnings.len(), + 1, + "generated summary must not inflate the count" + ); + assert!( + !json.warnings.iter().any(|w| w.contains("generated")), + "the generated summary line must not appear in the output" + ); + } + + #[test] + fn test_extract_json_diagnostics_counts_ice_as_error() { + let output = concat!( + r#"{"reason":"compiler-message","message":{"level":"error: internal compiler error","message":"unexpected panic","rendered":"error: internal compiler error: unexpected panic"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let json = extract_json_diagnostics(output); + assert_eq!(json.errors.len(), 1, "an ICE must count as an error"); + assert!( + json.errors[0].contains("internal compiler error"), + "got: {:?}", + json.errors[0] + ); + } + + #[test] + fn test_json_format_inline_eq() { + assert!(has_json_message_format(&["--message-format=json".into()])); + } + + #[test] + fn test_json_format_space_separated() { + assert!(has_json_message_format(&[ + "--message-format".into(), + "json".into(), + ])); + } + + #[test] + fn test_json_format_rendered_ansi() { + assert!(has_json_message_format(&[ + "--message-format=json-diagnostic-rendered-ansi".into() + ])); + } + + #[test] + fn test_json_format_render_diagnostics() { + assert!(has_json_message_format(&[ + "--message-format=json-render-diagnostics".into() + ])); + } + + #[test] + fn test_json_format_space_rendered() { + assert!(has_json_message_format(&[ + "--message-format".into(), + "json-diagnostic-rendered-ansi".into(), + ])); + } + + #[test] + fn test_non_json_format_not_detected() { + assert!(!has_json_message_format(&["--message-format=human".into()])); + } + + #[test] + fn test_no_format_flag() { + assert!(!has_json_message_format(&[ + "--release".into(), + "-p".into(), + "my-crate".into(), + ])); + } + + #[test] + fn test_json_format_among_other_args() { + assert!(has_json_message_format(&[ + "--release".into(), + "-p".into(), + "my-crate".into(), + "--message-format=json".into(), + ])); + } + + #[test] + fn test_json_format_trailing_message_format_no_value() { + assert!(!has_json_message_format(&["--message-format".into()])); + } + + #[test] + fn test_json_format_last_occurrence_wins() { + assert!(!has_json_message_format(&[ + "--message-format=json".into(), + "--message-format=human".into(), + ])); + assert!(has_json_message_format(&[ + "--message-format=human".into(), + "--message-format=json".into(), + ])); + } + + #[test] + fn test_filter_cargo_build_json_failure_not_compiled() { + let output = concat!( + r#"{"reason":"compiler-message","message":{"rendered":"error[E0277]: the trait bound is not satisfied","level":"error","message":"trait bound"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let result = filter_cargo_build(output); + assert!(result.contains("E0277"), "got: {}", result); + assert!(result.contains("1 errors"), "got: {}", result); + assert!(!result.contains("crates compiled"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_build_json_errors_capped() { + let total = CAP_ERRORS + 5; + let mut output = String::new(); + for i in 0..total { + output.push_str(&format!( + r#"{{"reason":"compiler-message","message":{{"code":{{"code":"E0308"}},"level":"error","message":"mismatched types","rendered":"error[E0308]: mismatched types ({})"}}}}"#, + i + )); + output.push('\n'); + } + output.push_str(r#"{"reason":"build-finished","success":false}"#); + output.push('\n'); + + let result = filter_cargo_build(&output); + let rendered = result.matches("error[E0308]").count(); + assert_eq!( + rendered, CAP_ERRORS, + "json errors must be capped: {}", + result + ); + assert!( + result.contains(&format!("… +{} more errors", total - CAP_ERRORS)), + "expected overflow hint: {}", + result + ); + assert!( + result.contains(&format!("{} errors", total)), + "summary must report the real total: {}", + result + ); + } + + #[test] + fn test_filter_cargo_build_json_savings() { + // Real --message-format=json lines carry a verbose envelope (spans, + // children, code, message) around the human `rendered` text. The filter + // keeps only `rendered` and caps the list, so savings come from both + // dropping the envelope and capping a large failing build. + let template = r#"{"reason":"compiler-message","package_id":"demo 0.1.0 (path+file:///tmp/demo)","manifest_path":"/tmp/demo/Cargo.toml","target":{"kind":["bin"],"crate_types":["bin"],"name":"demo","src_path":"/tmp/demo/src/main.rs","edition":"2021","doc":true,"doctest":false,"test":true},"message":{"rendered":"error[E0308]: mismatched types\n --> src/main.rs:IDX:18\n |\nIDX | let _xIDX: i32 = \"errIDX\";\n | --- ^^^^^^^ expected `i32`, found `&str`\n | |\n | expected due to this\n\n","$message_type":"diagnostic","children":[{"children":[],"code":null,"level":"note","message":"expected due to the type annotation here","rendered":null,"spans":[]}],"code":{"code":"E0308","explanation":null},"level":"error","message":"mismatched types","spans":[{"byte_end":40,"byte_start":33,"column_end":25,"column_start":18,"expansion":null,"file_name":"src/main.rs","is_primary":true,"label":"expected `i32`, found `&str`","line_end":IDX,"line_start":IDX,"suggested_replacement":null,"suggestion_applicability":null,"text":[{"highlight_end":25,"highlight_start":18,"text":" let _xIDX: i32 = \"errIDX\";"}]}]}}"#; + + let total = CAP_ERRORS + 30; + let mut input = String::new(); + for i in 0..total { + input.push_str(&template.replace("IDX", &i.to_string())); + input.push('\n'); + } + input.push_str(r#"{"reason":"build-finished","success":false}"#); + input.push('\n'); + + let result = filter_cargo_build(&input); + + // The cap is what bounds the output, so assert it holds here too. + let rendered = result.matches("error[E0308]").count(); + assert_eq!( + rendered, CAP_ERRORS, + "json errors must be capped: {}", + result + ); + assert!( + result.contains(&format!("… +{} more errors", total - CAP_ERRORS)), + "expected overflow hint: {}", + result + ); + + let raw = input.split_whitespace().count(); + let out = result.split_whitespace().count(); + let savings = 100.0 - (out as f64 / raw as f64) * 100.0; + assert!( + savings >= 60.0, + "token savings dropped below 60%: {savings:.1}%" + ); + } + + #[test] + fn test_cargo_test_stream_all_pass() { + let input = r#" Compiling rtk v0.5.0 + Finished test [unoptimized + debuginfo] target(s) in 2.53s + Running target/debug/deps/rtk-abc123 + +running 15 tests +test utils::tests::test_truncate_short_string ... ok +test utils::tests::test_truncate_long_string ... ok +test utils::tests::test_strip_ansi_simple ... ok + +test result: ok. 15 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s +"#; + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 0); + assert!( + result.contains("cargo test: 15 passed (1 suite, 0.01s)"), + "got: {}", + result + ); + assert!(!result.contains("Compiling"), "got: {}", result); + } + + #[test] + fn test_cargo_test_stream_failures() { + let input = r#"running 5 tests +test foo::test_a ... ok +test foo::test_b ... FAILED +test foo::test_c ... ok + +failures: + +---- foo::test_b stdout ---- +thread 'foo::test_b' panicked at 'assert_eq!(1, 2)' + +failures: + foo::test_b + +test result: FAILED. 4 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +"#; + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 1); + assert!(result.contains("test_b"), "got: {}", result); + assert!(result.contains("panicked"), "got: {}", result); + } + + #[test] + fn test_cargo_test_stream_multi_suite() { + let input = r#" Running unittests src/lib.rs (target/debug/deps/rtk-abc123) + +running 50 tests +test result: ok. 50 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.45s + + Running unittests src/main.rs (target/debug/deps/rtk-def456) + +running 30 tests +test result: ok. 30 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.30s +"#; + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 0); + assert!( + result.contains("cargo test: 80 passed (2 suites, 0.75s)"), + "got: {}", + result + ); + } + + #[test] + fn test_cargo_test_stream_compile_error() { + let input = r#" Compiling rtk v0.31.0 (/workspace/projects/rtk) +error[E0425]: cannot find value `missing_symbol` in this scope + --> tests/repro_compile_fail.rs:3:13 + | +3 | let _ = missing_symbol; + | ^^^^^^^^^^^^^^ not found in this scope + +For more information about this error, try `rustc --explain E0425`. +error: could not compile `rtk` (test "repro_compile_fail") due to 1 previous error +"#; + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 1); + assert!(result.contains("cargo test:"), "got: {}", result); + assert!(result.contains("1 errors"), "got: {}", result); + } + + #[test] + fn test_cargo_test_stream_json_compile_error() { + let input = concat!( + " Compiling v_cargo v0.1.0 (/tmp/v_cargo)\n", + r#"{"reason":"compiler-message","package_id":"v_cargo 0.1.0","message":{"code":{"code":"E0425"},"level":"error","message":"cannot find value `missing_symbol` in this scope","rendered":"error[E0425]: cannot find value `missing_symbol` in this scope\n --> tests/repro.rs:3:13"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 101); + assert!( + !result.trim().is_empty(), + "json compile error must not be silent" + ); + assert!(result.contains("cargo test:"), "got: {}", result); + assert!(result.contains("1 errors"), "got: {}", result); + assert!( + result.contains("E0425"), + "diagnostic must be surfaced: {}", + result + ); + } + + #[test] + fn test_cargo_test_stream_no_diagnostic_falls_back_to_raw() { + // "could not compile" sets has_compile_errors but is skipped by the build + // re-scan, so the reused filter yields a success-shaped line. We must not + // trust it — fall back to the raw tail instead of a bogus "compiled". + let input = concat!( + " Compiling foo v0.1.0 (/tmp/foo)\n", + "error: could not compile `foo` (test \"bar\") due to previous error\n", + ); + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 101); + assert!( + !result.contains("crates compiled"), + "must not report a failed test run as compiled: {}", + result + ); + assert!(result.contains("could not compile"), "got: {}", result); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/core/constants.rs b/crates/sparkplane-rtk/upstream/src/core/constants.rs new file mode 100644 index 0000000..e7cc757 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/constants.rs @@ -0,0 +1,38 @@ +pub const RTK_DATA_DIR: &str = "rtk"; +pub const HISTORY_DB: &str = "history.db"; +pub const RECALL_DB: &str = "recall.db"; +pub const CONFIG_TOML: &str = "config.toml"; +pub const FILTERS_TOML: &str = "filters.toml"; +pub const TRUSTED_FILTERS_JSON: &str = "trusted_filters.json"; +pub const DEFAULT_HISTORY_DAYS: i64 = 90; + +/// RTK-only subcommands that should never fall back to raw execution. +/// When adding a new RTK-only subcommand to `Commands`, add its clap name here. +pub const RTK_META_COMMANDS: &[&str] = &[ + "gain", + "discover", + "learn", + "init", + "config", + "proxy", + "recall", + "run", + "hook", + "hook-audit", + "pipe", + "cc-economics", + "verify", + "trust", + "untrust", + "session", + "rewrite", + "telemetry", + "smart", + "deps", + "json", + // `err` and `summary` name no real binary, and since they grew `--shell` + // they have a flag to get wrong. Falling through would try to exec a + // program called `err` instead of reporting the flag error (#4125 review). + "err", + "summary", +]; diff --git a/crates/sparkplane-rtk/upstream/src/core/guard.rs b/crates/sparkplane-rtk/upstream/src/core/guard.rs new file mode 100644 index 0000000..3824d44 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/guard.rs @@ -0,0 +1,71 @@ +//! Never-worse output guard: RTK never emits more tokens than the raw command. +//! +//! Two callers are allowed past it. `rtk diff` prints a one-line message for a +//! difference `str::lines()` cannot render — CRLF against LF, or a missing +//! final newline — where the raw fallback is two blobs that look identical and +//! answer the question worse at any size. The exception is bounded by the case +//! rather than by a token count: it fires only when the bytes differ and the +//! line vectors do not, so the message never competes with a change list. A +//! fixed allowance above raw was tried and could not be met by construction: +//! the shortest form of the message is ~20 tokens and a one-line pair is ~2, so +//! the ceiling sat under the message's own floor and dropped it on 90% of +//! one-line pairs. +//! +//! The other is a clean run of a command whose output format rtk injected +//! (`runner::RunOptions::clean_outputs`): ruff's `[]` answers a question the +//! user never asked, so the filter's summary is shown although it is longer. + +use crate::core::tracking::estimate_tokens; + +/// Returns `filtered`, or `raw` when `filtered` would emit more tokens. +pub fn never_worse<'a>(raw: &'a str, filtered: &'a str) -> &'a str { + if estimate_tokens(filtered) > estimate_tokens(raw) { + raw + } else { + filtered + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_filtered_when_smaller() { + let raw = "a".repeat(400); + assert_eq!(never_worse(&raw, "ok"), "ok"); + } + + #[test] + fn falls_back_to_raw_when_filtered_bigger() { + let raw = "{}"; + let filtered = "{\n \"pretty\": true\n}"; + assert_eq!(never_worse(raw, filtered), raw); + } + + #[test] + fn tie_keeps_filtered() { + assert_eq!(never_worse("abcd", "wxyz"), "wxyz"); + } + + #[test] + fn token_boundary_follows_estimate_tokens() { + assert_eq!(never_worse("abcd", "abcde"), "abcd"); + assert_eq!(never_worse("abcdefgh", "ijklmnop"), "ijklmnop"); + } + + #[test] + fn empty_raw_returns_raw() { + assert_eq!(never_worse("", "0 matches"), ""); + } + + #[test] + fn empty_filtered_returns_filtered() { + assert_eq!(never_worse("data", ""), ""); + } + + #[test] + fn both_empty_returns_filtered() { + assert_eq!(never_worse("", ""), ""); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/core/stream.rs b/crates/sparkplane-rtk/upstream/src/core/stream.rs new file mode 100644 index 0000000..986aa22 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/stream.rs @@ -0,0 +1,1469 @@ +use anyhow::{Context, Result}; +use std::borrow::Cow; +use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; +use std::process::{Command, Stdio}; +use std::sync::mpsc; + +#[cfg(test)] +use regex::Regex; + +/// Read `reader` line by line, decoding each line through the console code +/// page and falling back to lossy UTF-8 (invalid bytes become U+FFFD) instead +/// of erroring. +/// +/// `BufRead::lines()` returns `Err` for a non-UTF-8 line, and callers +/// commonly chain `.map_while(Result::ok)` to skip bad lines — but +/// `map_while` stops at the *first* `None`, so one invalid-UTF-8 line (e.g. +/// OEM/ANSI bytes from a non-English-locale Windows tool) silently discards +/// every line after it too, not just the bad one. This reads raw bytes and +/// never fails on the source encoding, so a garbled line still surfaces +/// instead of vanishing along with everything downstream of it. +/// +/// The OEM/ANSI lines this guards against are exactly what +/// [`decode_process_output`](super::utils::decode_process_output) exists to +/// read, so the streamed path decodes them the same way the captured path +/// does rather than going straight to U+FFFD. +fn read_lines_lossy(reader: impl Read) -> impl Iterator { + BufReader::new(reader).split(b'\n').filter_map(|res| { + let mut buf = match res { + Ok(buf) => buf, + Err(e) => { + eprintln!("rtk: stream read error: {}", e); + return None; + } + }; + if buf.last() == Some(&b'\r') { + buf.pop(); + } + Some(super::utils::decode_process_output(&buf)) + }) +} + +pub trait StreamFilter { + fn feed_line(&mut self, line: &str) -> Option; + fn flush(&mut self) -> String; + fn on_exit(&mut self, _exit_code: i32, _raw: &str) -> Option { + None + } +} + +pub trait BlockHandler { + /// Rewrite a raw line before it is matched or emitted — the place to + /// strip ANSI escapes for tools that colour by default. Identity unless + /// a handler opts in, so the rest of the pipeline sees raw bytes. + fn normalize_line<'a>(&self, line: &'a str) -> Cow<'a, str> { + Cow::Borrowed(line) + } + fn should_skip(&mut self, line: &str) -> bool; + fn is_block_start(&mut self, line: &str) -> bool; + fn is_block_continuation(&mut self, line: &str, block: &[String]) -> bool; + fn format_summary(&self, exit_code: i32, raw: &str) -> Option; +} + +pub struct BlockStreamFilter { + handler: H, + in_block: bool, + current_block: Vec, + blocks_emitted: usize, +} + +impl BlockStreamFilter { + pub fn new(handler: H) -> Self { + Self { + handler, + in_block: false, + current_block: Vec::new(), + blocks_emitted: 0, + } + } + + fn emit_block(&mut self) -> Option { + if self.current_block.is_empty() { + return None; + } + let block = self.current_block.join("\n"); + self.current_block.clear(); + self.blocks_emitted += 1; + Some(format!("{}\n", block)) + } +} + +impl StreamFilter for BlockStreamFilter { + fn feed_line(&mut self, line: &str) -> Option { + let line = self.handler.normalize_line(line); + let line = line.as_ref(); + if self.handler.should_skip(line) { + return None; + } + + if self.handler.is_block_start(line) { + let prev = self.emit_block(); + self.current_block.push(line.to_string()); + self.in_block = true; + prev + } else if self.in_block { + if self + .handler + .is_block_continuation(line, &self.current_block) + { + self.current_block.push(line.to_string()); + None + } else { + self.in_block = false; + self.emit_block() + } + } else { + None + } + } + + fn flush(&mut self) -> String { + self.emit_block().unwrap_or_default() + } + + fn on_exit(&mut self, exit_code: i32, raw: &str) -> Option { + self.handler.format_summary(exit_code, raw) + } +} + +/// Counterpart to [`BlockHandler`] for line-oriented streams. +/// +/// Default behaviour is KEEP — every line is emitted unchanged. Implementors +/// opt in to dropping noise via [`LineHandler::should_skip`] and may capture +/// state for the final summary via [`LineHandler::observe_line`]. +pub trait LineHandler { + fn should_skip(&mut self, _line: &str) -> bool { + false + } + + fn observe_line(&mut self, _line: &str) {} + + fn format_summary(&self, exit_code: i32, raw: &str) -> Option; +} + +pub struct LineStreamFilter { + handler: H, +} + +impl LineStreamFilter { + pub fn new(handler: H) -> Self { + Self { handler } + } +} + +impl StreamFilter for LineStreamFilter { + fn feed_line(&mut self, line: &str) -> Option { + if self.handler.should_skip(line) { + return None; + } + self.handler.observe_line(line); + Some(format!("{}\n", line)) + } + + fn flush(&mut self) -> String { + String::new() + } + + fn on_exit(&mut self, exit_code: i32, raw: &str) -> Option { + self.handler.format_summary(exit_code, raw) + } +} + +#[cfg(test)] // available for command modules; currently used in tests only +pub struct RegexBlockFilter { + start_re: Regex, + skip_prefixes: Vec, + tool_name: String, + block_count: usize, +} + +#[cfg(test)] +impl RegexBlockFilter { + pub fn new(tool_name: &str, start_pattern: &str) -> Self { + Self { + start_re: Regex::new(start_pattern).unwrap_or_else(|e| { + panic!("RegexBlockFilter: bad pattern '{}': {}", start_pattern, e) + }), + skip_prefixes: Vec::new(), + tool_name: tool_name.to_string(), + block_count: 0, + } + } + + pub fn skip_prefix(mut self, prefix: &str) -> Self { + self.skip_prefixes.push(prefix.to_string()); + self + } + + pub fn skip_prefixes(mut self, prefixes: &[&str]) -> Self { + self.skip_prefixes + .extend(prefixes.iter().map(|s| s.to_string())); + self + } +} + +#[cfg(test)] +impl BlockHandler for RegexBlockFilter { + fn should_skip(&mut self, line: &str) -> bool { + self.skip_prefixes.iter().any(|p| line.starts_with(p)) + } + + fn is_block_start(&mut self, line: &str) -> bool { + if self.start_re.is_match(line) { + self.block_count += 1; + true + } else { + false + } + } + + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + line.starts_with(' ') || line.starts_with('\t') + } + + fn format_summary(&self, _exit_code: i32, _raw: &str) -> Option { + if self.block_count == 0 { + Some(format!("{}: no errors found\n", self.tool_name)) + } else { + Some(format!( + "{}: {} blocks in output\n", + self.tool_name, self.block_count + )) + } + } +} + +pub trait StdinFilter: Send { + fn feed_line(&mut self, line: &str) -> Option; + fn flush(&mut self) -> String; +} + +pub enum FilterMode<'a> { + Streaming(Box), + #[allow(dead_code)] + Buffered(Box String + 'a>), + CaptureOnly, + Passthrough, +} + +pub enum StdinMode { + Inherit, + #[allow(dead_code)] // future API: stdin filtering for interactive commands + Filter(Box), + Null, +} + +pub struct StreamResult { + pub exit_code: i32, + pub raw: String, + pub raw_stdout: String, + pub raw_stderr: String, + pub filtered: String, +} + +impl StreamResult { + #[cfg(test)] + pub fn success(&self) -> bool { + self.exit_code == 0 + } +} + +// #2375 +#[cfg(unix)] +mod signal_relay { + use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; + use std::thread; + use std::time::Duration; + + const POLL: Duration = Duration::from_millis(25); + const KILL_GRACE: Duration = Duration::from_millis(750); + const EXIT_GRACE: Duration = Duration::from_millis(750); + + static CHILD_PID: AtomicU32 = AtomicU32::new(0); + static RELAYED: AtomicI32 = AtomicI32::new(0); + static FINISHED: AtomicBool = AtomicBool::new(false); + + #[allow(unsafe_code)] + unsafe extern "C" fn relay(sig: libc::c_int) { + let pid = CHILD_PID.load(Ordering::SeqCst); + if pid == 0 || RELAYED.swap(sig, Ordering::SeqCst) != 0 { + // nosemgrep: unsafe-block + unsafe { + libc::signal(sig, libc::SIG_DFL); + libc::raise(sig); + } + return; + } + // nosemgrep: unsafe-block + unsafe { + libc::kill(pid as libc::pid_t, sig); + } + } + + fn escalate(pid: u32) { + thread::sleep(KILL_GRACE); + if FINISHED.load(Ordering::SeqCst) { + return; + } + #[allow(unsafe_code)] + // nosemgrep: unsafe-block + unsafe { + libc::kill(pid as libc::pid_t, libc::SIGKILL); + } + thread::sleep(EXIT_GRACE); + if FINISHED.load(Ordering::SeqCst) { + return; + } + let sig = RELAYED.load(Ordering::SeqCst); + #[allow(unsafe_code)] + // nosemgrep: unsafe-block + unsafe { + libc::signal(sig, libc::SIG_DFL); + libc::raise(sig); + } + } + + pub fn relayed() -> Option { + match RELAYED.load(Ordering::SeqCst) { + 0 => None, + sig => Some(sig), + } + } + + pub struct Relay; + + impl Relay { + pub fn install(pid: u32) -> Self { + CHILD_PID.store(pid, Ordering::SeqCst); + RELAYED.store(0, Ordering::SeqCst); + FINISHED.store(false, Ordering::SeqCst); + #[allow(unsafe_code)] + // nosemgrep: unsafe-block + unsafe { + for sig in [libc::SIGINT, libc::SIGTERM] { + let previous = libc::signal(sig, relay as *const () as libc::sighandler_t); + if previous == libc::SIG_IGN { + libc::signal(sig, libc::SIG_IGN); + } + } + } + thread::spawn(move || { + while !FINISHED.load(Ordering::SeqCst) { + if RELAYED.load(Ordering::SeqCst) != 0 { + escalate(pid); + return; + } + thread::sleep(POLL); + } + }); + Relay + } + } + + impl Drop for Relay { + fn drop(&mut self) { + FINISHED.store(true, Ordering::SeqCst); + CHILD_PID.store(0, Ordering::SeqCst); + #[allow(unsafe_code)] + // nosemgrep: unsafe-block + unsafe { + for sig in [libc::SIGINT, libc::SIGTERM] { + if libc::signal(sig, libc::SIG_DFL) == libc::SIG_IGN { + libc::signal(sig, libc::SIG_IGN); + } + } + } + } + } +} + +#[cfg(not(unix))] +mod signal_relay { + pub struct Relay; + + impl Relay { + pub fn install(_pid: u32) -> Self { + Relay + } + } +} + +// #2375 +#[cfg(unix)] +pub fn die_by_relayed_signal() { + let Some(sig) = signal_relay::relayed() else { + return; + }; + #[allow(unsafe_code)] + // nosemgrep: unsafe-block + unsafe { + libc::signal(sig, libc::SIG_DFL); + libc::raise(sig); + } +} + +#[cfg(not(unix))] +pub fn die_by_relayed_signal() {} + +pub fn status_to_exit_code(status: std::process::ExitStatus) -> i32 { + if let Some(code) = status.code() { + return code; + } + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if let Some(sig) = status.signal() { + return 128 + sig; + } + } + 1 +} + +// ISSUE #897: ChildGuard RAII prevents zombie processes that caused kernel panic +pub const RAW_CAP: usize = 10_485_760; // 10 MiB + +pub fn run_streaming( + cmd: &mut Command, + stdin_mode: StdinMode, + stdout_mode: FilterMode<'_>, +) -> Result { + if matches!(stdout_mode, FilterMode::Passthrough) { + match &stdin_mode { + StdinMode::Inherit => { + cmd.stdin(Stdio::inherit()); + } + _ => { + cmd.stdin(Stdio::null()); + } + }; + cmd.stdout(Stdio::inherit()); + cmd.stderr(Stdio::inherit()); + let status = cmd.status().context("Failed to spawn process")?; + return Ok(StreamResult { + exit_code: status_to_exit_code(status), + raw: String::new(), + raw_stdout: String::new(), + raw_stderr: String::new(), + filtered: String::new(), + }); + } + + match &stdin_mode { + StdinMode::Inherit => { + cmd.stdin(Stdio::inherit()); + } + StdinMode::Filter(_) | StdinMode::Null => { + cmd.stdin(Stdio::piped()); + } + } + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + struct ChildGuard(std::process::Child); + impl Drop for ChildGuard { + fn drop(&mut self) { + self.0.wait().ok(); + } + } + + let is_streaming = matches!(stdout_mode, FilterMode::Streaming(_)); + + let mut child = ChildGuard(cmd.spawn().context("Failed to spawn process")?); + let _signal_relay = signal_relay::Relay::install(child.0.id()); + + let stdin_thread: Option> = match stdin_mode { + StdinMode::Filter(mut filter) => { + let child_stdin = child.0.stdin.take().context("No child stdin handle")?; + Some(std::thread::spawn(move || { + let mut writer = BufWriter::new(child_stdin); + let stdin_handle = io::stdin(); + for line in read_lines_lossy(stdin_handle.lock()) { + if let Some(out) = filter.feed_line(&line) + && writeln!(writer, "{}", out).is_err() + { + break; + } + } + let tail = filter.flush(); + if !tail.is_empty() { + write!(writer, "{}", tail).ok(); + } + })) + } + StdinMode::Null => { + child.0.stdin.take(); + None + } + StdinMode::Inherit => None, + }; + + let stdout = child.0.stdout.take().context("No child stdout handle")?; + let stderr = child.0.stderr.take().context("No child stderr handle")?; + let mut raw_stdout = String::new(); + let mut raw_stderr = String::new(); + let mut filtered = String::new(); + let mut capped_out = false; + let mut capped_err = false; + let mut saved_filter: Option> = None; + let mut filter_fd_is_stderr = false; + + if is_streaming { + enum StreamLine { + Stdout(String), + Stderr(String), + } + + let (tx, rx) = mpsc::channel(); + let tx_out = tx.clone(); + let stdout_thread = std::thread::spawn(move || { + for line in read_lines_lossy(stdout) { + if tx_out.send(StreamLine::Stdout(line)).is_err() { + break; + } + } + }); + let tx_err = tx; + let stderr_thread = std::thread::spawn(move || { + for line in read_lines_lossy(stderr) { + if tx_err.send(StreamLine::Stderr(line)).is_err() { + break; + } + } + }); + + if let FilterMode::Streaming(mut filter) = stdout_mode { + let stdout_handle = io::stdout(); + let mut out = stdout_handle.lock(); + let stderr_handle = io::stderr(); + let mut err_out = stderr_handle.lock(); + + for msg in rx { + let (line, is_stderr) = match msg { + StreamLine::Stderr(l) => (l, true), + StreamLine::Stdout(l) => (l, false), + }; + if is_stderr { + if !capped_err { + if raw_stderr.len() + line.len() < RAW_CAP { + raw_stderr.push_str(&line); + raw_stderr.push('\n'); + } else { + capped_err = true; + eprintln!("[rtk] warning: stderr exceeds 10 MiB — capture truncated"); + } + } + } else if !capped_out { + if raw_stdout.len() + line.len() < RAW_CAP { + raw_stdout.push_str(&line); + raw_stdout.push('\n'); + } else { + capped_out = true; + eprintln!("[rtk] warning: stdout exceeds 10 MiB — filter input truncated"); + } + } + filter_fd_is_stderr = is_stderr; + if let Some(output) = filter.feed_line(&line) { + filtered.push_str(&output); + let dest: &mut dyn Write = if is_stderr { &mut err_out } else { &mut out }; + match write!(dest, "{}", output) { + Err(e) if e.kind() == io::ErrorKind::BrokenPipe => break, + Err(e) => return Err(e.into()), + Ok(_) => {} + } + } + } + let tail = filter.flush(); + filtered.push_str(&tail); + let flush_dest: &mut dyn Write = if filter_fd_is_stderr { + &mut err_out + } else { + &mut out + }; + match write!(flush_dest, "{}", tail) { + Err(e) if e.kind() == io::ErrorKind::BrokenPipe => {} + Err(e) => return Err(e.into()), + Ok(_) => {} + } + saved_filter = Some(filter); + } + + stdout_thread.join().ok(); + stderr_thread.join().ok(); + } else { + let stderr_thread = std::thread::spawn(move || -> String { + let mut raw_err = String::new(); + let mut capped = false; + for line in read_lines_lossy(stderr) { + if raw_err.len() + line.len() < RAW_CAP { + raw_err.push_str(&line); + raw_err.push('\n'); + } else if !capped { + capped = true; + } + } + raw_err + }); + + { + let stdout_handle = io::stdout(); + let mut out = stdout_handle.lock(); + + match stdout_mode { + FilterMode::Passthrough => unreachable!("handled by early-return above"), + FilterMode::Streaming(_) => unreachable!("handled by is_streaming branch"), + FilterMode::Buffered(filter_fn) => { + for line in read_lines_lossy(stdout) { + if raw_stdout.len() + line.len() < RAW_CAP { + raw_stdout.push_str(&line); + raw_stdout.push('\n'); + } else if !capped_out { + capped_out = true; + eprintln!( + "[rtk] warning: output exceeds 10 MiB — filter input truncated" + ); + } + } + filtered = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + filter_fn(&raw_stdout) + })) + .unwrap_or_else(|_| { + eprintln!("[rtk] warning: filter panicked — passing through raw output"); + raw_stdout.clone() + }); + match write!(out, "{}", filtered) { + Err(e) if e.kind() == io::ErrorKind::BrokenPipe => {} + Err(e) => return Err(e.into()), + Ok(_) => {} + } + } + FilterMode::CaptureOnly => { + for line in read_lines_lossy(stdout) { + if raw_stdout.len() + line.len() < RAW_CAP { + raw_stdout.push_str(&line); + raw_stdout.push('\n'); + } else if !capped_out { + capped_out = true; + eprintln!( + "[rtk] warning: output exceeds 10 MiB — filter input truncated" + ); + } + } + filtered = raw_stdout.clone(); + } + } + } + + raw_stderr = stderr_thread.join().unwrap_or_else(|e| { + eprintln!("[rtk] warning: stderr reader thread panicked: {:?}", e); + String::new() + }); + } + if let Some(t) = stdin_thread { + t.join().ok(); + } + + let status = child.0.wait().context("Failed to wait for child")?; + let exit_code = status_to_exit_code(status); + let raw = format!("{}{}", raw_stdout, raw_stderr); + + if let Some(mut f) = saved_filter + && let Some(post) = f.on_exit(exit_code, &raw) + { + filtered.push_str(&post); + let mut dest: Box = if filter_fd_is_stderr { + Box::new(io::stderr().lock()) + } else { + Box::new(io::stdout().lock()) + }; + match write!(dest, "{}", post) { + Err(e) if e.kind() == io::ErrorKind::BrokenPipe => {} + Err(e) => return Err(e.into()), + Ok(_) => {} + } + } + + Ok(StreamResult { + exit_code, + raw, + raw_stdout, + raw_stderr, + filtered, + }) +} + +pub struct CaptureResult { + pub stdout: String, + pub stderr: String, + pub exit_code: i32, +} + +impl CaptureResult { + pub fn success(&self) -> bool { + self.exit_code == 0 + } + + pub fn combined(&self) -> String { + format!("{}{}", self.stdout, self.stderr) + } +} + +pub fn exec_capture(cmd: &mut Command) -> Result { + cmd.stdin(Stdio::null()); + capture(cmd) +} + +/// Like [`exec_capture`] but inherits stdin so a wrapped engine can read a piped stdin. +pub fn exec_capture_stdin(cmd: &mut Command) -> Result { + cmd.stdin(Stdio::inherit()); + capture(cmd) +} + +/// Run `cmd` to completion, decode what it wrote, and report the exit code. +/// +/// A process killed by a signal has no exit code of its own, and returning +/// only the synthesized `128 + signal` hides why it died. `exit_code_from_output` +/// announces that on stderr, so callers moving here from a hand-rolled +/// `.output()` keep the diagnostic instead of losing it. The program name is +/// used as the label so no call site has to pass one. +fn capture(cmd: &mut Command) -> Result { + let raw = capture_raw(cmd)?; + Ok(CaptureResult { + stdout: super::utils::decode_process_output(&raw.stdout), + stderr: super::utils::decode_process_output(&raw.stderr), + exit_code: raw.exit_code, + }) +} + +/// Run `cmd` to completion and return its raw bytes plus a signal-aware exit code. +/// The single spot that turns an `ExitStatus` into a code via +/// [`exit_code_from_output`](super::utils::exit_code_from_output) — so the +/// `process terminated by signal N` diagnostic is emitted uniformly whether the +/// caller decodes the bytes ([`capture`]) or keeps them raw ([`exec_capture_bytes`]), +/// instead of the raw path silently dropping it. +fn capture_raw(cmd: &mut Command) -> Result { + let program = cmd.get_program().to_string_lossy().into_owned(); + let output = cmd.output().context("Failed to execute command")?; + let exit_code = super::utils::exit_code_from_output(&output, &program); + Ok(CaptureBytes { + stdout: output.stdout, + stderr: output.stderr, + exit_code, + }) +} + +/// Raw-byte capture result, for callers that must control decoding themselves — +/// e.g. non-UTF-8 output that [`exec_capture`]'s `from_utf8_lossy` would corrupt. +pub struct CaptureBytes { + pub stdout: Vec, + pub stderr: Vec, + pub exit_code: i32, +} + +impl CaptureBytes { + pub fn success(&self) -> bool { + self.exit_code == 0 + } +} + +/// Like [`exec_capture`] but returns raw bytes so the caller decides how to decode. +pub fn exec_capture_bytes(cmd: &mut Command) -> Result { + cmd.stdin(Stdio::null()); + capture_raw(cmd) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::process::Command; + + #[test] + fn test_read_lines_lossy_preserves_lines_after_invalid_utf8() { + // Line 2 contains a lone 0xE3 byte (the cp850 case from the bug report). + // `BufRead::lines().map_while(Result::ok)` would stop at that Err and + // silently lose line 3 as well — not just the bad byte. + let mut data = Vec::new(); + data.extend_from_slice(b"ERRO A: ascii\n"); + data.extend_from_slice(&[b'E', b'R', b'R', b'O', b' ', b'B', b':', b' ', 0xE3, b'\n']); + data.extend_from_slice(b"ERRO C: ascii again\n"); + + let lines: Vec = read_lines_lossy(data.as_slice()).collect(); + assert_eq!(lines.len(), 3, "got: {:?}", lines); + assert_eq!(lines[0], "ERRO A: ascii"); + assert!(lines[1].starts_with("ERRO B: "), "got: {:?}", lines[1]); + assert!(lines[1].contains('\u{FFFD}'), "got: {:?}", lines[1]); + assert_eq!(lines[2], "ERRO C: ascii again"); + } + + #[test] + fn test_read_lines_lossy_strips_crlf() { + let lines: Vec = read_lines_lossy(&b"a\r\nb\n"[..]).collect(); + assert_eq!(lines, vec!["a".to_string(), "b".to_string()]); + } + + #[test] + fn test_read_lines_lossy_no_trailing_newline() { + let lines: Vec = read_lines_lossy(&b"only line"[..]).collect(); + assert_eq!(lines, vec!["only line".to_string()]); + } + + #[test] + fn test_read_lines_lossy_empty_input() { + let lines: Vec = read_lines_lossy(&b""[..]).collect(); + assert!(lines.is_empty()); + } + + /// A `Read` that yields some good lines, then a genuine I/O error -- + /// distinct from clean EOF (`Ok(0)`). Before this fix, `Ok(0) | Err(_)` + /// treated both the same way, silently truncating output on a real read + /// failure instead of surfacing it. + struct FailingReader { + data: std::io::Cursor>, + failed: bool, + } + + impl Read for FailingReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.data.position() as usize >= self.data.get_ref().len() { + if !self.failed { + self.failed = true; + return Err(io::Error::other("simulated read failure")); + } + return Ok(0); + } + std::io::Read::read(&mut self.data, buf) + } + } + + #[test] + fn test_read_lines_lossy_stops_on_io_error_without_panicking() { + let reader = FailingReader { + data: std::io::Cursor::new(b"line one\nline two\n".to_vec()), + failed: false, + }; + // The two good lines are still yielded; the simulated failure after + // them must not panic or hang -- it just ends the iterator, same as + // clean EOF would, but via the Err(_) arm instead of Ok(0). + let lines: Vec = read_lines_lossy(reader).collect(); + assert_eq!(lines, vec!["line one".to_string(), "line two".to_string()]); + } + + struct LineFilter Option> { + f: F, + } + + impl Option> LineFilter { + pub fn new(f: F) -> Self { + Self { f } + } + } + + impl Option> StreamFilter for LineFilter { + fn feed_line(&mut self, line: &str) -> Option { + (self.f)(line) + } + + fn flush(&mut self) -> String { + String::new() + } + } + + #[test] + fn test_exit_code_zero() { + let status = Command::new("true").status().unwrap(); + assert_eq!(status_to_exit_code(status), 0); + } + + #[test] + fn test_exit_code_nonzero() { + let status = Command::new("false").status().unwrap(); + assert_eq!(status_to_exit_code(status), 1); + } + + #[cfg(unix)] + #[test] + fn test_exit_code_signal_kill() { + let mut child = Command::new("sleep").arg("60").spawn().unwrap(); + child.kill().unwrap(); + let status = child.wait().unwrap(); + assert_eq!(status_to_exit_code(status), 137); + } + + #[test] + fn test_exec_capture_decodes_and_reports_exit_code() { + let captured = exec_capture(&mut Command::new("false")).expect("spawn"); + assert_eq!(captured.exit_code, 1); + assert!(!captured.success()); + } + + /// A signal-killed child keeps the `128 + signal` code that + /// `exit_code_from_output` reports, so callers that moved off a hand-rolled + /// `.output()` neither lose the code nor the stderr diagnostic with it. + #[cfg(unix)] + #[test] + fn test_exec_capture_reports_signal_exit_code() { + // `kill -TERM $$` makes the shell terminate itself by signal 15. + let mut cmd = Command::new("sh"); + cmd.arg("-c").arg("kill -TERM $$"); + let captured = exec_capture(&mut cmd).expect("spawn"); + assert_eq!(captured.exit_code, 128 + 15); + } + + #[test] + fn test_line_filter_passes_lines() { + let mut f = LineFilter::new(|l| Some(format!("{}\n", l.to_uppercase()))); + assert_eq!(f.feed_line("hello"), Some("HELLO\n".to_string())); + } + + #[test] + fn test_line_filter_drops_lines() { + let mut f = LineFilter::new(|l| { + if l.starts_with('#') { + None + } else { + Some(l.to_string()) + } + }); + assert_eq!(f.feed_line("# comment"), None); + assert_eq!(f.feed_line("code"), Some("code".to_string())); + } + + #[test] + fn test_line_filter_flush_empty() { + let mut f = LineFilter::new(|l| Some(l.to_string())); + assert_eq!(f.flush(), String::new()); + } + + #[test] + fn test_stream_result_success() { + let r = StreamResult { + exit_code: 0, + raw: String::new(), + raw_stdout: String::new(), + raw_stderr: String::new(), + filtered: String::new(), + }; + assert!(r.success()); + } + + #[test] + fn test_stream_result_failure() { + let r = StreamResult { + exit_code: 1, + raw: String::new(), + raw_stdout: String::new(), + raw_stderr: String::new(), + filtered: String::new(), + }; + assert!(!r.success()); + } + + #[test] + fn test_stream_result_signal_not_success() { + let r = StreamResult { + exit_code: 137, + raw: String::new(), + raw_stdout: String::new(), + raw_stderr: String::new(), + filtered: String::new(), + }; + assert!(!r.success()); + } + + #[test] + fn test_run_streaming_passthrough_echo() { + let mut cmd = Command::new("echo"); + cmd.arg("hello"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::Passthrough).unwrap(); + assert_eq!(result.exit_code, 0); + // Passthrough inherits TTY — raw/filtered are empty + assert!(result.raw.is_empty()); + } + + #[test] + fn test_run_streaming_exit_code_preserved() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + cmd.args(["-c", "exit 42"]); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::Passthrough).unwrap(); + assert_eq!(result.exit_code, 42); + } + + #[test] + fn test_run_streaming_exit_code_zero() { + let mut cmd = Command::new("true"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::Passthrough).unwrap(); + assert_eq!(result.exit_code, 0); + assert!(result.success()); + } + + #[test] + fn test_run_streaming_exit_code_one() { + let mut cmd = Command::new("false"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::Passthrough).unwrap(); + assert_eq!(result.exit_code, 1); + assert!(!result.success()); + } + + #[cfg(not(windows))] + #[test] + fn test_run_streaming_streaming_filter_drops_lines() { + let mut cmd = Command::new("printf"); + cmd.arg("a\nb\nc\n"); + let filter = LineFilter::new(|l| { + if l == "b" { + None + } else { + Some(format!("{}\n", l)) + } + }); + let result = run_streaming( + &mut cmd, + StdinMode::Null, + FilterMode::Streaming(Box::new(filter)), + ) + .unwrap(); + assert!(result.filtered.contains('a')); + assert!(!result.filtered.contains('b')); + assert!(result.filtered.contains('c')); + assert_eq!(result.exit_code, 0); + } + + #[cfg(not(windows))] + #[test] + fn test_run_streaming_buffered_filter() { + let mut cmd = Command::new("printf"); + cmd.arg("line1\nline2\nline3\n"); + let result = run_streaming( + &mut cmd, + StdinMode::Null, + FilterMode::Buffered(Box::new(|s: &str| s.to_uppercase())), + ) + .unwrap(); + assert!(result.filtered.contains("LINE1")); + assert!(result.filtered.contains("LINE2")); + assert_eq!(result.exit_code, 0); + } + + #[test] + fn test_run_streaming_raw_cap_at_10mb() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + // ~11 MiB of 80-char lines (fast: fewer lines than `yes | head -6M`) + cmd.args([ + "-c", + "dd if=/dev/zero bs=1024 count=11264 2>/dev/null | tr '\\0' 'a' | fold -w 80", + ]); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::CaptureOnly).unwrap(); + assert!( + result.raw.len() <= 10_485_760 + 100, + "raw should be capped at ~10 MiB, got {} bytes", + result.raw.len() + ); + assert!( + result.raw.len() > 1_000_000, + "Should have captured significant data" + ); + } + + #[test] + fn test_run_streaming_stderr_cap_at_10mb() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + // ~11 MiB on stderr, nothing on stdout + cmd.args([ + "-c", + "dd if=/dev/zero bs=1024 count=11264 2>/dev/null | tr '\\0' 'a' | fold -w 80 1>&2", + ]); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::CaptureOnly).unwrap(); + // raw = raw_stdout + raw_stderr; stdout is empty so raw ≈ stderr size + assert!( + result.raw.len() <= RAW_CAP + 200, + "stderr in raw should be capped at ~10 MiB, got {} bytes", + result.raw.len() + ); + } + + #[test] + fn test_child_guard_prevents_zombie() { + let mut cmd = Command::new("true"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::CaptureOnly); + assert!(result.is_ok()); + assert_eq!(result.unwrap().exit_code, 0); + } + + #[test] + fn test_run_streaming_null_stdin_cat() { + let mut cmd = Command::new("cat"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::Passthrough).unwrap(); + assert_eq!(result.exit_code, 0); + } + + #[test] + fn test_run_streaming_raw_contains_stdout() { + let mut cmd = Command::new("echo"); + cmd.arg("test_output_xyz"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::CaptureOnly).unwrap(); + assert!(result.raw.contains("test_output_xyz")); + } + + #[test] + fn test_run_streaming_capture_only_filtered_equals_raw() { + let mut cmd = Command::new("echo"); + cmd.arg("check_equality"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::CaptureOnly).unwrap(); + assert_eq!(result.filtered.trim(), result.raw_stdout.trim()); + } + + #[test] + fn test_exec_capture_success() { + let mut cmd = Command::new("echo"); + cmd.arg("hello_capture"); + let result = exec_capture(&mut cmd).unwrap(); + assert!(result.success()); + assert_eq!(result.exit_code, 0); + assert!(result.stdout.contains("hello_capture")); + } + + #[test] + fn test_exec_capture_failure() { + let mut cmd = Command::new("false"); + let result = exec_capture(&mut cmd).unwrap(); + assert!(!result.success()); + assert_eq!(result.exit_code, 1); + } + + #[test] + fn test_exec_capture_stderr() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo err_msg >&2"]); + let result = exec_capture(&mut cmd).unwrap(); + assert!(result.stderr.contains("err_msg")); + } + + #[test] + fn test_exec_capture_combined() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo out_msg; echo err_msg >&2"]); + let result = exec_capture(&mut cmd).unwrap(); + let combined = result.combined(); + assert!(combined.contains("out_msg")); + assert!(combined.contains("err_msg")); + } + + #[test] + fn test_capture_result_combined_empty() { + let r = CaptureResult { + stdout: String::new(), + stderr: String::new(), + exit_code: 0, + }; + assert_eq!(r.combined(), ""); + } + + pub fn run_block_filter(filter: &mut dyn StreamFilter, input: &str, exit_code: i32) -> String { + let mut output = String::new(); + for line in input.lines() { + if let Some(s) = filter.feed_line(line) { + output.push_str(&s); + } + } + output.push_str(&filter.flush()); + if let Some(post) = filter.on_exit(exit_code, input) { + output.push_str(&post); + } + output + } + + struct UpperHandler; + + impl BlockHandler for UpperHandler { + fn normalize_line<'a>(&self, line: &'a str) -> Cow<'a, str> { + Cow::Owned(line.to_uppercase()) + } + fn should_skip(&mut self, _line: &str) -> bool { + false + } + fn is_block_start(&mut self, line: &str) -> bool { + line.starts_with("ERR") + } + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + line.starts_with(" ") + } + fn format_summary(&self, _exit_code: i32, _raw: &str) -> Option { + None + } + } + + #[test] + fn block_handler_normalize_line_feeds_matching_and_emission() { + // Both the match and the emitted block see the normalized line. + let mut f = BlockStreamFilter::new(UpperHandler); + let out = run_block_filter(&mut f, "err: one\n detail\nnoise\n", 0); + assert_eq!(out, "ERR: ONE\n DETAIL\n"); + } + + struct TestHandler; + + impl BlockHandler for TestHandler { + fn should_skip(&mut self, line: &str) -> bool { + line.starts_with("SKIP") + } + fn is_block_start(&mut self, line: &str) -> bool { + line.starts_with("ERROR") + } + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + line.starts_with(" ") + } + fn format_summary(&self, _exit_code: i32, _raw: &str) -> Option { + Some("DONE\n".to_string()) + } + } + + #[test] + fn test_block_filter_emits_blocks() { + let mut f = BlockStreamFilter::new(TestHandler); + let input = "SKIP noise\nERROR first\n detail1\nnon-block\nERROR second\n detail2\n"; + let result = run_block_filter(&mut f, input, 0); + assert!(result.contains("ERROR first\n detail1"), "got: {}", result); + assert!( + result.contains("ERROR second\n detail2"), + "got: {}", + result + ); + assert!(!result.contains("SKIP"), "got: {}", result); + assert!(result.ends_with("DONE\n"), "got: {}", result); + } + + #[test] + fn test_block_filter_no_blocks() { + let mut f = BlockStreamFilter::new(TestHandler); + let result = run_block_filter(&mut f, "nothing here\njust text\n", 0); + assert_eq!(result, "DONE\n"); + } + + #[test] + fn test_regex_block_filter_emits_blocks() { + let handler = RegexBlockFilter::new("test", r"^error\["); + let mut f = BlockStreamFilter::new(handler); + let input = "ok line\nerror[E0308]: mismatched types\n expected `u32`\nok again\nerror[E0599]: no method\n help: try\n"; + let result = run_block_filter(&mut f, input, 1); + assert!( + result.contains("error[E0308]: mismatched types\n expected `u32`"), + "got: {}", + result + ); + assert!( + result.contains("error[E0599]: no method\n help: try"), + "got: {}", + result + ); + assert!( + result.contains("test: 2 blocks in output"), + "got: {}", + result + ); + } + + #[test] + fn test_regex_block_filter_skip_prefix() { + let handler = RegexBlockFilter::new("test", r"^error").skip_prefix("warning:"); + let mut f = BlockStreamFilter::new(handler); + let input = "warning: unused var\nerror: bad type\n detail\nwarning: dead code\n"; + let result = run_block_filter(&mut f, input, 1); + assert!(result.contains("error: bad type"), "got: {}", result); + assert!(!result.contains("warning:"), "got: {}", result); + } + + #[test] + fn test_regex_block_filter_no_blocks() { + let handler = RegexBlockFilter::new("mytest", r"^FAIL"); + let mut f = BlockStreamFilter::new(handler); + let result = run_block_filter(&mut f, "all passed\nok\n", 0); + assert_eq!(result, "mytest: no errors found\n"); + } + + #[test] + fn test_regex_block_filter_indent_continuation() { + let handler = RegexBlockFilter::new("test", r"^ERR"); + let mut f = BlockStreamFilter::new(handler); + let input = "ERR space indent\n two spaces\n\ttab indent\nnon-indent\n"; + let result = run_block_filter(&mut f, input, 1); + assert!( + result.contains("ERR space indent\n two spaces\n\ttab indent"), + "got: {}", + result + ); + assert!(!result.contains("non-indent"), "got: {}", result); + } + + #[test] + fn test_regex_block_filter_multiple_skip_prefixes() { + let handler = + RegexBlockFilter::new("test", r"^error").skip_prefixes(&["note:", "warning:", "help:"]); + let mut f = BlockStreamFilter::new(handler); + let input = "note: see docs\nwarning: unused\nhelp: try this\nerror: fatal\n details\n"; + let result = run_block_filter(&mut f, input, 1); + assert!(!result.contains("note:"), "got: {}", result); + assert!(!result.contains("warning:"), "got: {}", result); + assert!(!result.contains("help:"), "got: {}", result); + assert!( + result.contains("error: fatal\n details"), + "got: {}", + result + ); + } + + #[cfg(not(windows))] + #[test] + fn test_streaming_filters_both_fds_and_routes_to_correct_fd() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo 'error[E0308]: type mismatch'; echo ' Compiling foo v1.0' >&2; echo ' Downloading bar v2.0' >&2; echo ' Finished dev' >&2; echo 'real error on stderr' >&2"]); + + struct CargoLikeHandler; + impl BlockHandler for CargoLikeHandler { + fn should_skip(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + trimmed.starts_with("Compiling") + || trimmed.starts_with("Downloading") + || trimmed.starts_with("Finished") + } + fn is_block_start(&mut self, line: &str) -> bool { + line.starts_with("error") + } + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + line.starts_with(' ') + } + fn format_summary(&self, _: i32, _: &str) -> Option { + None + } + } + + let filter = BlockStreamFilter::new(CargoLikeHandler); + let result = run_streaming( + &mut cmd, + StdinMode::Null, + FilterMode::Streaming(Box::new(filter)), + ) + .unwrap(); + + assert!( + result.filtered.contains("error[E0308]"), + "filtered should contain stdout errors, got: {}", + result.filtered + ); + assert!( + !result.filtered.contains("Compiling"), + "cargo noise should be filtered out, got: {}", + result.filtered + ); + assert!( + !result.filtered.contains("Downloading"), + "cargo noise should be filtered out, got: {}", + result.filtered + ); + assert!( + result.raw_stderr.contains("Compiling"), + "raw_stderr should capture all stderr lines" + ); + assert!( + result.raw_stderr.contains("real error on stderr"), + "raw_stderr should capture all stderr lines" + ); + } + + struct CountingLineHandler { + observed: Vec, + skip_prefixes: Vec, + summary_tag: &'static str, + } + + impl LineHandler for CountingLineHandler { + fn should_skip(&mut self, line: &str) -> bool { + self.skip_prefixes.iter().any(|p| line.starts_with(p)) + } + + fn observe_line(&mut self, line: &str) { + self.observed.push(line.to_string()); + } + + fn format_summary(&self, exit_code: i32, _raw: &str) -> Option { + Some(format!( + "{}: {} kept, exit={}\n", + self.summary_tag, + self.observed.len(), + exit_code + )) + } + } + + fn run_line_filter(filter: &mut dyn StreamFilter, input: &str, exit_code: i32) -> String { + let mut out = String::new(); + for line in input.lines() { + if let Some(s) = filter.feed_line(line) { + out.push_str(&s); + } + } + out.push_str(&filter.flush()); + if let Some(post) = filter.on_exit(exit_code, input) { + out.push_str(&post); + } + out + } + + #[test] + fn test_line_filter_defaults_keep_all() { + struct DefaultHandler; + impl LineHandler for DefaultHandler { + fn format_summary(&self, _: i32, _: &str) -> Option { + None + } + } + let mut f = LineStreamFilter::new(DefaultHandler); + let result = run_line_filter(&mut f, "a\nb\nc\n", 0); + assert_eq!(result, "a\nb\nc\n"); + } + + #[test] + fn test_line_filter_skip_drops_matching_lines() { + let handler = CountingLineHandler { + observed: Vec::new(), + skip_prefixes: vec!["NOISE:".to_string()], + summary_tag: "demo", + }; + let mut f = LineStreamFilter::new(handler); + let input = "NOISE: progress 10%\nkeep me\nNOISE: progress 90%\nalso keep\n"; + let result = run_line_filter(&mut f, input, 0); + assert!(!result.contains("NOISE:"), "got: {}", result); + assert!(result.contains("keep me\n")); + assert!(result.contains("also keep\n")); + assert!(result.contains("demo: 2 kept, exit=0\n")); + } + + #[test] + fn test_line_filter_summary_propagates_exit_code() { + let handler = CountingLineHandler { + observed: Vec::new(), + skip_prefixes: Vec::new(), + summary_tag: "demo", + }; + let mut f = LineStreamFilter::new(handler); + let result = run_line_filter(&mut f, "one\n", 42); + assert!(result.contains("exit=42"), "got: {}", result); + } + + #[test] + fn test_line_filter_observe_only_called_for_kept_lines() { + let handler = CountingLineHandler { + observed: Vec::new(), + skip_prefixes: vec!["DROP".to_string()], + summary_tag: "demo", + }; + let mut f = LineStreamFilter::new(handler); + let result = run_line_filter(&mut f, "DROP a\nDROP b\nkeep\n", 0); + // Only "keep" was observed, so summary says "1 kept" + assert!(result.contains("demo: 1 kept"), "got: {}", result); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/core/toml_filter.rs b/crates/sparkplane-rtk/upstream/src/core/toml_filter.rs new file mode 100644 index 0000000..6850b59 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/toml_filter.rs @@ -0,0 +1,2229 @@ +//! Applies TOML-defined filter rules to command output. +/// +/// Provides a declarative pipeline of 8 stages that can be configured +/// via TOML files. Lookup priority (first match wins): +/// 1. `.rtk/filters.toml` — project-local, committable with the repo +/// 2. `~/.config/rtk/filters.toml` — user-global, applies to all projects +/// 3. Built-in TOML — `src/filters/*.toml`, concatenated by build.rs and embedded at compile time +/// 4. Passthrough — no match, handled by caller +/// +/// `rtk init` generates a commented template for both levels (project or global). +/// +/// Environment variables: +/// - `RTK_NO_TOML=1` — bypass TOML engine entirely +/// - `RTK_TOML_DEBUG=1` — print which filter matched and line counts to stderr +/// +/// Pipeline stages (applied in order): +/// 1. strip_ansi — remove ANSI escape codes +/// 2. replace — regex substitutions, line-by-line, chainable +/// 3. match_output — short-circuit: if blob matches a pattern, return message immediately +/// 4. strip/keep_lines — filter lines by regex +/// 5. truncate_lines_at — truncate each line to N chars +/// 6. head/tail_lines — keep first/last N lines +/// 7. max_lines — absolute line cap +/// 8. on_empty — message if result is empty +use super::constants::RTK_META_COMMANDS; +use regex::{Regex, RegexSet}; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::sync::LazyLock; + +// Built-in filters: concatenated from src/filters/*.toml by build.rs at compile time. +const BUILTIN_TOML: &str = include_str!(concat!(env!("OUT_DIR"), "/builtin_filters.toml")); + +// --------------------------------------------------------------------------- +// Deserialization types (TOML schema) +// --------------------------------------------------------------------------- + +/// A match-output rule: if `pattern` matches anywhere in the full output blob, +/// the filter short-circuits and returns `message` immediately. +/// First matching rule wins; remaining rules are not evaluated. +/// Optional `unless`: if this regex also matches the blob, the rule is skipped +/// (prevents short-circuiting when errors or warnings are present). +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct MatchOutputRule { + pattern: String, + message: String, + #[serde(default)] + unless: Option, +} + +/// A regex substitution applied line-by-line. Rules are chained sequentially: +/// rule N+1 operates on the output of rule N. +/// Backreferences (`$1`, `$2`, ...) are supported via the `regex` crate. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ReplaceRule { + pattern: String, + replacement: String, +} + +/// An inline test case attached to a filter in the TOML. +/// Lives in `[[tests.]]` sections, separate from `[filters.*]`. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TomlFilterTestDef { + pub name: String, + pub input: String, + pub expected: String, +} + +#[derive(Deserialize)] +struct TomlFilterFile { + schema_version: u32, + #[serde(default)] + filters: BTreeMap, + /// Inline tests keyed by filter name. Kept separate from `filters` so that + /// `TomlFilterDef` can keep `deny_unknown_fields` without touching test data. + #[serde(default)] + tests: BTreeMap>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TomlFilterDef { + description: Option, + match_command: String, + #[serde(default)] + strip_ansi: bool, + /// Regex substitutions, applied line-by-line before match_output (stage 2). + #[serde(default)] + replace: Vec, + /// Short-circuit rules: if the full output blob matches, return the message (stage 3). + #[serde(default)] + match_output: Vec, + #[serde(default)] + strip_lines_matching: Vec, + #[serde(default)] + keep_lines_matching: Vec, + truncate_lines_at: Option, + head_lines: Option, + tail_lines: Option, + max_lines: Option, + on_empty: Option, + /// When true, stderr is captured and merged with stdout before filtering. + /// Use for tools like liquibase that emit banners/logs to stderr. + #[serde(default)] + filter_stderr: bool, +} + +// --------------------------------------------------------------------------- +// Compiled types (post-validation, ready to use) +// --------------------------------------------------------------------------- + +#[derive(Debug)] +struct CompiledMatchOutputRule { + pattern: Regex, + message: String, + /// If set and matches the blob, this rule is skipped (prevents swallowing errors). + unless: Option, +} + +#[derive(Debug)] +struct CompiledReplaceRule { + pattern: Regex, + replacement: String, +} + +#[derive(Debug)] +enum LineFilter { + None, + Strip(RegexSet), + Keep(RegexSet), +} + +/// A filter that has been parsed and compiled — all regexes are ready. +#[derive(Debug)] +pub struct CompiledFilter { + pub name: String, + #[allow(dead_code)] + pub description: Option, + match_regex: Regex, + strip_ansi: bool, + replace: Vec, + match_output: Vec, + line_filter: LineFilter, + truncate_lines_at: Option, + head_lines: Option, + tail_lines: Option, + pub max_lines: Option, + on_empty: Option, + /// When true, the runner should capture stderr and merge it with stdout. + pub filter_stderr: bool, +} + +// --------------------------------------------------------------------------- +// Results for `rtk verify` +// --------------------------------------------------------------------------- + +/// Outcome of running a single inline test. +pub struct TestOutcome { + pub filter_name: String, + pub test_name: String, + pub passed: bool, + pub actual: String, + pub expected: String, +} + +/// Aggregated results from `run_filter_tests`. +pub struct VerifyResults { + /// Individual test outcomes (all filters, or just the requested one). + pub outcomes: Vec, + /// Filter names that have no inline tests (used by `--require-all`). + pub filters_without_tests: Vec, +} + +// --------------------------------------------------------------------------- +// Registry +// --------------------------------------------------------------------------- + +pub struct TomlFilterRegistry { + pub filters: Vec, +} + +impl TomlFilterRegistry { + /// Load registry from disk + built-in. Emits warnings to stderr on parse + /// errors but never panics — bad files are silently ignored. + fn load() -> Self { + let mut filters = Vec::new(); + + for path in crate::hooks::trust::gated_filter_paths() { + Self::extend_with_trusted(&mut filters, &path); + } + + match Self::parse_and_compile(BUILTIN_TOML, "builtin") { + Ok(f) => filters.extend(f), + Err(e) => eprintln!("[rtk] warning: builtin filters: {}", e), + } + + TomlFilterRegistry { filters } + } + + fn extend_with_trusted(filters: &mut Vec, path: &std::path::Path) { + if !path.exists() { + return; + } + let label = path.display().to_string(); + let (status, content) = crate::hooks::trust::check_trust_with_content(path) + .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); + match status { + crate::hooks::trust::TrustStatus::Trusted + | crate::hooks::trust::TrustStatus::EnvOverride => { + if let Some(content) = content { + match Self::parse_and_compile(&content, &label) { + Ok(f) => filters.extend(f), + Err(e) => eprintln!("[rtk] warning: {}: {}", label, e), + } + } + } + crate::hooks::trust::TrustStatus::Untrusted + | crate::hooks::trust::TrustStatus::ContentChanged { .. } => {} + } + } + + fn parse_and_compile(content: &str, source: &str) -> Result, String> { + let file: TomlFilterFile = toml::from_str(content) + .map_err(|e| format!("TOML parse error in {}: {}", source, e))?; + + if file.schema_version != 1 { + return Err(format!( + "unsupported schema_version {} in {} (expected 1)", + file.schema_version, source + )); + } + + let mut compiled = Vec::new(); + for (name, def) in file.filters { + if !is_fully_anchored(&def.match_command) { + eprintln!( + "[rtk] warning: filter '{}' in {}: match_command '{}' has a top-level branch \ + that does not start with '^'; it would match a path component mid-command. \ + Filter ignored.", + name, source, def.match_command + ); + continue; + } + match compile_filter(name.clone(), def) { + Ok(f) => compiled.push(f), + Err(e) => eprintln!("[rtk] warning: filter '{}' in {}: {}", name, source, e), + } + } + Ok(compiled) + } +} + +/// Commands already handled by dedicated Rust modules (routed by Clap before TOML). +/// A TOML filter whose match_command matches one of these will never activate — +/// Clap routes the command before `run_fallback()` is reached. +const RUST_HANDLED_COMMANDS: &[&str] = &[ + "ls", + "tree", + "read", + "smart", + "git", + "gh", + "aws", + "psql", + "pnpm", + "err", + "test", + "json", + "deps", + "env", + "find", + "diff", + "log", + "docker", + "kubectl", + "summary", + "grep", + "init", + "wget", + "wc", + "gain", + "config", + "ctest", + "vitest", + "prisma", + "tsc", + "next", + "lint", + "prettier", + "format", + "playwright", + "cargo", + "npm", + "npx", + "curl", + "discover", + "ruff", + "pytest", + "mypy", + "pip", + "sqlfluff", + "go", + "golangci-lint", + "rewrite", + "proxy", + "verify", + "learn", +]; + +pub fn is_rtk_reserved_command(name: &str) -> bool { + RUST_HANDLED_COMMANDS.contains(&name) || RTK_META_COMMANDS.contains(&name) +} + +fn compile_filter(name: String, def: TomlFilterDef) -> Result { + // Mutual exclusion: strip and keep cannot both be set + if !def.strip_lines_matching.is_empty() && !def.keep_lines_matching.is_empty() { + return Err("strip_lines_matching and keep_lines_matching are mutually exclusive".into()); + } + + let match_regex = Regex::new(&def.match_command) + .map_err(|e| format!("invalid match_command regex: {}", e))?; + + // Shadow warning: if match_command matches a Rust-handled command, this filter + // will never activate (Clap routes before run_fallback). Warn the author. + for cmd in RUST_HANDLED_COMMANDS { + if match_regex.is_match(cmd) { + eprintln!( + "[rtk] warning: filter '{}' match_command matches '{}' which is already \ + handled by a Rust module — this filter will never activate for that command", + name, cmd + ); + break; + } + } + + let replace = def + .replace + .into_iter() + .map(|r| { + let pat = r.pattern.clone(); + Regex::new(&r.pattern) + .map(|pattern| CompiledReplaceRule { + pattern, + replacement: r.replacement, + }) + .map_err(|e| format!("invalid replace pattern '{}': {}", pat, e)) + }) + .collect::, _>>()?; + + let match_output = def + .match_output + .into_iter() + .map(|r| -> Result { + let pat = r.pattern.clone(); + let pattern = Regex::new(&r.pattern) + .map_err(|e| format!("invalid match_output pattern '{}': {}", pat, e))?; + let unless = r + .unless + .as_deref() + .map(|u| { + Regex::new(u) + .map_err(|e| format!("invalid match_output unless pattern '{}': {}", u, e)) + }) + .transpose()?; + Ok(CompiledMatchOutputRule { + pattern, + message: r.message, + unless, + }) + }) + .collect::, _>>()?; + + let line_filter = if !def.strip_lines_matching.is_empty() { + let set = RegexSet::new(&def.strip_lines_matching) + .map_err(|e| format!("invalid strip_lines_matching regex: {}", e))?; + LineFilter::Strip(set) + } else if !def.keep_lines_matching.is_empty() { + let set = RegexSet::new(&def.keep_lines_matching) + .map_err(|e| format!("invalid keep_lines_matching regex: {}", e))?; + LineFilter::Keep(set) + } else { + LineFilter::None + }; + + Ok(CompiledFilter { + name, + description: def.description, + match_regex, + strip_ansi: def.strip_ansi, + replace, + match_output, + line_filter, + truncate_lines_at: def.truncate_lines_at, + head_lines: def.head_lines, + tail_lines: def.tail_lines, + max_lines: def.max_lines, + on_empty: def.on_empty, + filter_stderr: def.filter_stderr, + }) +} + +// --------------------------------------------------------------------------- +// Singleton (lazy-loaded, one-time cost) +// --------------------------------------------------------------------------- + +static REGISTRY: LazyLock = LazyLock::new(TomlFilterRegistry::load); + +pub fn toml_disabled() -> bool { + std::env::var("RTK_NO_TOML").ok().as_deref() == Some("1") +} + +pub fn active_filter_summaries(content: &str) -> Vec<(String, String)> { + toml::from_str::(content) + .map(|f| { + f.filters + .into_iter() + .map(|(name, def)| (name, def.match_command)) + .collect() + }) + .unwrap_or_default() +} + +pub fn filter_parse_error(content: &str) -> Option { + toml::from_str::(content) + .err() + .map(|e| e.to_string()) +} + +static MATCH_SET: LazyLock = LazyLock::new(build_match_set); + +pub fn command_matches_filter(command: &str) -> bool { + MATCH_SET.is_match(command) +} + +fn build_match_set() -> RegexSet { + let patterns = collect_match_patterns(); + RegexSet::new(&patterns).unwrap_or_else(|_| { + let valid: Vec = patterns + .into_iter() + .filter(|p| Regex::new(p).is_ok()) + .collect(); + RegexSet::new(&valid).unwrap_or_else(|_| RegexSet::empty()) + }) +} + +fn collect_match_patterns() -> Vec { + let mut patterns: Vec = Vec::new(); + for path in crate::hooks::trust::gated_filter_paths() { + if !path.exists() { + continue; + } + let (status, content) = crate::hooks::trust::check_trust_with_content(&path) + .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); + if matches!( + status, + crate::hooks::trust::TrustStatus::Trusted + | crate::hooks::trust::TrustStatus::EnvOverride + ) && let Some(content) = content + { + patterns.extend(match_patterns_in(&content)); + } + } + patterns.extend(match_patterns_in(BUILTIN_TOML)); + patterns +} + +/// Strip a leading inline-flag group such as `(?i)` so the anchor check sees the +/// pattern body. `(?:` opens a non-capturing group and is left alone. +fn strip_inline_flags(branch: &str) -> &str { + let Some(rest) = branch.strip_prefix("(?") else { + return branch; + }; + let Some(end) = rest.find(')') else { + return branch; + }; + if rest[..end].chars().all(|c| "imsuxU-".contains(c)) { + &rest[end + 1..] + } else { + branch + } +} + +/// Split `pattern` on top-level `|`, ignoring alternations nested inside groups +/// or character classes. +fn top_level_branches(pattern: &str) -> Vec<&str> { + let mut branches = Vec::new(); + let mut depth = 0usize; + let mut escaped = false; + let mut in_class = false; + let mut start = 0usize; + + for (idx, character) in pattern.char_indices() { + if escaped { + escaped = false; + continue; + } + match character { + '\\' => escaped = true, + '[' if !in_class => in_class = true, + ']' if in_class => in_class = false, + '(' if !in_class => depth += 1, + ')' if !in_class => depth = depth.saturating_sub(1), + '|' if !in_class && depth == 0 => { + branches.push(&pattern[start..idx]); + start = idx + 1; + } + _ => {} + } + } + branches.push(&pattern[start..]); + branches +} + +/// A filter selects on argv[0], but its regex runs against the whole command +/// line. A top-level branch that does not start with `^` therefore matches a +/// path component or wrapper argument mid-line, so `timeout 5 /usr/bin/liquibase +/// update` activates the liquibase filter and rewrites the wrapper instead. +fn is_fully_anchored(pattern: &str) -> bool { + top_level_branches(pattern) + .iter() + .all(|branch| strip_inline_flags(branch).starts_with('^')) +} + +fn match_patterns_in(content: &str) -> Vec { + match toml::from_str::(content) { + Ok(file) if file.schema_version == 1 => file + .filters + .into_values() + .map(|def| def.match_command) + .filter(|pattern| is_fully_anchored(pattern)) + .collect(), + _ => Vec::new(), + } +} + +// --------------------------------------------------------------------------- +// Public API — pure functions (testable without global state) +// --------------------------------------------------------------------------- + +/// Find the first matching filter in a slice. O(N) on the number of filters. +/// Tests should call this directly with a local filter list. +pub fn find_filter_in<'a>( + command: &str, + filters: &'a [CompiledFilter], +) -> Option<&'a CompiledFilter> { + filters.iter().find(|f| f.match_regex.is_match(command)) +} + +/// Apply a compiled filter pipeline to raw stdout. Pure String -> String. +/// +/// Pipeline stages (in order): +/// 1. strip_ansi — remove ANSI escape codes +/// 2. replace — regex substitutions, line-by-line, chainable +/// 3. match_output — short-circuit if blob matches a pattern +/// 4. strip/keep_lines — filter lines by regex +/// 5. truncate_lines_at — truncate each line to N chars +/// 6. head/tail_lines — keep first/last N lines +/// 7. max_lines — absolute line cap +/// 8. on_empty — message if result is empty +pub fn apply_filter(filter: &CompiledFilter, stdout: &str) -> String { + apply_filter_with_info(filter, stdout).0 +} + +#[derive(Debug, PartialEq)] +pub enum Lossiness { + None, + /// `tail -n +{tail_offset}` over `tee_payload` reproduces the dropped lines, + /// up to the tee `max_file_size` cap (larger payloads are truncated in the tee file). + Tail { + tee_payload: String, + tail_offset: usize, + }, + Whole, +} + +pub fn apply_filter_with_info(filter: &CompiledFilter, stdout: &str) -> (String, Lossiness) { + let mut lines: Vec = stdout.lines().map(String::from).collect(); + + // 1. strip_ansi + if filter.strip_ansi { + lines = lines + .into_iter() + .map(|l| crate::core::utils::strip_ansi(&l)) + .collect(); + } + + // 2. replace — line-by-line, rules chained sequentially + if !filter.replace.is_empty() { + lines = lines + .into_iter() + .map(|mut line| { + for rule in &filter.replace { + line = rule + .pattern + .replace_all(&line, rule.replacement.as_str()) + .into_owned(); + } + line + }) + .collect(); + } + + // 3. match_output — short-circuit on full blob match (first rule wins) + // If `unless` is set and also matches the blob, the rule is skipped. + if !filter.match_output.is_empty() { + let blob = lines.join("\n"); + for rule in &filter.match_output { + if rule.pattern.is_match(&blob) { + if let Some(ref unless_re) = rule.unless + && unless_re.is_match(&blob) + { + continue; // errors/warnings present — skip this rule + } + return (rule.message.clone(), Lossiness::Whole); + } + } + } + + // 4. strip OR keep (mutually exclusive) + match &filter.line_filter { + LineFilter::Strip(set) => lines.retain(|l| !set.is_match(l)), + LineFilter::Keep(set) => lines.retain(|l| set.is_match(l)), + LineFilter::None => {} + } + + // 5. truncate_lines_at — uses utils::truncate (unicode-safe) + let mut intra_line_loss = false; + if let Some(max_chars) = filter.truncate_lines_at { + lines = lines + .into_iter() + .map(|line| { + let truncated = crate::core::utils::truncate(&line, max_chars); + if truncated != line { + intra_line_loss = true; + } + truncated + }) + .collect(); + } + + let snapshot_for_tail = !intra_line_loss + && filter.tail_lines.is_none() + && (filter.head_lines.is_some() || filter.max_lines.is_some()); + let pre_cut = snapshot_for_tail.then(|| lines.clone()); + + // 6. head + tail + let total = lines.len(); + let mut noncontiguous_drop = false; + let mut head_cut: Option = None; + if let (Some(head), Some(tail)) = (filter.head_lines, filter.tail_lines) { + if total > head + tail { + let mut result = lines[..head].to_vec(); + result.push(format!("... ({} lines omitted)", total - head - tail)); + result.extend_from_slice(&lines[total - tail..]); + lines = result; + noncontiguous_drop = true; + } + } else if let Some(head) = filter.head_lines { + if total > head { + lines.truncate(head); + lines.push(format!("... ({} lines omitted)", total - head)); + head_cut = Some(head); + } + } else if let Some(tail) = filter.tail_lines + && total > tail + { + let omitted = total - tail; + lines = lines[omitted..].to_vec(); + lines.insert(0, format!("... ({} lines omitted)", omitted)); + noncontiguous_drop = true; + } + + // 7. max_lines — absolute cap applied after head/tail (includes omit messages) + let mut max_cut: Option = None; + if let Some(max) = filter.max_lines + && lines.len() > max + { + let dropped = lines.len() - max; + lines.truncate(max); + lines.push(format!("... ({} lines truncated)", dropped)); + max_cut = Some(max); + } + + // 8. on_empty + let result = lines.join("\n"); + if result.trim().is_empty() + && let Some(ref msg) = filter.on_empty + { + return (msg.clone(), Lossiness::None); + } + + let loss = if let Some(snapshot) = pre_cut { + match (head_cut, max_cut) { + (Some(_), Some(_)) => Lossiness::Whole, + (Some(head), None) => Lossiness::Tail { + tee_payload: snapshot.join("\n"), + tail_offset: head + 1, + }, + (None, Some(max)) => Lossiness::Tail { + tee_payload: snapshot.join("\n"), + tail_offset: max + 1, + }, + (None, None) => Lossiness::None, + } + } else if noncontiguous_drop || intra_line_loss || head_cut.is_some() || max_cut.is_some() { + Lossiness::Whole + } else { + Lossiness::None + }; + + (result, loss) +} + +// --------------------------------------------------------------------------- +// rtk verify — inline test execution +// --------------------------------------------------------------------------- + +/// Run inline tests from loaded TOML files (builtin + project-local). +/// +/// - `filter_name_opt`: if `Some`, only run tests for that filter name. +/// - Returns `VerifyResults` with all outcomes and filters that have no tests. +pub fn run_filter_tests(filter_name_opt: Option<&str>) -> VerifyResults { + let mut outcomes = Vec::new(); + let mut all_filter_names: Vec = Vec::new(); + let mut tested_filter_names: std::collections::HashSet = + std::collections::HashSet::new(); + + let builtin = BUILTIN_TOML; + collect_test_outcomes( + builtin, + filter_name_opt, + &mut outcomes, + &mut all_filter_names, + &mut tested_filter_names, + ); + + // Trust-gated: only verify project-local filters if trusted (SA-2025-RTK-002) + let project_path = std::path::Path::new(".rtk/filters.toml"); + if project_path.exists() { + let (trust_status, verified_content) = + crate::hooks::trust::check_trust_with_content(project_path) + .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); + match trust_status { + crate::hooks::trust::TrustStatus::Trusted + | crate::hooks::trust::TrustStatus::EnvOverride => { + if let Some(content) = verified_content { + collect_test_outcomes( + &content, + filter_name_opt, + &mut outcomes, + &mut all_filter_names, + &mut tested_filter_names, + ); + } + } + _ => { + eprintln!("[rtk] WARNING: untrusted project filters skipped in verify"); + } + } + } + + let filters_without_tests = all_filter_names + .into_iter() + .filter(|name| { + // When a specific filter is requested, only report that one as missing tests + filter_name_opt.is_none_or(|f| name == f) + }) + .filter(|name| !tested_filter_names.contains(name)) + .collect(); + + VerifyResults { + outcomes, + filters_without_tests, + } +} + +fn collect_test_outcomes( + content: &str, + filter_name_opt: Option<&str>, + outcomes: &mut Vec, + all_filter_names: &mut Vec, + tested_filter_names: &mut std::collections::HashSet, +) { + let file: TomlFilterFile = match toml::from_str(content) { + Ok(f) => f, + Err(e) => { + eprintln!("[rtk] warning: TOML parse error during verify: {}", e); + return; + } + }; + + // Compile all filters and track their names + let mut compiled_filters: BTreeMap = BTreeMap::new(); + for (name, def) in file.filters { + all_filter_names.push(name.clone()); + match compile_filter(name.clone(), def) { + Ok(f) => { + compiled_filters.insert(name, f); + } + Err(e) => eprintln!("[rtk] warning: filter '{}' compilation error: {}", name, e), + } + } + + // Run tests + for (filter_name, tests) in file.tests { + if let Some(name) = filter_name_opt + && filter_name != name + { + continue; + } + + tested_filter_names.insert(filter_name.clone()); + + let compiled = match compiled_filters.get(&filter_name) { + Some(f) => f, + None => { + eprintln!( + "[rtk] warning: [[tests.{}]] references unknown filter", + filter_name + ); + continue; + } + }; + + for test in tests { + let actual = apply_filter(compiled, &test.input); + // Trim trailing newlines: TOML multiline strings end with a newline + let actual_cmp = actual.trim_end_matches('\n').to_string(); + let expected_cmp = test.expected.trim_end_matches('\n').to_string(); + outcomes.push(TestOutcome { + filter_name: filter_name.clone(), + test_name: test.name, + passed: actual_cmp == expected_cmp, + actual: actual_cmp, + expected: expected_cmp, + }); + } + } +} + +// --------------------------------------------------------------------------- +// Convenience wrapper (uses singleton — for run_fallback) +// --------------------------------------------------------------------------- + +/// Find a matching filter from the global registry. Initialises the registry +/// lazily on first call. Returns `None` if no filter matches. +pub fn find_matching_filter(command: &str) -> Option<&'static CompiledFilter> { + if std::env::var("RTK_TOML_DEBUG").is_ok() { + eprintln!( + "[rtk:toml] looking up filter for: {:?} ({} filters loaded)", + command, + REGISTRY.filters.len() + ); + } + let result = find_filter_in(command, ®ISTRY.filters); + if std::env::var("RTK_TOML_DEBUG").is_ok() { + match result { + Some(f) => eprintln!("[rtk:toml] matched filter: '{}'", f.name), + None => eprintln!("[rtk:toml] no filter matched — passthrough"), + } + } + result +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // Helper: build a CompiledFilter from inline TOML for tests. + // Never touches the lazy registry. + fn make_filters(toml: &str) -> Vec { + TomlFilterRegistry::parse_and_compile(toml, "test").expect("test TOML should be valid") + } + + fn first_filter(toml: &str) -> CompiledFilter { + make_filters(toml) + .into_iter() + .next() + .expect("expected at least one filter") + } + + #[test] + fn command_matches_filter_agrees_with_find_matching_filter() { + for cmd in ["jj log", "jq .", "frobnicate xyz", "cd /tmp"] { + assert_eq!( + command_matches_filter(cmd), + find_matching_filter(cmd).is_some(), + "match-set disagreed with registry for {cmd:?}" + ); + } + } + + #[test] + fn test_loss_tail_configured_unfired_max_fires_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntail_lines = 100\nmax_lines = 2\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc\nd\ne"); + assert!(out.contains("lines truncated")); + assert_ne!(loss, Lossiness::None); + } + + #[test] + fn test_match_patterns_in_honors_schema_version() { + let v1 = "schema_version = 1\n[filters.a]\nmatch_command = \"^aaa\"\n"; + assert_eq!(match_patterns_in(v1), vec!["^aaa".to_string()]); + + let v2 = "schema_version = 2\n[filters.a]\nmatch_command = \"^aaa\"\n"; + assert!(match_patterns_in(v2).is_empty()); + + assert!(match_patterns_in("not valid toml {{{").is_empty()); + } + + #[test] + fn test_filter_parse_error() { + assert!( + filter_parse_error("schema_version = 1\n[filters.a]\nmatch_command = \"^a\"\n") + .is_none() + ); + assert!(filter_parse_error("[filters.a]\nmatch_command = \"^a\"\n").is_some()); + assert!(filter_parse_error("this is { not toml").is_some()); + } + + #[test] + fn test_toml_parse_tolerates_utf8_bom() { + // Hand-edited filter files on Windows often carry a BOM. The toml + // crate tolerates it natively — this pin is why the TOML file-read + // sites (config.rs, toml_filter.rs) deliberately skip + // strip_leading_bom; if a crate upgrade regresses this, add it there. + let bom = "\u{feff}schema_version = 1\n[filters.a]\nmatch_command = \"^a\"\n"; + assert!(filter_parse_error(bom).is_none()); + assert_eq!(match_patterns_in(bom), vec!["^a".to_string()]); + } + + #[test] + fn test_is_rtk_reserved_command() { + assert!(is_rtk_reserved_command("git")); + assert!(is_rtk_reserved_command("cargo")); + assert!(is_rtk_reserved_command("json")); + assert!(is_rtk_reserved_command("rewrite")); + assert!(!is_rtk_reserved_command("jj")); + assert!(!is_rtk_reserved_command("jq")); + assert!(!is_rtk_reserved_command("just")); + assert!(!is_rtk_reserved_command("frobnicate")); + } + + fn loss_of(toml: &str, input: &str) -> Lossiness { + apply_filter_with_info(&first_filter(toml), input).1 + } + + #[test] + fn test_loss_head_lines_is_tail() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc\nd\ne"); + assert!(out.starts_with("a\nb\n")); + match loss { + Lossiness::Tail { + tee_payload, + tail_offset, + } => { + assert_eq!(tail_offset, 3); + let recovered: Vec<&str> = tee_payload.lines().skip(tail_offset - 1).collect(); + assert_eq!(recovered, vec!["c", "d", "e"]); + } + other => panic!("expected Tail, got {:?}", other), + } + } + + #[test] + fn test_loss_max_lines_is_tail() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nmax_lines = 2\n"; + match loss_of(toml, "a\nb\nc\nd\ne") { + Lossiness::Tail { + tee_payload, + tail_offset, + } => { + assert_eq!(tail_offset, 3); + let recovered: Vec<&str> = tee_payload.lines().skip(tail_offset - 1).collect(); + assert_eq!(recovered, vec!["c", "d", "e"]); + } + other => panic!("expected Tail, got {:?}", other), + } + } + + #[test] + fn test_loss_tail_lines_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntail_lines = 2\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_head_then_max_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\nmax_lines = 2\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_head_plus_tail_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 1\ntail_lines = 1\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_truncate_lines_at_is_whole() { + let toml = + "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntruncate_lines_at = 3\n"; + assert_eq!(loss_of(toml, "abcdefgh\nshort"), Lossiness::Whole); + } + + #[test] + fn test_loss_match_output_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\n[[filters.f.match_output]]\npattern = \"ok\"\nmessage = \"all good\"\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "everything ok here\nmore"); + assert_eq!(out, "all good"); + assert_eq!(loss, Lossiness::Whole); + } + + #[test] + fn test_loss_strip_only_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nstrip_lines_matching = [\"^noise\"]\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "keep\nnoise line\nkeep2"); + assert_eq!(out, "keep\nkeep2"); + assert_eq!(loss, Lossiness::None); + } + + #[test] + fn test_loss_on_empty_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nstrip_lines_matching = [\".\"]\non_empty = \"nothing\"\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc"); + assert_eq!(out, "nothing"); + assert_eq!(loss, Lossiness::None); + } + + #[test] + fn test_loss_no_truncation_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 10\n"; + assert_eq!(loss_of(toml, "a\nb\nc"), Lossiness::None); + } + + #[test] + fn test_apply_filter_wrapper_matches_with_info() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\n"; + let f = first_filter(toml); + let input = "a\nb\nc\nd"; + assert_eq!(apply_filter(&f, input), apply_filter_with_info(&f, input).0); + } + + // --- Pipeline primitives (existing) --- + + #[test] + fn test_strip_ansi_removes_codes() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +"#, + ); + let out = apply_filter(&f, "\x1b[31mError\x1b[0m\nnormal"); + assert_eq!(out, "Error\nnormal"); + } + + #[test] + fn test_strip_lines_matching_basic() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["^noise", "^verbose"] +"#, + ); + let input = "noise line\nkeep this\nverbose stuff\nalso keep"; + let out = apply_filter(&f, input); + assert_eq!(out, "keep this\nalso keep"); + } + + #[test] + fn test_keep_lines_matching_basic() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +keep_lines_matching = ["^PASS", "^FAIL"] +"#, + ); + let input = "PASS test_a\nsome noise\nFAIL test_b\nmore noise"; + let out = apply_filter(&f, input); + assert_eq!(out, "PASS test_a\nFAIL test_b"); + } + + #[test] + fn test_truncate_lines_at_unicode_safe() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +truncate_lines_at = 5 +"#, + ); + // utils::truncate(s, 5) takes 2 chars + "..." when len > 5 + // "hello" = 5 chars exactly, stays unchanged + // "日本語xyz" = 6 chars, truncated to "日本..." (take 2 + "...") + let out = apply_filter(&f, "hello\n日本語xyz"); + assert_eq!(out, "hello\n日本..."); + } + + #[test] + fn test_head_lines() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +head_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + assert!(out.starts_with("a\nb\n")); + assert!(out.contains("3 lines omitted")); + } + + #[test] + fn test_tail_lines() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +tail_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + assert!(out.contains("3 lines omitted")); + assert!(out.ends_with("d\ne")); + } + + #[test] + fn test_head_and_tail_combined() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +head_lines = 2 +tail_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne\nf"; + let out = apply_filter(&f, input); + assert!(out.starts_with("a\nb\n")); + assert!(out.contains("2 lines omitted")); + assert!(out.ends_with("e\nf")); + } + + #[test] + fn test_max_lines_counts_omit_message() { + // max_lines applied AFTER head — the "omitted" message counts as a line + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +max_lines = 3 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + let line_count = out.lines().count(); + // 3 content lines + 1 truncated message = 4 lines output + assert_eq!(line_count, 4); + assert!(out.contains("lines truncated")); + } + + #[test] + fn test_on_empty_when_all_filtered() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = [".*"] +on_empty = "nothing left" +"#, + ); + let out = apply_filter(&f, "line1\nline2"); + assert_eq!(out, "nothing left"); + } + + #[test] + fn test_on_empty_not_triggered_when_output_remains() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +keep_lines_matching = ["keep"] +on_empty = "nothing left" +"#, + ); + let out = apply_filter(&f, "keep this\nnoise"); + assert_eq!(out, "keep this"); + } + + #[test] + fn test_full_pipeline_order() { + // Verify all 8 stages fire in order on a single input + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +strip_lines_matching = ["^noise"] +truncate_lines_at = 10 +head_lines = 3 +max_lines = 4 +on_empty = "empty" +"#, + ); + let input = + "\x1b[31mred line\x1b[0m\nnoise skip\nkeep one\nkeep two\nkeep three\nkeep four"; + let out = apply_filter(&f, input); + // After strip_ansi: "red line", strip noise: removed, head 3 from remaining 4 lines + assert!(out.contains("red line")); + assert!(!out.contains("noise skip")); + assert!(out.contains("lines omitted") || out.contains("lines truncated")); + } + + // --- Validation --- + + #[test] + fn test_mutual_exclusion_strip_keep_errors() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["a"] +keep_lines_matching = ["b"] +"#, + ); + // The filter should be skipped (warning emitted), resulting in empty list + assert!(result.is_empty()); + } + + #[test] + fn test_invalid_regex_returns_err() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "[" +"#, + ); + assert!(result.is_empty()); + } + + #[test] + fn test_schema_version_mismatch_errors() { + let result = TomlFilterRegistry::parse_and_compile( + r#"schema_version = 99 +[filters.f] +match_command = "^cmd" +"#, + "test", + ); + assert!(result.is_err()); + } + + #[test] + fn test_unknown_field_typo_errors() { + // deny_unknown_fields should catch this + let result = TomlFilterRegistry::parse_and_compile( + r#"schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi_typo = true +"#, + "test", + ); + assert!(result.is_err()); + } + + #[test] + fn test_empty_filter_passthrough() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +"#, + ); + let input = "line1\nline2\nline3"; + let out = apply_filter(&f, input); + assert_eq!(out, input); + } + + // --- Registry / find --- + + #[test] + fn test_builtin_filters_compile() { + // Compile-time safety: panics if any src/filters/*.toml is broken + let builtin = BUILTIN_TOML; + let result = TomlFilterRegistry::parse_and_compile(builtin, "builtin"); + assert!( + result.is_ok(), + "builtin filters failed to compile: {:?}", + result + ); + assert!(!result.unwrap().is_empty()); + } + + #[test] + fn test_find_filter_matches_terraform() { + let filters = make_filters( + r#" +schema_version = 1 +[filters.terraform-plan] +match_command = "^terraform\\s+plan" +strip_ansi = true +"#, + ); + let found = find_filter_in("terraform plan -out=tfplan", &filters); + assert!(found.is_some()); + assert_eq!(found.unwrap().name, "terraform-plan"); + } + + #[test] + fn test_find_filter_no_match_returns_none() { + let filters = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^terraform" +"#, + ); + let found = find_filter_in("kubectl get pods", &filters); + assert!(found.is_none()); + } + + #[test] + fn test_project_filters_priority_over_builtin() { + // Project filter has same name but different max_lines — project wins + let project = make_filters( + r#" +schema_version = 1 +[filters.make] +match_command = "^make\\b" +max_lines = 999 +"#, + ); + let builtin = make_filters(BUILTIN_TOML); + + // Simulate the registry: project first + let mut all = project; + all.extend(builtin); + + let found = find_filter_in("make all", &all).expect("should match"); + assert_eq!(found.name, "make"); + // The first (project) match has max_lines=999 + assert_eq!(found.max_lines, Some(999)); + } + + // --- Token savings --- + + #[test] + fn test_terraform_savings_above_60pct() { + let filters = make_filters(BUILTIN_TOML); + let filter = find_filter_in("terraform plan", &filters).expect("terraform-plan built-in"); + + // Inline fixture: realistic terraform plan with many Refreshing state lines (noise). + // Real infra refreshes 30+ resources; the plan section is small. + // All Refreshing/lock/blank/unchanged lines are stripped -> >60% savings. + let input = concat!( + "Acquiring state lock. This may take a few moments...\n", + "Refreshing state... [id=vpc-0a1b2c3d]\n", + "Refreshing state... [id=subnet-11111111]\n", + "Refreshing state... [id=subnet-22222222]\n", + "Refreshing state... [id=subnet-33333333]\n", + "Refreshing state... [id=subnet-44444444]\n", + "Refreshing state... [id=igw-aabbccdd]\n", + "Refreshing state... [id=rtb-aabbccdd]\n", + "Refreshing state... [id=rtb-11223344]\n", + "Refreshing state... [id=sg-00112233]\n", + "Refreshing state... [id=sg-44556677]\n", + "Refreshing state... [id=sg-88990011]\n", + "Refreshing state... [id=nacl-00aabbcc]\n", + "Refreshing state... [id=acm-arn:us-east-1:cert/abc]\n", + "Refreshing state... [id=Z1234567890ABC]\n", + "Refreshing state... [id=alb-arn:my-alb]\n", + "Refreshing state... [id=tg-arn:my-tg]\n", + "Refreshing state... [id=db-ABCDEFGHIJKLMNO]\n", + "Refreshing state... [id=rds-cluster:my-aurora]\n", + "Refreshing state... [id=elasticache:my-cluster]\n", + "Refreshing state... [id=lambda:my-api-function]\n", + "Refreshing state... [id=lambda:my-worker]\n", + "Refreshing state... [id=iam-role:my-lambda-role]\n", + "Refreshing state... [id=iam-role:my-ecs-role]\n", + "Refreshing state... [id=s3:::my-app-assets]\n", + "Refreshing state... [id=s3:::my-app-logs]\n", + "Refreshing state... [id=cloudfront:ABCDEFGHIJK]\n", + "Refreshing state... [id=ssm:/my/app/db-url]\n", + "Refreshing state... [id=ssm:/my/app/api-key]\n", + "Refreshing state... [id=secretsmanager:my-secret]\n", + "Releasing state lock. This may take a few moments...\n", + "\n", + "Terraform will perform the following actions:\n", + "\n", + " # aws_instance.web will be created\n", + " + resource \"aws_instance\" \"web\" {\n", + " + ami = \"ami-0c55b159cbfafe1f0\"\n", + " + instance_type = \"t3.micro\"\n", + " }\n", + "\n", + "Plan: 1 to add, 0 to change, 0 to destroy.\n", + ); + let out = apply_filter(filter, input); + let input_words = input.split_whitespace().count(); + let out_words = out.split_whitespace().count(); + let savings = 100.0 - (out_words as f64 / input_words as f64 * 100.0); + assert!( + savings >= 60.0, + "terraform-plan filter: expected >=60% savings, got {:.1}% (in={} out={})", + savings, + input_words, + out_words + ); + } + + #[test] + fn test_make_savings_above_60pct() { + let filters = make_filters(BUILTIN_TOML); + let filter = find_filter_in("make all", &filters).expect("make built-in"); + + let input = r#"make[1]: Entering directory '/home/user/project' +make[2]: Entering directory '/home/user/project/src' +gcc -O2 -Wall -c foo.c -o foo.o + +make[2]: Nothing to be done for 'install'. +make[3]: Entering directory '/home/user/project/src/lib' +ar rcs libfoo.a foo.o bar.o baz.o +make[3]: Leaving directory '/home/user/project/src/lib' +make[2]: Leaving directory '/home/user/project/src' + +make[1]: Leaving directory '/home/user/project' +gcc -O2 -Wall -c bar.c -o bar.o + +gcc -O2 -Wall -c baz.c -o baz.o + +make[1]: Entering directory '/home/user/project/test' +make[2]: Entering directory '/home/user/project/test/unit' +./run_tests --verbose +make[2]: Nothing to be done for 'check'. +make[2]: Leaving directory '/home/user/project/test/unit' +make[1]: Leaving directory '/home/user/project/test' + +ld -o myapp foo.o bar.o baz.o -lfoo + +make[1]: Entering directory '/home/user/project/docs' +doxygen Doxyfile +make[1]: Leaving directory '/home/user/project/docs' +"#; + let out = apply_filter(filter, input); + let input_words = input.split_whitespace().count(); + let out_words = out.split_whitespace().count(); + let savings = 100.0 - (out_words as f64 / input_words as f64 * 100.0); + assert!( + savings >= 60.0, + "make filter: expected >=60% savings, got {:.1}% (in={} out={})", + savings, + input_words, + out_words + ); + } + + #[test] + fn test_spring_boot_match_command_requires_spring_named_jar() { + let filters = make_filters(BUILTIN_TOML); + let spring_boot = filters + .iter() + .find(|f| f.name == "spring-boot") + .expect("spring-boot filter must exist"); + + assert!( + !spring_boot + .match_regex + .is_match("java -jar build/libs/my-other-tool.jar"), + "a non-Spring jar must not activate the spring-boot filter" + ); + + let spring_jar = find_filter_in("java -jar build/libs/my-spring-app.jar", &filters) + .expect("a jar with 'spring' in its filename must still match"); + assert_eq!(spring_jar.name, "spring-boot"); + + let mvn_run = find_filter_in("mvn spring-boot:run", &filters) + .expect("mvn spring-boot:run must still match"); + assert_eq!(mvn_run.name, "spring-boot"); + + let capitalized = find_filter_in("java -jar build/libs/MySpringApp.jar", &filters) + .expect("a jar with 'Spring' capitalized in its filename must still match"); + assert_eq!(capitalized.name, "spring-boot"); + + assert!( + !spring_boot + .match_regex + .is_match("java -jar /opt/spring-cache/other-tool.jar"), + "'spring' appearing only in a directory segment (not the jar filename itself) must not activate the spring-boot filter" + ); + + // Windows paths: '\' must be treated as a path separator too, not swallowed + // into the filename-only match (the exact bug this guard exists to catch). + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\spring-cache\other.jar"), + "'spring' appearing only in a Windows directory segment must not activate the spring-boot filter" + ); + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\dev\spring-workspace\build\other-tool.jar"), + "'spring' appearing only in a nested Windows directory segment must not activate the spring-boot filter" + ); + assert!( + spring_boot + .match_regex + .is_match(r"java -jar C:\dev\my-spring-app.jar"), + "a Windows-path jar with 'spring' in its own filename must still match" + ); + + // argv reaches this regex as one space-joined string, so a path containing spaces + // is indistinguishable from a following argument. Widening the prefix across + // whitespace would let a later 'spring'-named jar argument re-trigger the filter, + // so jars under such a path stay on full passthrough instead. + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\Program Files\app\my-spring-app.jar"), + "a jar under a path containing spaces is deliberately left to full passthrough" + ); + } + + #[test] + fn test_liquibase_match_command_ignores_path_substring() { + let filters = make_filters(BUILTIN_TOML); + let liquibase = filters + .iter() + .find(|f| f.name == "liquibase") + .expect("liquibase filter must exist"); + + assert!( + !liquibase.match_regex.is_match("rm -rf /opt/liquibase"), + "'liquibase' appearing only as a path argument must not activate the liquibase filter" + ); + + let bare = find_filter_in("liquibase status", &filters) + .expect("bare liquibase invocation must still match"); + assert_eq!(bare.name, "liquibase"); + + // Production callers (run_fallback in src/main.rs, strip_absolute_path in + // src/discover/registry.rs) always basename argv[0] before this regex runs, + // so a raw path-qualified string must NOT match on its own — the regex has + // no path-prefix branch to fall back on. + assert!( + !liquibase + .match_regex + .is_match("/usr/local/bin/liquibase update"), + "a raw path-qualified invocation must not match — callers basename argv[0] before matching" + ); + + for wrapped in [ + "timeout 5 /usr/bin/liquibase update", + "nohup /opt/tools/liquibase update", + ] { + assert!( + !liquibase.match_regex.is_match(wrapped), + "a wrapper command must not activate the liquibase filter: {wrapped}" + ); + } + } + + #[test] + fn test_ssh_match_command_excludes_ssh_dash_utilities() { + let filters = make_filters(BUILTIN_TOML); + let ssh = filters + .iter() + .find(|f| f.name == "ssh") + .expect("ssh filter must exist"); + + assert!( + !ssh.match_regex.is_match("ssh-keygen -t ed25519"), + "ssh-keygen must not activate the plain ssh connection filter" + ); + assert!( + !ssh.match_regex.is_match("ssh-add id_ed25519"), + "ssh-add must not activate the plain ssh connection filter" + ); + + let plain = find_filter_in("ssh user@host", &filters) + .expect("plain ssh invocation must still match"); + assert_eq!(plain.name, "ssh"); + + let bare = find_filter_in("ssh", &filters).expect("bare ssh with no args must still match"); + assert_eq!(bare.name, "ssh"); + } + + // --- Edge cases --- + + #[test] + fn test_empty_input() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = [".*"] +"#, + ); + let out = apply_filter(&f, ""); + assert_eq!(out, ""); + } + + #[test] + fn test_unicode_preserved() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["^noise"] +"#, + ); + let out = apply_filter(&f, "日本語テスト\nnoise\n中文内容"); + assert_eq!(out, "日本語テスト\n中文内容"); + } + + // --- match_output tests (PR1) --- + + #[test] + fn test_match_output_basic_short_circuit() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +"#, + ); + let out = apply_filter(&f, "Switched to branch 'main'"); + assert_eq!(out, "ok"); + } + + #[test] + fn test_match_output_second_rule_matches() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "switched" }, + { pattern = "Already on", message = "already" }, +] +"#, + ); + let out = apply_filter(&f, "Already on 'main'"); + assert_eq!(out, "already"); + } + + #[test] + fn test_match_output_no_match_pipeline_continues() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +strip_lines_matching = ["^noise"] +"#, + ); + let out = apply_filter(&f, "noise\nkeep this"); + // No match_output match, pipeline continues and strips noise + assert_eq!(out, "keep this"); + } + + #[test] + fn test_match_output_strip_ansi_before_match() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +"#, + ); + // ANSI stripped before match_output check (stage 1 before stage 3) + let out = apply_filter(&f, "\x1b[32mSwitched to branch\x1b[0m 'main'"); + assert_eq!(out, "ok"); + } + + #[test] + fn test_match_output_no_match_then_on_empty() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched", message = "ok" }, +] +strip_lines_matching = [".*"] +on_empty = "nothing" +"#, + ); + // No match_output match; pipeline strips all lines; on_empty fires + let out = apply_filter(&f, "foo bar baz"); + assert_eq!(out, "nothing"); + } + + #[test] + fn test_match_output_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "[invalid", message = "ok" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- match_output unless tests (PR3) --- + + #[test] + fn test_match_output_unless_blocks_short_circuit_when_errors_present() { + // "total size is" matches, but "error" also matches — unless fires, rule is skipped. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^rsync" +match_output = [ + { pattern = "total size is", message = "ok (synced)", unless = "error|failed" }, +] +"#, + ); + let input = "rsync: [sender] error\ntotal size is 1000 speedup is 3.33\n"; + let out = apply_filter(&f, input); + // Should NOT return "ok (synced)" because "error" matches the unless pattern + assert_ne!( + out.trim(), + "ok (synced)", + "unless should have blocked short-circuit when errors are present" + ); + // The raw lines should pass through (no further strip rules in this filter) + assert!(out.contains("error")); + } + + #[test] + fn test_match_output_unless_allows_short_circuit_when_no_errors() { + // "total size is" matches and "error" does NOT appear — unless does not fire, rule wins. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^rsync" +match_output = [ + { pattern = "total size is", message = "ok (synced)", unless = "error|failed" }, +] +"#, + ); + let input = "file.txt\ntotal size is 98765 speedup is 77.31\n"; + let out = apply_filter(&f, input); + assert_eq!(out.trim(), "ok (synced)"); + } + + #[test] + fn test_match_output_unless_falls_through_to_next_rule() { + // First rule blocked by unless; second rule (no unless) should match. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "success", message = "ok", unless = "error" }, + { pattern = "success", message = "ok with warnings" }, +] +"#, + ); + let input = "success\nerror: something went wrong\n"; + let out = apply_filter(&f, input); + // First rule skipped (unless matched), second rule (no unless) fires + assert_eq!(out.trim(), "ok with warnings"); + } + + #[test] + fn test_match_output_unless_no_field_behaves_like_before() { + // When unless is absent, behaviour is identical to original (no regression). + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Build complete", message = "ok (build complete)" }, +] +"#, + ); + let out = apply_filter(&f, "Build complete!\n"); + assert_eq!(out.trim(), "ok (build complete)"); + } + + #[test] + fn test_match_output_unless_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "success", message = "ok", unless = "[invalid" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- replace tests (PR1) --- + + #[test] + fn test_replace_basic_all_occurrences() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "foo", replacement = "bar" }, +] +"#, + ); + let out = apply_filter(&f, "foo baz foo\nfoo"); + assert_eq!(out, "bar baz bar\nbar"); + } + + #[test] + fn test_replace_chaining_sequential() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "aaa", replacement = "bbb" }, + { pattern = "bbb", replacement = "ccc" }, +] +"#, + ); + // Rule 2 operates on the output of rule 1 + let out = apply_filter(&f, "aaa"); + assert_eq!(out, "ccc"); + } + + #[test] + fn test_replace_backreferences() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "(\\w+):(\\w+)", replacement = "$2:$1" }, +] +"#, + ); + let out = apply_filter(&f, "hello:world"); + assert_eq!(out, "world:hello"); + } + + #[test] + fn test_replace_then_strip_interaction() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "noise", replacement = "DROPPED" }, +] +strip_lines_matching = ["^DROPPED"] +"#, + ); + // replace transforms "noise line" -> "DROPPED line", strip removes it + let out = apply_filter(&f, "noise line\nkeep this"); + assert_eq!(out, "keep this"); + } + + #[test] + fn test_replace_empty_input_noop() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "foo", replacement = "bar" }, +] +"#, + ); + let out = apply_filter(&f, ""); + assert_eq!(out, ""); + } + + #[test] + fn test_replace_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "[invalid", replacement = "bar" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- verify (PR2) --- + + #[test] + fn test_run_filter_tests_passes_on_correct_expected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +strip_lines_matching = ["^make\\[\\d+\\]:"] + +[[tests.make]] +name = "strips entering/leaving lines" +input = """ +make[1]: Entering directory '/home/user' +gcc -O2 foo.c +make[1]: Leaving directory '/home/user' +""" +expected = """ +gcc -O2 foo.c +""" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + assert_eq!(outcomes.len(), 1); + assert!( + outcomes[0].passed, + "test should pass: {:?}", + outcomes[0].actual + ); + } + + #[test] + fn test_run_filter_tests_fails_on_wrong_expected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +strip_lines_matching = ["^make\\[\\d+\\]:"] + +[[tests.make]] +name = "wrong expected" +input = "make[1]: Entering\ngcc foo.c" +expected = "wrong output" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + assert_eq!(outcomes.len(), 1); + assert!(!outcomes[0].passed); + } + + #[test] + fn test_filters_without_tests_detected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + // No tests defined, but filter exists + assert_eq!(outcomes.len(), 0); + assert!(all_names.contains(&"make".to_string())); + assert!(!tested.contains("make")); + } + + // --- Multi-file architecture tests (build.rs) --- + + /// Verify BUILTIN_TOML was generated with the correct schema_version header. + /// build.rs injects it — if the const is somehow stale this fails immediately. + #[test] + fn test_builtin_toml_has_schema_version() { + assert!( + BUILTIN_TOML.contains("schema_version = 1"), + "BUILTIN_TOML must start with 'schema_version = 1' (injected by build.rs)" + ); + } + + /// Verify every expected filter name is present in BUILTIN_TOML. + /// This is the safeguard against accidentally deleting a filter file. + #[test] + fn test_builtin_all_expected_filters_present() { + let filters = make_filters(BUILTIN_TOML); + let names: std::collections::HashSet<&str> = + filters.iter().map(|f| f.name.as_str()).collect(); + + let expected = [ + "ansible-playbook", + "brew-install", + "composer-install", + "df", + "dotnet-build", + "du", + "fail2ban-client", + "gcloud", + "hadolint", + "helm", + "iptables", + "liquibase", + "make", + "markdownlint", + "mix-compile", + "mix-format", + "ping", + "pio-run", + "poetry-install", + "pre-commit", + "ps", + "pulumi-destroy", + "pulumi-preview", + "pulumi-refresh", + "pulumi-stack", + "pulumi-up", + "quarto-render", + "rsync", + "shellcheck", + "shopify-theme", + "sops", + "swift-build", + "systemctl-status", + "terraform-plan", + "tofu-fmt", + "tofu-init", + "tofu-plan", + "tofu-validate", + "trunk-build", + "uv-sync", + "yamllint", + ]; + + for name in &expected { + assert!( + names.contains(name), + "Built-in filter '{}' is missing — was its .toml file deleted from src/filters/?", + name + ); + } + } + + /// Verify the exact count of built-in filters. + /// Fails if a file is added/removed without updating this test. + #[test] + fn test_builtin_filter_count() { + let filters = make_filters(BUILTIN_TOML); + assert_eq!( + filters.len(), + 62, + "Expected exactly 62 built-in filters, got {}. \ + Update this count when adding/removing filters in src/filters/.", + filters.len() + ); + } + + /// Every built-in filter must be anchored on every top-level branch: the + /// match runs against the whole command line, so an unanchored branch + /// activates the filter on a path component or wrapper argument. + #[test] + fn test_builtin_all_filters_match_command_anchored() { + // Read the raw patterns rather than the compiled registry: both loaders + // drop unanchored filters, so a compiled view cannot observe one. + let file: TomlFilterFile = + toml::from_str(BUILTIN_TOML).expect("built-in filters should parse"); + assert!(!file.filters.is_empty()); + for (name, def) in &file.filters { + assert!( + is_fully_anchored(&def.match_command), + "Filter '{}' has match_command '{}' with a top-level branch that does not start with '^'", + name, + def.match_command + ); + } + + assert!(is_fully_anchored(r"^liquibase(?:\s|$)")); + assert!(is_fully_anchored(r"^(liquibase\b|/liquibase\b)")); + assert!(is_fully_anchored(r"^pnpm\b|^npm\b")); + assert!(is_fully_anchored(r"(?i)^liquibase\b")); + assert!(!is_fully_anchored(r"^liquibase\b|/liquibase\b")); + assert!(!is_fully_anchored(r"(?:^|/)liquibase(?:\s|$)")); + } + + /// The invariant above covers `BUILTIN_TOML`; project-local and global + /// filters are user-authored, so both loaders drop unanchored patterns + /// rather than letting them match mid-command. + #[test] + fn test_unanchored_user_filter_is_rejected_by_both_loaders() { + let unanchored = + "schema_version = 1\n[filters.mytool]\nmatch_command = \"(?:^|/)mytool\\\\b\"\n"; + assert!(match_patterns_in(unanchored).is_empty()); + assert!( + TomlFilterRegistry::parse_and_compile(unanchored, "test") + .expect("schema is valid") + .is_empty() + ); + + let anchored = "schema_version = 1\n[filters.mytool]\nmatch_command = \"^mytool\\\\b\"\n"; + assert_eq!(match_patterns_in(anchored), vec!["^mytool\\b".to_string()]); + assert_eq!( + TomlFilterRegistry::parse_and_compile(anchored, "test") + .expect("schema is valid") + .len(), + 1 + ); + } + + /// Verify that every built-in filter has at least one inline test. + /// Prevents shipping filters with zero test coverage. + #[test] + fn test_builtin_all_filters_have_inline_tests() { + let mut all_names: Vec = Vec::new(); + let mut tested: std::collections::HashSet = std::collections::HashSet::new(); + let mut outcomes = Vec::new(); + collect_test_outcomes( + BUILTIN_TOML, + None, + &mut outcomes, + &mut all_names, + &mut tested, + ); + + let untested: Vec<&str> = all_names + .iter() + .filter(|name| !tested.contains(name.as_str())) + .map(|s| s.as_str()) + .collect(); + + assert!( + untested.is_empty(), + "The following built-in filters have no inline tests: {:?}\n\ + Add [[tests.]] entries to the corresponding src/filters/.toml file.", + untested + ); + } + + /// Verify that adding a new filter entry to any TOML content makes it + /// immediately discoverable via find_filter_in — simulating how a new + /// src/filters/my-tool.toml would work after cargo build. + #[test] + fn test_new_filter_discoverable_after_concat() { + // Simulate build.rs: concat BUILTIN_TOML with a brand-new filter block + let new_filter = r#" +[filters.my-new-tool] +description = "Compact my-new-tool output" +match_command = "^my-new-tool\\b" +strip_lines_matching = ["^\\s*$"] +max_lines = 30 +on_empty = "my-new-tool: ok" + +[[tests.my-new-tool]] +name = "strips blank lines" +input = "output line 1\n\noutput line 2" +expected = "output line 1\noutput line 2" +"#; + let combined = format!("{}\n\n{}", BUILTIN_TOML, new_filter); + let filters = make_filters(&combined); + + // All 62 existing filters still present + 1 new = 63 + assert_eq!( + filters.len(), + 63, + "Expected 63 filters after concat (62 built-in + 1 new)" + ); + + // New filter is discoverable + let found = find_filter_in("my-new-tool --verbose", &filters); + assert!( + found.is_some(), + "Newly added filter must be discoverable via find_filter_in" + ); + assert_eq!(found.unwrap().name, "my-new-tool"); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/core/truncate.rs b/crates/sparkplane-rtk/upstream/src/core/truncate.rs new file mode 100644 index 0000000..58285f6 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/truncate.rs @@ -0,0 +1,60 @@ +//! Global truncation caps shared by every filter. See `src/core/README.md` +//! ("Truncation Caps") for the cap classes, config policy, and deviation rules. + +/// Errors: most actionable, shown the most. +pub const CAP_ERRORS: usize = 20; +/// Warnings and test failures: lower signal density than errors. +pub const CAP_WARNINGS: usize = 10; +/// Flat lists (PRs, services, packages): one line per item. +pub const CAP_LIST: usize = 20; +/// Inventories (`pip list`, `docker images`): exhaustive lookups. +pub const CAP_INVENTORY: usize = 50; + +/// A cap reduced for a verbose data class. Falls back to `cap` when `by >= cap` +/// so a deviation can never empty the list; `0` stays `0`. `const fn`, underflow-safe. +pub const fn reduced(cap: usize, by: usize) -> usize { + if by < cap { cap - by } else { cap } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reduced_preserves_current_values() { + assert_eq!(reduced(CAP_WARNINGS, 5), 5); + assert_eq!(reduced(CAP_LIST, 5), 15); + } + + #[test] + fn reduced_falls_back_to_cap_when_offset_too_large() { + assert_eq!(reduced(4, 5), 4); + assert_eq!(reduced(5, 5), 5); + } + + #[test] + fn reduced_honors_zero_cap() { + assert_eq!(reduced(0, 5), 0); + } + + // Sweep every plausible (cap, by) a future config could produce and assert + // the invariants that make caps safe: the result never wraps past `cap`, and + // the offset never empties a non-zero cap. `usize::MAX` covers a wraparound bug. + #[test] + fn reduced_is_underflow_safe_across_all_inputs() { + for cap in 0..=64usize { + for by in [0usize, 1, 5, 10, 64, usize::MAX] { + let r = reduced(cap, by); + assert!(r <= cap, "reduced({cap}, {by}) = {r} exceeds cap (wrapped)"); + if cap == 0 { + assert_eq!(r, 0, "zero cap must stay zero"); + } else { + assert!(r >= 1, "reduced({cap}, {by}) = {r} emptied a non-zero cap"); + } + if by < cap { + assert_eq!(r, cap - by, "exact deviation must be preserved"); + } + } + } + } +} diff --git a/crates/sparkplane-rtk/upstream/src/core/utils.rs b/crates/sparkplane-rtk/upstream/src/core/utils.rs new file mode 100644 index 0000000..809f5bb --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/utils.rs @@ -0,0 +1,1842 @@ +//! Utility functions for text processing and command execution. +//! +//! Provides common helpers used across rtk commands: +//! - ANSI color code stripping +//! - Text truncation +//! - Command execution with error context + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use regex::Regex; +use serde_json::Value; +use std::ffi::OsStr; +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use std::sync::LazyLock; +use std::sync::OnceLock; + +/// Compute `days` ago from now, clamped instead of panicking on overflow. +/// +/// `days` is typically user-supplied (`--since `) and unbounded. Naively +/// building `chrono::Duration::days(days as i64)` and subtracting it from +/// `Utc::now()` panics for a large enough value ("DateTime - TimeDelta +/// overflowed") — this happened identically in `rtk discover --since` and `rtk +/// hook audit --since` before both were routed through this helper. `days` is +/// capped to `i64::MAX` before the `as i64` cast (a `u64` past `i64::MAX` would +/// otherwise reinterpret as negative, turning "look back" into "look forward"), +/// and any remaining overflow building or applying the `Duration` falls back to +/// `DateTime::::MIN_UTC` — an arbitrarily distant cutoff means "no lower +/// bound", which is exactly what an absurdly large `--since` should behave like. +pub fn days_ago_cutoff(days: u64) -> DateTime { + let days = days.min(i64::MAX as u64) as i64; + chrono::Duration::try_days(days) + .and_then(|d| Utc::now().checked_sub_signed(d)) + .unwrap_or(DateTime::::MIN_UTC) +} + +/// Truncates a string to `max_len` characters, appending `...` if needed. +/// +/// # Arguments +/// * `s` - The string to truncate +/// * `max_len` - Maximum length before truncation (minimum 3 to include "...") +/// +/// # Examples +/// ``` +/// use rtk::utils::truncate; +/// assert_eq!(truncate("hello world", 8), "hello..."); +/// assert_eq!(truncate("hi", 10), "hi"); +/// ``` +pub fn truncate(s: &str, max_len: usize) -> String { + let char_count = s.chars().count(); + if char_count <= max_len { + s.to_string() + } else if max_len < 3 { + // If max_len is too small, just return "..." + "...".to_string() + } else { + format!("{}...", s.chars().take(max_len - 3).collect::()) + } +} + +/// Strip ANSI escape codes (colors, styles) from a string. +/// +/// # Arguments +/// * `text` - Text potentially containing ANSI escape codes +/// +/// # Examples +/// ``` +/// use rtk::utils::strip_ansi; +/// let colored = "\x1b[31mError\x1b[0m"; +/// assert_eq!(strip_ansi(colored), "Error"); +/// ``` +pub fn strip_ansi(text: &str) -> String { + static ANSI_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\x1b\[[0-9;]*[a-zA-Z]").unwrap()); + + ANSI_RE.replace_all(text, "").to_string() +} + +/// Strip one or more leading UTF-8 BOMs (`EF BB BF`, sometimes doubled), +/// which serde_json refuses to parse. Windows editors (Notepad, PowerShell +/// 5.1 `Out-File -Encoding utf8`) prepend one to hand-edited config files, +/// and Cursor on Windows ships hook payloads with them. Strip defensively +/// wherever RTK parses JSON a human or another tool may have written. +pub fn strip_leading_bom(input: &str) -> &str { + let mut s = input; + while let Some(rest) = s.strip_prefix('\u{feff}') { + s = rest; + } + s +} + +/// BOM-tolerant `serde_json::from_str`. Prefer this over `serde_json::from_str` +/// anywhere the JSON may have been written by a human, an editor, or another tool. +pub fn from_json_str<'a, T: serde::Deserialize<'a>>(s: &'a str) -> serde_json::Result { + serde_json::from_str(strip_leading_bom(s)) +} + +/// Executes a command and returns cleaned stdout/stderr. +/// +/// # Arguments +/// * `cmd` - Command to execute (e.g., "eslint") +/// * `args` - Command arguments +/// +/// # Returns +/// `(stdout: String, stderr: String, exit_code: i32)` +/// Formats a token count with K/M suffixes for readability. +/// +/// # Arguments +/// * `n` - Number of tokens +/// +/// # Returns +/// Formatted string (e.g., "1.2M", "59.2K", "694") +/// +/// # Examples +/// ``` +/// use rtk::utils::format_tokens; +/// assert_eq!(format_tokens(1_234_567), "1.2M"); +/// assert_eq!(format_tokens(59_234), "59.2K"); +/// assert_eq!(format_tokens(694), "694"); +/// ``` +pub fn format_tokens(n: usize) -> String { + if n >= 1_000_000 { + format!("{:.1}M", n as f64 / 1_000_000.0) + } else if n >= 1_000 { + format!("{:.1}K", n as f64 / 1_000.0) + } else { + format!("{}", n) + } +} + +/// Formats a USD amount with adaptive precision. +/// +/// # Arguments +/// * `amount` - Amount in dollars +/// +/// # Returns +/// Formatted string with $ prefix +/// +/// # Examples +/// ``` +/// use rtk::utils::format_usd; +/// assert_eq!(format_usd(1234.567), "$1234.57"); +/// assert_eq!(format_usd(12.345), "$12.35"); +/// assert_eq!(format_usd(0.123), "$0.12"); +/// assert_eq!(format_usd(0.0096), "$0.0096"); +/// ``` +pub fn format_usd(amount: f64) -> String { + if !amount.is_finite() { + return "$0.00".to_string(); + } + if amount >= 0.01 { + format!("${:.2}", amount) + } else { + format!("${:.4}", amount) + } +} + +/// Format cost-per-token as $/MTok (e.g., "$3.86/MTok") +/// +/// # Arguments +/// * `cpt` - Cost per token (not per million tokens) +/// +/// # Returns +/// Formatted string like "$3.86/MTok" +/// +/// # Examples +/// ``` +/// use rtk::utils::format_cpt; +/// assert_eq!(format_cpt(0.000003), "$3.00/MTok"); +/// assert_eq!(format_cpt(0.0000038), "$3.80/MTok"); +/// assert_eq!(format_cpt(0.00000386), "$3.86/MTok"); +/// ``` +pub fn format_cpt(cpt: f64) -> String { + if !cpt.is_finite() || cpt <= 0.0 { + return "$0.00/MTok".to_string(); + } + let cpt_per_million = cpt * 1_000_000.0; + format!("${:.2}/MTok", cpt_per_million) +} + +/// Join items into a newline-separated string, appending an overflow hint when total > max. +/// +/// # Examples +/// ``` +/// use rtk::utils::join_with_overflow; +/// let items = vec!["a".to_string(), "b".to_string()]; +/// assert_eq!(join_with_overflow(&items, 5, 3, "items"), "a\nb\n... +2 more items"); +/// assert_eq!(join_with_overflow(&items, 2, 3, "items"), "a\nb"); +/// ``` +pub fn join_with_overflow(items: &[String], total: usize, max: usize, label: &str) -> String { + let mut out = items.join("\n"); + if total > max { + out.push_str(&format!("\n… +{} more {}", total - max, label)); + } + out +} + +/// Truncate an ISO 8601 datetime string to just the date portion (first 10 chars). +/// +/// # Examples +/// ``` +/// use rtk::utils::truncate_iso_date; +/// assert_eq!(truncate_iso_date("2024-01-15T10:30:00Z"), "2024-01-15"); +/// assert_eq!(truncate_iso_date("2024-01-15"), "2024-01-15"); +/// assert_eq!(truncate_iso_date("short"), "short"); +/// ``` +pub fn truncate_iso_date(date: &str) -> &str { + if date.len() >= 10 { &date[..10] } else { date } +} + +/// Format a confirmation message: "ok \ \" +/// Used for write operations (merge, create, comment, edit, etc.) +/// +/// # Examples +/// ``` +/// use rtk::utils::ok_confirmation; +/// assert_eq!(ok_confirmation("merged", "#42"), "ok merged #42"); +/// assert_eq!(ok_confirmation("created", "PR #5 https://..."), "ok created PR #5 https://..."); +/// ``` +pub fn ok_confirmation(action: &str, detail: &str) -> String { + if detail.is_empty() { + format!("ok {}", action) + } else { + format!("ok {} {}", action, detail) + } +} + +/// Extract exit code from a process output. Returns the actual exit code, or +/// `128 + signal` per Unix convention when terminated by a signal (no exit code +/// available). Falls back to 1 on non-Unix platforms. +pub fn exit_code_from_output(output: &std::process::Output, label: &str) -> i32 { + match output.status.code() { + Some(code) => code, + None => { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if let Some(sig) = output.status.signal() { + eprintln!("[rtk] {}: process terminated by signal {}", label, sig); + return 128 + sig; + } + } + eprintln!("[rtk] {}: process terminated by signal", label); + 1 + } + } +} + +/// Extract exit code from an ExitStatus (for `.status()` calls, not `.output()`). +/// Returns the actual exit code, or `128 + signal` per Unix convention when +/// terminated by a signal. Falls back to 1 on non-Unix platforms. +pub fn exit_code_from_status(status: &std::process::ExitStatus, label: &str) -> i32 { + match status.code() { + Some(code) => code, + None => { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if let Some(sig) = status.signal() { + eprintln!("[rtk] {}: process terminated by signal {}", label, sig); + return 128 + sig; + } + } + eprintln!("[rtk] {}: process terminated by signal", label); + 1 + } + } +} + +/// Return the last `n` lines of output with a label, for use as a fallback +/// when filter parsing fails. Logs a diagnostic to stderr. +pub fn fallback_tail(output: &str, label: &str, n: usize) -> String { + eprintln!( + "[rtk] {}: output format not recognized, showing last {} lines", + label, n + ); + let lines: Vec<&str> = output.lines().collect(); + let start = lines.len().saturating_sub(n); + lines[start..].join("\n") +} + +/// Create a directory owner-only (0700 on Unix), tightening one that already exists. +/// Serializes tests across modules that mutate process-global recall env vars +/// (`RTK_RECALL`, `RTK_TEE`): a static declared inside one test module is not +/// shared with other modules, so those tests would not actually serialize. +#[cfg(test)] +pub(crate) static TEST_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +pub fn create_private_dir(path: &std::path::Path) -> std::io::Result<()> { + fs::create_dir_all(path)?; + set_owner_only(path, 0o700); + Ok(()) +} + +/// Restrict an existing file to owner-only access (0600 on Unix). +pub fn restrict_file(path: &std::path::Path) { + set_owner_only(path, 0o600); +} + +/// Open a file owner-only (0600 on Unix), applied at creation so content is +/// never briefly readable under a permissive umask. `mode` is ignored for a +/// file that already exists, so an older one is still tightened afterwards. +pub fn open_private( + opts: &mut fs::OpenOptions, + path: &std::path::Path, +) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let file = opts.open(path)?; + set_owner_only(path, 0o600); + Ok(file) +} + +/// Pure core of `set_owner_only`'s "is a chmod even needed" check, taking the raw +/// `st_mode` reading directly so it's deterministically testable without touching +/// the filesystem (an unprivileged `chmod` setting setuid/setgid isn't reliably +/// honored across every environment — e.g. some sandboxed/containerized +/// filesystems silently drop it — so a test that round-trips through a real file +/// would be flaky rather than actually exercising this logic). +/// +/// Masks with `0o7777`, NOT `0o777`: setuid/setgid/sticky live in the `0o7000` +/// range, above the rwx bits but below the file-type bits `st_mode` also carries. +/// A `0o777` mask would strip those dangerous bits out of the comparison too, so +/// a file with e.g. setuid set (`0o4600`) would read as "already correct" against +/// a `target` of `0o600` and permanently skip the chmod that would clear it — +/// defeating the self-heal `set_owner_only` exists to provide, for exactly the +/// files (external tampering, restored backups) it's meant to catch. +#[cfg(unix)] +fn mode_already_correct(actual: u32, target: u32) -> bool { + actual & 0o7777 == target +} + +#[cfg(unix)] +fn set_owner_only(path: &std::path::Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + // Skip the chmod syscall once permissions already match. This runs on every + // `Tracker::new()` call, which is now on the PreToolUse hook's hot path for + // *every* Bash tool call (not just RTK-covered ones — see rtk-ai/rtk#3206's + // hook_decisions ground-truth logging), under the project's <10ms latency + // budget, so a redundant chmod on the already-correct common case adds up. + // Falls through to chmod on any metadata-read failure, erring toward + // enforcing the permission rather than silently skipping it. + // + if let Ok(meta) = fs::metadata(path) + && mode_already_correct(meta.permissions().mode(), mode) + { + return; + } + let _ = fs::set_permissions(path, fs::Permissions::from_mode(mode)); +} + +#[cfg(not(unix))] +fn set_owner_only(_path: &std::path::Path, _mode: u32) {} + +/// Build a Command for Ruby tools, auto-detecting bundle exec. +/// Uses `bundle exec ` when a Gemfile exists (transitive deps like rake +/// won't appear in the Gemfile but still need bundler for version isolation). +pub fn ruby_exec(tool: &str) -> Command { + if std::path::Path::new("Gemfile").exists() { + let mut c = Command::new("bundle"); + c.arg("exec").arg(tool); + return c; + } + Command::new(tool) +} + +/// Count whitespace-delimited tokens in text. Used by filter tests to verify +/// token savings claims. +#[cfg(test)] +pub fn count_tokens(text: &str) -> usize { + text.split_whitespace().count() +} + +/// Detect the package manager used in the current directory. +/// Returns "pnpm", "yarn", "bun", or "npm" based on lockfile presence. +/// +/// # Examples +/// ```no_run +/// use rtk::utils::detect_package_manager; +/// let pm = detect_package_manager(); +/// // "pnpm" for pnpm-lock.yaml, "yarn" for yarn.lock, "bun" for bun.lock(b), else "npm" +/// ``` +#[allow(dead_code)] +pub fn detect_package_manager() -> &'static str { + detect_package_manager_in(std::path::Path::new(".")) +} + +/// Lockfile detection against an explicit directory, so callers (and tests) do +/// not have to move the process's current directory to ask the question. +pub fn detect_package_manager_in(dir: &std::path::Path) -> &'static str { + if dir.join("pnpm-lock.yaml").exists() { + "pnpm" + } else if dir.join("yarn.lock").exists() { + "yarn" + } else if dir.join("bun.lockb").exists() || dir.join("bun.lock").exists() { + "bun" + } else { + "npm" + } +} + +/// Build a Command using the detected package manager's exec mechanism. +/// Returns a Command ready to have tool-specific args appended. +pub fn package_manager_exec(tool: &str) -> Command { + tool_exec(None, tool, MissingTool::Fail) +} + +/// What the npm arm does when `tool` is not installed. `npx` fetches on demand +/// by default, which is right for a tool the user named themselves and +/// surprising for one rtk picked on their behalf. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum MissingTool { + Fetch, + Fail, +} + +/// The runner used to resolve a tool that is not on PATH. A runner the user +/// named wins outright. Otherwise lockfile detection picks one, except that a +/// detected bun resolves through npx: bunx always fetches a missing tool into +/// its cache and cannot be told not to, so `MissingTool` could not be honoured +/// and the presence of a lockfile alone would decide what gets downloaded. +pub fn exec_runner(runner: Option<&str>, missing: MissingTool) -> &str { + match runner { + Some(named) => named, + // Nothing was named and rtk may fetch: npx is the only runner that can, + // and lockfile detection would hand `pnpm exec` a tool the project + // does not have. + None if missing == MissingTool::Fetch => "npx", + None => match detect_package_manager() { + // bunx always fetches; npm's runner is npx. + "bun" | "npm" => "npx", + other => other, + }, + } +} + +/// Build a Command that runs `tool`, preferring the package runner the user +/// actually named over lockfile detection. +/// +/// RTK forwards what was typed rather than substituting an engine for it: a +/// user who types `bunx tsc` must not get `pnpm` because a lockfile is present. +/// Detection applies only when nothing was named, as with a bare `rtk tsc`. +/// +/// A tool already on PATH is run directly only when no runner was named. +pub fn tool_exec(runner: Option<&str>, tool: &str, missing: MissingTool) -> Command { + // Only when nothing was named: `bunx tsc` must resolve the project's tsc, + // not a global one that happens to be on PATH. Both bunx and npx prefer + // node_modules/.bin before fetching, so naming one is a real choice. + if runner.is_none() && tool_exists(tool) { + resolved_command(tool) + } else { + match exec_runner(runner, missing) { + "pnpm" => { + let mut c = resolved_command("pnpm"); + c.arg("exec").arg("--").arg(tool); + c + } + "yarn" => { + let mut c = resolved_command("yarn"); + c.arg("exec").arg("--").arg(tool); + c + } + "bun" | "bunx" => { + let mut c = resolved_command("bunx"); + c.arg(tool); + c + } + _ => { + let mut c = resolved_command("npx"); + if missing == MissingTool::Fail { + c.arg("--no-install"); + } + c.arg("--").arg(tool); + c + } + } + } +} + +/// Encode one argument for a child's raw command line the way libuv's +/// `quote_cmd_arg` does: wrap it in `"`, escape an inner `"` as `\"`, and double +/// every backslash run that ends up in front of a quote so it stays literal. +/// +/// std's encoder emits the same bytes but wraps only for a space or a tab, which +/// is what leaves MSYS children with a mangled command line (#3727). +// Windows-only in production; the rules stay unit-tested on every platform. +#[cfg_attr(not(windows), allow(dead_code))] +pub fn quote_arg_for_child(arg: &str) -> String { + let mut out = String::with_capacity(arg.len() + 2); + out.push('"'); + let mut backslashes = 0usize; + for ch in arg.chars() { + match ch { + '\\' => backslashes += 1, + '"' => { + for _ in 0..backslashes * 2 + 1 { + out.push('\\'); + } + backslashes = 0; + out.push('"'); + } + _ => { + for _ in 0..backslashes { + out.push('\\'); + } + backslashes = 0; + out.push(ch); + } + } + } + // The closing quote is a quote too, so a trailing run is doubled as well. + for _ in 0..backslashes * 2 { + out.push('\\'); + } + out.push('"'); + out +} + +/// Pass caller-supplied arguments to a child so that MSYS/Cygwin children on +/// Windows receive them intact. +/// +/// Use instead of `Command::arg`/`args` for anything that arrived on rtk's own +/// command line — a pattern, a path, a flag value. +pub trait ChildArgExt { + fn child_arg>(&mut self, arg: S) -> &mut Command; + + fn child_args(&mut self, args: I) -> &mut Command + where + I: IntoIterator, + S: AsRef; +} + +impl ChildArgExt for Command { + fn child_arg>(&mut self, arg: S) -> &mut Command { + push_child_arg(self, arg.as_ref()); + self + } + + fn child_args(&mut self, args: I) -> &mut Command + where + I: IntoIterator, + S: AsRef, + { + for arg in args { + push_child_arg(self, arg.as_ref()); + } + self + } +} + +/// Windows: `"` is the only character std encodes differently from libuv, so +/// re-encode just those arguments and leave the rest on std's path. `.bat`/`.cmd` +/// shims stay on it too — cmd.exe parses by its own rules and `raw_arg` would +/// bypass the escaping std applies for them. +#[cfg(windows)] +fn push_child_arg(cmd: &mut Command, arg: &OsStr) { + match arg.to_str() { + Some(s) if s.contains('"') && !is_batch_program(cmd) => { + std::os::windows::process::CommandExt::raw_arg(cmd, quote_arg_for_child(s)); + } + _ => { + cmd.arg(arg); + } + } +} + +#[cfg(windows)] +fn is_batch_program(cmd: &Command) -> bool { + std::path::Path::new(cmd.get_program()) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("bat") || ext.eq_ignore_ascii_case("cmd")) +} + +/// Unix: the argument vector reaches `execvp` verbatim, so there is nothing to +/// encode. +#[cfg(not(windows))] +fn push_child_arg(cmd: &mut Command, arg: &OsStr) { + cmd.arg(arg); +} + +/// Resolve a binary name to its full path, honoring PATHEXT on Windows. +/// +/// On Windows, Node.js tools are installed as `.CMD`/`.BAT`/`.PS1` shims. +/// Rust's `std::process::Command::new()` does NOT honor PATHEXT, so +/// `Command::new("vitest")` fails even when `vitest.CMD` is on PATH. +/// +/// This function uses the `which` crate to perform proper PATH+PATHEXT resolution. +/// +/// # Arguments +/// * `name` - Binary name (e.g., "vitest", "eslint", "tsc") +/// +/// # Returns +/// Full path to the resolved binary, or error if not found. +pub fn resolve_binary(name: &str) -> Result { + which::which(name).context(format!("Binary '{}' not found on PATH", name)) +} + +/// Create a `Command` with PATHEXT-aware binary resolution. +/// +/// Drop-in replacement for `Command::new(name)` that works on Windows +/// with `.CMD`/`.BAT`/`.PS1` wrappers. +/// +/// Falls back to `Command::new(name)` if resolution fails, so native +/// commands (git, cargo) still work even if `which` can't find them. +/// +/// # Arguments +/// * `name` - Binary name (e.g., "vitest", "eslint") +/// +/// # Returns +/// A `Command` configured with the resolved binary path. +pub fn resolved_command(name: &str) -> Command { + match resolve_binary(name) { + Ok(path) => Command::new(path), + Err(e) => { + // On Windows, resolution failure likely means a .CMD/.BAT wrapper + // wasn't found — always warn so users have a signal. + // On Unix, this is less common; only log in debug builds. + if cfg!(any(target_os = "windows", debug_assertions)) { + eprintln!( + "rtk: Failed to resolve '{}' via PATH, falling back to direct exec: {}", + name, e + ); + } + + Command::new(name) + } + } +} + +/// Return Composer bin directories in precedence order. +/// +/// Composer allows overriding the default `vendor/bin` via `COMPOSER_BIN_DIR` +/// or `composer.json` `config.bin-dir`. Keep the default as a fallback so we +/// continue recognizing the common layout even when the repo is not configured. +pub fn composer_bin_dirs() -> Vec { + // Resolution depends only on the process's env + cwd composer.json, both + // constant for a single rtk invocation. The rewrite hot path queries this + // several times per command segment, so read the file once and cache. + static CACHE: OnceLock> = OnceLock::new(); + CACHE + .get_or_init(|| { + let env_bin_dir = std::env::var("COMPOSER_BIN_DIR").ok(); + let composer_json = fs::read_to_string("composer.json").ok(); + composer_bin_dirs_from(env_bin_dir.as_deref(), composer_json.as_deref()) + }) + .clone() +} + +pub fn composer_tool_paths(tool: &str) -> Vec { + composer_bin_dirs() + .into_iter() + .map(|dir| dir.join(tool)) + .collect() +} + +fn composer_bin_dirs_from(env_bin_dir: Option<&str>, composer_json: Option<&str>) -> Vec { + let mut dirs = Vec::new(); + + if let Some(dir) = env_bin_dir + .map(str::trim) + .filter(|dir| !dir.is_empty()) + .map(PathBuf::from) + .or_else(|| composer_json.and_then(read_composer_bin_dir)) + { + dirs.push(dir); + } + + let default_dir = PathBuf::from("vendor/bin"); + if !dirs.iter().any(|dir| dir == &default_dir) { + dirs.push(default_dir); + } + + dirs +} + +fn read_composer_bin_dir(composer_json: &str) -> Option { + let parsed: Value = from_json_str(composer_json).ok()?; + let bin_dir = parsed.get("config")?.get("bin-dir")?.as_str()?.trim(); + if bin_dir.is_empty() { + None + } else { + Some(PathBuf::from(bin_dir)) + } +} + +/// Join lines with newlines, or return "ok" when there is nothing left to show. +/// +/// Shared by output filters that collapse fully to a success marker. +pub fn join_or_ok(lines: &[&str]) -> String { + if lines.is_empty() { + "ok".to_string() + } else { + lines.join("\n") + } +} + +/// Check if a tool exists on PATH (PATHEXT-aware on Windows). +/// +/// Replaces manual `Command::new("which").arg(tool)` checks that fail on Windows. +pub fn tool_exists(name: &str) -> bool { + which::which(name).is_ok() +} + +/// Check if a compile-time environment variable was set to a non-empty value. +/// +/// `option_env!` yields `Some("")` when the build environment exports the +/// variable with an empty value, which a CI `env:` block fed by an unset +/// repository variable does. `.is_some()` alone then reports a feature as +/// configured when it is not, so pair every `option_env!` gate with this. +/// +/// # Examples +/// ``` +/// use rtk::utils::env_is_some; +/// assert!(env_is_some(Some("https://example.com"))); +/// assert!(!env_is_some(Some(""))); +/// assert!(!env_is_some(None)); +/// ``` +pub fn env_is_some(value: Option<&str>) -> bool { + value.is_some_and(|v| !v.is_empty()) +} + +/// Extract short name from AWS ARN. +/// Example: `arn:aws:ecs:region:acct:service/cluster/name` -> `name` +/// For simple ARNs like `arn:aws:iam::123:user/alice`, returns `alice`. +pub fn shorten_arn(arn: &str) -> &str { + // ARNs use "/" or ":" as separators. Try "/" first (service/cluster/name pattern), + // then fall back to ":" for Lambda/IAM ARNs. + let slash_result = arn.rsplit('/').next().unwrap_or(arn); + // If rsplit('/') returned the whole string (no '/' found), try ':' + if slash_result == arn { + arn.rsplit(':').next().unwrap_or(arn) + } else { + slash_result + } +} + +/// Convert bytes to human-readable format (KB, MB, GB, TB). +/// Used for S3 object sizes. +pub fn human_bytes(bytes: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = KB * 1024; + const GB: u64 = MB * 1024; + const TB: u64 = GB * 1024; + + if bytes >= TB { + format!("{:.1} TB", bytes as f64 / TB as f64) + } else if bytes >= GB { + format!("{:.1} GB", bytes as f64 / GB as f64) + } else if bytes >= MB { + format!("{:.1} MB", bytes as f64 / MB as f64) + } else if bytes >= KB { + format!("{:.1} KB", bytes as f64 / KB as f64) + } else { + format!("{} B", bytes) + } +} + +/// Decode child process output bytes, respecting the Windows console code page. +/// +/// Valid UTF-8 is returned untouched — the overwhelmingly common case, and the +/// only one on Unix. Otherwise the buffer is decoded a line at a time: lines +/// that are valid UTF-8 keep their bytes, and only the lines that are not get +/// re-decoded through the console code page (GBK, Big5, CP850, …). Decoding +/// the whole buffer as a legacy code page at the first bad byte would mangle +/// output that was almost entirely valid UTF-8. +/// +/// Falls back to lossy UTF-8 (`U+FFFD`) when no code page applies, matching the +/// previous behavior. +pub fn decode_process_output(bytes: &[u8]) -> String { + // Fast path: fully valid UTF-8 needs no scanning and no allocation beyond + // the copy. This is every Unix run and every UTF-8 console on Windows. + match std::str::from_utf8(bytes) { + Ok(s) => s.to_owned(), + Err(_) => decode_mixed(bytes, output_codepage()), + } +} + +/// Decode `bytes` line by line, keeping valid UTF-8 lines verbatim and passing +/// the rest through code page `cp` (lossy UTF-8 when `cp` is `None`). +/// +/// The line is the decoding unit because a byte run is not one: GB18030's +/// four-byte sequences embed bytes in the ASCII digit range, so any rule that +/// stops a run at the first byte under `0x80` splits them. `\n` is unambiguous +/// in every encoding handled here — none of UTF-8, GBK, gb18030, Big5, +/// Shift_JIS, EUC-KR or the single-byte pages can produce it as a trail byte — +/// and a process does not switch encoding mid-line, so the whole line can be +/// handed to one decoder. +/// +/// Takes the code page as a parameter so the walk and the mapping are +/// unit-testable on every platform, not only Windows. +fn decode_mixed(bytes: &[u8], cp: Option) -> String { + let mut out = String::with_capacity(bytes.len()); + // `split_inclusive` keeps the terminator, so line endings survive intact + // and an empty input yields no chunks at all. + for line in bytes.split_inclusive(|&b| b == b'\n') { + match std::str::from_utf8(line) { + Ok(valid) => out.push_str(valid), + Err(_) => out.push_str(&decode_line(line, cp)), + } + } + out +} + +/// Decode one line that failed UTF-8 validation using code page `cp`. +/// +/// A code page result is only accepted when it decodes cleanly. A line that is +/// really UTF-8 with a corrupt byte usually fails the code page decoder too, +/// and lossy UTF-8 preserves its valid characters where the code page would +/// turn all of them into mojibake. +fn decode_line(line: &[u8], cp: Option) -> String { + if let Some(cp) = cp { + // encoding_rs covers the ANSI and DBCS pages (1252, GBK, gb18030, + // Shift_JIS, Big5, …); `codepage` maps the Windows page number onto it. + if let Some(encoding) = codepage::to_encoding(cp) { + let (decoded, _, had_errors) = encoding.decode(line); + if !had_errors { + return decoded.into_owned(); + } + // encoding_rs implements only WHATWG encodings, which exclude the + // legacy OEM/DOS pages (437, 850, 852, …) that plain cmd.exe still + // defaults to in many locales. `oem_cp` supplies those tables. + } else if let Some(table) = oem_cp::code_table::DECODING_TABLE_CP_MAP.get(&cp) { + if let Some(decoded) = table.decode_string_checked(line) { + return decoded; + } + } else { + warn_unmapped_codepage(cp); + } + } + String::from_utf8_lossy(line).into_owned() +} + +/// Warn once that output is being decoded lossily because the console code +/// page has no known table — previously this fell back silently. +fn warn_unmapped_codepage(cp: u16) { + static WARNED: std::sync::Once = std::sync::Once::new(); + WARNED.call_once(|| { + eprintln!( + "[rtk] warning: no decoder for console code page {}; \ + non-UTF-8 output will be shown with replacement characters", + cp + ); + }); +} + +/// The code page child output should be decoded with, or `None` when the +/// platform has no such concept (every Unix) and lossy UTF-8 should be used. +#[cfg(not(windows))] +fn output_codepage() -> Option { + None +} + +/// Windows: the console output code page, cached after the first lookup. +/// +/// `GetConsoleOutputCP` describes the console rtk is attached to, which the +/// child inherits. It returns 0 when there is no console — rtk running under a +/// hook with its output piped, the common case flagged in review — and a +/// console program writing to a pipe uses the ANSI code page instead, so +/// `GetACP` is the fallback rather than giving up and decoding lossily. +/// +/// This remains a best guess: a child is free to emit any encoding regardless +/// of either code page. It is only ever consulted for bytes that already +/// failed UTF-8 validation, so a wrong guess degrades to the same replacement +/// characters that the previous lossy conversion produced. +#[cfg(windows)] +fn output_codepage() -> Option { + static CODEPAGE: OnceLock> = OnceLock::new(); + *CODEPAGE.get_or_init(|| { + #[allow(unsafe_code)] + // nosemgrep: unsafe-block — read-only Win32 APIs, no memory or thread safety risk + let cp = unsafe { + let console = windows_sys::Win32::System::Console::GetConsoleOutputCP(); + if console != 0 { + console + } else { + windows_sys::Win32::Globalization::GetACP() + } + }; + u16::try_from(cp).ok().filter(|&cp| cp != 0) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_strip_leading_bom_helper() { + // Direct unit test on the helper so future refactors can't + // regress the loop semantics without a clear failure signal. + assert_eq!(strip_leading_bom(""), ""); + assert_eq!(strip_leading_bom("hello"), "hello"); + assert_eq!(strip_leading_bom("\u{feff}hello"), "hello"); + assert_eq!(strip_leading_bom("\u{feff}\u{feff}hello"), "hello"); + assert_eq!(strip_leading_bom("\u{feff}\u{feff}\u{feff}hello"), "hello"); + // BOM in the middle is preserved (not "leading"). + assert_eq!(strip_leading_bom("a\u{feff}b"), "a\u{feff}b"); + } + + #[test] + fn test_from_json_str_strips_bom() { + let v: Value = from_json_str("\u{feff}{\"foo\": 1}").unwrap(); + assert_eq!(v["foo"], 1); + } + + #[test] + fn test_from_json_str_no_bom() { + let v: Value = from_json_str("{\"foo\": 1}").unwrap(); + assert_eq!(v["foo"], 1); + } + + #[test] + fn test_from_json_str_borrowed_type() { + // Deserialize<'a> (not DeserializeOwned) must keep zero-copy + // borrowing working through the wrapper. + #[derive(serde::Deserialize)] + struct Borrowed<'a> { + name: &'a str, + } + let b: Borrowed = from_json_str("\u{feff}{\"name\": \"rtk\"}").unwrap(); + assert_eq!(b.name, "rtk"); + } + + #[test] + fn env_is_some_rejects_unset_and_empty() { + assert!(env_is_some(Some("https://telemetry.example"))); + // An unset CI repository variable still exports the env var, so the + // empty string reaches `option_env!` as `Some("")`. + assert!(!env_is_some(Some(""))); + assert!(!env_is_some(None)); + } + + #[test] + fn test_truncate_short_string() { + assert_eq!(truncate("hello", 10), "hello"); + } + + #[test] + fn test_days_ago_cutoff_normal_value_is_in_the_past() { + let cutoff = days_ago_cutoff(30); + assert!(cutoff < Utc::now()); + } + + #[test] + fn test_days_ago_cutoff_does_not_panic_on_huge_since_days() { + // rtk-ai/rtk#3206 review: `rtk discover --since 100000000` and `rtk hook + // audit --since 100000000` both panicked with "DateTime - TimeDelta + // overflowed". Must clamp instead of crashing. + assert_eq!(days_ago_cutoff(100_000_000), DateTime::::MIN_UTC); + } + + #[test] + fn test_days_ago_cutoff_does_not_panic_on_u64_max() { + assert_eq!(days_ago_cutoff(u64::MAX), DateTime::::MIN_UTC); + } + + #[test] + fn test_truncate_long_string() { + let result = truncate("hello world", 8); + assert_eq!(result, "hello..."); + } + + #[test] + fn test_truncate_exact_length() { + assert_eq!(truncate("hello", 5), "hello"); + } + + #[test] + fn test_truncate_edge_case() { + // max_len < 3 returns just "..." + assert_eq!(truncate("hello", 2), "..."); + // When string length equals max_len, return as is + assert_eq!(truncate("abc", 3), "abc"); + // When string is longer and max_len is exactly 3, return "..." + assert_eq!(truncate("hello world", 3), "..."); + } + + #[test] + fn test_composer_bin_dirs_use_default_when_unconfigured() { + assert_eq!( + composer_bin_dirs_from(None, None), + vec![PathBuf::from("vendor/bin")] + ); + } + + #[test] + fn test_composer_bin_dirs_prefer_env_override() { + let composer_json = r#"{"config":{"bin-dir":"tools/bin"}}"#; + assert_eq!( + composer_bin_dirs_from(Some("custom/bin"), Some(composer_json)), + vec![PathBuf::from("custom/bin"), PathBuf::from("vendor/bin")] + ); + } + + #[test] + fn test_composer_bin_dirs_read_composer_config() { + let composer_json = r#"{"config":{"bin-dir":"tools/bin"}}"#; + assert_eq!( + composer_bin_dirs_from(None, Some(composer_json)), + vec![PathBuf::from("tools/bin"), PathBuf::from("vendor/bin")] + ); + } + + #[test] + fn test_strip_ansi_simple() { + let input = "\x1b[31mError\x1b[0m"; + assert_eq!(strip_ansi(input), "Error"); + } + + #[test] + fn test_strip_ansi_multiple() { + let input = "\x1b[1m\x1b[32mSuccess\x1b[0m\x1b[0m"; + assert_eq!(strip_ansi(input), "Success"); + } + + #[test] + fn test_strip_ansi_no_codes() { + assert_eq!(strip_ansi("plain text"), "plain text"); + } + + #[test] + fn test_strip_ansi_complex() { + let input = "\x1b[32mGreen\x1b[0m normal \x1b[31mRed\x1b[0m"; + assert_eq!(strip_ansi(input), "Green normal Red"); + } + + #[test] + fn test_format_tokens_millions() { + assert_eq!(format_tokens(1_234_567), "1.2M"); + assert_eq!(format_tokens(12_345_678), "12.3M"); + } + + #[test] + fn test_format_tokens_thousands() { + assert_eq!(format_tokens(59_234), "59.2K"); + assert_eq!(format_tokens(1_000), "1.0K"); + } + + #[test] + fn test_format_tokens_small() { + assert_eq!(format_tokens(694), "694"); + assert_eq!(format_tokens(0), "0"); + } + + #[test] + fn test_format_usd_large() { + assert_eq!(format_usd(1234.567), "$1234.57"); + assert_eq!(format_usd(1000.0), "$1000.00"); + } + + #[test] + fn test_format_usd_medium() { + assert_eq!(format_usd(12.345), "$12.35"); + assert_eq!(format_usd(0.99), "$0.99"); + } + + #[test] + fn test_format_usd_small() { + assert_eq!(format_usd(0.0096), "$0.0096"); + assert_eq!(format_usd(0.0001), "$0.0001"); + } + + #[test] + fn test_format_usd_edge() { + assert_eq!(format_usd(0.01), "$0.01"); + assert_eq!(format_usd(0.009), "$0.0090"); + } + + #[test] + fn test_ok_confirmation_with_detail() { + assert_eq!(ok_confirmation("merged", "#42"), "ok merged #42"); + assert_eq!( + ok_confirmation("created", "PR #5 https://github.com/foo/bar/pull/5"), + "ok created PR #5 https://github.com/foo/bar/pull/5" + ); + } + + #[test] + fn test_ok_confirmation_no_detail() { + assert_eq!(ok_confirmation("commented", ""), "ok commented"); + } + + #[test] + fn test_format_cpt_normal() { + assert_eq!(format_cpt(0.000003), "$3.00/MTok"); + assert_eq!(format_cpt(0.0000038), "$3.80/MTok"); + assert_eq!(format_cpt(0.00000386), "$3.86/MTok"); + } + + #[test] + fn test_format_cpt_edge_cases() { + assert_eq!(format_cpt(0.0), "$0.00/MTok"); // zero + assert_eq!(format_cpt(-0.000001), "$0.00/MTok"); // negative + assert_eq!(format_cpt(f64::INFINITY), "$0.00/MTok"); // infinite + assert_eq!(format_cpt(f64::NAN), "$0.00/MTok"); // NaN + } + + #[test] + fn test_detect_package_manager_default() { + // In the test environment (rtk repo), there's no JS lockfile + // so it should default to "npm" + let pm = detect_package_manager(); + assert!(["pnpm", "yarn", "bun", "npm"].contains(&pm)); + } + + #[test] + fn test_truncate_multibyte_thai() { + // Thai characters are 3 bytes each + let thai = "สวัสดีครับ"; + let result = truncate(thai, 5); + // Should not panic, should produce valid UTF-8 + assert!(result.len() <= thai.len()); + assert!(result.ends_with("...")); + } + + #[test] + fn test_truncate_multibyte_emoji() { + let emoji = "🎉🎊🎈🎁🎂🎄🎃🎆🎇✨"; + let result = truncate(emoji, 5); + assert!(result.ends_with("...")); + } + + #[test] + fn test_truncate_multibyte_cjk() { + let cjk = "你好世界测试字符串"; + let result = truncate(cjk, 6); + assert!(result.ends_with("...")); + } + + #[test] + fn test_truncate_multibyte_cyrillic_issue_2787() { + // Regression: `rtk gain --history` panicked slicing this at byte 22, + // which falls inside 'н' (Cyrillic chars are 2 bytes each) + let cmd = "rtk ls -la Ародинамический расчёт Новый"; + let result = truncate(cmd, 25); + assert_eq!(result.chars().count(), 25); + assert!(result.ends_with("...")); + assert_eq!(result, "rtk ls -la Ародинамиче..."); + } + + // ===== resolve_binary tests (issue #212) ===== + + #[test] + fn test_resolve_binary_finds_known_command() { + // "cargo" must be on PATH in any Rust dev environment + let result = resolve_binary("cargo"); + assert!( + result.is_ok(), + "resolve_binary('cargo') should succeed, got: {:?}", + result.err() + ); + } + + #[test] + fn test_resolve_binary_returns_absolute_path() { + let path = resolve_binary("cargo").expect("cargo should be resolvable"); + assert!( + path.is_absolute(), + "resolve_binary should return absolute path, got: {:?}", + path + ); + } + + #[test] + fn test_resolve_binary_fails_for_unknown() { + let result = resolve_binary("nonexistent_binary_xyz_99999"); + assert!( + result.is_err(), + "resolve_binary should fail for nonexistent binary" + ); + } + + #[test] + fn test_resolve_binary_path_contains_binary_name() { + let path = resolve_binary("cargo").expect("cargo should be resolvable"); + let filename = path + .file_name() + .expect("should have filename") + .to_string_lossy(); + // On Windows this could be "cargo.exe", on Unix just "cargo" + assert!( + filename.starts_with("cargo"), + "resolved path filename should start with 'cargo', got: {}", + filename + ); + } + + // ===== resolved_command tests (issue #212) ===== + + #[test] + fn test_resolved_command_executes_known_command() { + let output = resolved_command("cargo") + .arg("--version") + .output() + .expect("resolved_command('cargo') should execute"); + assert!( + output.status.success(), + "cargo --version should succeed via resolved_command" + ); + } + + // ===== tool_exists tests (issue #212) ===== + + #[test] + fn test_tool_exists_finds_cargo() { + assert!( + tool_exists("cargo"), + "tool_exists('cargo') should return true" + ); + } + + #[test] + fn test_tool_exists_rejects_unknown() { + assert!( + !tool_exists("nonexistent_binary_xyz_99999"), + "tool_exists should return false for nonexistent binary" + ); + } + + #[test] + fn test_tool_exists_finds_git() { + assert!(tool_exists("git"), "tool_exists('git') should return true"); + } + + // ===== Child-process argument quoting (issue #3727) ===== + + #[test] + fn test_quote_arg_wraps_a_quote_with_no_space_around_it() { + assert_eq!(quote_arg_for_child(r#"a"b"#), r#""a\"b""#); + } + + #[test] + fn test_quote_arg_wraps_the_reported_json_key_pattern() { + // `rtk grep -c '"type"' file`, the shape reported in #3727. + assert_eq!(quote_arg_for_child(r#""type""#), r#""\"type\"""#); + } + + #[test] + fn test_quote_arg_wraps_repeated_quotes() { + assert_eq!(quote_arg_for_child(r#"a""b"#), r#""a\"\"b""#); + } + + #[test] + fn test_quote_arg_doubles_backslash_runs_before_a_quote() { + assert_eq!(quote_arg_for_child(r#"a\"b"#), r#""a\\\"b""#); + assert_eq!(quote_arg_for_child(r#"a\\"b"#), r#""a\\\\\"b""#); + } + + #[test] + fn test_quote_arg_doubles_a_trailing_backslash_run() { + assert_eq!(quote_arg_for_child(r#"a"b\"#), r#""a\"b\\""#); + } + + #[test] + fn test_quote_arg_keeps_backslashes_that_precede_ordinary_text() { + assert_eq!(quote_arg_for_child(r#"C:\a\b "x""#), r#""C:\a\b \"x\"""#); + } + + #[test] + fn test_quote_arg_keeps_a_space_inside_the_wrapping() { + assert_eq!(quote_arg_for_child(r#"a b "c""#), r#""a b \"c\"""#); + } + + #[cfg(target_os = "windows")] + #[test] + fn test_child_arg_re_encodes_only_arguments_holding_a_quote() { + let mut cmd = Command::new("grep"); + cmd.child_args(["-c", r#""type""#, "q.jsonl"]); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!(args, ["-c", r#""\"type\"""#, "q.jsonl"]); + } + + #[cfg(target_os = "windows")] + #[test] + fn test_child_arg_leaves_batch_shims_on_stds_encoding() { + // cmd.exe parses by its own rules, so `raw_arg` must not be used there. + let mut cmd = Command::new("gradlew.bat"); + cmd.child_arg(r#""type""#); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!(args, [r#""type""#]); + } + + // ===== Windows-specific PATHEXT resolution tests (issue #212) ===== + + #[cfg(target_os = "windows")] + mod windows_tests { + use super::super::*; + use std::fs; + + /// Create a temporary .cmd wrapper to simulate Node.js tool installation + fn create_temp_cmd_wrapper(dir: &std::path::Path, name: &str) -> std::path::PathBuf { + let cmd_path = dir.join(format!("{}.cmd", name)); + fs::write(&cmd_path, "@echo off\r\necho fake-tool-output\r\n") + .expect("failed to create .cmd wrapper"); + cmd_path + } + + /// Build a PATH string that includes the temp dir + fn path_with_dir(dir: &std::path::Path) -> std::ffi::OsString { + let original = std::env::var_os("PATH").unwrap_or_default(); + let mut new_path = std::ffi::OsString::from(dir.as_os_str()); + new_path.push(";"); + new_path.push(&original); + new_path + } + + #[test] + fn test_resolve_binary_finds_cmd_wrapper() { + let temp_dir = tempfile::tempdir().expect("failed to create temp dir"); + create_temp_cmd_wrapper(temp_dir.path(), "fake-tool-test"); + + // Use which::which_in to avoid mutating global PATH (thread-safe) + let search_path = path_with_dir(temp_dir.path()); + let result = which::which_in( + "fake-tool-test", + Some(search_path), + std::env::current_dir().unwrap(), + ); + + assert!( + result.is_ok(), + "which_in should find .cmd wrapper on Windows, got: {:?}", + result.err() + ); + + let path = result.unwrap(); + let ext = path + .extension() + .unwrap_or_default() + .to_string_lossy() + .to_lowercase(); + assert!( + ext == "cmd" || ext == "bat", + "resolved path should have .cmd/.bat extension, got: {:?}", + path + ); + } + + #[test] + fn test_resolve_binary_finds_bat_wrapper() { + let temp_dir = tempfile::tempdir().expect("failed to create temp dir"); + let bat_path = temp_dir.path().join("fake-bat-tool.bat"); + fs::write(&bat_path, "@echo off\r\necho bat-output\r\n") + .expect("failed to create .bat wrapper"); + + let search_path = path_with_dir(temp_dir.path()); + let result = which::which_in( + "fake-bat-tool", + Some(search_path), + std::env::current_dir().unwrap(), + ); + + assert!( + result.is_ok(), + "which_in should find .bat wrapper on Windows, got: {:?}", + result.err() + ); + } + + #[test] + fn test_resolved_command_executes_cmd_wrapper() { + let temp_dir = tempfile::tempdir().expect("failed to create temp dir"); + create_temp_cmd_wrapper(temp_dir.path(), "fake-exec-test"); + + // Resolve the full path, then execute it directly (no PATH mutation) + let search_path = path_with_dir(temp_dir.path()); + let resolved = which::which_in( + "fake-exec-test", + Some(search_path), + std::env::current_dir().unwrap(), + ) + .expect("should resolve fake-exec-test"); + + let output = Command::new(&resolved).output(); + + assert!( + output.is_ok(), + "Command with resolved path should execute .cmd wrapper on Windows" + ); + let output = output.unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("fake-tool-output"), + "should get output from .cmd wrapper, got: {}", + stdout + ); + } + + #[test] + fn test_resolved_command_fallback_on_unknown_binary() { + // When resolve_binary fails, resolved_command should fall back to + // Command::new(name) instead of panicking. On Windows this also + // prints a warning to stderr. + let mut cmd = resolved_command("nonexistent_binary_xyz_99999"); + // The Command should be created (not panic). Attempting to run it + // will fail, but that's expected — we just verify the fallback path + // produces a usable Command. + let result = cmd.output(); + assert!( + result.is_err() || !result.unwrap().status.success(), + "nonexistent binary should fail to execute, but resolved_command must not panic" + ); + } + + #[test] + fn test_tool_exists_finds_cmd_wrapper() { + let temp_dir = tempfile::tempdir().expect("failed to create temp dir"); + create_temp_cmd_wrapper(temp_dir.path(), "fake-exists-test"); + + let search_path = path_with_dir(temp_dir.path()); + let result = which::which_in( + "fake-exists-test", + Some(search_path), + std::env::current_dir().unwrap(), + ); + + assert!( + result.is_ok(), + "which_in should find .cmd wrapper on Windows" + ); + } + } + + // ===== AWS helper function tests ===== + + #[test] + fn test_shorten_arn_ecs_service() { + assert_eq!( + shorten_arn("arn:aws:ecs:us-east-1:123:service/cluster/api-service"), + "api-service" + ); + } + + #[test] + fn test_shorten_arn_iam_user() { + assert_eq!(shorten_arn("arn:aws:iam::123456789012:user/alice"), "alice"); + } + + #[test] + fn test_shorten_arn_lambda() { + assert_eq!( + shorten_arn("arn:aws:lambda:us-west-2:123:function:my-function"), + "my-function" + ); + } + + #[test] + fn test_shorten_arn_fallback() { + // Non-ARN string - return as-is + assert_eq!(shorten_arn("simple-name"), "simple-name"); + } + + #[test] + fn test_human_bytes_bytes() { + assert_eq!(human_bytes(0), "0 B"); + assert_eq!(human_bytes(512), "512 B"); + assert_eq!(human_bytes(1023), "1023 B"); + } + + #[test] + fn test_human_bytes_kb() { + assert_eq!(human_bytes(1024), "1.0 KB"); + assert_eq!(human_bytes(2048), "2.0 KB"); + assert_eq!(human_bytes(1536), "1.5 KB"); + } + + #[test] + fn test_human_bytes_mb() { + assert_eq!(human_bytes(1_048_576), "1.0 MB"); + assert_eq!(human_bytes(5_242_880), "5.0 MB"); + } + + #[test] + fn test_human_bytes_gb() { + assert_eq!(human_bytes(1_073_741_824), "1.0 GB"); + assert_eq!(human_bytes(2_147_483_648), "2.0 GB"); + } + + #[test] + fn test_human_bytes_tb() { + assert_eq!(human_bytes(1_099_511_627_776), "1.0 TB"); + } + + #[test] + fn test_count_tokens_basic() { + assert_eq!(count_tokens("hello world"), 2); + assert_eq!(count_tokens("one two three four"), 4); + } + + #[test] + fn test_count_tokens_empty() { + assert_eq!(count_tokens(""), 0); + assert_eq!(count_tokens(" "), 0); + } + + #[test] + fn test_count_tokens_multiple_spaces() { + assert_eq!(count_tokens("hello world"), 2); + assert_eq!(count_tokens(" hello world "), 2); + } + + #[cfg(unix)] + fn mode_of(path: &std::path::Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + fs::metadata(path).unwrap().permissions().mode() & 0o777 + } + + #[test] + #[cfg(unix)] + fn test_create_private_dir_is_owner_only() { + let tmp = tempfile::tempdir().unwrap(); + let nested = tmp.path().join("rtk").join("tee"); + create_private_dir(&nested).unwrap(); + assert_eq!(mode_of(&nested), 0o700); + } + + #[test] + #[cfg(unix)] + fn test_create_private_dir_tightens_existing_dir() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("legacy"); + fs::create_dir_all(&dir).unwrap(); + set_owner_only(&dir, 0o755); + + create_private_dir(&dir).unwrap(); + assert_eq!(mode_of(&dir), 0o700); + } + + #[test] + fn test_create_private_dir_is_idempotent() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("a").join("b"); + create_private_dir(&dir).unwrap(); + create_private_dir(&dir).expect("second call must succeed"); + assert!(dir.is_dir()); + } + + #[test] + #[cfg(unix)] + fn test_restrict_file_is_owner_only() { + let tmp = tempfile::tempdir().unwrap(); + let file = tmp.path().join("history.db"); + fs::write(&file, b"x").unwrap(); + set_owner_only(&file, 0o644); + + restrict_file(&file); + assert_eq!(mode_of(&file), 0o600); + } + + #[test] + fn test_restrict_file_ignores_missing_path() { + let tmp = tempfile::tempdir().unwrap(); + restrict_file(&tmp.path().join("absent.db-wal")); + } + + #[test] + fn test_decode_process_output_valid_utf8() { + assert_eq!(decode_process_output(b"hello world"), "hello world"); + } + + #[test] + fn test_decode_process_output_chinese_utf8() { + let input = "测试中文".as_bytes(); + assert_eq!(decode_process_output(input), "测试中文"); + } + + #[test] + fn test_decode_process_output_empty() { + assert_eq!(decode_process_output(b""), ""); + } + + #[test] + fn test_decode_process_output_invalid_utf8_no_panic() { + let bytes: &[u8] = &[0xFF, 0xFE, 0x41, 0x42]; + let result = decode_process_output(bytes); + assert!(!result.is_empty()); + } + + // `decode_mixed` takes the code page as a parameter so these run on every + // platform, not only Windows — the CI runners that would exercise the + // Windows-gated versions do not exist. + + /// The bug this fix targets: GBK output from a Chinese-locale console. + #[test] + fn test_decode_mixed_gbk_run() { + assert_eq!(decode_mixed(&[0xB2, 0xE2, 0xCA, 0xD4], Some(936)), "测试"); + } + + /// GB18030 (54936) must not be treated as plain GBK: it has to keep its + /// own table so 4-byte sequences decode. + #[test] + fn test_decode_mixed_gb18030_is_not_gbk() { + // 0x81 0x35 0xF4 0x37 is a 4-byte GB18030 sequence. + let decoded = decode_mixed(&[0x81, 0x35, 0xF4, 0x37], Some(54936)); + assert_eq!( + decoded.chars().count(), + 1, + "expected one char: {:?}", + decoded + ); + assert!( + !decoded.contains('\u{FFFD}'), + "got replacement: {:?}", + decoded + ); + assert_eq!( + codepage::to_encoding(54936).map(|e| e.name()), + Some("gb18030") + ); + } + + /// Legacy OEM/DOS pages are cmd.exe's default in many locales and are not + /// WHATWG encodings, so they come from `oem_cp` rather than encoding_rs. + #[test] + fn test_decode_mixed_oem_codepages() { + assert_eq!(decode_mixed(&[0xB0, 0xDB], Some(437)), "░█"); + // 850 and 852 are likewise absent from encoding_rs. + assert!(codepage::to_encoding(437).is_none()); + assert!(!decode_mixed(&[0xE1], Some(850)).contains('\u{FFFD}')); + } + + /// The regression the review flagged: a buffer that is almost entirely + /// valid UTF-8 must not be reinterpreted wholesale because of one bad + /// line. Only the GBK line is re-decoded; the UTF-8 lines keep their bytes. + #[test] + fn test_decode_mixed_only_redecodes_invalid_lines() { + let mut bytes = "héllo wörld\n".as_bytes().to_vec(); + bytes.extend_from_slice(&[0xB2, 0xE2, 0xCA, 0xD4, b'\n']); // GBK 测试 + bytes.extend_from_slice("grüße\n".as_bytes()); + + assert_eq!( + decode_mixed(&bytes, Some(936)), + "héllo wörld\n测试\ngrüße\n" + ); + } + + /// A corrupt byte inside an otherwise-UTF-8 line falls back to lossy UTF-8 + /// rather than mojibake, because the code page decode does not come out + /// clean. The surrounding lines are untouched either way. + #[test] + fn test_decode_mixed_corrupt_byte_prefers_lossy_utf8() { + let mut bytes = "first\n".as_bytes().to_vec(); + bytes.extend_from_slice("naïve".as_bytes()); + bytes.push(0xC3); // dangling lead byte: invalid UTF-8 and invalid GBK + bytes.extend_from_slice(b"\nlast\n"); + + let decoded = decode_mixed(&bytes, Some(936)); + assert!(decoded.starts_with("first\n"), "{:?}", decoded); + assert!(decoded.contains("naïve"), "{:?}", decoded); + assert!(decoded.ends_with("\nlast\n"), "{:?}", decoded); + } + + /// Line endings and the final line without a terminator must survive. + #[test] + fn test_decode_mixed_preserves_line_structure() { + let mut bytes = b"a\r\n".to_vec(); + bytes.extend_from_slice(&[0xB2, 0xE2, b'\n']); // GBK 测 + bytes.extend_from_slice(b"tail"); // no trailing newline + + assert_eq!(decode_mixed(&bytes, Some(936)), "a\r\n测\ntail"); + } + + /// No code page (every Unix run) keeps the old lossy behavior. + #[test] + fn test_decode_mixed_without_codepage_is_lossy() { + let mut bytes = b"ok ".to_vec(); + bytes.push(0xFF); + assert_eq!(decode_mixed(&bytes, None), "ok \u{FFFD}"); + } + + /// An unmapped code page must degrade to lossy rather than panic. + #[test] + fn test_decode_mixed_unknown_codepage_is_lossy() { + assert_eq!(decode_mixed(&[0xFF], Some(60000)), "\u{FFFD}"); + } + + /// Truncated trailing UTF-8 must terminate rather than loop forever. + #[test] + fn test_decode_mixed_truncated_utf8_terminates() { + // First two bytes of a three-byte character, cut short. + let decoded = decode_mixed(&[b'a', 0xE6, 0xB5], None); + assert!(decoded.starts_with('a'), "{:?}", decoded); + } + + /// Every byte value must survive both paths without panicking. + #[test] + fn test_decode_mixed_all_byte_values_no_panic() { + let all: Vec = (0u8..=255).collect(); + for cp in [None, Some(936), Some(437), Some(65001), Some(1252)] { + let _ = decode_mixed(&all, cp); + } + } + + /// Without a code page — every Unix run — the result must stay identical to + /// the `String::from_utf8_lossy` this replaced. Splitting on `\n` cannot + /// change it: a newline is valid ASCII, so no ill-formed sequence spans one. + #[test] + fn test_decode_process_output_unchanged_without_codepage() { + let cases: [&[u8]; 6] = [ + b"", + b"plain ascii\n", + "utf8 ünïcödé\nsecond ライン\n".as_bytes(), + &[0xFF, 0xFE, b'\n', b'o', b'k'], + &[b'a', b'\n', 0xE6, 0xB5, b'\n', b'b'], + &[0xB2, 0xE2, 0xCA, 0xD4], + ]; + for bytes in cases { + assert_eq!( + decode_mixed(bytes, None), + String::from_utf8_lossy(bytes), + "lossy mismatch for {:?}", + bytes + ); + } + } + + // rtk-ai/rtk#3206 review: set_owner_only now skips the chmod syscall once + // permissions already match, since Tracker::new() (which calls this via + // create_private_dir/open_private/restrict_db_files) is on the PreToolUse + // hook's hot path for every Bash tool call. These don't observe the skipped + // syscall directly, but confirm the already-correct case stays correct + // (idempotent) across repeated calls, on both a directory and a file. + + #[test] + #[cfg(unix)] + fn test_set_owner_only_idempotent_when_already_correct_dir() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("already-private"); + create_private_dir(&dir).unwrap(); + assert_eq!(mode_of(&dir), 0o700); + + // Second call hits the already-correct fast path — must stay 0o700. + create_private_dir(&dir).unwrap(); + assert_eq!(mode_of(&dir), 0o700); + } + + #[test] + #[cfg(unix)] + fn test_restrict_file_idempotent_when_already_correct() { + let tmp = tempfile::tempdir().unwrap(); + let file = tmp.path().join("history.db"); + fs::write(&file, b"x").unwrap(); + restrict_file(&file); + assert_eq!(mode_of(&file), 0o600); + + // Second call hits the already-correct fast path — must stay 0o600. + restrict_file(&file); + assert_eq!(mode_of(&file), 0o600); + } + + // Regression: the fast-path comparison must mask with 0o7777 (including + // setuid/setgid/sticky), not 0o777 — a 0o777 mask would make a file with e.g. + // setuid set (0o4600) read as "already 0o600" and permanently skip the chmod + // that clears the dangerous bit. Tested against the pure mask logic directly + // (not round-tripped through a real chmod) since an unprivileged chmod setting + // setuid isn't reliably honored across every environment — some sandboxed/ + // containerized filesystems silently drop it, which would make a + // filesystem-based version of this test flaky rather than exercise the logic. + #[test] + #[cfg(unix)] + fn test_mode_already_correct_rejects_setuid_bit() { + assert!( + !mode_already_correct(0o4600, 0o600), + "setuid set (0o4600) must not read as already-correct against target 0o600" + ); + assert!( + !mode_already_correct(0o2600, 0o600), + "setgid set (0o2600) must not read as already-correct against target 0o600" + ); + assert!( + !mode_already_correct(0o1600, 0o600), + "sticky bit set (0o1600) must not read as already-correct against target 0o600" + ); + } + + #[test] + #[cfg(unix)] + fn test_mode_already_correct_accepts_exact_match() { + assert!(mode_already_correct(0o600, 0o600)); + assert!(mode_already_correct(0o700, 0o700)); + assert!(!mode_already_correct(0o644, 0o600)); + } + + #[test] + #[cfg(unix)] + fn test_mode_already_correct_ignores_file_type_bits() { + // `st_mode` carries file-type bits (e.g. S_IFREG = 0o100000) above the + // 0o7777 permission range this function masks to — those must not affect + // the comparison either way. + assert!(mode_already_correct(0o100600, 0o600)); + } + + #[test] + fn test_detect_package_manager_recognizes_bun() { + // Asks about an explicit directory: chdir is process-global and would + // race the other tests in this binary that assert on the real cwd. + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(dir.path().join("bun.lockb"), "").expect("write lockfile"); + assert_eq!(detect_package_manager_in(dir.path()), "bun"); + } + + #[test] + fn test_missing_tool_policy_controls_npx_no_install() { + // Semantics are per caller: a tool the user named may be fetched, one + // rtk chose may not. Changing either would change what runs, not what + // is printed, which is not rtk's job. + let args: Vec = + tool_exec(Some("npm"), "definitely-not-installed", MissingTool::Fetch) + .get_args() + .map(|a| a.to_string_lossy().into_owned()) + .collect(); + assert!(!args.contains(&"--no-install".to_string()), "{args:?}"); + + let args: Vec = + tool_exec(Some("npm"), "definitely-not-installed", MissingTool::Fail) + .get_args() + .map(|a| a.to_string_lossy().into_owned()) + .collect(); + assert!(args.contains(&"--no-install".to_string()), "{args:?}"); + } + + #[test] + fn test_exec_runner_prefers_the_named_runner_and_never_detects_bunx() { + // A named runner is used as given, whatever the fetch policy. + for missing in [MissingTool::Fetch, MissingTool::Fail] { + assert_eq!(exec_runner(Some("bunx"), missing), "bunx"); + assert_eq!(exec_runner(Some("pnpm"), missing), "pnpm"); + } + + // Detection never resolves through bunx: it always fetches a missing + // tool, so MissingTool::Fail could not be honoured. + assert_ne!(exec_runner(None, MissingTool::Fail), "bun"); + assert_ne!(exec_runner(None, MissingTool::Fail), "bunx"); + + // Nothing named and rtk may fetch: npx is the only runner that can. + assert_eq!(exec_runner(None, MissingTool::Fetch), "npx"); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/filters/README.md b/crates/sparkplane-rtk/upstream/src/filters/README.md new file mode 100644 index 0000000..5b9bbb9 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/README.md @@ -0,0 +1,142 @@ +# Built-in Filters + +> See also [docs/contributing/TECHNICAL.md](../../docs/contributing/TECHNICAL.md) for the full architecture overview + +Each `.toml` file in this directory defines one filter and its inline tests. +Files are concatenated alphabetically by `build.rs` into a single TOML blob embedded in the binary. + +## When to Use a TOML Filter + +TOML filters strip noise lines — they don't reformat output. The filtered result must still look like real command output (see [Design Philosophy](../../CONTRIBUTING.md#design-philosophy)). For the full TOML-vs-Rust decision criteria, see [CONTRIBUTING.md](../../CONTRIBUTING.md#toml-vs-rust-which-one). + +TOML works well for commands with **predictable, line-by-line text output** where regex filtering achieves 60%+ savings: +- Install/update logs (brew, composer, poetry) — strip `Using ...` / `Already installed` lines +- System monitoring (df, ps, systemctl) — keep essential rows, drop headers/decorations +- Simple linters (shellcheck, yamllint, hadolint) — strip context, keep findings +- Infra tools (terraform plan, helm, rsync) — strip progress, keep summary + +For the full contribution checklist (including `discover/rules.rs` registration), see [src/cmds/README.md — Adding a New Command Filter](../cmds/README.md#adding-a-new-command-filter). + +## Adding a filter + +1. Copy any existing `.toml` file and rename it (e.g. `my-tool.toml`) +2. Update the three required fields: `description`, `match_command`, and at least one action field +3. Add `[[tests.my-tool]]` entries to verify the filter behaves correctly +4. Run `cargo test` — the build step validates TOML syntax and runs inline tests + +## File format + +```toml +[filters.my-tool] +description = "Short description of what this filter does" +match_command = "^my-tool\\b" # regex matched against the full command string +strip_ansi = true # optional: strip ANSI escape codes first +strip_lines_matching = [ # optional: drop lines matching any of these regexes + "^\\s*$", + "^noise pattern", +] +max_lines = 40 # optional: keep only the first N lines after filtering +on_empty = "my-tool: ok" # optional: message to emit when output is empty after filtering + +[[tests.my-tool]] +name = "descriptive test name" +input = "raw command output here" +expected = "expected filtered output" +``` + +## Available filter fields + +| Field | Type | Description | +|-------|------|-------------| +| `description` | string | Human-readable description | +| `match_command` | regex | Matches the command string (e.g. `"^docker\\s+inspect"`) | +| `strip_ansi` | bool | Strip ANSI escape codes before processing | +| `filter_stderr` | bool | Capture and merge stderr into stdout before filtering (use for tools like liquibase that emit banners to stderr) | +| `strip_lines_matching` | regex[] | Drop lines matching any regex | +| `keep_lines_matching` | regex[] | Keep only lines matching at least one regex | +| `replace` | array | Regex substitutions (`{ pattern, replacement }`) | +| `match_output` | array | Short-circuit rules (`{ pattern, message }`) | +| `truncate_lines_at` | int | Truncate lines longer than N characters | +| `max_lines` | int | Keep only the first N lines | +| `tail_lines` | int | Keep only the last N lines (applied after other filters) | +| `on_empty` | string | Fallback message when filtered output is empty | + +## Naming convention + +Use the command name as the filename: `terraform-plan.toml`, `docker-inspect.toml`, `mix-compile.toml`. +For commands with subcommands, prefer `-.toml` over grouping multiple filters in one file. + +## Build and runtime pipeline + +How a `.toml` file goes from contributor → binary → filtered output. + +```mermaid +flowchart TD + A[["src/filters/my-tool.toml\n(new file)"]] --> B + + subgraph BUILD ["cargo build"] + B["build.rs\n1. ls src/filters/*.toml\n2. sort alphabetically\n3. concat → BUILTIN_TOML"] --> C + C{"TOML valid?\nDuplicate names?"} -->|"fail"| D[["Build fails\nerror points to bad file"]] + C -->|"ok"| E[["OUT_DIR/builtin_filters.toml\n(generated)"]] + E --> F["rustc embeds via include_str!"] + F --> G[["rtk binary\nBUILTIN_TOML embedded"]] + end + + subgraph TESTS ["cargo test"] + H["test_builtin_filter_count\nassert_eq!(filters.len(), N)"] -->|"wrong count"| I[["FAIL"]] + J["test_builtin_all_filters_present\nassert!(names.contains('my-tool'))"] -->|"name missing"| K[["FAIL"]] + L["test_builtin_all_filters_have_inline_tests\nassert!(tested.contains(name))"] -->|"no tests"| M[["FAIL"]] + end + + subgraph RUNTIME ["rtk my-tool args"] + R["TomlFilterRegistry::load()\n1. .rtk/filters.toml\n2. ~/.config/rtk/filters.toml\n3. BUILTIN_TOML\n4. passthrough"] --> S + S{"match_command\nmatches?"} -->|"no match"| T[["exec raw (passthrough)"]] + S -->|"match"| U["exec command\ncapture stdout"] + U --> V["8-stage pipeline\nstrip_ansi → replace → match_output\n→ strip/keep_lines → truncate\n→ tail_lines → max_lines → on_empty"] + V --> W[["print filtered output + exit code"]] + end + + G --> H & J & L & R +``` + +## Filter lookup priority + +```mermaid +flowchart LR + CMD["rtk my-tool args"] --> P1 + P1{"1. .rtk/filters.toml\n(project-local)"} + P1 -->|"match"| WIN["apply filter"] + P1 -->|"no match"| P2 + P2{"2. ~/.config/rtk/filters.toml\n(user-global)"} + P2 -->|"match"| WIN + P2 -->|"no match"| P3 + P3{"3. BUILTIN_TOML\n(binary)"} + P3 -->|"match"| WIN + P3 -->|"no match"| P4[["exec raw (passthrough)"]] +``` + +First match wins. A project filter with the same name as a built-in shadows the built-in and triggers a warning: + +``` +[rtk] warning: filter 'make' is shadowing a built-in filter +``` + +## Custom filters and trust + +You can add your own filters in two places (both use the format above): + +- **Project-local** — `.rtk/filters.toml` (committed with a repo, applies in that project) +- **User-global** — `~/.config/rtk/filters.toml` (applies in every project) + +Because a filter can rewrite or hide the command output an agent sees, **custom filter files are not applied until you trust them**. An untrusted (or edited) filter file is skipped **silently** on the command path — RTK never prints a warning around a rewritten command. You discover and enable untrusted filters through commands you run deliberately: + +```bash +rtk trust # lists each detected filter (labelled project/global) + a risk summary, then asks to confirm ([y/N], or --yes) +rtk untrust # revokes trust +``` + +- Trust is recorded as a SHA-256 of the file's contents, so **editing a trusted file requires re-running `rtk trust`** — a content change invalidates trust. +- `rtk init` detects existing custom filters and lets you enable them — an interactive `[y/N]`, or `--trust-filters` / `--no-trust-filters` for scripts. It stays silent for an empty template (comments only) and, when run non-interactively, leaves filters disabled. +- Built-in filters (this directory) are compiled into the binary and always trusted; only the on-disk project and user-global files are gated. + +> **Honest limitation:** this is consent + tamper-evidence, not a sandbox. An attacker who can write your filter file can usually also write the trust store (`~/.local/share/rtk/trusted_filters.json`) and bypass the gate. It defends the common case — a filter dropped in by a script, a dotfile sync, or an untrusted repo — not a same-user attacker who specifically targets RTK. diff --git a/crates/sparkplane-rtk/upstream/src/filters/ansible-playbook.toml b/crates/sparkplane-rtk/upstream/src/filters/ansible-playbook.toml new file mode 100644 index 0000000..6294379 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ansible-playbook.toml @@ -0,0 +1,34 @@ +[filters.ansible-playbook] +description = "Compact ansible-playbook output" +match_command = "^ansible-playbook\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ok: \\[", + "^skipping: \\[", +] +max_lines = 60 + +[[tests.ansible-playbook]] +name = "strips ok and skipping lines, keeps changed and failures" +input = """ +PLAY [all] ********************************************************************* + +TASK [Gathering Facts] ********************************************************* +ok: [web01] +ok: [web02] + +TASK [Install nginx] *********************************************************** +changed: [web01] +skipping: [web02] + +PLAY RECAP ********************************************************************* +web01 : ok=2 changed=1 unreachable=0 failed=0 +web02 : ok=1 changed=0 unreachable=0 failed=0 +""" +expected = "PLAY [all] *********************************************************************\nTASK [Gathering Facts] *********************************************************\nTASK [Install nginx] ***********************************************************\nchanged: [web01]\nPLAY RECAP *********************************************************************\nweb01 : ok=2 changed=1 unreachable=0 failed=0\nweb02 : ok=1 changed=0 unreachable=0 failed=0" + +[[tests.ansible-playbook]] +name = "failed task preserved" +input = "TASK [Start service] ***\nfailed: [web01] => {\"msg\": \"Service not found\"}\nPLAY RECAP ***\nweb01 : ok=1 failed=1" +expected = "TASK [Start service] ***\nfailed: [web01] => {\"msg\": \"Service not found\"}\nPLAY RECAP ***\nweb01 : ok=1 failed=1" diff --git a/crates/sparkplane-rtk/upstream/src/filters/basedpyright.toml b/crates/sparkplane-rtk/upstream/src/filters/basedpyright.toml new file mode 100644 index 0000000..0826cdb --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/basedpyright.toml @@ -0,0 +1,47 @@ +[filters.basedpyright] +description = "Compact basedpyright type checker output — strip blank lines, keep errors" +match_command = "^basedpyright\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Searching for source files", + "^Found \\d+ source file", + "^Pyright \\d+\\.\\d+", + "^basedpyright \\d+\\.\\d+", +] +max_lines = 50 +on_empty = "basedpyright: ok" + +[[tests.basedpyright]] +name = "strips noise, keeps errors and summary" +input = """ +basedpyright 1.22.0 +Searching for source files +Found 42 source files + +/home/user/app/main.py + /home/user/app/main.py:10:5 - error: "foo" is not defined (reportUndefinedVariable) + /home/user/app/main.py:25:1 - error: Type "str" is not assignable to type "int" (reportAssignmentType) + +/home/user/app/utils.py + /home/user/app/utils.py:8:9 - warning: Variable "x" is not accessed (reportUnusedVariable) + +3 errors, 1 warning, 0 informations +""" +expected = "/home/user/app/main.py\n /home/user/app/main.py:10:5 - error: \"foo\" is not defined (reportUndefinedVariable)\n /home/user/app/main.py:25:1 - error: Type \"str\" is not assignable to type \"int\" (reportAssignmentType)\n/home/user/app/utils.py\n /home/user/app/utils.py:8:9 - warning: Variable \"x\" is not accessed (reportUnusedVariable)\n3 errors, 1 warning, 0 informations" + +[[tests.basedpyright]] +name = "clean output" +input = """ +basedpyright 1.22.0 +Searching for source files +Found 10 source files + +0 errors, 0 warnings, 0 informations +""" +expected = "0 errors, 0 warnings, 0 informations" + +[[tests.basedpyright]] +name = "empty input returns on_empty message" +input = "" +expected = "basedpyright: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/biome.toml b/crates/sparkplane-rtk/upstream/src/filters/biome.toml new file mode 100644 index 0000000..800f778 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/biome.toml @@ -0,0 +1,45 @@ +[filters.biome] +description = "Compact Biome lint/format output — strip blank lines, keep diagnostics" +match_command = "^biome\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Checked \\d+ file", + "^Fixed \\d+ file", + "^The following command", + "^Run it with", +] +max_lines = 50 +on_empty = "biome: ok" + +[[tests.biome]] +name = "lint strips noise, keeps diagnostics" +input = """ +Checked 42 files in 0.5s + +src/app.tsx:5:3 lint/suspicious/noExplicitAny ━━━━━━━━━━━━━━━━━━━━ + × Unexpected any. Specify a different type. + 3 │ interface Props { + 4 │ data: any; + 5 │ ^^^ + +src/utils.ts:12:1 lint/complexity/noForEach ━━━━━━━━━━━━━━━━━━━━ + × Prefer for...of instead of forEach. + 12 │ items.forEach(item => process(item)); + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Found 2 errors. +""" +expected = "src/app.tsx:5:3 lint/suspicious/noExplicitAny ━━━━━━━━━━━━━━━━━━━━\n × Unexpected any. Specify a different type.\n 3 │ interface Props {\n 4 │ data: any;\n 5 │ ^^^\nsrc/utils.ts:12:1 lint/complexity/noForEach ━━━━━━━━━━━━━━━━━━━━\n × Prefer for...of instead of forEach.\n 12 │ items.forEach(item => process(item));\n │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nFound 2 errors." + +[[tests.biome]] +name = "clean check" +input = """ +Checked 42 files in 0.3s +""" +expected = "biome: ok" + +[[tests.biome]] +name = "empty input returns on_empty message" +input = "" +expected = "biome: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/brew-install.toml b/crates/sparkplane-rtk/upstream/src/filters/brew-install.toml new file mode 100644 index 0000000..3c8893b --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/brew-install.toml @@ -0,0 +1,37 @@ +[filters.brew-install] +description = "Compact brew install/upgrade output — strip downloads, short-circuit when already installed" +match_command = "^brew\\s+(install|upgrade)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^==> Downloading", + "^==> Pouring", + "^Already downloaded:", + "^###", + "^==> Fetching", +] +match_output = [ + { pattern = "already installed", message = "ok (already installed)" }, +] +max_lines = 20 + +[[tests.brew-install]] +name = "already installed short-circuits" +input = """ +Warning: rtk 0.27.1 is already installed and up-to-date. +To reinstall 0.27.1, run: + brew reinstall rtk +""" +expected = "ok (already installed)" + +[[tests.brew-install]] +name = "install strips download lines" +input = """ +==> Fetching jq +==> Downloading https://homebrew.bintray.com/bottles/jq-1.7.1.arm64_sonoma.bottle.tar.gz +######################################################################## 100.0% +==> Pouring jq-1.7.1.arm64_sonoma.bottle.tar.gz +==> Summary +/opt/homebrew/Cellar/jq/1.7.1: 18 files, 1.2MB +""" +expected = "==> Summary\n/opt/homebrew/Cellar/jq/1.7.1: 18 files, 1.2MB" diff --git a/crates/sparkplane-rtk/upstream/src/filters/bundle-install.toml b/crates/sparkplane-rtk/upstream/src/filters/bundle-install.toml new file mode 100644 index 0000000..80e0748 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/bundle-install.toml @@ -0,0 +1,61 @@ +[filters.bundle-install] +description = "Compact bundle install/update — strip 'Using' lines, keep installs and errors" +match_command = "^bundle\\s+(install|update)\\b" +strip_ansi = true +strip_lines_matching = [ + "^Using ", + "^\\s*$", + "^Fetching gem metadata", + "^Resolving dependencies", +] +match_output = [ + { pattern = "Bundle complete!", message = "ok bundle: complete" }, + { pattern = "Bundle updated!", message = "ok bundle: updated" }, +] +max_lines = 30 + +[[tests.bundle-install]] +name = "all cached short-circuits" +input = """ +Using bundler 2.5.6 +Using rake 13.1.0 +Using ast 2.4.2 +Using base64 0.2.0 +Using minitest 5.22.2 +Bundle complete! 85 Gemfile dependencies, 200 gems now installed. +Use `bundle info [gemname]` to see where a bundled gem is installed. +""" +expected = "ok bundle: complete" + +[[tests.bundle-install]] +name = "mixed install keeps Fetching and Installing lines" +input = """ +Fetching gem metadata from https://rubygems.org/......... +Resolving dependencies... +Using rake 13.1.0 +Using ast 2.4.2 +Fetching rspec 3.13.0 +Installing rspec 3.13.0 +Using rubocop 1.62.0 +Fetching simplecov 0.22.0 +Installing simplecov 0.22.0 +Bundle complete! 85 Gemfile dependencies, 202 gems now installed. +""" +expected = "ok bundle: complete" + +[[tests.bundle-install]] +name = "update output" +input = """ +Fetching gem metadata from https://rubygems.org/......... +Resolving dependencies... +Using rake 13.1.0 +Fetching rspec 3.14.0 (was 3.13.0) +Installing rspec 3.14.0 (was 3.13.0) +Bundle updated! +""" +expected = "ok bundle: updated" + +[[tests.bundle-install]] +name = "empty output" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/composer-install.toml b/crates/sparkplane-rtk/upstream/src/filters/composer-install.toml new file mode 100644 index 0000000..84d42de --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/composer-install.toml @@ -0,0 +1,40 @@ +[filters.composer-install] +description = "Compact composer install/update/require output — strip downloads, short-circuit when up-to-date" +match_command = "^composer\\s+(install|update|require)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ - Downloading ", + "^ - Installing ", + "^Loading composer", + "^Updating dependencies", +] +match_output = [ + { pattern = "Nothing to install, update or remove", message = "ok (up to date)" }, +] +max_lines = 30 + +[[tests.composer-install]] +name = "nothing to do short-circuits" +input = """ +Loading composer repositories with package information +Updating dependencies +Lock file operations: 0 installs, 0 updates, 0 removals +Nothing to install, update or remove +Generating autoload files +""" +expected = "ok (up to date)" + +[[tests.composer-install]] +name = "install strips download lines" +input = """ +Loading composer repositories with package information +Updating dependencies + - Downloading symfony/console (v6.4.0) + - Installing symfony/console (v6.4.0): Extracting archive + - Downloading psr/log (3.0.0) + - Installing psr/log (3.0.0): Extracting archive +Writing lock file +Generating autoload files +""" +expected = "Writing lock file\nGenerating autoload files" diff --git a/crates/sparkplane-rtk/upstream/src/filters/df.toml b/crates/sparkplane-rtk/upstream/src/filters/df.toml new file mode 100644 index 0000000..f2edf1e --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/df.toml @@ -0,0 +1,16 @@ +[filters.df] +description = "Compact df output — truncate wide columns, limit rows" +match_command = "^df(\\s|$)" +strip_ansi = true +truncate_lines_at = 80 +max_lines = 20 + +[[tests.df]] +name = "short output passes through unchanged" +input = "Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/sda1 4096000 123456 3972544 4% /" +expected = "Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/sda1 4096000 123456 3972544 4% /" + +[[tests.df]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/dotnet-build.toml b/crates/sparkplane-rtk/upstream/src/filters/dotnet-build.toml new file mode 100644 index 0000000..46d9beb --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/dotnet-build.toml @@ -0,0 +1,64 @@ +[filters.dotnet-build] +description = "Compact dotnet build output — short-circuit on success, strip banners" +match_command = "^dotnet\\s+build\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Microsoft \\(R\\)", + "^Copyright \\(C\\)", + "^ Determining projects", +] +match_output = [ + { pattern = "0 Warning\\(s\\)\\n\\s+0 Error\\(s\\)", message = "ok (build succeeded)" }, +] +max_lines = 40 + +[[tests.dotnet-build]] +name = "successful build short-circuits to ok" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... + All projects are up-to-date for restore. + MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll + +Build succeeded. + 0 Warning(s) + 0 Error(s) + +Time Elapsed 00:00:02.34 +""" +expected = "ok (build succeeded)" + +[[tests.dotnet-build]] +name = "build with warnings not short-circuited" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... + MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll + +Build succeeded. + 3 Warning(s) + 0 Error(s) + +Time Elapsed 00:00:01.87 +""" +expected = " MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll\nBuild succeeded.\n 3 Warning(s)\n 0 Error(s)\nTime Elapsed 00:00:01.87" + +[[tests.dotnet-build]] +name = "build errors pass through" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... +src/Program.cs(10,5): error CS1002: ; expected [/home/user/MyApp/MyApp.csproj] + +Build FAILED. + 0 Warning(s) + 1 Error(s) +""" +expected = "src/Program.cs(10,5): error CS1002: ; expected [/home/user/MyApp/MyApp.csproj]\nBuild FAILED.\n 0 Warning(s)\n 1 Error(s)" diff --git a/crates/sparkplane-rtk/upstream/src/filters/du.toml b/crates/sparkplane-rtk/upstream/src/filters/du.toml new file mode 100644 index 0000000..378927d --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/du.toml @@ -0,0 +1,16 @@ +[filters.du] +description = "Compact du output" +match_command = "^du\\b" +strip_lines_matching = ["^\\s*$"] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.du]] +name = "preserves sizes, strips blank lines" +input = "4.0K\t./src\n\n8.0K\t./tests\n16K\t." +expected = "4.0K\t./src\n8.0K\t./tests\n16K\t." + +[[tests.du]] +name = "single line passthrough" +input = "128K\t." +expected = "128K\t." diff --git a/crates/sparkplane-rtk/upstream/src/filters/fail2ban-client.toml b/crates/sparkplane-rtk/upstream/src/filters/fail2ban-client.toml new file mode 100644 index 0000000..f2107cf --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/fail2ban-client.toml @@ -0,0 +1,15 @@ +[filters.fail2ban-client] +description = "Compact fail2ban-client output" +match_command = "^fail2ban-client\\b" +strip_lines_matching = ["^\\s*$"] +max_lines = 30 + +[[tests.fail2ban-client]] +name = "strips blank lines" +input = "Status for the jail: sshd\n|- Filter\n| |- Currently failed: 3\n\n|- Actions\n `- Total banned: 42" +expected = "Status for the jail: sshd\n|- Filter\n| |- Currently failed: 3\n|- Actions\n `- Total banned: 42" + +[[tests.fail2ban-client]] +name = "single line passthrough" +input = "Shutdown successful" +expected = "Shutdown successful" diff --git a/crates/sparkplane-rtk/upstream/src/filters/gcc.toml b/crates/sparkplane-rtk/upstream/src/filters/gcc.toml new file mode 100644 index 0000000..cf5f983 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/gcc.toml @@ -0,0 +1,49 @@ +[filters.gcc] +description = "Compact gcc/g++ compiler output — strip notes, keep errors and warnings" +match_command = "^g(cc|\\+\\+)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s+\\|\\s*$", + "^In file included from", + "^\\s+from\\s", + "^\\d+ warnings? generated", + "^\\d+ errors? generated", +] +max_lines = 50 +on_empty = "gcc: ok" + +[[tests.gcc]] +name = "strips include chain, keeps errors and warnings" +input = """ +In file included from /usr/include/stdio.h:42: + from main.c:1: +main.c:10:5: error: use of undeclared identifier 'foo' + foo(); + ^ +main.c:15:12: warning: unused variable 'x' [-Wunused-variable] + int x = 42; + ^ +2 warnings generated. +1 error generated. +""" +expected = "main.c:10:5: error: use of undeclared identifier 'foo'\n foo();\n ^\nmain.c:15:12: warning: unused variable 'x' [-Wunused-variable]\n int x = 42;\n ^" + +[[tests.gcc]] +name = "clean compilation" +input = """ +""" +expected = "gcc: ok" + +[[tests.gcc]] +name = "linker error kept" +input = """ +/usr/bin/ld: /tmp/main.o: undefined reference to 'missing_func' +collect2: error: ld returned 1 exit status +""" +expected = "/usr/bin/ld: /tmp/main.o: undefined reference to 'missing_func'\ncollect2: error: ld returned 1 exit status" + +[[tests.gcc]] +name = "empty input returns on_empty message" +input = "" +expected = "gcc: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/gcloud.toml b/crates/sparkplane-rtk/upstream/src/filters/gcloud.toml new file mode 100644 index 0000000..275d7c3 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/gcloud.toml @@ -0,0 +1,22 @@ +[filters.gcloud] +description = "Compact gcloud output" +match_command = "^gcloud\\b" +strip_ansi = true +strip_lines_matching = ["^\\s*$"] +truncate_lines_at = 120 +max_lines = 30 + +[[tests.gcloud]] +name = "strips blank lines, preserves output" +input = """ +Updated property [core/project]. + +NAME REGION STATUS +my-cluster us-central1 RUNNING +""" +expected = "Updated property [core/project].\nNAME REGION STATUS\nmy-cluster us-central1 RUNNING" + +[[tests.gcloud]] +name = "single line passthrough" +input = "Listed 0 items." +expected = "Listed 0 items." diff --git a/crates/sparkplane-rtk/upstream/src/filters/hadolint.toml b/crates/sparkplane-rtk/upstream/src/filters/hadolint.toml new file mode 100644 index 0000000..141a9be --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/hadolint.toml @@ -0,0 +1,24 @@ +[filters.hadolint] +description = "Compact hadolint Dockerfile linting output" +match_command = "^hadolint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.hadolint]] +name = "Dockerfile warnings kept, blank lines stripped" +input = """ +Dockerfile:3 DL3008 warning: Pin versions in apt-get install +Dockerfile:5 DL3009 info: Delete apt-get lists after installing + +Dockerfile:8 DL4006 warning: Set SHELL option -o pipefail before RUN with pipe +""" +expected = "Dockerfile:3 DL3008 warning: Pin versions in apt-get install\nDockerfile:5 DL3009 info: Delete apt-get lists after installing\nDockerfile:8 DL4006 warning: Set SHELL option -o pipefail before RUN with pipe" + +[[tests.hadolint]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/helm.toml b/crates/sparkplane-rtk/upstream/src/filters/helm.toml new file mode 100644 index 0000000..790ddd3 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/helm.toml @@ -0,0 +1,103 @@ +[filters.helm] +description = "Compact helm output" +match_command = "^helm\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^W\\d{4}", +] +truncate_lines_at = 120 + +[[tests.helm]] +name = "strips blank lines, preserves release info" +input = """ +NAME: my-release +LAST DEPLOYED: Mon Jan 15 10:30:00 2024 +NAMESPACE: default +STATUS: deployed +REVISION: 3 + +NOTES: +Application is running. +""" +expected = "NAME: my-release\nLAST DEPLOYED: Mon Jan 15 10:30:00 2024\nNAMESPACE: default\nSTATUS: deployed\nREVISION: 3\nNOTES:\nApplication is running." + +[[tests.helm]] +name = "strips glog W-prefix warnings" +input = "W0115 10:30:00 warning message from internal\nNAME: my-chart\nSTATUS: deployed" +expected = "NAME: my-chart\nSTATUS: deployed" + +[[tests.helm]] +name = "helm template output is not truncated" +input = """ +# Source: my-chart/templates/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: my-ns +--- +# Source: my-chart/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-secret +--- +# Source: my-chart/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-cm +--- +# Source: my-chart/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc +--- +# Source: my-chart/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: my-svc +--- +# Source: my-chart/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-deploy +""" +expected = """# Source: my-chart/templates/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: my-ns +--- +# Source: my-chart/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-secret +--- +# Source: my-chart/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-cm +--- +# Source: my-chart/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc +--- +# Source: my-chart/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: my-svc +--- +# Source: my-chart/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-deploy""" diff --git a/crates/sparkplane-rtk/upstream/src/filters/iptables.toml b/crates/sparkplane-rtk/upstream/src/filters/iptables.toml new file mode 100644 index 0000000..95f417d --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/iptables.toml @@ -0,0 +1,27 @@ +[filters.iptables] +description = "Compact iptables output" +match_command = "^iptables\\b" +strip_lines_matching = [ + "^\\s*$", + "^Chain DOCKER", + "^Chain BR-", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.iptables]] +name = "strips Docker chains, preserves real rules" +input = """ +Chain INPUT (policy ACCEPT) +num target prot opt source destination +1 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 +Chain DOCKER (1 references) + DOCKER all -- 0.0.0.0/0 0.0.0.0/0 +Chain BR-abcdef (0 references) +""" +expected = "Chain INPUT (policy ACCEPT)\nnum target prot opt source destination\n1 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0\n DOCKER all -- 0.0.0.0/0 0.0.0.0/0" + +[[tests.iptables]] +name = "preserves FORWARD and OUTPUT chains" +input = "Chain FORWARD (policy DROP)\n1 ACCEPT tcp\nChain OUTPUT (policy ACCEPT)\n1 ACCEPT all" +expected = "Chain FORWARD (policy DROP)\n1 ACCEPT tcp\nChain OUTPUT (policy ACCEPT)\n1 ACCEPT all" diff --git a/crates/sparkplane-rtk/upstream/src/filters/jira.toml b/crates/sparkplane-rtk/upstream/src/filters/jira.toml new file mode 100644 index 0000000..9de5ad3 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/jira.toml @@ -0,0 +1,20 @@ +[filters.jira] +description = "Compact Jira CLI output — strip verbose metadata, keep essentials" +match_command = "^jira\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*--", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.jira]] +name = "strips blank lines from issue list" +input = "TYPE\tKEY\tSUMMARY\tSTATUS\n\nStory\tPROJ-123\tAdd login feature\tIn Progress\n\nBug\tPROJ-456\tFix crash on startup\tOpen" +expected = "TYPE\tKEY\tSUMMARY\tSTATUS\nStory\tPROJ-123\tAdd login feature\tIn Progress\nBug\tPROJ-456\tFix crash on startup\tOpen" + +[[tests.jira]] +name = "single issue view" +input = "KEY: PROJ-123\nSummary: Add login feature\nStatus: In Progress\nAssignee: john@example.com" +expected = "KEY: PROJ-123\nSummary: Add login feature\nStatus: In Progress\nAssignee: john@example.com" diff --git a/crates/sparkplane-rtk/upstream/src/filters/jj.toml b/crates/sparkplane-rtk/upstream/src/filters/jj.toml new file mode 100644 index 0000000..6b330a3 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/jj.toml @@ -0,0 +1,28 @@ +[filters.jj] +description = "Compact Jujutsu (jj) output — strip blank lines, truncate" +match_command = "^jj\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Hint:", + "^Working copy now at:", +] +max_lines = 30 +truncate_lines_at = 120 + +[[tests.jj]] +name = "log output stripped of hints" +input = """ +@ qpvuntsm patrick@example.com 2026-03-10 12:00 abc123 +│ feat: add new feature +◉ zzzzzzzz root() + +Working copy now at: qpvuntsm abc123 feat: add new feature +Hint: use `jj log` to see the full history +""" +expected = "@ qpvuntsm patrick@example.com 2026-03-10 12:00 abc123\n│ feat: add new feature\n◉ zzzzzzzz root()" + +[[tests.jj]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/jq.toml b/crates/sparkplane-rtk/upstream/src/filters/jq.toml new file mode 100644 index 0000000..49f68ab --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/jq.toml @@ -0,0 +1,24 @@ +[filters.jq] +description = "Compact jq output — truncate large JSON results" +match_command = "^jq\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 40 +truncate_lines_at = 120 + +[[tests.jq]] +name = "short output passes through" +input = """ +{ + "name": "test", + "version": "1.0" +} +""" +expected = "{\n \"name\": \"test\",\n \"version\": \"1.0\"\n}" + +[[tests.jq]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/just.toml b/crates/sparkplane-rtk/upstream/src/filters/just.toml new file mode 100644 index 0000000..31e58a5 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/just.toml @@ -0,0 +1,26 @@ +[filters.just] +description = "Compact just task runner output — strip recipe headers, keep command output" +match_command = "^just\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Available recipes:", + "^\\s*just --list", +] +truncate_lines_at = 150 +max_lines = 50 + +[[tests.just]] +name = "preserves command output" +input = "cargo test\n\ntest result: ok. 42 passed; 0 failed\n" +expected = "cargo test\ntest result: ok. 42 passed; 0 failed" + +[[tests.just]] +name = "preserves error output" +input = "error: Compilation failed\nsrc/main.rs:10: expected `;`" +expected = "error: Compilation failed\nsrc/main.rs:10: expected `;`" + +[[tests.just]] +name = "empty input" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/liquibase.toml b/crates/sparkplane-rtk/upstream/src/filters/liquibase.toml new file mode 100644 index 0000000..12d068f --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/liquibase.toml @@ -0,0 +1,84 @@ +[filters.liquibase] +description = "Compact liquibase output — strip headers and generic info" +match_command = "^liquibase(?:\\s|$)" +strip_ansi = true +filter_stderr = true +strip_lines_matching = [ + "^\\s*$", + "^Starting Liquibase at", + "^Liquibase (?:Community|Open Source)", + "^Liquibase Home:", + "^Java Home", + "^Libraries:", + "^\\s*-\\s+\\S+\\.jar", + "^INFO \\[liquibase\\.integration\\]", + "^INFO \\[liquibase\\.core\\] Reading resource", + "^INFO \\[liquibase\\.core\\] Parsing", + "^(?:\\[?INFO\\]?\\s*)?#+$", + "^\\s*##" +] +on_empty = "liquibase: ok" +max_lines = 200 + +[[tests.liquibase]] +name = "strip ascii banner and info logs from subcommand" +input = ''' +#################################################### +## _ _ _ _ ## +## | | (_) (_) | ## +#################################################### +Starting Liquibase at 10:12:11 (version 4.29.1) +Liquibase Version: 4.29.1 +Liquibase Open Source 4.29.1 by Liquibase +INFO [liquibase.integration] Starting command +INFO [liquibase.core] Reading resource db/changelog.xml +INFO [liquibase.core] Parsing db/changelog.xml +Running Changeset: filepath::id::author +Changeset filepath::id::author ran successfully +''' +expected = ''' +Liquibase Version: 4.29.1 +Running Changeset: filepath::id::author +Changeset filepath::id::author ran successfully''' + +[[tests.liquibase]] +name = "strip --version noise, keep only version line" +input = ''' +#################################################### +## _ _ _ _ ## +#################################################### +Starting Liquibase at 13:45:24 using Java 17.0.15 (version 4.30.0 #4943 built at 2024-10-31 17:00+0000) +Liquibase Home: D:\mcp\bash\lbr\third-party +Java Home C:\Program Files\Java\jdk-17.0.15 (Version 17.0.15) +Libraries: + - internal\lib\commons-io.jar: Apache Commons IO 2.17.0 By The Apache Software Foundation + - internal\lib\picocli.jar: picocli 4.7.6 By Remko Popma + - lib\ojdbc10-19.30.0.0.jar: JDBC 19.30.0.0.0 By Oracle Corporation + +Liquibase Version: 4.30.0 +Liquibase Open Source 4.30.0 by Liquibase +''' +expected = ''' +Liquibase Version: 4.30.0''' + +[[tests.liquibase]] +name = "keep status and error lines" +input = ''' +#################################################### +## _ _ _ _ ## +#################################################### +Starting Liquibase at 10:00:00 (version 4.30.0) +Liquibase Version: 4.30.0 +Liquibase Open Source 4.30.0 by Liquibase +HR@jdbc:oracle:thin:@localhost:1523:XE is up to date +Liquibase command 'status' was executed successfully. +''' +expected = ''' +Liquibase Version: 4.30.0 +HR@jdbc:oracle:thin:@localhost:1523:XE is up to date +Liquibase command 'status' was executed successfully.''' + +[[tests.liquibase]] +name = "empty input" +input = "" +expected = "liquibase: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/make.toml b/crates/sparkplane-rtk/upstream/src/filters/make.toml new file mode 100644 index 0000000..63925d4 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/make.toml @@ -0,0 +1,41 @@ +[filters.make] +description = "Compact make output" +match_command = "^make\\b" +strip_lines_matching = [ + "^make\\[\\d+\\]:", + "^\\s*$", + "^Nothing to be done", +] +max_lines = 50 +on_empty = "make: ok" + +[[tests.make]] +name = "strips entering/leaving lines" +input = """ +make[1]: Entering directory '/home/user' +gcc -O2 foo.c +make[1]: Leaving directory '/home/user' +""" +expected = """ +gcc -O2 foo.c +""" + +[[tests.make]] +name = "strips blank lines" +input = """ +gcc -O2 foo.c + +gcc -O2 bar.c +""" +expected = """ +gcc -O2 foo.c +gcc -O2 bar.c +""" + +[[tests.make]] +name = "on_empty when all stripped" +input = """ +make[1]: Entering directory '/home/user' +make[1]: Leaving directory '/home/user' +""" +expected = "make: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/markdownlint.toml b/crates/sparkplane-rtk/upstream/src/filters/markdownlint.toml new file mode 100644 index 0000000..2b25d8c --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/markdownlint.toml @@ -0,0 +1,24 @@ +[filters.markdownlint] +description = "Compact markdownlint output — strip blank lines, limit rows" +match_command = "^markdownlint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.markdownlint]] +name = "linting errors stripped of blank lines" +input = """ +README.md:1:1 MD041/first-line-heading/first-line-h1 First line in file should be a top level heading +README.md:10:1 MD022/blanks-around-headings Headings should be surrounded by blank lines + +README.md:15:80 MD013/line-length Line length [Expected: 80; Actual: 95] +""" +expected = "README.md:1:1 MD041/first-line-heading/first-line-h1 First line in file should be a top level heading\nREADME.md:10:1 MD022/blanks-around-headings Headings should be surrounded by blank lines\nREADME.md:15:80 MD013/line-length Line length [Expected: 80; Actual: 95]" + +[[tests.markdownlint]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/mise.toml b/crates/sparkplane-rtk/upstream/src/filters/mise.toml new file mode 100644 index 0000000..7223d12 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/mise.toml @@ -0,0 +1,30 @@ +[filters.mise] +description = "Compact mise task runner output — strip status lines, keep task results" +match_command = "^mise\\s+(run|exec|install|upgrade)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^mise\\s+(trust|install|upgrade).*✓", + "^mise\\s+Installing\\s", + "^mise\\s+Downloading\\s", + "^mise\\s+Extracting\\s", + "^mise\\s+\\w+@[\\d.]+ installed", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "mise: ok" + +[[tests.mise]] +name = "strips install noise, keeps task output" +input = "mise Installing node@20.0.0\nmise Downloading node@20.0.0\nmise Extracting node@20.0.0\nmise node@20.0.0 installed\n\nlint check passed\n2 warnings found" +expected = "lint check passed\n2 warnings found" + +[[tests.mise]] +name = "preserves error output" +input = "mise run lint\nError: biome check failed\nsrc/index.ts:5 — unused variable" +expected = "mise run lint\nError: biome check failed\nsrc/index.ts:5 — unused variable" + +[[tests.mise]] +name = "empty after stripping" +input = "mise trust ~/dev/.mise.toml ✓\nmise install node@20 ✓\n" +expected = "mise: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/mix-compile.toml b/crates/sparkplane-rtk/upstream/src/filters/mix-compile.toml new file mode 100644 index 0000000..14072f6 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/mix-compile.toml @@ -0,0 +1,27 @@ +[filters.mix-compile] +description = "Compact mix compile output" +match_command = "^mix\\s+compile(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^Compiling \\d+ file", + "^\\s*$", + "^Generated\\s", +] +max_lines = 40 +on_empty = "mix compile: ok" + +[[tests.mix-compile]] +name = "strips compile noise, preserves warnings" +input = """ +Compiling 12 files (.ex) +Generated my_app app + +warning: variable "conn" is unused + lib/router.ex:42 +""" +expected = "warning: variable \"conn\" is unused\n lib/router.ex:42" + +[[tests.mix-compile]] +name = "on_empty when only noise" +input = "Compiling 3 files (.ex)\nGenerated my_app app\n" +expected = "mix compile: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/mix-format.toml b/crates/sparkplane-rtk/upstream/src/filters/mix-format.toml new file mode 100644 index 0000000..460e23a --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/mix-format.toml @@ -0,0 +1,15 @@ +[filters.mix-format] +description = "Compact mix format output" +match_command = "^mix\\s+format(\\s|$)" +on_empty = "mix format: ok" +max_lines = 20 + +[[tests.mix-format]] +name = "empty output returns ok" +input = "" +expected = "mix format: ok" + +[[tests.mix-format]] +name = "changed files pass through" +input = "lib/my_app.ex\ntest/my_app_test.exs" +expected = "lib/my_app.ex\ntest/my_app_test.exs" diff --git a/crates/sparkplane-rtk/upstream/src/filters/nx.toml b/crates/sparkplane-rtk/upstream/src/filters/nx.toml new file mode 100644 index 0000000..d42dfb7 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/nx.toml @@ -0,0 +1,25 @@ +[filters.nx] +description = "Compact Nx monorepo output — strip task graph noise, keep results" +match_command = "^(pnpm\\s+)?nx\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*>\\s*NX\\s+Running target", + "^\\s*>\\s*NX\\s+Nx read the output", + "^\\s*>\\s*NX\\s+View logs", + "^———————", + "^—————————", + "^\\s+Nx \\(powered by", +] +truncate_lines_at = 150 +max_lines = 60 + +[[tests.nx]] +name = "strips Nx noise, keeps build output" +input = "\n > NX Running target build for project myapp\n\n———————————————————————————————————————\nCompiled successfully.\nOutput: dist/apps/myapp\n\n > NX View logs at /tmp/.nx/runs/abc123\n\n Nx (powered by computation caching)\n" +expected = "Compiled successfully.\nOutput: dist/apps/myapp" + +[[tests.nx]] +name = "preserves error output" +input = "ERROR: Cannot find module '@myapp/shared'\n\n > NX Running target build for project myapp\n\nFailed at step: build" +expected = "ERROR: Cannot find module '@myapp/shared'\nFailed at step: build" diff --git a/crates/sparkplane-rtk/upstream/src/filters/ollama.toml b/crates/sparkplane-rtk/upstream/src/filters/ollama.toml new file mode 100644 index 0000000..e325ec9 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ollama.toml @@ -0,0 +1,23 @@ +[filters.ollama] +description = "Strip ANSI spinners and cursor control from ollama output, keep final text" +match_command = "^ollama\\s+run\\b" +strip_ansi = true +strip_lines_matching = [ + "^[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏\\s]*$", + "^\\s*$", +] + +[[tests.ollama]] +name = "strips spinner lines, keeps response" +input = "⠋ \n⠙ \n⠹ \nHello! How can I help you today?" +expected = "Hello! How can I help you today?" + +[[tests.ollama]] +name = "preserves multi-line response" +input = "⠋ \n⠙ \nLine one of the response.\nLine two of the response." +expected = "Line one of the response.\nLine two of the response." + +[[tests.ollama]] +name = "empty input" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/oxlint.toml b/crates/sparkplane-rtk/upstream/src/filters/oxlint.toml new file mode 100644 index 0000000..74326ca --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/oxlint.toml @@ -0,0 +1,43 @@ +[filters.oxlint] +description = "Compact oxlint output — strip blank lines, keep diagnostics" +match_command = "^oxlint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Finished in \\d+", + "^Found \\d+ warning", +] +max_lines = 50 +on_empty = "oxlint: ok" + +[[tests.oxlint]] +name = "strips noise, keeps diagnostics" +input = """ + × eslint(no-console): Unexpected console statement. + ╭─[src/app.ts:5:3] + 5 │ console.log("debug"); + │ ^^^^^^^^^^^ + ╰──── + + × eslint(no-unused-vars): 'x' is defined but never used. + ╭─[src/utils.ts:2:7] + 2 │ let x = 42; + │ ^ + ╰──── + +Found 2 warnings on 2 files. +Finished in 12ms on 100 files. +""" +expected = " × eslint(no-console): Unexpected console statement.\n ╭─[src/app.ts:5:3]\n 5 │ console.log(\"debug\");\n │ ^^^^^^^^^^^\n ╰────\n × eslint(no-unused-vars): 'x' is defined but never used.\n ╭─[src/utils.ts:2:7]\n 2 │ let x = 42;\n │ ^\n ╰────" + +[[tests.oxlint]] +name = "clean output" +input = """ +Finished in 5ms on 100 files. +""" +expected = "oxlint: ok" + +[[tests.oxlint]] +name = "empty input returns on_empty message" +input = "" +expected = "oxlint: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/ping.toml b/crates/sparkplane-rtk/upstream/src/filters/ping.toml new file mode 100644 index 0000000..793b121 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ping.toml @@ -0,0 +1,63 @@ +[filters.ping] +description = "Compact ping output — strip per-packet lines, keep summary" +match_command = "^ping\\b" +strip_ansi = true +strip_lines_matching = [ + "^PING ", + "^Pinging ", + "^\\d+ bytes from ", + "^Reply from .+: bytes=", + "^\\s*$", +] +tail_lines = 4 + +[[tests.ping]] +name = "success keeps summary only" +input = """ +PING example.com (93.184.216.34): 56 data bytes +64 bytes from 93.184.216.34: icmp_seq=0 ttl=56 time=14.2 ms +64 bytes from 93.184.216.34: icmp_seq=1 ttl=56 time=13.8 ms +64 bytes from 93.184.216.34: icmp_seq=2 ttl=56 time=14.1 ms +64 bytes from 93.184.216.34: icmp_seq=3 ttl=56 time=13.9 ms + +--- example.com ping statistics --- +4 packets transmitted, 4 packets received, 0.0% packet loss +round-trip min/avg/max/stddev = 13.8/14.0/14.2/0.2 ms +""" +expected = """--- example.com ping statistics --- +4 packets transmitted, 4 packets received, 0.0% packet loss +round-trip min/avg/max/stddev = 13.8/14.0/14.2/0.2 ms""" + +[[tests.ping]] +name = "windows format keeps stats block only" +input = """ +Pinging 192.0.2.1 with 32 bytes of data: +Reply from 192.0.2.1: bytes=32 time=14ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=13ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=14ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=13ms TTL=56 + +Ping statistics for 192.0.2.1: + Packets: Sent = 4, Received = 4, Lost = 0 (0% loss), +Approximate round trip times in milli-seconds: + Minimum = 13ms, Maximum = 14ms, Average = 13ms +""" +expected = """Ping statistics for 192.0.2.1: + Packets: Sent = 4, Received = 4, Lost = 0 (0% loss), +Approximate round trip times in milli-seconds: + Minimum = 13ms, Maximum = 14ms, Average = 13ms""" + +[[tests.ping]] +name = "unreachable host passes error through" +input = """ +PING unreachable.example.com (192.0.2.1): 56 data bytes +Request timeout for icmp_seq 0 +Request timeout for icmp_seq 1 + +--- unreachable.example.com ping statistics --- +2 packets transmitted, 0 packets received, 100.0% packet loss +""" +expected = """Request timeout for icmp_seq 0 +Request timeout for icmp_seq 1 +--- unreachable.example.com ping statistics --- +2 packets transmitted, 0 packets received, 100.0% packet loss""" diff --git a/crates/sparkplane-rtk/upstream/src/filters/pio-run.toml b/crates/sparkplane-rtk/upstream/src/filters/pio-run.toml new file mode 100644 index 0000000..54d2471 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/pio-run.toml @@ -0,0 +1,40 @@ +[filters.pio-run] +description = "Compact PlatformIO build output" +match_command = "^pio\\s+run(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Verbose mode", + "^CONFIGURATION:", + "^LDF:", + "^Library Manager:", + "^Compiling\\s", + "^Linking\\s", + "^Building\\s", + "^Checking size", +] +max_lines = 30 +on_empty = "pio run: ok" + +[[tests.pio-run]] +name = "strips build noise, preserves errors" +input = """ +Verbose mode can be enabled via `-v, --verbose` option +CONFIGURATION: https://docs.platformio.org/page/boards/espressif32/esp32dev.html +LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf +Compiling .pio/build/esp32dev/src/main.cpp.o +Building .pio/build/esp32dev/firmware.elf +Linking .pio/build/esp32dev/firmware.elf +Checking size .pio/build/esp32dev/firmware.elf +src/main.cpp:10:3: error: 'LED_BUILTINN' was not declared +""" +expected = "src/main.cpp:10:3: error: 'LED_BUILTINN' was not declared" + +[[tests.pio-run]] +name = "on_empty when clean build with only noise" +input = """ +Verbose mode can be enabled via `-v, --verbose` option +Compiling .pio/build/esp32dev/src/main.cpp.o +Linking .pio/build/esp32dev/firmware.elf +""" +expected = "pio run: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/poetry-install.toml b/crates/sparkplane-rtk/upstream/src/filters/poetry-install.toml new file mode 100644 index 0000000..bceb4dd --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/poetry-install.toml @@ -0,0 +1,50 @@ +[filters.poetry-install] +description = "Compact poetry install/lock/update output — strip downloads, short-circuit when up-to-date" +match_command = "^poetry\\s+(install|lock|update)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ [-•] Downloading ", + "^ [-•] Installing .* \\(", + "^Creating virtualenv", + "^Using virtualenv", +] +match_output = [ + { pattern = "No dependencies to install or update|No changes\\.", message = "ok (up to date)" }, +] +max_lines = 30 + +[[tests.poetry-install]] +name = "up to date short-circuits" +input = """ +Installing dependencies from lock file + +No dependencies to install or update +""" +expected = "ok (up to date)" + +[[tests.poetry-install]] +name = "poetry 2.x bullet syntax short-circuits to ok" +input = """ +• Installing requests (2.31.0) +• Installing certifi (2023.11.17) + +No changes. +""" +expected = "ok (up to date)" + +[[tests.poetry-install]] +name = "install strips download lines" +input = """ +Installing dependencies from lock file + + - Downloading requests-2.31.0-py3-none-any.whl (62.6 kB) + - Installing certifi (2023.11.17) + - Installing charset-normalizer (3.3.2) + - Installing idna (3.6) + - Installing urllib3 (2.1.0) + - Installing requests (2.31.0) + +Writing lock file +""" +expected = "Installing dependencies from lock file\nWriting lock file" diff --git a/crates/sparkplane-rtk/upstream/src/filters/pre-commit.toml b/crates/sparkplane-rtk/upstream/src/filters/pre-commit.toml new file mode 100644 index 0000000..04fcc33 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/pre-commit.toml @@ -0,0 +1,35 @@ +[filters.pre-commit] +description = "Compact pre-commit output" +match_command = "^pre-commit\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\[INFO\\] Installing environment", + "^\\[INFO\\] Once installed this environment will be reused", + "^\\[INFO\\] This may take a few minutes", + "^\\s*$", +] +max_lines = 40 + +[[tests.pre-commit]] +name = "strips INFO install noise, keeps hook results" +input = """ +[INFO] Installing environment for https://github.com/psf/black. +[INFO] Once installed this environment will be reused. +[INFO] This may take a few minutes... +Trim Trailing Whitespace.................................................Passed +Fix End of Files.........................................................Passed +Check Yaml...............................................................Failed +- hook id: check-yaml +- exit code: 1 +""" +expected = "Trim Trailing Whitespace.................................................Passed\nFix End of Files.........................................................Passed\nCheck Yaml...............................................................Failed\n- hook id: check-yaml\n- exit code: 1" + +[[tests.pre-commit]] +name = "all passed — no INFO noise" +input = """ +[INFO] Installing environment for https://github.com/pre-commit/mirrors-isort. +[INFO] Once installed this environment will be reused. +isort....................................................................Passed +black....................................................................Passed +""" +expected = "isort....................................................................Passed\nblack....................................................................Passed" diff --git a/crates/sparkplane-rtk/upstream/src/filters/ps.toml b/crates/sparkplane-rtk/upstream/src/filters/ps.toml new file mode 100644 index 0000000..685ef93 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ps.toml @@ -0,0 +1,16 @@ +[filters.ps] +description = "Compact ps output — truncate wide lines, limit rows" +match_command = "^ps(\\s|$)" +strip_ansi = true +truncate_lines_at = 120 +max_lines = 30 + +[[tests.ps]] +name = "short process list passes through unchanged" +input = "USER PID %CPU %MEM COMMAND\nroot 1 0.0 0.0 /sbin/launchd\nflorian 42 0.1 0.2 bash" +expected = "USER PID %CPU %MEM COMMAND\nroot 1 0.0 0.0 /sbin/launchd\nflorian 42 0.1 0.2 bash" + +[[tests.ps]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/pulumi-destroy.toml b/crates/sparkplane-rtk/upstream/src/filters/pulumi-destroy.toml new file mode 100644 index 0000000..1052c0f --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/pulumi-destroy.toml @@ -0,0 +1,60 @@ +[filters.pulumi-destroy] +description = "Compact Pulumi destroy output" +match_command = "^pulumi\\s+destroy(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Destroying \\(", + "^Previewing (update|destroy)", + "^View in Browser", + "^View Live:", + "^Duration:", + "^Permalink:", + "^\\s*Type\\s+Name\\s+", + "^Loading policy packs", + "^@ (Previewing (update|destroy)|Destroying)", + "^More information at:", + "^Use `pulumi ", + "^The resources in the stack have been deleted", + "^If you want to remove the stack completely", + "^\\s+-\\s+.*\\bdeleting\\s+\\(", + "^\\s{4,}at\\s+\\S+\\s*\\(", + "^\\s{4,}at\\s+/", + "^\\s+at\\s+processTicksAndRejections", + "^\\s+promise:\\s+Promise", + "^\\s+\\[Circular", + "^\\s*-.jar name, and jars under a path containing spaces, are deliberately left to full passthrough rather than risk misclassifying an unrelated tool." +match_command = "^(mvn\\s+spring-boot:run|java\\s+-jar\\s+(?:\\S*[/\\\\])?[^/\\\\\\s]*(?i:spring)[^/\\\\\\s]*\\.jar|gradle\\s+.*bootRun)" +strip_ansi = true +keep_lines_matching = [ + "Started\\s.*\\sin\\s", + "Tomcat started on port", + "ERROR", + "WARN", + "Exception", + "Caused by:", + "Application run failed", + "BUILD\\s", + "Tests run:", + "FAILURE", + "listening on port", +] +max_lines = 30 + +[[tests.spring-boot]] +name = "keeps startup summary and errors" +input = " . ____ _ \n /\\\\ / ___'_ __ _ _(_)_ __ \n( ( )\\___ | '_ | '_| | '_ \\ \n \\/ ___)| |_)| | | | | || )\n ' |____| .__|_| |_|_| |_\\__|\n :: Spring Boot :: (v3.2.0)\n2024-01-01 INFO Initializing Spring\n2024-01-01 INFO Bean 'dataSource' created\n2024-01-01 INFO Tomcat started on port 8080\n2024-01-01 INFO Started MyApp in 3.2 seconds" +expected = "2024-01-01 INFO Tomcat started on port 8080\n2024-01-01 INFO Started MyApp in 3.2 seconds" + +[[tests.spring-boot]] +name = "preserves errors" +input = " :: Spring Boot :: (v3.2.0)\n2024-01-01 INFO Initializing Spring\n2024-01-01 ERROR Application run failed\nCaused by: java.lang.NullPointerException" +expected = "2024-01-01 ERROR Application run failed\nCaused by: java.lang.NullPointerException" diff --git a/crates/sparkplane-rtk/upstream/src/filters/ssh.toml b/crates/sparkplane-rtk/upstream/src/filters/ssh.toml new file mode 100644 index 0000000..71c953f --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ssh.toml @@ -0,0 +1,44 @@ +[filters.ssh] +description = "Compact ssh output — strip connection banners, keep command output" +match_command = "^ssh(?:\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Warning: Permanently added", + "^Connection to .+ closed", + "^Authenticated to", + "^debug1:", + "^OpenSSH_", + "^Pseudo-terminal", +] +max_lines = 200 +truncate_lines_at = 120 + +[[tests.ssh]] +name = "strips connection banners, keeps command output" +input = """ +Warning: Permanently added '192.168.1.10' (ED25519) to the list of known hosts. + +total 32 +drwxr-xr-x 4 user user 4096 Mar 10 12:00 app +-rw-r--r-- 1 user user 1234 Mar 10 11:00 config.yaml + +Connection to 192.168.1.10 closed. +""" +expected = "total 32\ndrwxr-xr-x 4 user user 4096 Mar 10 12:00 app\n-rw-r--r-- 1 user user 1234 Mar 10 11:00 config.yaml" + +[[tests.ssh]] +name = "verbose debug lines stripped" +input = """ +debug1: Connecting to host.example.com port 22. +debug1: Connection established. +Authenticated to host.example.com ([1.2.3.4]:22). +uptime: 12:00:00 up 42 days, load average: 0.10, 0.15, 0.12 +Connection to host.example.com closed. +""" +expected = "uptime: 12:00:00 up 42 days, load average: 0.10, 0.15, 0.12" + +[[tests.ssh]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/stat.toml b/crates/sparkplane-rtk/upstream/src/filters/stat.toml new file mode 100644 index 0000000..8c240c0 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/stat.toml @@ -0,0 +1,44 @@ +[filters.stat] +description = "Compact stat output — strip device/inode/birth noise" +match_command = "^stat\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Device:", + "^\\s*Birth:", +] +truncate_lines_at = 120 +max_lines = 20 + +[[tests.stat]] +name = "linux stat output strips device and birth" +input = """ + File: main.rs + Size: 12345 Blocks: 24 IO Block: 4096 regular file +Device: 801h/2049d Inode: 1234567 Links: 1 +Access: (0644/-rw-r--r--) Uid: ( 1000/ patrick) Gid: ( 1000/ patrick) +Access: 2026-03-10 12:00:00.000000000 +0100 +Modify: 2026-03-10 11:00:00.000000000 +0100 +Change: 2026-03-10 11:00:00.000000000 +0100 + Birth: 2026-03-09 10:00:00.000000000 +0100 +""" +expected = " File: main.rs\n Size: 12345 Blocks: 24 IO Block: 4096 regular file\nAccess: (0644/-rw-r--r--) Uid: ( 1000/ patrick) Gid: ( 1000/ patrick)\nAccess: 2026-03-10 12:00:00.000000000 +0100\nModify: 2026-03-10 11:00:00.000000000 +0100\nChange: 2026-03-10 11:00:00.000000000 +0100" + +[[tests.stat]] +name = "macOS stat -x strips device and birth" +input = """ + File: "main.rs" + Size: 82848 FileType: Regular File + Mode: (0644/-rw-r--r--) Uid: ( 501/ patrick) Gid: ( 20/ staff) +Device: 1,15 Inode: 66302332 Links: 1 +Access: Wed Mar 18 21:21:15 2026 +Modify: Wed Mar 18 20:56:11 2026 +Change: Wed Mar 18 20:56:11 2026 + Birth: Wed Mar 18 20:56:11 2026 +""" +expected = " File: \"main.rs\"\n Size: 82848 FileType: Regular File\n Mode: (0644/-rw-r--r--) Uid: ( 501/ patrick) Gid: ( 20/ staff)\nAccess: Wed Mar 18 21:21:15 2026\nModify: Wed Mar 18 20:56:11 2026\nChange: Wed Mar 18 20:56:11 2026" + +[[tests.stat]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/swift-build.toml b/crates/sparkplane-rtk/upstream/src/filters/swift-build.toml new file mode 100644 index 0000000..fb866e5 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/swift-build.toml @@ -0,0 +1,41 @@ +[filters.swift-build] +description = "Compact swift build output — short-circuit on success, strip Compiling/Linking" +match_command = "^swift\\s+build\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Compiling ", + "^Linking ", +] +match_output = [ + { pattern = "Build complete!", message = "ok (build complete)", unless = "warning:|error:" }, +] +max_lines = 40 + +[[tests.swift-build]] +name = "successful build short-circuits to ok" +input = """ +Build complete! +""" +expected = "ok (build complete)" + +[[tests.swift-build]] +name = "build errors pass through after stripping noise" +input = """ +Compiling MyApp MyApp.swift +/home/user/MyApp/Sources/MyApp/main.swift:5:1: error: use of unresolved identifier 'foo' +foo() +^~~ +Linking MyApp +error: build had 1 command failure +""" +expected = "/home/user/MyApp/Sources/MyApp/main.swift:5:1: error: use of unresolved identifier 'foo'\nfoo()\n^~~\nerror: build had 1 command failure" + +[[tests.swift-build]] +name = "warnings not swallowed when Build complete present" +input = """ +CompileSwift normal x86_64 MyFile.swift +/path/to/MyFile.swift:42:10: warning: unused variable 'x' +Build complete! (with warnings) +""" +expected = "CompileSwift normal x86_64 MyFile.swift\n/path/to/MyFile.swift:42:10: warning: unused variable 'x'\nBuild complete! (with warnings)" diff --git a/crates/sparkplane-rtk/upstream/src/filters/systemctl-status.toml b/crates/sparkplane-rtk/upstream/src/filters/systemctl-status.toml new file mode 100644 index 0000000..4a8a084 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/systemctl-status.toml @@ -0,0 +1,33 @@ +[filters.systemctl-status] +description = "Compact systemctl status output — strip blank lines, limit to 20 lines" +match_command = "^systemctl\\s+status\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 20 + +[[tests.systemctl-status]] +name = "verbose unit status stripped of blank lines" +input = """ +● nginx.service - A high performance web server + Loaded: loaded (/lib/systemd/system/nginx.service; enabled) + Active: active (running) since Mon 2024-01-15 10:30:00 UTC; 2h ago + Docs: man:nginx(8) + Main PID: 1234 (nginx) + Tasks: 3 (limit: 4915) + Memory: 8.5M + + CGroup: /system.slice/nginx.service + ├─1234 nginx: master process /usr/sbin/nginx + └─1235 nginx: worker process + +Jan 15 10:30:00 host nginx[1234]: nginx/1.24.0 +Jan 15 10:30:00 host systemd[1]: Started nginx.service +""" +expected = "● nginx.service - A high performance web server\n Loaded: loaded (/lib/systemd/system/nginx.service; enabled)\n Active: active (running) since Mon 2024-01-15 10:30:00 UTC; 2h ago\n Docs: man:nginx(8)\n Main PID: 1234 (nginx)\n Tasks: 3 (limit: 4915)\n Memory: 8.5M\n CGroup: /system.slice/nginx.service\n ├─1234 nginx: master process /usr/sbin/nginx\n └─1235 nginx: worker process\nJan 15 10:30:00 host nginx[1234]: nginx/1.24.0\nJan 15 10:30:00 host systemd[1]: Started nginx.service" + +[[tests.systemctl-status]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/task.toml b/crates/sparkplane-rtk/upstream/src/filters/task.toml new file mode 100644 index 0000000..31868fc --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/task.toml @@ -0,0 +1,27 @@ +[filters.task] +description = "Compact go-task output — strip task headers, keep command results" +match_command = "^task\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^task: \\[.*\\] ", + "^task: Task .* is up to date", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "task: ok" + +[[tests.task]] +name = "strips task headers, keeps output" +input = "task: [build] go build ./...\n\ntask: [test] go test ./...\nok myapp 0.5s\n\ntask: Task \"lint\" is up to date" +expected = "ok myapp 0.5s" + +[[tests.task]] +name = "preserves error output" +input = "task: [build] go build ./...\n./main.go:10: undefined: foo\ntask: Failed to run task \"build\": exit status 1" +expected = "./main.go:10: undefined: foo\ntask: Failed to run task \"build\": exit status 1" + +[[tests.task]] +name = "all up to date" +input = "task: Task \"build\" is up to date\ntask: Task \"lint\" is up to date\n" +expected = "task: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/terraform-plan.toml b/crates/sparkplane-rtk/upstream/src/filters/terraform-plan.toml new file mode 100644 index 0000000..7ee22f3 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/terraform-plan.toml @@ -0,0 +1,35 @@ +[filters.terraform-plan] +description = "Compact Terraform plan output" +match_command = "^terraform\\s+plan(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^Refreshing state", + "^\\s*#.*unchanged", + "^\\s*$", + "^Acquiring state lock", + "^Releasing state lock", +] +max_lines = 80 +on_empty = "terraform plan: no changes detected" + +[[tests.terraform-plan]] +name = "strips Refreshing state lines and blank lines" +input = """ +Acquiring state lock. This may take a few moments... +Refreshing state... [id=vpc-abc] +Refreshing state... [id=sg-123] +Releasing state lock. This may take a few moments... + +Terraform will perform the following actions: + + # aws_instance.web will be created + + resource "aws_instance" "web" {} + +Plan: 1 to add, 0 to change, 0 to destroy. +""" +expected = "Terraform will perform the following actions:\n # aws_instance.web will be created\n + resource \"aws_instance\" \"web\" {}\nPlan: 1 to add, 0 to change, 0 to destroy." + +[[tests.terraform-plan]] +name = "strips noise, preserves non-blank content" +input = "Refreshing state... [id=vpc-abc]\nNo changes. Your infrastructure matches the configuration." +expected = "No changes. Your infrastructure matches the configuration." diff --git a/crates/sparkplane-rtk/upstream/src/filters/tofu-fmt.toml b/crates/sparkplane-rtk/upstream/src/filters/tofu-fmt.toml new file mode 100644 index 0000000..bfb78d9 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/tofu-fmt.toml @@ -0,0 +1,16 @@ +[filters.tofu-fmt] +description = "Compact OpenTofu fmt output" +match_command = "^tofu\\s+fmt(\\s|$)" +strip_ansi = true +on_empty = "tofu fmt: ok (no changes)" +max_lines = 30 + +[[tests.tofu-fmt]] +name = "empty output returns on_empty message" +input = "" +expected = "tofu fmt: ok (no changes)" + +[[tests.tofu-fmt]] +name = "changed files pass through" +input = "main.tf\nvariables.tf" +expected = "main.tf\nvariables.tf" diff --git a/crates/sparkplane-rtk/upstream/src/filters/tofu-init.toml b/crates/sparkplane-rtk/upstream/src/filters/tofu-init.toml new file mode 100644 index 0000000..d983d94 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/tofu-init.toml @@ -0,0 +1,38 @@ +[filters.tofu-init] +description = "Compact OpenTofu init output" +match_command = "^tofu\\s+init(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^- Downloading", + "^- Installing", + "^- Using previously-installed", + "^\\s*$", + "^Initializing provider", + "^Initializing the backend", + "^Initializing modules", +] +max_lines = 20 +on_empty = "tofu init: ok" + +[[tests.tofu-init]] +name = "strips downloading/installing lines" +input = """ +Initializing the backend... +Initializing provider plugins... +- Downloading hashicorp/aws 5.0.0... +- Installing hashicorp/aws 5.0.0... +- Using previously-installed hashicorp/random 3.5.1 + +OpenTofu has been successfully initialized! +""" +expected = "OpenTofu has been successfully initialized!" + +[[tests.tofu-init]] +name = "on_empty when all noise stripped" +input = """ +Initializing the backend... +Initializing provider plugins... +- Using previously-installed hashicorp/aws 5.0.0 + +""" +expected = "tofu init: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/tofu-plan.toml b/crates/sparkplane-rtk/upstream/src/filters/tofu-plan.toml new file mode 100644 index 0000000..72310e7 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/tofu-plan.toml @@ -0,0 +1,35 @@ +[filters.tofu-plan] +description = "Compact OpenTofu plan output" +match_command = "^tofu\\s+plan(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^Refreshing state", + "^\\s*#.*unchanged", + "^\\s*$", + "^Acquiring state lock", + "^Releasing state lock", +] +max_lines = 80 +on_empty = "tofu plan: no changes detected" + +[[tests.tofu-plan]] +name = "strips Refreshing state and lock lines" +input = """ +Acquiring state lock. This may take a few moments... +Refreshing state... [id=vpc-abc123] +Refreshing state... [id=sg-def456] +Releasing state lock. This may take a few moments... + +OpenTofu will perform the following actions: + + # aws_instance.web will be created + + resource "aws_instance" "web" {} + +Plan: 1 to add, 0 to change, 0 to destroy. +""" +expected = "OpenTofu will perform the following actions:\n # aws_instance.web will be created\n + resource \"aws_instance\" \"web\" {}\nPlan: 1 to add, 0 to change, 0 to destroy." + +[[tests.tofu-plan]] +name = "on_empty when all noise stripped" +input = "Refreshing state... [id=vpc-abc]\nAcquiring state lock. This may take a few moments...\nReleasing state lock. This may take a few moments..." +expected = "tofu plan: no changes detected" diff --git a/crates/sparkplane-rtk/upstream/src/filters/tofu-validate.toml b/crates/sparkplane-rtk/upstream/src/filters/tofu-validate.toml new file mode 100644 index 0000000..f6aaed8 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/tofu-validate.toml @@ -0,0 +1,17 @@ +[filters.tofu-validate] +description = "Compact OpenTofu validate output" +match_command = "^tofu\\s+validate(\\s|$)" +strip_ansi = true +match_output = [ + { pattern = "Success! The configuration is valid", message = "ok (valid)" }, +] + +[[tests.tofu-validate]] +name = "success short-circuits to ok" +input = "Success! The configuration is valid." +expected = "ok (valid)" + +[[tests.tofu-validate]] +name = "error passes through unchanged" +input = "Error: Invalid resource type\n on main.tf line 3: resource \"aws_instancee\" \"web\"" +expected = "Error: Invalid resource type\n on main.tf line 3: resource \"aws_instancee\" \"web\"" diff --git a/crates/sparkplane-rtk/upstream/src/filters/trunk-build.toml b/crates/sparkplane-rtk/upstream/src/filters/trunk-build.toml new file mode 100644 index 0000000..2baf5b4 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/trunk-build.toml @@ -0,0 +1,39 @@ +[filters.trunk-build] +description = "Compact trunk build output" +match_command = "^trunk\\s+build(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Compiling\\s", + "^\\s*Downloading\\s", + "^\\s*Fetching\\s", + "^\\s*Fresh\\s", + "^\\s*Checking\\s", +] +tail_lines = 10 +max_lines = 30 +on_empty = "trunk build: ok" + +[[tests.trunk-build]] +name = "strips compile noise, keeps tail summary" +input = """ + Compiling tokio v1.35.0 + Compiling hyper v0.14.28 + Compiling my-crate v0.1.0 + Downloading serde v1.0.195 + Fresh regex v1.10.2 + + Finished release [optimized] target(s) in 45.23s + Binary: target/release/my-crate (5.2MB) +""" +expected = " Finished release [optimized] target(s) in 45.23s\n Binary: target/release/my-crate (5.2MB)" + +[[tests.trunk-build]] +name = "on_empty when all noise stripped" +input = """ + Compiling my-crate v0.1.0 + Fresh serde v1.0 + Checking tokio v1.35.0 + +""" +expected = "trunk build: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/turbo.toml b/crates/sparkplane-rtk/upstream/src/filters/turbo.toml new file mode 100644 index 0000000..c5a09ac --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/turbo.toml @@ -0,0 +1,30 @@ +[filters.turbo] +description = "Compact Turborepo output — strip cache status noise, keep task results" +match_command = "^turbo\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*cache (hit|miss|bypass)", + "^\\s*\\d+ packages in scope", + "^\\s*Tasks:\\s+\\d+", + "^\\s*Duration:\\s+", + "^\\s*Remote caching (enabled|disabled)", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "turbo: ok" + +[[tests.turbo]] +name = "strips cache noise, keeps task output" +input = " cache hit, replaying logs abc123\n cache miss, executing abc456\n\n3 packages in scope\n\n> myapp:build\n\nCompiled successfully.\n\nTasks: 2 successful, 2 total (1 cached)\nDuration: 3.2s" +expected = "> myapp:build\nCompiled successfully." + +[[tests.turbo]] +name = "preserves error output" +input = "> myapp:lint\n\nError: src/index.ts(5,1): error TS2304\n\nTasks: 0 successful, 1 total\nDuration: 1.1s" +expected = "> myapp:lint\nError: src/index.ts(5,1): error TS2304" + +[[tests.turbo]] +name = "empty after stripping" +input = " cache hit, replaying logs abc\n\n" +expected = "turbo: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/ty.toml b/crates/sparkplane-rtk/upstream/src/filters/ty.toml new file mode 100644 index 0000000..689fde9 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ty.toml @@ -0,0 +1,50 @@ +[filters.ty] +description = "Compact ty type checker output — strip blank lines, keep errors" +match_command = "^ty\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Checking \\d+ file", + "^ty \\d+\\.\\d+", +] +max_lines = 50 +on_empty = "ty: ok" + +[[tests.ty]] +name = "strips noise, keeps diagnostics" +input = """ +ty 0.1.0 +Checking 15 files + +error[unresolved-reference]: Name `foo` used when not defined + --> app/main.py:10:5 + | +10 | foo() + | ^^^ + | + +warning[unused-variable]: Variable `x` is not used + --> app/utils.py:8:9 + | + 8 | x = 42 + | ^ + | + +Found 1 error, 1 warning +""" +expected = "error[unresolved-reference]: Name `foo` used when not defined\n --> app/main.py:10:5\n |\n10 | foo()\n | ^^^\n |\nwarning[unused-variable]: Variable `x` is not used\n --> app/utils.py:8:9\n |\n 8 | x = 42\n | ^\n |\nFound 1 error, 1 warning" + +[[tests.ty]] +name = "clean output" +input = """ +ty 0.1.0 +Checking 10 files + +All checks passed! +""" +expected = "All checks passed!" + +[[tests.ty]] +name = "empty input returns on_empty message" +input = "" +expected = "ty: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/uv-sync.toml b/crates/sparkplane-rtk/upstream/src/filters/uv-sync.toml new file mode 100644 index 0000000..b9eee32 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/uv-sync.toml @@ -0,0 +1,37 @@ +[filters.uv-sync] +description = "Compact uv sync/pip install output — strip downloads, short-circuit when up-to-date" +match_command = "^uv\\s+(sync|pip\\s+install)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s+Downloading ", + "^\\s+Using cached ", + "^\\s+Preparing ", +] +match_output = [ + { pattern = "Audited \\d+ package", message = "ok (up to date)" }, +] +max_lines = 20 + +[[tests.uv-sync]] +name = "audited packages short-circuits to ok" +input = """ +Resolved 42 packages in 123ms +Audited 42 packages in 0.05ms +""" +expected = "ok (up to date)" + +[[tests.uv-sync]] +name = "install strips download and cached lines" +input = """ + Downloading requests-2.31.0-py3-none-any.whl (62.6 kB) + Using cached certifi-2023.11.17-py3-none-any.whl (162 kB) + Preparing packages... +Installed 5 packages in 23ms + + certifi==2023.11.17 + + charset-normalizer==3.3.2 + + idna==3.6 + + requests==2.31.0 + + urllib3==2.1.0 +""" +expected = "Installed 5 packages in 23ms\n + certifi==2023.11.17\n + charset-normalizer==3.3.2\n + idna==3.6\n + requests==2.31.0\n + urllib3==2.1.0" diff --git a/crates/sparkplane-rtk/upstream/src/filters/xcodebuild.toml b/crates/sparkplane-rtk/upstream/src/filters/xcodebuild.toml new file mode 100644 index 0000000..f959de9 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/xcodebuild.toml @@ -0,0 +1,99 @@ +[filters.xcodebuild] +description = "Compact xcodebuild output — strip build phases, keep errors/warnings/summary" +match_command = "^xcodebuild\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^CompileC\\s", + "^CompileSwift\\s", + "^Ld\\s", + "^CreateBuildDirectory\\s", + "^MkDir\\s", + "^ProcessInfoPlistFile\\s", + "^CopySwiftLibs\\s", + "^CodeSign\\s", + "^Signing Identity:", + "^RegisterWithLaunchServices", + "^Validate\\s", + "^ProcessProductPackaging", + "^Touch\\s", + "^LinkStoryboards", + "^CompileStoryboard", + "^CompileAssetCatalog", + "^GenerateDSYMFile", + "^PhaseScriptExecution", + "^PBXCp\\s", + "^SetMode\\s", + "^SetOwnerAndGroup\\s", + "^Ditto\\s", + "^CpResource\\s", + "^CpHeader\\s", + "^\\s+cd\\s+/", + "^\\s+export\\s", + "^\\s+/Applications/Xcode", + "^\\s+/usr/bin/", + "^\\s+builtin-", + "^note: Using new build system", +] +max_lines = 60 +on_empty = "xcodebuild: ok" + +[[tests.xcodebuild]] +name = "strips build phases, keeps errors and summary" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/App/ViewController.swift + cd /Users/dev/App + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-frontend -c +CompileSwift normal arm64 /Users/dev/App/AppDelegate.swift + cd /Users/dev/App + export DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer +Ld /Users/dev/Build/Products/Debug/App normal arm64 + cd /Users/dev/App + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang +CodeSign /Users/dev/Build/Products/Debug/App.app + cd /Users/dev/App + builtin-codesign --force --sign + +/Users/dev/App/ViewController.swift:42:9: error: use of unresolved identifier 'foo' +/Users/dev/App/Model.swift:18:5: warning: variable 'x' was never used + +** BUILD FAILED ** +""" +expected = "/Users/dev/App/ViewController.swift:42:9: error: use of unresolved identifier 'foo'\n/Users/dev/App/Model.swift:18:5: warning: variable 'x' was never used\n** BUILD FAILED **" + +[[tests.xcodebuild]] +name = "clean build success" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/App/Main.swift + cd /Users/dev/App +Ld /Users/dev/Build/Products/Debug/App normal arm64 + cd /Users/dev/App +CodeSign /Users/dev/Build/Products/Debug/App.app + cd /Users/dev/App + builtin-codesign --force --sign + +** BUILD SUCCEEDED ** +""" +expected = "** BUILD SUCCEEDED **" + +[[tests.xcodebuild]] +name = "test output keeps test results" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/AppTests/Tests.swift + cd /Users/dev/App +Test Suite 'All tests' started at 2026-03-10 12:00:00 +Test Suite 'AppTests' started at 2026-03-10 12:00:00 +Test Case '-[AppTests testExample]' passed (0.001 seconds). +Test Case '-[AppTests testFailing]' failed (0.002 seconds). +Test Suite 'AppTests' passed at 2026-03-10 12:00:01 +Executed 2 tests, with 1 failure in 0.003 seconds +""" +expected = "Test Suite 'All tests' started at 2026-03-10 12:00:00\nTest Suite 'AppTests' started at 2026-03-10 12:00:00\nTest Case '-[AppTests testExample]' passed (0.001 seconds).\nTest Case '-[AppTests testFailing]' failed (0.002 seconds).\nTest Suite 'AppTests' passed at 2026-03-10 12:00:01\nExecuted 2 tests, with 1 failure in 0.003 seconds" + +[[tests.xcodebuild]] +name = "empty input returns on_empty message" +input = "" +expected = "xcodebuild: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/yadm.toml b/crates/sparkplane-rtk/upstream/src/filters/yadm.toml new file mode 100644 index 0000000..f2cd3d1 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/yadm.toml @@ -0,0 +1,21 @@ +[filters.yadm] +description = "Compact yadm (git wrapper) output — same filtering as git" +match_command = "^yadm\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*\\(use \"git ", + "^\\s*\\(use \"yadm ", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.yadm]] +name = "strips hint lines" +input = "On branch main\nYour branch is up to date with 'origin/main'.\n\n (use \"yadm add\" to update what will be committed)\n\nChanges not staged for commit:\n modified: .bashrc" +expected = "On branch main\nYour branch is up to date with 'origin/main'.\nChanges not staged for commit:\n modified: .bashrc" + +[[tests.yadm]] +name = "short output preserved" +input = "Already up to date." +expected = "Already up to date." diff --git a/crates/sparkplane-rtk/upstream/src/filters/yamllint.toml b/crates/sparkplane-rtk/upstream/src/filters/yamllint.toml new file mode 100644 index 0000000..8a33d51 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/yamllint.toml @@ -0,0 +1,25 @@ +[filters.yamllint] +description = "Compact yamllint output — strip blank lines, limit rows" +match_command = "^yamllint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.yamllint]] +name = "multi-warning output stripped of blank lines" +input = """ +config.yml + 3:1 warning missing document start "---" (document-start) + 5:12 error too many spaces inside braces (braces) + + 8:1 error wrong indentation: expected 2 but found 4 (indentation) +""" +expected = "config.yml\n 3:1 warning missing document start \"---\" (document-start)\n 5:12 error too many spaces inside braces (braces)\n 8:1 error wrong indentation: expected 2 but found 4 (indentation)" + +[[tests.yamllint]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/data/economics-prices.json b/data/economics-prices.json new file mode 100644 index 0000000..fa97ceb --- /dev/null +++ b/data/economics-prices.json @@ -0,0 +1,40 @@ +{ + "version": "2026-09-30.1", + "currency": "USD", + "tariff": "standard_uncached", + "models": [ + { + "model": "gpt-6.1-sol", + "source": "https://developers.openai.com/api/docs/models/gpt-6.1-sol", + "verified_at": "2026-09-30", + "effective_from": "2026-09-30", + "input_microusd_per_million": 2000000, + "output_microusd_per_million": 10000000, + "long_context_threshold": 272000, + "long_input_microusd_per_million": 4000000, + "long_output_microusd_per_million": 15000000 + }, + { + "model": "claude-sonnet-4-6", + "source": "https://platform.claude.com/docs/en/models/sonnet-4-6/overview", + "verified_at": "2026-09-30", + "effective_from": "2026-09-30", + "input_microusd_per_million": 3000000, + "output_microusd_per_million": 15000000, + "long_context_threshold": null, + "long_input_microusd_per_million": null, + "long_output_microusd_per_million": null + }, + { + "model": "gemini-3.5-flash", + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "verified_at": "2026-09-30", + "effective_from": "2026-09-30", + "input_microusd_per_million": 1500000, + "output_microusd_per_million": 9000000, + "long_context_threshold": null, + "long_input_microusd_per_million": null, + "long_output_microusd_per_million": null + } + ] +} diff --git a/docs/how-to/develop-spark.md b/docs/how-to/develop-spark.md index fd72d0a..ddc5101 100644 --- a/docs/how-to/develop-spark.md +++ b/docs/how-to/develop-spark.md @@ -418,6 +418,15 @@ The generated `sparkplane-launch-models.json` must include both `text` and `image`. Relaunch Codex so it reads that catalog. Do not hand-edit the catalog or its ownership receipt to make an old session accept attachments. +Updating the Qwen image-count policy from one to 16 also requires a managed +stop and re-serve: the count belongs to the signed engine fingerprint. The +model artifacts are unchanged, so a model download refresh is unnecessary. +Keep the same instance name, context, launch arguments and resources. Verify +successive `view_image` results and a 16-image request through the public +gateway after activation. The shared 512 KiB engine image budget remains; +normalization divides it among the frames, and the gateway enforces the +combined source-pixel limit before resizing. + If regeneration reports an ownership conflict, preserve a private backup of the affected profile and its receipt, move the indicated profile aside, and rerun `launch ... --config --json`. Additional tables such as `[tui]` in an earlier diff --git a/docs/reference/spark.md b/docs/reference/spark.md index f30a07a..b2c8430 100644 --- a/docs/reference/spark.md +++ b/docs/reference/spark.md @@ -37,7 +37,8 @@ sparkplane status --json sparkplane doctor --json sparkplane qualify --manifest --signature (--dry-run | --yes) [--detach] [--json] sparkplane serve [--name ] [--detach] [--dry-run] [--json] -sparkplane launch [--mode auto|inherit] [--model ] [--allow-network] [--config] [--restore] [-y] [-- ...] +sparkplane launch [--mode auto|inherit] [--eco-mode max|none] [--model ] [--allow-network] [--config] [--restore] [-y] [-- ...] +sparkplane economics [--session ] [--json] sparkplane ls [--json] sparkplane ps [--json] sparkplane logs [--limit N] @@ -204,8 +205,10 @@ committed only after exact-container cleanup is acknowledged. ## Engine profiles and modalities - Ornith accepts bounded inline JPEG, PNG, or WebP through OpenAI - Responses and Anthropic Messages. The adapters validate media - type, file magic, decoded bytes, image count, and dimensions + Responses and Anthropic Messages. The gateway resizes and re-encodes oversized + images to the selected qualified engine profile, preserving aspect ratio and + transparency. Images already within the profile's limits keep their original + bytes. The adapters validate media type, file magic, decoded bytes, image count, and dimensions before contacting vLLM. Remote URLs, local paths, traversal, unsupported media, and images sent to text-only instances are rejected. @@ -265,11 +268,145 @@ Existing model records retain their verified artifact traits across upgrades. When a catalog adds vision support, refresh the record through the managed `download --update-alias` operation and re-serve it with the updated engine policy before relaunching Codex. -The Qwen3.8 vLLM policy accepts one inline PNG, JPEG, or WebP image per request, -at most 512 KiB of image data, with width and height at most 4096 pixels. +### Image history and upload limits + +The Qwen3.8 vLLM policy accepts up to 16 inline PNG, JPEG, or WebP images per request, +sharing at most 512 KiB of image data, with width and height at most 4096 pixels. +The gateway accepts larger source uploads and normalizes them before either +native or translated forwarding; the model's image-count limit still applies. +Each frame receives an equal share of the total image byte budget so earlier +screenshots cannot consume the space needed by subsequent `view_image` results. +Long sessions retain a rolling window of images. At 12 images, older images +already followed by a model response are replaced with explicit text references, +leaving six recent frames. Large source payloads trigger earlier eviction of +processed frames to keep their combined inline payload near 16 MiB; this may +retain fewer than six frames. Frames awaiting their first inspection are +protected. A batch of 16 new images remains supported, and larger new batches +must be split. +Tool calls, call IDs, accompanying text and written observations are retained. +The same policy handles Responses, Chat Completions and nested Anthropic tool +results. Text-only profiles continue rejecting image input. + +Vision-enabled `launch` invocations run a private, authenticated loopback adapter. +It streams the historical JSON one item at a time and removes old image payloads +before uploading to the appliance, so repeatedly viewing images does not make +the appliance upload grow with every old screenshot. Source histories may exceed +32 MiB; individual items and the projected request remain capped at 32 MiB, and +history text is capped at 8 MiB. Upload and projection have 120-second deadlines, +with two workers. The adapter uses the pinned appliance CA, refuses redirects, +and exposes only inference routes for the selected instance. Its listener and +invocation credential disappear when the child exits. + +Evicted originals are retained privately in +`/spark/image-archive/`, under stable content-derived filenames. +The model receives a path it can reopen with `view_image`; inherited agent +filesystem permissions still apply. The archive uses 0700 directories and 0600 +files, deduplicates copies, and caps storage at 512 MiB and 4096 images. Writes +remove the least recently used originals when necessary and expire unused +originals after 90 days. Older references can therefore expire; existing source +files remain another way to inspect those images. Direct gateway clients get +the retention policy, but must prune or compact their own uploads before ingress. + +No caption inference is added: descriptions come from the agent's existing +observations. Codex launches compact the conversation at 75% of the qualified +context window, preserving room for new tool results and compaction. Image +retention remains active with `--eco-mode=none`; that switch controls RTK command +compression. These bounds do not change engine context, sampling or resources. +Codex `view_image` results and other typed function/custom tool outputs preserve +inline images as multimodal tool content, including resizing and the shared +request image budget. Their base64 is never converted into prompt text. Regular +string and JSON tool outputs retain their text representation. +The root-owned agent configuration has a separate `[gateway]` upload policy: + +```toml +[gateway] +request_body_bytes = 33554432 # 32 MiB, including JSON and base64 +image_bytes = 16777216 # 16 MiB per decoded source file +image_pixels = 32000000 # all source images combined, before resizing +image_dimension = 16384 # maximum source width or height +max_parallel_requests = 2 # concurrent uploads and image workers +``` + +These defaults also apply to older configurations without a `[gateway]` section. +Configuration ceilings are 64 MiB requests, 32 MiB source files, 32 million +total source pixels, 16384 pixels per dimension, and four concurrent workers. Uploads have a +30-second deadline; image decoding has a 128 MiB allocation limit. The normalized +inference body and other inference POST bodies are capped at 8 MiB; control API +bodies retain their 1 MiB limit. Saturated upload workers return JSON 503 with +`Retry-After: 1`; oversized uploads return protocol-specific JSON 413. Gateway +rejections do not forward images to the engine. Resizing may reduce fine detail +to fit the qualified profile's byte budget. + +Engine request rejections use safe protocol-specific errors: invalid requests +return 400, body limits return 413, and engine capacity limits return 429 with +`Retry-After: 1`. Context overflow returns OpenAI code `context_length_exceeded` +so clients can compact the conversation. Engine error bodies and private request +contents are not forwarded. Unavailable engines and transport failures return +503. + See the [live capability activation procedure](../how-to/develop-spark.md#activate-a-capability-change-on-an-existing-instance) for signing, cached-model refresh, managed restart, and attachment verification. +### Eco mode and session economics + +`--eco-mode=max` is the default. Sparkplane embeds an Apache-2.0 RTK library, +pinned to upstream revision `6d4b77eadee1c66dc1f68466ad77e96d1b6e4989`. +There is no RTK executable download or installer. Invocation-local Codex and +Claude hooks, and an OpenCode plugin, compact supported shell output. The +initial profiles cover Git status, Cargo build/check/clippy/test, pytest and +RTK's bundled declarative filters. Custom RTK configuration is not loaded. +Unknown commands, shell scripts, pipes, redirects, explicit structured output, +binary output and unsuccessful commands pass through unchanged. Captures over +8 MiB per stream pass through; stdout, stderr and the command exit code remain +separate. For an explicit raw escape, use `sparkplane __eco raw -- `. + +`--eco-mode=none` disables Sparkplane's compression hooks while retaining +inference accounting. Compression stays inactive with `--mode inherit`, so +rewriting cannot circumvent the agent's own command permissions. Agent hook +support and trust still apply: for Codex, review the Sparkplane hooks in +`/hooks`. Sparkplane does not grant hook trust. Disabled, unsupported or +untrusted hooks leave the session usable and are reported as inactive. +Persistent agent hook settings are not modified. + +On a supporting appliance, every launch gets a unique inference-only +credential, valid for at most seven days. It is passed only in the child's +environment and revoked when the coding agent exits. Usage comes from engine +responses before protocol translation, including streamed and native Responses +usage; repeated final usage events count once. Other clients' credentials do +not contribute. The session API is administrator-only. Missing engine usage, +interrupted streams or unavailable accounting produce an explicit incomplete +or unavailable result. Older appliances can still launch agents with usage +shown as unavailable. + +The report appears on stderr after normal exit, failure or a handled interrupt. +Finalization has a five-second client timeout. The managed model keeps running; +`stop ` continues to stop only that model. A session report includes +input/output tokens, duration, compressed command counts, estimated output +reduction and two separate USD columns: + +- **Cloud equivalent:** known local inference tokens valued at published + standard uncached input/output rates for GPT-6.1 Sol, Claude Sonnet 4.6 and + Gemini 3.5 Flash. The GPT comparison applies its longer-context tier per + request above 272,000 input tokens. +- **Estimated RTK savings:** removed tool-output bytes divided by four, + valued once at standard input rates. This estimate does not assume repeated + context reuse and is unavailable when compression metrics are incomplete. + +These columns are not added together. Different tokenizers, prompt caching, +hardware and electricity can change actual costs; these are comparisons, not +bills or measured cash savings. The bundled price catalog records official +source links, verification dates, rates and a version. Launch never fetches +prices from the network; each report retains the catalog it used. + +`sparkplane economics` displays the most recent report; +`economics --session --json` emits the structured report on stdout. +Reports are scoped by host under the Sparkplane config directory, with +mode-0700 directories and mode-0600 files. They contain only aggregate usage +and comparison data, never credentials, prompts, command text or output. +Reports older than 90 days are pruned on save or read; the appliance prunes +session/request ledgers when creating a session. Invocation-local compression +metrics are removed when the launcher exits. + ### Internet access for development Launches default to `--mode auto` (`SPARKPLANE_LAUNCH_MODE=auto`): full diff --git a/specs/openapi/sparkplane-control-v1.json b/specs/openapi/sparkplane-control-v1.json index 7aac4cc..ba97ea8 100644 --- a/specs/openapi/sparkplane-control-v1.json +++ b/specs/openapi/sparkplane-control-v1.json @@ -10,6 +10,9 @@ "/api/sparkplane/v1/instances", "/api/sparkplane/v1/instances/{id}", "/api/sparkplane/v1/instances/{id}/logs", + "/api/sparkplane/v1/launch-sessions", + "/api/sparkplane/v1/launch-sessions/{id}", + "/api/sparkplane/v1/launch-sessions/{id}/finish", "/api/sparkplane/v1/metrics", "/api/sparkplane/v1/models", "/api/sparkplane/v1/models/{id}", @@ -48,6 +51,9 @@ "InstanceDocument", "InstanceListDocument", "InstanceObservedState", + "LaunchSessionCreated", + "LaunchSessionDocument", + "LaunchSessionRequest", "ModelArtifactFileDocument", "ModelArtifactFormat", "ModelArtifactRole", @@ -83,6 +89,7 @@ "ResourcePolicy", "ServeAdmissionRequest", "ServeRequest", + "SessionUsage", "StatusDocument", "StopRequest", "TokenCreateRequest", diff --git a/src/main.rs b/src/main.rs index b20e915..09d095b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,6 +14,16 @@ struct Cli { } fn main() { + if std::env::args_os().nth(1).is_some_and(|a| a == "__eco") { + let code = match sparkplane::spark::eco::run(std::env::args_os().skip(2).collect()) { + Ok(code) => code, + Err(error) => { + eprintln!("eco: {error}"); + 1 + } + }; + std::process::exit(code); + } if std::env::args_os().skip(1).eq(["--bridge-protocol"]) { println!("sparkplane.bridge/v1"); return; diff --git a/src/spark/agent.rs b/src/spark/agent.rs index 8a1840e..f498b1b 100644 --- a/src/spark/agent.rs +++ b/src/spark/agent.rs @@ -89,6 +89,8 @@ pub struct AgentConfig { pub resources: ResourcePolicyConfig, pub retention: RetentionConfig, pub models: ModelsConfig, + #[serde(default)] + pub gateway: gateway::IngressLimits, } #[derive(Debug, Clone, Deserialize)] @@ -130,6 +132,8 @@ pub struct AgentState { download_slots: Arc, start_slots: Arc, inference_slots: Arc>, + gateway_limits: gateway::IngressLimits, + ingress_slots: Arc, admission: TransitionCoordinator, routes: RouteRegistry, qualification_authority: Option>, @@ -198,6 +202,10 @@ impl AgentState { download_slots: Arc::new(tokio::sync::Semaphore::new(1)), start_slots: Arc::new(tokio::sync::Semaphore::new(1)), inference_slots: Arc::new(std::sync::Mutex::new(BTreeMap::new())), + gateway_limits: gateway::IngressLimits::default(), + ingress_slots: Arc::new(tokio::sync::Semaphore::new( + gateway::IngressLimits::default().max_parallel_requests, + )), admission: TransitionCoordinator::new(), routes: RouteRegistry::default(), qualification_authority: None, @@ -270,6 +278,12 @@ impl AgentState { self } + fn with_gateway_limits(mut self, limits: gateway::IngressLimits) -> Self { + self.ingress_slots = Arc::new(tokio::sync::Semaphore::new(limits.max_parallel_requests)); + self.gateway_limits = limits; + self + } + fn with_models(mut self, models: HubAcquirer, max_parallel: usize) -> Self { self.models = Some(Arc::new(models)); self.download_slots = Arc::new(tokio::sync::Semaphore::new(max_parallel)); @@ -341,6 +355,9 @@ impl Cidr { operation_events, cancel_operation, create_token, + create_launch_session, + get_launch_session, + finish_launch_session, list_tokens, revoke_token, list_models, @@ -371,6 +388,10 @@ impl Cidr { OperationEvent, TokenCreateRequest, TokenCreatedDocument, + super::wire::LaunchSessionRequest, + super::wire::LaunchSessionCreated, + super::wire::LaunchSessionDocument, + super::wire::SessionUsage, TokenListDocument, DownloadRequest, DownloadPlanDocument, @@ -403,6 +424,18 @@ pub fn router(state: AgentState) -> Router { let authenticated = Router::new() .route(&format!("{API_BASE}/status"), get(status)) .route(&format!("{API_BASE}/doctor"), get(doctor)) + .route( + &format!("{API_BASE}/launch-sessions"), + axum::routing::post(create_launch_session), + ) + .route( + &format!("{API_BASE}/launch-sessions/{{id}}"), + get(get_launch_session), + ) + .route( + &format!("{API_BASE}/launch-sessions/{{id}}/finish"), + axum::routing::post(finish_launch_session), + ) .route(&format!("{API_BASE}/metrics"), get(metrics)) .route( &format!("{API_BASE}/certificates/status"), @@ -451,6 +484,8 @@ pub fn router(state: AgentState) -> Router { get(list_tokens).post(create_token), ) .route(&format!("{API_BASE}/tokens/{{id}}"), delete(revoke_token)) + .layer(DefaultBodyLimit::max(1024 * 1024)); + let inference = Router::new() .route("/openai/{instance}/v1/models", get(gateway_models)) .route( "/openai/{instance}/v1/completions", @@ -478,11 +513,160 @@ pub fn router(state: AgentState) -> Router { axum::routing::post(gateway_anthropic_count_tokens), ) .route("/anthropic/{instance}/{*path}", any(anthropic_not_found)) - .layer(DefaultBodyLimit::max(gateway::MAX_COMPLETION_BODY_BYTES)) + .layer(DefaultBodyLimit::max(gateway::HARD_REQUEST_BYTES)) + .route_layer(middleware::from_fn_with_state( + state.clone(), + inference_ingress, + )); + let authenticated = authenticated + .merge(inference) .route_layer(middleware::from_fn_with_state(state.clone(), authenticate)); authenticated.fallback(not_found).with_state(state) } +// The worker owns a clone so cancellation cannot free its CPU/memory slot early. +#[derive(Clone)] +struct IngressLease(Arc); + +fn ingress_error( + path: &str, + status: StatusCode, + code: &'static str, + message: &'static str, +) -> Response { + if path.starts_with("/anthropic/") { + anthropic_error( + status, + gateway::AnthropicError { + error_type: if status == StatusCode::SERVICE_UNAVAILABLE { + "overloaded_error" + } else { + "invalid_request_error" + }, + message, + }, + ) + } else { + openai_error(status, gateway::OpenAiError { code, message }) + } +} + +async fn inference_ingress( + State(state): State, + request: Request, + next: Next, +) -> Response { + if request.method() != Method::POST { + return next.run(request).await; + } + let path = request.uri().path().to_owned(); + let limit = if path.ends_with("/v1/responses") + || path.ends_with("/v1/messages") + || path.ends_with("/v1/chat/completions") + { + state.gateway_limits.request_body_bytes + } else { + gateway::MAX_COMPLETION_BODY_BYTES.min(state.gateway_limits.request_body_bytes) + }; + if request + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|h| h.to_str().ok()) + .and_then(|h| h.parse::().ok()) + .is_some_and(|size| size > limit as u64) + { + return ingress_error( + &path, + StatusCode::PAYLOAD_TOO_LARGE, + "request_too_large", + "request exceeds the gateway upload byte limit", + ); + } + let permit = match state.ingress_slots.clone().try_acquire_owned() { + Ok(permit) => IngressLease(Arc::new(permit)), + Err(_) => { + let mut response = ingress_error( + &path, + StatusCode::SERVICE_UNAVAILABLE, + "gateway_busy", + "gateway upload and image workers are busy; retry shortly", + ); + response + .headers_mut() + .insert(header::RETRY_AFTER, HeaderValue::from_static("1")); + return response; + } + }; + let (mut parts, body) = request.into_parts(); + let body = match tokio::time::timeout( + Duration::from_secs(30), + axum::body::to_bytes(body, limit), + ) + .await + { + Ok(Ok(body)) => body, + Ok(Err(error)) => { + if !std::error::Error::source(&error) + .is_some_and(|source| source.is::()) + { + return ingress_error( + &path, + StatusCode::BAD_REQUEST, + "invalid_request_error", + "gateway upload body cannot be read", + ); + } + return ingress_error( + &path, + StatusCode::PAYLOAD_TOO_LARGE, + "request_too_large", + "request exceeds the gateway upload byte limit", + ); + } + Err(_) => { + return ingress_error( + &path, + StatusCode::REQUEST_TIMEOUT, + "request_timeout", + "gateway upload deadline exceeded", + ); + } + }; + parts.extensions.insert(permit); + next.run(Request::from_parts(parts, Body::from(body))).await +} + +async fn normalize_gateway_images( + body: Bytes, + protocol: gateway::ImageProtocol, + profile: gateway::GatewayProfile, + limits: gateway::IngressLimits, + lease: IngressLease, +) -> Result, gateway::OpenAiError> { + tokio::task::spawn_blocking(move || { + let _permit = lease.0; + gateway::normalize_image_request(&body, protocol, &profile, &limits) + }) + .await + .unwrap_or(Err(gateway::OpenAiError { + code: "invalid_request_error", + message: "image preprocessing failed", + })) +} + +async fn meter_inference_route( + state: &AgentState, + auth: &AuthenticatedToken, + instance: &str, + route: Arc, +) -> Result, StateError> { + let mut metered_route = (*route).clone(); + metered_route.upstream = metered_route.upstream.with_meter(Arc::new( + super::sessions::UsageMeter::begin(state.database.as_ref(), &auth.id, instance).await?, + )); + Ok(Arc::new(metered_route)) +} + async fn openai_not_found() -> Response { openai_error( StatusCode::NOT_FOUND, @@ -551,6 +735,7 @@ async fn gateway_anthropic_messages( AxumPath(instance): AxumPath, RawQuery(query): RawQuery, Extension(auth): Extension, + Extension(lease): Extension, headers: HeaderMap, body: Result, ) -> Response { @@ -583,6 +768,30 @@ async fn gateway_anthropic_messages( if !route.profile.allows(PublicAction::Responses) { return anthropic_not_found().await; } + let body = match normalize_gateway_images( + body, + gateway::ImageProtocol::Anthropic, + route.profile.clone(), + state.gateway_limits.clone(), + lease, + ) + .await + { + Ok(body) => body, + Err(error) => { + return anthropic_error( + if error.code == "request_too_large" { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_REQUEST + }, + gateway::AnthropicError { + error_type: "invalid_request_error", + message: error.message, + }, + ); + } + }; let request = match gateway::rewrite_anthropic_request_with_profile( &body, &route.served_model, @@ -591,6 +800,10 @@ async fn gateway_anthropic_messages( Ok(request) => request, Err(error) => return anthropic_error(StatusCode::BAD_REQUEST, error), }; + let route = match meter_inference_route(&state, &auth, &instance, route).await { + Ok(route) => route, + Err(error) => return state_problem(error), + }; let token_permit = match auth.acquire_inference().await { Ok(permit) => permit, Err(()) => return anthropic_upstream_unavailable(), @@ -605,7 +818,7 @@ async fn gateway_anthropic_messages( .await { Ok(upstream) => upstream, - Err(_) => return anthropic_upstream_unavailable(), + Err(error) => return anthropic_generation_error(error), }; let encoder = if request.omit_reasoning { gateway::AnthropicEncoder::with_omitted_reasoning(route.public_model.clone()) @@ -834,6 +1047,15 @@ async fn gateway_completions( RouteLookup::Warming => return gateway_warming(), RouteLookup::Missing => return not_found().await, }; + let mut metered_route = (*route).clone(); + metered_route.upstream = metered_route.upstream.with_meter(Arc::new( + match super::sessions::UsageMeter::begin(state.database.as_ref(), &auth.id, &instance).await + { + Ok(meter) => meter, + Err(error) => return state_problem(error), + }, + )); + let route = Arc::new(metered_route); if !route.profile.allows(PublicAction::Completions) { return not_found().await; } @@ -855,7 +1077,7 @@ async fn gateway_completions( if streaming { let upstream = match route.upstream.completion_stream(&upstream_body).await { Ok(stream) => stream, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; let public_model = route.public_model.clone(); let events = futures_util::stream::unfold( @@ -913,7 +1135,7 @@ async fn gateway_completions( .request("POST", "/v1/completions", upstream_body.len()) { Ok(request) => request, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; match route.upstream.send(&request, &upstream_body).await { Ok(response) if (200..300).contains(&response.status) => { @@ -922,21 +1144,74 @@ async fn gateway_completions( Err(()) => gateway_upstream_unavailable(), } } - _ => gateway_upstream_unavailable(), + Ok(response) => gateway_upstream_response_error(response), + Err(error) => gateway_generation_error(error), } } fn openai_error(status: StatusCode, error: gateway::OpenAiError) -> Response { - let error_type = if error.code == "server_error" { - "server_error" - } else { - "invalid_request_error" + let error_type = match error.code { + "server_error" => "server_error", + "rate_limit_error" => "rate_limit_error", + _ => "invalid_request_error", }; let body = serde_json::json!({"error": {"message": error.message, "type": error_type, "param": null, "code": error.code}}); (status, Json(body)).into_response() } +fn gateway_request_rejected(rejection: super::upstream::RequestRejection) -> Response { + let mut response = openai_error( + StatusCode::from_u16(rejection.status()).expect("finite engine rejection status"), + gateway::OpenAiError { + code: rejection.code(), + message: rejection.message(), + }, + ); + if rejection == super::upstream::RequestRejection::RateLimited { + response + .headers_mut() + .insert(header::RETRY_AFTER, HeaderValue::from_static("1")); + } + response +} + +fn gateway_generation_error(error: super::upstream::UpstreamError) -> Response { + error + .request_rejection() + .map(gateway_request_rejected) + .unwrap_or_else(gateway_upstream_unavailable) +} + +fn gateway_upstream_response_error(response: super::upstream::UpstreamResponse) -> Response { + super::upstream::RequestRejection::from_response(response.status, &response.bytes) + .map(gateway_request_rejected) + .unwrap_or_else(gateway_upstream_unavailable) +} + +fn anthropic_generation_error(error: super::upstream::UpstreamError) -> Response { + let Some(rejection) = error.request_rejection() else { + return anthropic_upstream_unavailable(); + }; + let mut response = anthropic_error( + StatusCode::from_u16(rejection.status()).expect("finite engine rejection status"), + gateway::AnthropicError { + error_type: if rejection == super::upstream::RequestRejection::RateLimited { + "rate_limit_error" + } else { + "invalid_request_error" + }, + message: rejection.message(), + }, + ); + if rejection == super::upstream::RequestRejection::RateLimited { + response + .headers_mut() + .insert(header::RETRY_AFTER, HeaderValue::from_static("1")); + } + response +} + fn valid_inference_headers(headers: &HeaderMap) -> bool { headers.len() <= 64 && headers @@ -962,6 +1237,7 @@ async fn gateway_responses( AxumPath(instance): AxumPath, RawQuery(query): RawQuery, Extension(auth): Extension, + Extension(lease): Extension, headers: HeaderMap, body: Bytes, ) -> Response { @@ -985,6 +1261,27 @@ async fn gateway_responses( if !route.profile.allows(PublicAction::Responses) { return openai_not_found().await; } + let body = match normalize_gateway_images( + body, + gateway::ImageProtocol::Responses, + route.profile.clone(), + state.gateway_limits.clone(), + lease, + ) + .await + { + Ok(body) => body, + Err(error) => { + return openai_error( + if error.code == "request_too_large" { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_REQUEST + }, + error, + ); + } + }; if route.profile.native_responses { let request = match gateway::rewrite_native_responses_request_with_profile( &body, @@ -994,6 +1291,10 @@ async fn gateway_responses( Ok(request) => request, Err(error) => return openai_error(StatusCode::BAD_REQUEST, error), }; + let route = match meter_inference_route(&state, &auth, &instance, route).await { + Ok(route) => route, + Err(error) => return state_problem(error), + }; let token_permit = match auth.acquire_inference().await { Ok(permit) => permit, Err(()) => return gateway_upstream_unavailable(), @@ -1008,6 +1309,10 @@ async fn gateway_responses( Ok(request) => request, Err(error) => return openai_error(StatusCode::BAD_REQUEST, error), }; + let route = match meter_inference_route(&state, &auth, &instance, route).await { + Ok(route) => route, + Err(error) => return state_problem(error), + }; let token_permit = match auth.acquire_inference().await { Ok(permit) => permit, Err(()) => return gateway_upstream_unavailable(), @@ -1031,7 +1336,7 @@ async fn gateway_native_responses( .await { Ok(upstream) => upstream, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; let stream = futures_util::stream::unfold( (upstream, permit, token_permit), @@ -1054,7 +1359,7 @@ async fn gateway_native_responses( .request("POST", "/v1/responses", request.body.len()) { Ok(request) => request, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; let response = route .upstream @@ -1067,6 +1372,9 @@ async fn gateway_native_responses( drop(permit); drop(token_permit); match response { + Ok(response) if !(200..300).contains(&response.status) => { + gateway_upstream_response_error(response) + } Ok(response) => Response::builder() .status( StatusCode::from_u16(response.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), @@ -1150,8 +1458,9 @@ async fn gateway_chat_completions( AxumPath(instance): AxumPath, RawQuery(query): RawQuery, Extension(auth): Extension, + Extension(lease): Extension, headers: HeaderMap, - body: Bytes, + body: Result, ) -> Response { if reject_query(query).is_some() || !valid_inference_headers(&headers) @@ -1170,9 +1479,50 @@ async fn gateway_chat_completions( RouteLookup::Warming => return gateway_warming(), RouteLookup::Missing => return not_found().await, }; + let mut metered_route = (*route).clone(); + metered_route.upstream = metered_route.upstream.with_meter(Arc::new( + match super::sessions::UsageMeter::begin(state.database.as_ref(), &auth.id, &instance).await + { + Ok(meter) => meter, + Err(error) => return state_problem(error), + }, + )); + let route = Arc::new(metered_route); if !route.profile.allows(PublicAction::Chat) { return openai_not_found().await; } + let body = match body { + Ok(body) => body, + Err(_) => { + return ingress_error( + "/openai/v1/chat/completions", + StatusCode::PAYLOAD_TOO_LARGE, + "request_too_large", + "request body is too large", + ); + } + }; + let body = match normalize_gateway_images( + body, + gateway::ImageProtocol::Chat, + route.profile.clone(), + state.gateway_limits.clone(), + lease, + ) + .await + { + Ok(body) => body, + Err(error) => { + return openai_error( + if error.code == "request_too_large" { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_REQUEST + }, + error, + ); + } + }; let request = match gateway::rewrite_chat_request_with_profile( &body, &route.served_model, @@ -1204,7 +1554,7 @@ async fn gateway_chat( .await { Ok(upstream) => upstream, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; return chat_sse(upstream, route.public_model.clone(), permit, token_permit); } @@ -1214,7 +1564,7 @@ async fn gateway_chat( .request("POST", "/v1/chat/completions", request.body.len()) { Ok(request) => request, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; let response = route.upstream.send(&upstream_request, &request.body).await; drop(permit); @@ -1225,7 +1575,8 @@ async fn gateway_chat( .map(IntoResponse::into_response) .unwrap_or_else(|error| openai_error(StatusCode::BAD_GATEWAY, error)) } - _ => gateway_upstream_unavailable(), + Ok(response) => gateway_upstream_response_error(response), + Err(error) => gateway_generation_error(error), } } @@ -1272,7 +1623,7 @@ async fn gateway_responses_stream( .await { Ok(upstream) => upstream, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; let encoder = gateway::ResponsesEncoder::new(route.public_model.clone(), request.custom_tools); if request.stream { @@ -2323,6 +2674,14 @@ async fn reconcile_once( .await; continue; } + if let Some(routes) = routes { + routes.mark_warming(&instance.name, instance.generation); + } + // Admission can be temporarily closed while startup telemetry settles. + // Only an admitted restart attempt consumes the engine failure budget. + if !persistent_restart_allowed(database, executor).await? { + continue; + } let failed = database .record_restart_failure(&instance.id, instance.generation, unix_millis() / 1_000) .await @@ -2340,9 +2699,6 @@ async fn reconcile_once( else { continue; }; - if !persistent_restart_allowed(database, executor).await? { - continue; - } let Ok(prepared) = executor .prepare_instance(StartInstanceInput { instance_id: instance.id.clone(), @@ -2489,7 +2845,7 @@ async fn reconcile_running_engine( if let Some(routes) = routes { routes.mark_warming(&instance.name, instance.generation); } - if !observed.running || !persistent_restart_allowed(database, executor).await? { + if !observed.running { let _ = executor .disable_restart(StopInstanceInput { instance_id: instance.id.clone(), @@ -2503,6 +2859,20 @@ async fn reconcile_running_engine( .map_err(|_| ())?; return Ok(()); } + if !persistent_restart_allowed(database, executor).await? { + // Keep the route unpublished and Docker restart disabled, but do not + // classify an admission deferral as a failure of this running engine. + if observed.restart_policy != "no" { + let _ = executor + .disable_restart(StopInstanceInput { + instance_id: instance.id.clone(), + generation: instance.generation, + grace_seconds: 0, + }) + .await; + } + return Ok(()); + } let address = observed.address.parse().map_err(|_| ())?; let route = ObservedRoute::new( &observed.instance_id, @@ -4216,6 +4586,95 @@ fn qualification_executor_problem(error: ExecutorClientError) -> Response { problem(status, code, detail) } +#[utoipa::path(post, path = "/api/sparkplane/v1/launch-sessions", request_body = super::wire::LaunchSessionRequest, responses((status = 200, body = super::wire::LaunchSessionCreated)))] +async fn create_launch_session( + State(state): State, + Extension(auth): Extension, + RawQuery(query): RawQuery, + Json(body): Json, +) -> Response { + if let Some(response) = reject_query(query) { + return response; + } + if let Err(response) = require_executor_for_mutation(&state).await { + return response; + } + let Some(db) = &state.database else { + return database_unavailable(); + }; + match db.create_launch_session(&auth.id, body).await { + Ok(created) => { + match db.auth_snapshot().await { + Ok(snapshot) => state.store_auth(snapshot), + Err(error) => return state_problem(error), + } + let mut response = Json(created).into_response(); + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + response + } + Err(error) => state_problem(error), + } +} + +#[utoipa::path(get, path = "/api/sparkplane/v1/launch-sessions/{id}", params(("id" = String, Path)), responses((status = 200, body = super::wire::LaunchSessionDocument)))] +async fn get_launch_session( + State(state): State, + AxumPath(id): AxumPath, + RawQuery(query): RawQuery, +) -> Response { + if let Some(response) = reject_query(query) { + return response; + } + if id.parse::().is_err() { + return state_problem(StateError::Invalid("invalid session id".into())); + } + let Some(db) = &state.database else { + return database_unavailable(); + }; + match db.launch_session(&id, false, "").await { + Ok(session) => Json(session).into_response(), + Err(error) => state_problem(error), + } +} + +#[utoipa::path(post, path = "/api/sparkplane/v1/launch-sessions/{id}/finish", params(("id" = String, Path)), responses((status = 200, body = super::wire::LaunchSessionDocument)))] +async fn finish_launch_session( + State(state): State, + AxumPath(id): AxumPath, + Extension(auth): Extension, + RawQuery(query): RawQuery, +) -> Response { + if let Some(response) = reject_query(query) { + return response; + } + if id.parse::().is_err() { + return state_problem(StateError::Invalid("invalid session id".into())); + } + let Some(db) = &state.database else { + return database_unavailable(); + }; + match db.launch_session(&id, true, &auth.id).await { + Ok(mut session) => { + match db.auth_snapshot().await { + Ok(snapshot) => state.store_auth(snapshot), + Err(error) => return state_problem(error), + } + let deadline = tokio::time::Instant::now() + Duration::from_secs(4); + while session.usage.pending_requests > 0 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(20)).await; + match db.launch_session(&id, false, "").await { + Ok(value) => session = value, + Err(_) => break, + } + } + Json(session).into_response() + } + Err(error) => state_problem(error), + } +} + #[utoipa::path(post, path = "/api/sparkplane/v1/tokens", request_body = TokenCreateRequest, responses((status = 202, body = TokenCreatedDocument)))] async fn create_token( State(state): State, @@ -4520,6 +4979,14 @@ async fn authenticate( let Some(verifier) = snapshot.tokens.get(id) else { return auth_failed(&request_path); }; + if verifier.token.revoked_at.is_some() + || verifier.token.expires_at.as_ref().is_some_and(|expiry| { + chrono::DateTime::parse_from_rfc3339(expiry) + .map_or(true, |expiry| expiry <= chrono::Utc::now()) + }) + { + return auth_failed(&request_path); + } if !verifier.verify( &state .database @@ -4615,7 +5082,8 @@ fn required_scope(method: &Method, path: &str) -> Option { TokenScope::InstancesWrite }); } - if path.starts_with(&format!("{API_BASE}/tokens")) + if path.starts_with(&format!("{API_BASE}/launch-sessions")) + || path.starts_with(&format!("{API_BASE}/tokens")) || path == format!("{API_BASE}/doctor") || path.starts_with(&format!("{API_BASE}/certificates")) { @@ -4877,6 +5345,7 @@ pub async fn serve( config.schema == "sparkplane.agent/v1", "unsupported Spark agent configuration schema" ); + config.gateway.validate().map_err(anyhow::Error::msg)?; anyhow::ensure!( config.engine_catalog == Path::new("/etc/sparkplane/engines"), "engine catalog must use the fixed root-owned path" @@ -4989,7 +5458,8 @@ pub async fn serve( )?, config.operations.max_parallel_downloads, ) - .with_start_slots(config.operations.max_parallel_starts); + .with_start_slots(config.operations.max_parallel_starts) + .with_gateway_limits(config.gateway); state = state.with_qualification_authority(Arc::::from(qualification_authority)); if let Some(executor) = executor { state = state.with_executor(executor); @@ -5191,6 +5661,44 @@ mod tests { const TOKEN: &str = "test-bootstrap-token-with-at-least-256-bits-of-random-material"; + #[test] + fn gateway_limits_default_for_existing_configs_and_have_hard_ceilings() { + let mut config: toml::Value = + toml::from_str(include_str!("../../configs/sparkplane/agent.toml")).unwrap(); + let shipped: super::AgentConfig = config.clone().try_into().unwrap(); + shipped.gateway.validate().unwrap(); + config.as_table_mut().unwrap().remove("gateway"); + let legacy: super::AgentConfig = config.try_into().unwrap(); + assert_eq!( + serde_json::to_value(shipped.gateway).unwrap(), + serde_json::to_value(legacy.gateway).unwrap() + ); + for field in [ + "request_body_bytes", + "image_bytes", + "image_pixels", + "image_dimension", + "max_parallel_requests", + ] { + let mut limits = + serde_json::to_value(crate::spark::gateway::IngressLimits::default()).unwrap(); + limits[field] = 0.into(); + assert!( + serde_json::from_value::(limits.clone()) + .unwrap() + .validate() + .is_err() + ); + limits[field] = u32::MAX.into(); + assert!( + serde_json::from_value::(limits) + .unwrap() + .validate() + .is_err() + ); + } + } + fn llama_chat_sse() -> String { [ r#"{"choices":[{"delta":{"reasoning_content":"inspect"},"finish_reason":null}]}"#, @@ -5435,14 +5943,550 @@ mod tests { } #[tokio::test] - async fn native_responses_route_preserves_tool_history_and_engine_sse() { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.unwrap(); - let mut request = vec![0; 16_384]; - let count = socket.read(&mut request).await.unwrap(); + async fn inference_upload_rejections_are_json_and_authentication_runs_first() { + let state = AgentState::new(TOKEN, Vec::new(), Vec::new()); + for path in [ + "/openai/fixture/v1/responses", + "/anthropic/fixture/v1/messages", + ] { + let request = || { + Request::post(path) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(vec![b' '; 33 * 1024 * 1024])) + .unwrap() + }; + let denied = router(state.clone()).oneshot(request()).await.unwrap(); + assert_eq!(denied.status(), axum::http::StatusCode::UNAUTHORIZED); + let mut request = request(); + request.headers_mut().insert( + header::AUTHORIZATION, + format!("Bearer {TOKEN}").parse().unwrap(), + ); + let response = router(state.clone()).oneshot(request).await.unwrap(); + assert_eq!(response.status(), axum::http::StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(response.headers()[header::CONTENT_TYPE], "application/json"); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + let document: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert!( + document["error"]["message"] + .as_str() + .unwrap() + .contains("gateway") + ); + } + } + + #[tokio::test] + async fn large_image_uploads_fit_the_qualified_profile_over_https() { + use base64::Engine as _; + use sha2::{Digest, Sha256}; + let (sent, mut received) = tokio::sync::mpsc::unbounded_channel(); + let engine = axum::Router::new().route("/v1/{*path}", axum::routing::post( + move |uri: axum::http::Uri, axum::Json(body): axum::Json| { + let sent = sent.clone(); + async move { + let native = uri.path() == "/v1/responses"; + sent.send(body).unwrap(); + let body = if native { + "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":7,\"output_tokens\":3}}}\n\n".to_owned() + } else {llama_chat_sse()}; + ([(header::CONTENT_TYPE, "text/event-stream")], body) + } + } + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let engine_address = listener.local_addr().unwrap(); + let engine_server = + tokio::spawn(async move { axum::serve(listener, engine).await.unwrap() }); + let state = AgentState::new(TOKEN, Vec::new(), Vec::new()); + let profile = crate::spark::engine::EnginePolicy::parse(include_str!( + "../../configs/sparkplane/engines/vllm-qwen38-mmap.toml" + )) + .unwrap() + .gateway_profile(None); + for (name, native) in [ + ("native", true), + ("translated", false), + ("anthropic", false), + ("tool-native", true), + ("tool-translated", false), + ] { + let mut profile = profile.clone(); + profile.native_responses = native; + let upstream = crate::spark::upstream::ObservedRoute::new( + "i_11111111111111111111111111111111", + 1, + engine_address.ip(), + engine_address.port(), + [("POST", "/v1/responses"), ("POST", "/v1/chat/completions")], + ) + .unwrap(); + state.routes.publish_with_profile( + name, + "public-model".into(), + "served-model".into(), + profile, + upstream, + ); + } + let rcgen::CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let certificate = cert.pem(); + let tls = super::tls13_config( + certificate.as_bytes().to_vec(), + signing_key.serialize_pem().into_bytes(), + ) + .await + .unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let handle = axum_server::Handle::new(); + let server_handle = handle.clone(); + let server = tokio::spawn(async move { + axum_server::from_tcp_rustls(listener, tls) + .unwrap() + .handle(server_handle) + .serve(router(state).into_make_service_with_connect_info::()) + .await + .unwrap(); + }); + let client = reqwest::Client::builder() + .tls_built_in_root_certs(false) + .add_root_certificate(reqwest::Certificate::from_pem(certificate.as_bytes()).unwrap()) + .build() + .unwrap(); + let mut seed = 0x12345678_u32; + let pixels = (0..1024 * 512 * 3) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed as u8 + }) + .collect(); + let image = + image::DynamicImage::ImageRgb8(image::RgbImage::from_raw(1024, 512, pixels).unwrap()); + let mut png = std::io::Cursor::new(Vec::new()); + image.write_to(&mut png, image::ImageFormat::Png).unwrap(); + let encoded = base64::engine::general_purpose::STANDARD.encode(png.into_inner()); + for name in [ + "native", + "translated", + "anthropic", + "tool-native", + "tool-translated", + ] { + let mut request = if name == "anthropic" { + serde_json::json!({"model":"public-model","max_tokens":8,"stream":true,"messages":[{"role":"user","content":[{"type":"text","text":"describe"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":encoded}}]}]}) + } else { + serde_json::json!({"input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"describe"},{"type":"input_image","image_url":format!("data:image/png;base64,{encoded}"),"detail":"high"}]}],"stream":true,"store":false}) + }; + if name.starts_with("tool-") { + let image = request["input"][0]["content"][1].clone(); + request["input"] = serde_json::json!([ + {"type":"message","role":"user","content":"inspect the screenshot"}, + {"type":"function_call","call_id":"call_view","name":"view_image","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_view","output":[{"type":"input_text","text":"Screenshot from the tool."},image]} + ]); + } + assert!(serde_json::to_vec(&request).unwrap().len() > 1024 * 1024); + let path = if name == "anthropic" { + format!("/anthropic/{name}/v1/messages") + } else { + format!("/openai/{name}/v1/responses") + }; + let response = client + .post(format!("https://localhost:{}{path}", address.port())) + .bearer_auth(TOKEN) + .header("anthropic-version", "2023-06-01") + .json(&request) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.bytes().await.unwrap(); + assert_eq!( + status, + reqwest::StatusCode::OK, + "{}", + String::from_utf8_lossy(&body) + ); + let forwarded = received.recv().await.unwrap(); + let url = if name == "tool-native" { + forwarded["input"][2]["output"][1]["image_url"] + .as_str() + .unwrap() + } else if name == "tool-translated" { + assert_eq!(forwarded["messages"][2]["role"], "tool"); + assert_eq!(forwarded["messages"][2]["tool_call_id"], "call_view"); + forwarded["messages"][2]["content"][1]["image_url"]["url"] + .as_str() + .unwrap() + } else if name == "native" { + forwarded["input"][0]["content"][1]["image_url"] + .as_str() + .unwrap() + } else { + forwarded["messages"][0]["content"][1]["image_url"]["url"] + .as_str() + .unwrap() + }; + let (media, data) = url + .strip_prefix("data:") + .unwrap() + .split_once(";base64,") + .unwrap(); + let bytes = base64::engine::general_purpose::STANDARD + .decode(data) + .unwrap(); + let vision = profile.vision.as_ref().unwrap(); + assert!(vision.media_types.contains(media)); + assert!(bytes.len() <= vision.max_bytes); + let decoded = image::ImageReader::new(std::io::Cursor::new(bytes)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + assert!(decoded.width() <= vision.max_width && decoded.height() <= vision.max_height); + assert!(decoded.width() <= 1024 && decoded.height() <= 512); + assert_eq!(decoded.width(), decoded.height() * 2); + } + let mut small_png = std::io::Cursor::new(Vec::new()); + image + .resize(256, 128, image::imageops::FilterType::Lanczos3) + .write_to(&mut small_png, image::ImageFormat::Png) + .unwrap(); + let small_encoded = + base64::engine::general_purpose::STANDARD.encode(small_png.into_inner()); + for count in [2, 3, 16] { + for name in ["tool-native", "tool-translated", "anthropic"] { + let encoded = if count == 16 { + &small_encoded + } else { + &encoded + }; + let request = if name == "anthropic" { + let image = serde_json::json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":encoded}}); + serde_json::json!({"model":"public-model","max_tokens":8,"stream":true,"messages":[{"role":"user","content":vec![image;count]}]}) + } else { + let image = serde_json::json!({"type":"input_image","image_url":format!("data:image/png;base64,{encoded}")}); + let mut input = vec![ + serde_json::json!({"type":"message","role":"user","content":"compare the screenshots"}), + ]; + for index in 0..count { + let id = format!("view_{index}"); + input.push(serde_json::json!({"type":"function_call","name":"view_image","call_id":id,"arguments":"{}"})); + input.push(serde_json::json!({"type":"function_call_output","call_id":id,"output":[{"type":"input_text","text":format!("Frame {index}")},image]})); + if index + 1 < count { + input.push(serde_json::json!({"type":"message","role":"assistant","content":"Inspect the next screenshot."})); + } + } + serde_json::json!({"input":input,"stream":true}) + }; + let path = if name == "anthropic" { + format!("/anthropic/{name}/v1/messages") + } else { + format!("/openai/{name}/v1/responses") + }; + let response = client + .post(format!("https://localhost:{}{path}", address.port())) + .bearer_auth(TOKEN) + .header("anthropic-version", "2023-06-01") + .json(&request) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.bytes().await.unwrap(); + assert_eq!( + status, + reqwest::StatusCode::OK, + "{}", + String::from_utf8_lossy(&body) + ); + let forwarded = received.recv().await.unwrap(); + let messages = forwarded[if name == "tool-native" { + "input" + } else { + "messages" + }] + .as_array() + .unwrap(); + let mut images = 0; + let mut total = 0; + let retained = if count == 16 && name != "anthropic" { + 10 + } else { + count + }; + for message in messages { + let parts = message[if message["type"] == "function_call_output" { + "output" + } else { + "content" + }] + .as_array(); + for part in parts.into_iter().flatten() { + let url = if part["type"] == "input_image" { + part["image_url"].as_str() + } else { + part["image_url"]["url"].as_str() + }; + if let Some(url) = url { + let bytes = base64::engine::general_purpose::STANDARD + .decode(url.split_once(";base64,").unwrap().1) + .unwrap(); + assert!( + bytes.len() + <= profile.vision.as_ref().unwrap().max_total_bytes / retained + ); + total += bytes.len(); + images += 1; + } + } + } + assert_eq!(images, retained); + assert!(total <= profile.vision.as_ref().unwrap().max_total_bytes); + } + } + // The launch adapter streams a source history larger than ingress allows, + // then forwards only its recent window over the same pinned HTTPS route. + let config = tempfile::tempdir().unwrap(); + std::fs::create_dir(config.path().join("spark")).unwrap(); + std::fs::write(config.path().join("spark/fixture.ca.pem"), &certificate).unwrap(); + let pin = format!("sha256:{:x}", Sha256::digest(certificate.as_bytes())); + std::fs::write(config.path().join("spark.toml"), format!("[hosts.fixture]\nurl='https://localhost:{}'\nca_cert_sha256='{pin}'\ncredential='spark/unused'\nrequest_timeout_seconds=30\n", address.port())).unwrap(); + let adapter = crate::spark::inference_adapter::Adapter::start( + config.path(), + "fixture", + "tool-native", + TOKEN, + ) + .unwrap(); + let image = serde_json::json!({"type":"input_image","image_url":format!("data:image/png;base64,{encoded}")}); + let mut input = vec![ + serde_json::json!({"type":"message","role":"user","content":"Inspect the screenshots."}), + ]; + for index in 0..25 { + input.push(serde_json::json!({"type":"function_call","name":"view_image","call_id":format!("long_{index}"),"arguments":"{}"})); + input.push(serde_json::json!({"type":"function_call_output","call_id":format!("long_{index}"),"output":[{"type":"input_text","text":format!("Frame {index}")},image]})); + if index < 24 { + input.push(serde_json::json!({"type":"message","role":"assistant","content":"Observed the button."})); + } + } + let payload = + serde_json::to_vec(&serde_json::json!({"input":input,"stream":true})).unwrap(); + assert!(payload.len() > crate::spark::gateway::IngressLimits::default().request_body_bytes); + let response = client + .post(format!("{}/openai/v1/responses", adapter.base_url)) + .bearer_auth(adapter.token()) + .header("content-type", "application/json") + .body(payload) + .send() + .await + .unwrap(); + let status = response.status(); + let text = response.text().await.unwrap(); + assert_eq!(status, reqwest::StatusCode::OK, "{text}"); + assert!(text.contains("response.completed")); + let mut forwarded = received.recv().await.unwrap(); + let mut count = 0; + crate::spark::image_history::visit_parts( + &mut forwarded["input"], + crate::spark::image_history::Protocol::Responses, + &mut |_| count += 1, + ); + assert!(count <= 12); + assert_eq!(forwarded["input"][74]["call_id"], "long_24"); + assert_eq!(forwarded["input"][74]["output"][0]["text"], "Frame 24"); + drop(adapter); + handle.shutdown(); + server.await.unwrap(); + engine_server.abort(); + } + + #[tokio::test] + async fn upstream_client_rejections_keep_safe_status_codes_and_context_errors() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for (status, body, expected, code) in [ + ( + 400, + r#"{"error":{"code":"context_length_exceeded","message":"private prompt SECRET"}}"#, + 400, + "context_length_exceeded", + ), + ( + 400, + r#"{"error":{"message":"This model's maximum context length is 262144 tokens. private prompt SECRET"}}"#, + 400, + "context_length_exceeded", + ), + ( + 400, + r#"{"error":{"message":"private prompt SECRET"}}"#, + 400, + "invalid_request_error", + ), + (413, "private prompt SECRET", 413, "request_too_large"), + (429, "private prompt SECRET", 429, "rate_limit_error"), + (500, "private prompt SECRET", 503, "server_error"), + ] { + for kind in [ + "responses", + "native_stream", + "native_json", + "chat_json", + "anthropic", + ] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 16384]; + assert!(socket.read(&mut request).await.unwrap() > 0); + socket.write_all(format!("HTTP/1.1 {status} Error\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",body.len()).as_bytes()).await.unwrap(); + }); + let state = AgentState::new(TOKEN, Vec::new(), Vec::new()); + let upstream = crate::spark::upstream::ObservedRoute::new( + "i_11111111111111111111111111111111", + 1, + address.ip(), + address.port(), + [("POST", "/v1/chat/completions"), ("POST", "/v1/responses")], + ) + .unwrap(); + let mut profile = crate::spark::gateway::GatewayProfile::text(); + profile.native_responses = kind.starts_with("native"); + profile.native_response_timeout_seconds = 5; + state.routes.publish_with_profile( + "fixture", + "public-model".into(), + "served-model".into(), + profile, + upstream, + ); + let (path, body) = match kind { + "anthropic" => ( + "/anthropic/fixture/v1/messages", + r#"{"model":"public-model","max_tokens":8,"messages":[{"role":"user","content":"work"}]}"#, + ), + "chat_json" => ( + "/openai/fixture/v1/chat/completions", + r#"{"messages":[{"role":"user","content":"work"}],"stream":false}"#, + ), + "native_json" => ( + "/openai/fixture/v1/responses", + r#"{"input":"work","stream":false}"#, + ), + _ => ( + "/openai/fixture/v1/responses", + r#"{"input":"work","stream":true}"#, + ), + }; + let request = Request::post(path) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .header("anthropic-version", "2023-06-01") + .body(Body::from(body)) + .unwrap(); + let response = router(state).oneshot(request).await.unwrap(); + assert_eq!( + response.status().as_u16(), + expected, + "{kind}, engine status {status}" + ); + if expected == 429 { + assert_eq!(response.headers()[header::RETRY_AFTER], "1"); + } + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("SECRET")); + assert!(!String::from_utf8_lossy(&bytes).contains("private prompt")); + let response: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + if kind == "anthropic" { + assert_eq!(response["type"], "error"); + assert_eq!( + response["error"]["type"], + if expected == 503 { + "api_error" + } else if expected == 429 { + "rate_limit_error" + } else { + "invalid_request_error" + } + ); + } else { + assert_eq!(response["error"]["code"], code); + } + server.await.unwrap(); + } + } + } + + #[tokio::test] + async fn gateway_busy_rejections_are_bounded_and_release_after_cancellation() { + let state = AgentState::new(TOKEN, Vec::new(), Vec::new()); + let first = state.ingress_slots.clone().try_acquire_owned().unwrap(); + let second = state.ingress_slots.clone().try_acquire_owned().unwrap(); + for path in [ + "/openai/fixture/v1/responses", + "/anthropic/fixture/v1/messages", + ] { + let response = router(state.clone()) + .oneshot( + Request::post(path) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .body(Body::from("{}")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + response.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE + ); + assert_eq!(response.headers()[header::RETRY_AFTER], "1"); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + assert!( + serde_json::from_slice::(&bytes).unwrap()["error"]["message"] + .as_str() + .unwrap() + .contains("busy") + ); + } + drop((first, second)); + // A cancelled upload frees its slot without waiting for the 30-second deadline. + let request = Request::post("/openai/fixture/v1/responses") + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .body(Body::from_stream(futures_util::stream::pending::< + Result, + >())) + .unwrap(); + assert!( + tokio::time::timeout( + Duration::from_millis(50), + router(state.clone()).oneshot(request) + ) + .await + .is_err() + ); + assert_eq!(state.ingress_slots.available_permits(), 2); + } + + #[tokio::test] + async fn native_responses_route_preserves_tool_history_and_engine_sse() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 16_384]; + let count = socket.read(&mut request).await.unwrap(); let request = String::from_utf8_lossy(&request[..count]); assert!(request.starts_with("POST /v1/responses ")); assert!(request.contains(r#""model":"public-model""#)); @@ -6317,6 +7361,62 @@ mod tests { mutations: Arc, } + struct DeferredReconcileExecutor { + matched: bool, + swap_in_pages_delta: Option, + guard_heartbeat: bool, + disables: Arc, + unexpected_mutations: Arc, + } + + impl sparkplane_ipc::Handler for DeferredReconcileExecutor { + async fn handle(&self, request: sparkplane_ipc::Request) -> sparkplane_ipc::Response { + let action = &request.params["action"]; + if let Some(expected) = action.get("reconcile_scan") { + let identity = &expected[0]; + let matched = if self.matched { + vec![serde_json::json!({ + "instance_id":identity["instance_id"],"generation":identity["generation"], + "container_id":"container-g1","network_id":"network","address":"172.30.0.2","port":8000, + "running":true,"restart_policy":"unless-stopped","health_method":"GET","health_path":"/health", + "allowed_routes":[["GET","/health"]],"gateway_profile":crate::spark::gateway::GatewayProfile::text(), + "served_model":"Ornith-1.5-9B","semantic_prompt":"health","semantic_max_tokens":1, + "startup_deadline_seconds":900,"init_pid":1,"pid_start_time_ticks":1, + "cgroup_path":"/system.slice/docker-container-g1.scope" + })] + } else { + Vec::new() + }; + return sparkplane_ipc::Response::Ok { + schema_version: sparkplane_ipc::SCHEMA_VERSION, + request_id: request.request_id, + result: serde_json::json!({"action":"reconcile_scan","scan":{ + "matched":matched,"missing":if self.matched { serde_json::json!([]) } else { expected.clone() },"quarantined":[]}}), + blob: None, + }; + } + if action.get("disable_restart_policy").is_some() { + self.disables.fetch_add(1, Ordering::SeqCst); + } else if action.get("prepare_instance").is_some() + || action.get("start_instance").is_some() + || action.get("promote_restart_policy").is_some() + || action.get("stop_instance").is_some() + { + self.unexpected_mutations.fetch_add(1, Ordering::SeqCst); + } + let mut response = ResourceExecutor.handle(request).await; + if let sparkplane_ipc::Response::Ok { result, .. } = &mut response { + if result["action"] == "inspect_resources" { + result["snapshot"]["swap_in_pages_delta"] = + serde_json::json!(self.swap_in_pages_delta); + } else if result["action"] == "health" { + result["health"]["guard_heartbeat"] = serde_json::json!(self.guard_heartbeat); + } + } + response + } + } + struct StopRaceExecutor { stops: Arc, removed: Arc, @@ -6659,6 +7759,84 @@ mod tests { database.shutdown().unwrap(); } + #[tokio::test] + async fn deferred_reconciliation_keeps_admission_closed_without_consuming_restart_budget() { + for (matched, swap_in_pages_delta, guard_heartbeat) in [ + (true, None, true), + (true, Some(1), true), + (true, Some(0), false), + (false, Some(1), true), + ] { + let (state, database, root) = durable_state().await; + let socket = root.join("executor.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let disables = Arc::new(AtomicUsize::new(0)); + let unexpected_mutations = Arc::new(AtomicUsize::new(0)); + let server = tokio::spawn( + sparkplane_ipc::Server::new(DeferredReconcileExecutor { + matched, + swap_in_pages_delta, + guard_heartbeat, + disables: Arc::clone(&disables), + unexpected_mutations: Arc::clone(&unexpected_mutations), + }) + .serve(listener), + ); + let model = ornith_model(); + database.promote_model(model.clone(), false).await.unwrap(); + database + .accept_operation( + "bootstrap", + "instance.serve", + "01K00000000000000000000000", + &"a".repeat(64), + Some("ornith".into()), + ) + .await + .unwrap(); + let instance = database + .begin_serve(creating_instance(&model)) + .await + .unwrap() + .instance; + if !matched { + let route = crate::spark::upstream::ObservedRoute::new( + &instance.id, + instance.generation, + "127.0.0.1".parse().unwrap(), + 9, + [("GET", "/health")], + ) + .unwrap(); + state.routes.publish( + &instance.name, + instance.model.clone(), + "served-model".into(), + route, + ); + } + let executor = crate::spark::executor::ExecutorClient::new(socket); + for _ in 0..6 { + super::reconcile_once(&database, &executor, Some(&state.routes), &state.admission) + .await + .unwrap(); + } + let current = database.instance(&instance.id).await.unwrap(); + assert_eq!(current.restart_failures, 0); + assert!(!current.restart_suppressed); + assert_eq!(current.engine_fingerprint, instance.engine_fingerprint); + assert_eq!(current.generation, instance.generation); + assert!(!matches!( + state.routes.lookup(&instance.name), + crate::spark::gateway::RouteLookup::Healthy(_) + )); + assert_eq!(disables.load(Ordering::SeqCst), if matched { 6 } else { 0 }); + assert_eq!(unexpected_mutations.load(Ordering::SeqCst), 0); + database.shutdown().unwrap(); + server.abort(); + } + } + #[tokio::test] async fn reconcile_restarts_the_same_engine_and_artifact_generation() { let (state, database, root) = durable_state().await; @@ -7201,6 +8379,313 @@ mod tests { server.abort(); } + #[tokio::test] + async fn launch_session_usage_and_revocation_round_trip_over_https() { + use crate::spark::wire::{LaunchSessionCreated, LaunchSessionDocument}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let (state, database, root) = durable_state().await; + let model = ornith_model(); + database.promote_model(model.clone(), false).await.unwrap(); + let instance = database + .begin_serve(creating_instance(&model)) + .await + .unwrap() + .instance; + let rcgen::CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let certificate = cert.pem(); + let tls = super::tls13_config( + certificate.as_bytes().to_vec(), + signing_key.serialize_pem().into_bytes(), + ) + .await + .unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let handle = axum_server::Handle::new(); + let server_handle = handle.clone(); + let server_state = state.clone(); + let server = tokio::spawn(async move { + axum_server::from_tcp_rustls(listener, tls) + .unwrap() + .handle(server_handle) + .serve(router(server_state).into_make_service_with_connect_info::()) + .await + .unwrap(); + }); + let client = reqwest::Client::builder() + .tls_built_in_root_certs(false) + .add_root_certificate(reqwest::Certificate::from_pem(certificate.as_bytes()).unwrap()) + .build() + .unwrap(); + let base = format!("https://localhost:{}", address.port()); + let id = ulid::Ulid::new().to_string(); + let create = + serde_json::json!({"id":id,"instance":"ornith","integration":"codex","eco_mode":"max"}); + let response = client + .post(format!("{base}{API_BASE}/launch-sessions")) + .bearer_auth(TOKEN) + .json(&create) + .send() + .await + .unwrap() + .error_for_status() + .unwrap(); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + let created: LaunchSessionCreated = response.json().await.unwrap(); + let bearer = created.bearer_token.unwrap(); + let repeated: LaunchSessionCreated = client + .post(format!("{base}{API_BASE}/launch-sessions")) + .bearer_auth(TOKEN) + .json(&create) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert!(repeated.bearer_token.is_none()); + assert_eq!( + client + .get(format!("{base}{API_BASE}/launch-sessions/{id}")) + .bearer_auth(&bearer) + .send() + .await + .unwrap() + .status(), + reqwest::StatusCode::UNAUTHORIZED + ); + for (path, body, missing, shared) in [ + ( + "/openai/ornith/v1/responses", + serde_json::json!({"input":"private fixture prompt","stream":false}), + false, + false, + ), + ( + "/anthropic/ornith/v1/messages", + serde_json::json!({"model":"sparkplane","messages":[{"role":"user","content":"fixture"}],"max_tokens":10,"stream":true}), + false, + false, + ), + ( + "/openai/ornith/v1/chat/completions", + serde_json::json!({"messages":[{"role":"user","content":"fixture"}],"stream":true}), + false, + false, + ), + ( + "/openai/ornith/v1/responses", + serde_json::json!({"input":"fixture","stream":true}), + true, + false, + ), + ( + "/openai/ornith/v1/responses", + serde_json::json!({"input":"another client","stream":false}), + false, + true, + ), + ] { + let fixture = if missing { + llama_chat_sse() + .lines() + .filter(|line| !line.contains("usage")) + .collect::>() + .join("\n") + + "\n" + } else { + llama_chat_sse() + }; + let (engine, task) = llama_chat_server_with(fixture).await; + let upstream = crate::spark::upstream::ObservedRoute::new( + &instance.id, + instance.generation, + engine.ip(), + engine.port(), + [("POST", "/v1/chat/completions")], + ) + .unwrap(); + state.routes.publish( + "ornith", + model.canonical.clone(), + "Ornith-1.5-9B".into(), + upstream, + ); + let response = client + .post(format!("{base}{path}")) + .header("anthropic-version", "2023-06-01") + .bearer_auth(if shared { TOKEN } else { &bearer }) + .json(&body) + .send() + .await + .unwrap(); + let status = response.status(); + let output = response.bytes().await.unwrap(); + assert!( + status.is_success(), + "{path}: {status}: {}", + String::from_utf8_lossy(&output) + ); + assert!(!output.is_empty()); + task.await.unwrap(); + } + for stream in [false, true] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let engine = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = [0; 16384]; + let count = socket.read(&mut request).await.unwrap(); + assert!( + String::from_utf8_lossy(&request[..count]).starts_with("POST /v1/responses ") + ); + let document = r#"{"id":"resp_fixture","object":"response","status":"completed","output":[],"usage":{"input_tokens":11,"output_tokens":5,"total_tokens":16}}"#; + let body = if stream { + format!( + "data: {{\"type\":\"response.completed\",\"response\":{document}}}\r\n\r\n" + ) + } else { + document.into() + }; + let content = if stream { + "text/event-stream" + } else { + "application/json" + }; + socket.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: {content}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",body.len()).as_bytes()).await.unwrap(); + for chunk in body.as_bytes().chunks(3) { + socket.write_all(chunk).await.unwrap(); + tokio::task::yield_now().await; + } + }); + let upstream = crate::spark::upstream::ObservedRoute::new( + &instance.id, + instance.generation, + engine.ip(), + engine.port(), + [("POST", "/v1/responses")], + ) + .unwrap(); + let mut profile = crate::spark::gateway::GatewayProfile::text(); + profile.native_responses = true; + profile.native_response_timeout_seconds = 5; + state.routes.publish_with_profile( + "ornith", + model.canonical.clone(), + "Ornith-1.5-9B".into(), + profile, + upstream, + ); + let output = client + .post(format!("{base}/openai/ornith/v1/responses")) + .bearer_auth(&bearer) + .json(&serde_json::json!({"input":"fixture","stream":stream})) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .bytes() + .await + .unwrap(); + assert!(String::from_utf8_lossy(&output).contains("resp_fixture")); + task.await.unwrap(); + } + let finish = format!("{base}{API_BASE}/launch-sessions/{id}/finish"); + let finished: LaunchSessionDocument = client + .post(&finish) + .bearer_auth(TOKEN) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(finished.usage.requests, 6); + assert_eq!(finished.usage.input_tokens, 43); + assert_eq!(finished.usage.output_tokens, 19); + assert_eq!(finished.usage.unknown_usage_requests, 1); + assert_eq!(finished.usage.pending_requests, 0); + let repeated: LaunchSessionDocument = client + .post(&finish) + .bearer_auth(TOKEN) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(finished.finished_at, repeated.finished_at); + assert_eq!(repeated.usage.input_tokens, 43); + assert_eq!( + client + .get(format!("{base}/openai/ornith/v1/models")) + .bearer_auth(&bearer) + .send() + .await + .unwrap() + .status(), + reqwest::StatusCode::UNAUTHORIZED + ); + let expired_create = serde_json::json!({"id":ulid::Ulid::new().to_string(),"instance":"ornith","integration":"claude","eco_mode":"none"}); + let expiring: LaunchSessionCreated = client + .post(format!("{base}{API_BASE}/launch-sessions")) + .bearer_auth(TOKEN) + .json(&expired_create) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + let expired_bearer = expiring.bearer_token.unwrap(); + let expired_id = expired_bearer + .strip_prefix("sparkplane_") + .unwrap() + .split('_') + .next() + .unwrap(); + let mut snapshot = database.auth_snapshot().await.unwrap(); + snapshot + .tokens + .get_mut(expired_id) + .unwrap() + .token + .expires_at = Some((chrono::Utc::now() - chrono::Duration::seconds(1)).to_rfc3339()); + state.store_auth(snapshot); + assert_eq!( + client + .get(format!("{base}/openai/ornith/v1/models")) + .bearer_auth(expired_bearer) + .send() + .await + .unwrap() + .status(), + reqwest::StatusCode::UNAUTHORIZED + ); + let document = serde_json::to_string(&finished).unwrap(); + assert!(!document.contains("private fixture prompt")); + assert!(!document.contains(&bearer)); + assert_eq!( + database.list_instances().await.unwrap()[0].desired, + crate::spark::wire::InstanceDesiredState::Running + ); + handle.graceful_shutdown(None); + server.await.unwrap(); + database.shutdown().unwrap(); + std::fs::remove_dir_all(root).unwrap(); + } + async fn durable_state() -> (super::AgentState, crate::spark::state::DbActor, PathBuf) { let root = tempfile::tempdir().unwrap().keep(); let database = crate::spark::state::DbActor::open( diff --git a/src/spark/cli.rs b/src/spark/cli.rs index b5d1247..45a9f02 100644 --- a/src/spark/cli.rs +++ b/src/spark/cli.rs @@ -126,6 +126,8 @@ pub enum SparkCommand { after_help = "Examples:\n sparkplane dgx-spark launch codex --model ornith-1.5:35b\n sparkplane dgx-spark launch codex --mode inherit --allow-network -- --sandbox workspace-write\n sparkplane dgx-spark launch claude --mode inherit --model ornith-1.5:35b -- --permission-mode plan\n sparkplane dgx-spark launch opencode --config --model ornith-1.5:35b\n sparkplane dgx-spark launch codex --model ornith-1.5:35b --dry-run --json\n\nArguments after `--` are passed directly to the selected local agent without a shell. The Spark administrator credential is never given to the child process.\n\nAuto mode (default) gives agents full access without action approvals. Use --mode inherit to retain agent permissions. --allow-network requests invocation-only network access: Codex workspace-write networking, or Claude sandbox domains. OpenCode already has network access; its tool permissions remain unchanged. Filesystem sandboxing and approval policies are not disabled. Managed restrictions still apply. Claude --settings cannot be forwarded with this flag.\n\nEnvironment:\n SPARKPLANE_LAUNCH_MODE, SPARKPLANE_LAUNCH_MODEL, SPARKPLANE_LAUNCH_ALLOW_NETWORK, SPARKPLANE_LAUNCH_CONFIG, SPARKPLANE_LAUNCH_RESTORE, SPARKPLANE_YES, SPARKPLANE_DRY_RUN, SPARKPLANE_JSON, SPARKPLANE_CONFIG_DIR" )] Launch(LaunchArgs), + /// View a retained private session economics report. + Economics(EconomicsArgs), /// List currently active managed model processes. #[command( after_help = "Examples:\n sparkplane dgx-spark ps\n sparkplane dgx-spark ps --json\n\nThe default output is a compact table. Absent and failed historical instances are omitted.\n\nEnvironment:\n SPARKPLANE_JSON, SPARKPLANE_CONFIG_DIR" @@ -430,6 +432,9 @@ pub struct LaunchArgs { /// Agent permissions: auto grants full access without action approvals; inherit keeps agent settings. #[arg(long, default_value = "auto", value_parser = ["auto", "inherit"], env = "SPARKPLANE_LAUNCH_MODE")] pub mode: String, + /// Embedded RTK compression: max enables supported filters; none disables it. + #[arg(long, default_value = "max", value_parser = ["max", "none"], env = "SPARKPLANE_LAUNCH_ECO_MODE")] + pub eco_mode: String, /// Allow this agent session's sandbox network access; keep filesystem and approval policies. #[arg(long, env = "SPARKPLANE_LAUNCH_ALLOW_NETWORK", conflicts_with_all = ["configure", "restore"])] pub allow_network: bool, @@ -451,6 +456,16 @@ pub struct LaunchArgs { pub extra_args: Vec, } +#[derive(Debug, Args)] +pub struct EconomicsArgs { + #[arg(long)] + pub session: Option, + #[arg(long, env = "SPARKPLANE_JSON")] + pub json: bool, + #[arg(long, env = "SPARKPLANE_CONFIG_DIR")] + pub config_dir: Option, +} + #[derive(Debug, Args)] pub struct StopArgs { pub instance: String, @@ -747,6 +762,19 @@ pub fn dispatch(cli: SparkCli) -> anyhow::Result<()> { SparkCommand::Download(args) => dispatch_download(&cli.host, args), SparkCommand::Serve(args) => dispatch_serve(&cli.host, args), SparkCommand::Launch(args) => dispatch_launch(&cli.host, args), + SparkCommand::Economics(args) => { + let report = super::economics::load( + &args.config_dir.unwrap_or_else(default_config_dir), + &cli.host, + args.session.as_deref(), + )?; + if args.json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + super::economics::display(&report); + } + Ok(()) + } SparkCommand::Ps(args) => dispatch_instances(&cli.host, args), SparkCommand::Logs(args) => dispatch_logs(&cli.host, args), SparkCommand::Stop(args) => dispatch_stop(&cli.host, args), diff --git a/src/spark/client.rs b/src/spark/client.rs index 9b870cc..1a3169e 100644 --- a/src/spark/client.rs +++ b/src/spark/client.rs @@ -73,6 +73,39 @@ pub struct ClaudeCodeClientConfig { pub shell: String, } +/// Inference launched through the local adapter keeps the appliance's pinned TLS +/// authority and never follows redirects carrying its private session credential. +pub(super) fn inference_transport( + config_dir: &Path, + host: &str, +) -> Result<(reqwest::Client, Url), ClientError> { + let profiles: Profiles = toml::from_str(&read_text(&config_dir.join("spark.toml"))?) + .map_err(|_| usage("invalid Spark host profiles"))?; + let profile = profiles + .hosts + .get(host) + .ok_or_else(|| usage("Spark host is not configured"))?; + let base = Url::parse(&profile.url).map_err(|_| usage("invalid Spark URL"))?; + reject_plaintext_lan(&base)?; + let ca = fs::read(config_dir.join("spark").join(format!("{host}.ca.pem"))) + .map_err(|_| unreachable("pinned Spark CA certificate is unavailable"))?; + if format!("sha256:{:x}", Sha256::digest(&ca)) != profile.ca_cert_sha256 { + return Err(unreachable("pinned Spark CA fingerprint does not match")); + } + let certificate = reqwest::Certificate::from_pem(&ca) + .map_err(|_| unreachable("pinned Spark CA certificate is invalid"))?; + let http = reqwest::Client::builder() + .tls_built_in_root_certs(false) + .add_root_certificate(certificate) + .https_only(base.scheme() == "https") + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(1800)) + .build() + .map_err(|_| unreachable("could not construct pinned inference transport"))?; + Ok((http, base)) +} + pub fn codex_client_config( config_dir: &Path, host: &str, @@ -230,6 +263,54 @@ impl Drop for SparkClient { } impl SparkClient { + pub fn create_launch_session( + &self, + request: &super::wire::LaunchSessionRequest, + ) -> Result, ClientError> { + let url = self + .base + .join("api/sparkplane/v1/launch-sessions") + .map_err(|_| usage("invalid session route"))?; + let response = self + .http + .post(url) + .bearer_auth(&self.token) + .timeout(Duration::from_secs(5)) + .json(request) + .send() + .map_err(|_| unreachable("session accounting unavailable"))?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + if !response.status().is_success() { + return Err(map_problem(response.status(), response.json().ok())); + } + decode_response(response).map(Some) + } + + pub fn finish_launch_session( + &self, + id: &str, + ) -> Result { + if id.parse::().is_err() { + return Err(usage("invalid session id")); + } + let url = self + .base + .join(&format!("api/sparkplane/v1/launch-sessions/{id}/finish")) + .map_err(|_| usage("invalid session route"))?; + let response = self + .http + .post(url) + .bearer_auth(&self.token) + .timeout(Duration::from_secs(5)) + .send() + .map_err(|_| unreachable("session finalization unavailable"))?; + if !response.status().is_success() { + return Err(map_problem(response.status(), response.json().ok())); + } + decode_response(response) + } pub fn load(config_dir: &Path, host: &str) -> Result { let profiles: Profiles = toml::from_str(&read_text(&config_dir.join("spark.toml"))?) .map_err(|error| usage(format!("invalid Spark host profiles: {error}")))?; @@ -658,7 +739,7 @@ fn valid_model_id(id: &str) -> bool { .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) } -fn validate_instance_reference(value: &str) -> Result<(), ClientError> { +pub(super) fn validate_instance_reference(value: &str) -> Result<(), ClientError> { if !value.is_empty() && value.len() <= 96 && value diff --git a/src/spark/eco.rs b/src/spark/eco.rs new file mode 100644 index 0000000..8c0d06d --- /dev/null +++ b/src/spark/eco.rs @@ -0,0 +1,568 @@ +//! Invocation-local agent hooks backed by the embedded RTK library. +use super::{ + cli::LaunchIntegration, + economics::{self, Compression}, +}; +use anyhow::{Result, ensure}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::{ + ffi::OsString, + fs, + io::{self, Read, Write}, + path::{Path, PathBuf}, + process::{Command, ExitStatus, Stdio}, +}; +use tokio::io::AsyncReadExt; + +const MAX_CAPTURE: usize = 8 * 1024 * 1024; + +#[derive(Serialize, Deserialize)] +struct Metric { + category: String, + raw_bytes: u64, + filtered_bytes: u64, + error: bool, +} + +pub struct EcoSession { + path: PathBuf, + enabled: bool, + policy_enabled: bool, +} +impl EcoSession { + pub fn create(id: &str, enabled: bool, mode: &str) -> Result { + ensure!(id.parse::().is_ok(), "invalid session id"); + let path = std::env::temp_dir().join(format!("sparkplane-eco-{id}")); + ensure!(!path.exists(), "eco session directory already exists"); + economics::private_dir(&path)?; + Ok(Self { + path, + enabled, + policy_enabled: mode == "auto", + }) + } + pub fn configure(&self, command: &mut Command, integration: LaunchIntegration) -> Result<()> { + command.env("SPARKPLANE_ECO_SESSION_DIR", &self.path).env( + "SPARKPLANE_ECO_REWRITE_ENABLED", + if self.enabled && self.policy_enabled { + "1" + } else { + "0" + }, + ); + if !self.enabled || !self.policy_enabled { + return Ok(()); + } + let binary = std::env::current_exe()?; + let hook = format!( + "{} __eco hook {}", + shell_quote(&binary.to_string_lossy()), + integration.as_str() + ); + let hooks = json!({"PreToolUse":[{"matcher":"Bash","hooks":[{"type":"command","command":hook,"timeout":3}]}],"SessionStart":[{"hooks":[{"type":"command","command":hook,"timeout":3}]}]}); + match integration { + LaunchIntegration::Codex => { + for event in ["PreToolUse", "SessionStart"] { + let table: toml::Value = serde_json::from_value(hooks[event].clone())?; + command.arg("-c").arg(format!("hooks.{event}={}", table)); + } + } + LaunchIntegration::Claude => { + // Auto mode owns invocation settings. Carry its permission settings into the + // combined object; user settings files remain loaded by Claude itself. + let args = command + .get_args() + .map(|v| v.to_string_lossy().into_owned()) + .collect::>(); + let mut settings = args + .windows(2) + .rfind(|pair| pair[0] == "--settings") + .and_then(|pair| serde_json::from_str::(&pair[1]).ok()) + .unwrap_or_else(|| json!({})); + settings["hooks"] = hooks; + let retained = command.get_args().map(OsString::from).collect::>(); + // Command has no remove-argument API. Rebuild with the same program, + // environment and inherited stdio, replacing only owned settings. + let mut replacement = Command::new(command.get_program()); + for (key, value) in command.get_envs() { + if let Some(value) = value { + replacement.env(key, value); + } else { + replacement.env_remove(key); + } + } + if let Some(dir) = command.get_current_dir() { + replacement.current_dir(dir); + } + let mut index = 0; + while index < retained.len() { + if retained[index] == "--settings" { + index += 2; + } else { + replacement.arg(&retained[index]); + index += 1; + } + } + replacement + .arg("--settings") + .arg(serde_json::to_string(&settings)?); + *command = replacement; + } + LaunchIntegration::Opencode => { + let plugin = self.path.join("eco.ts"); + fs::write(&plugin, opencode_plugin(&binary))?; + let content = command + .get_envs() + .find(|(k, _)| *k == "OPENCODE_CONFIG_CONTENT") + .and_then(|(_, v)| v) + .ok_or_else(|| anyhow::anyhow!("missing OpenCode session config"))?; + let mut settings: Value = serde_json::from_str(&content.to_string_lossy())?; + let plugins = settings + .as_object_mut() + .ok_or_else(|| anyhow::anyhow!("invalid OpenCode config"))? + .entry("plugin") + .or_insert_with(|| json!([])); + plugins + .as_array_mut() + .ok_or_else(|| anyhow::anyhow!("invalid OpenCode plugins"))? + .push(json!(format!("file://{}", plugin.display()))); + command.env("OPENCODE_CONFIG_CONTENT", serde_json::to_string(&settings)?); + } + } + Ok(()) + } + pub fn summary(&self) -> Compression { + let mut result = Compression { + status: if !self.enabled { + "disabled" + } else if !self.policy_enabled { + "inactive (inherited permissions)" + } else { + "inactive (hook unavailable or awaiting trust)" + } + .into(), + ..Compression::default() + }; + if !self.enabled || !self.policy_enabled { + return result; + } + let Ok(entries) = fs::read_dir(&self.path) else { + result.metrics_errors += 1; + return result; + }; + for entry in entries { + let Ok(entry) = entry else { + result.metrics_errors += 1; + continue; + }; + let path = entry.path(); + if path.extension().is_none_or(|v| v != "json") { + continue; + } + match economics::read_record::(&path) { + Ok(metric) if metric.category == "hook" => result.status = "active".into(), + Ok(metric) => { + result.commands += 1; + result.status = "active".into(); + result.raw_bytes = result.raw_bytes.saturating_add(metric.raw_bytes); + result.filtered_bytes = + result.filtered_bytes.saturating_add(metric.filtered_bytes); + if metric.error { + result.metrics_errors += 1; + } + } + Err(_) => result.metrics_errors += 1, + } + } + result + } +} +impl Drop for EcoSession { + fn drop(&mut self) { + if fs::symlink_metadata(&self.path).is_ok() && economics::private_dir(&self.path).is_ok() { + let _ = fs::remove_dir_all(&self.path); + } + } +} + +fn opencode_plugin(binary: &Path) -> String { + format!( + r#"export const SparkplaneEco = async () => {{ + const binary = {}; + const activate = Bun.spawn([binary, "__eco", "activate"], {{stdout:"ignore",stderr:"ignore"}}); + await activate.exited; + return {{"tool.execute.before": async (input, output) => {{ + if (!["bash", "shell"].includes(String(input.tool).toLowerCase())) return; + if (typeof output.args?.command !== "string") return; + const child = Bun.spawn([binary, "__eco", "rewrite", output.args.command], {{stdout:"pipe",stderr:"ignore"}}); + const command = (await new Response(child.stdout).text()).trim(); + if (await child.exited === 0 && command) output.args.command = command; + }}}}; +}}; +"#, + serde_json::to_string(&binary.to_string_lossy()).expect("path JSON") + ) +} + +pub fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) +} + +pub fn rewrite(command: &str) -> Result> { + if command.len() > 64 * 1024 + || command.chars().any(|c| { + matches!( + c, + '\n' | '\r' + | '\0' + | '$' + | '`' + | '|' + | '&' + | ';' + | '<' + | '>' + | '*' + | '?' + | '(' + | ')' + ) + }) + { + return Ok(None); + } + let Some(args) = shlex::split(command) else { + return Ok(None); + }; + if sparkplane_rtk::select(&args).is_none() { + return Ok(None); + } + let binary = std::env::current_exe()?; + Ok(Some(format!( + "{} __eco exec -- {}", + shell_quote(&binary.to_string_lossy()), + args.iter() + .map(|a| shell_quote(a)) + .collect::>() + .join(" ") + ))) +} + +fn metric(metric: &Metric) { + metric_named(&uuid::Uuid::new_v4().to_string(), metric); +} + +fn metric_named(id: &str, metric: &Metric) { + let Some(path) = std::env::var_os("SPARKPLANE_ECO_SESSION_DIR").map(PathBuf::from) else { + return; + }; + if economics::private_dir(&path).is_err() { + eprintln!("Sparkplane eco metrics unavailable"); + return; + } + if economics::write_record(&path.join(format!("{id}.json")), metric).is_err() { + eprintln!("Sparkplane eco metrics unavailable"); + } +} + +fn activate() { + metric(&Metric { + category: "hook".into(), + raw_bytes: 0, + filtered_bytes: 0, + error: false, + }); +} + +pub fn run(args: Vec) -> Result { + let Some(action) = args.first().and_then(|a| a.to_str()) else { + anyhow::bail!("missing eco action"); + }; + match action { + "activate" => { + activate(); + Ok(0) + } + "rewrite" => { + ensure!(args.len() == 2, "rewrite expects one command"); + if std::env::var("SPARKPLANE_ECO_REWRITE_ENABLED") + .ok() + .as_deref() + != Some("0") + && let Some(command) = rewrite(&args[1].to_string_lossy())? + { + println!("{command}"); + } + Ok(0) + } + "hook" => { + let mut input = String::new(); + io::stdin() + .take(1024 * 1024 + 1) + .read_to_string(&mut input)?; + if input.len() > 1024 * 1024 { + return Ok(0); + } + let Ok(payload) = serde_json::from_str::(&input) else { + return Ok(0); + }; + if std::env::var("SPARKPLANE_ECO_REWRITE_ENABLED") + .ok() + .as_deref() + != Some("1") + { + return Ok(0); + } + activate(); + if payload.get("hook_event_name").and_then(Value::as_str) == Some("SessionStart") { + return Ok(0); + } + if payload.get("tool_name").and_then(Value::as_str) != Some("Bash") { + return Ok(0); + } + let Some(command) = payload + .pointer("/tool_input/command") + .and_then(Value::as_str) + else { + return Ok(0); + }; + if let Some(rewritten) = rewrite(command)? { + println!( + "{}", + json!({"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"allow","updatedInput":{"command":rewritten}}}) + ); + } + Ok(0) + } + "exec" | "raw" => { + let operands = &args[1..]; + let operands = operands + .strip_prefix(&[OsString::from("--")]) + .unwrap_or(operands); + ensure!(!operands.is_empty(), "missing command"); + let mut command = Command::new(&operands[0]); + command.args(&operands[1..]); + if action == "raw" { + return Ok(exit_code(wait(command)?)); + } + execute(command, operands) + } + _ => anyhow::bail!("unknown eco action"), + } +} + +pub fn exit_code(status: ExitStatus) -> i32 { + use std::os::unix::process::ExitStatusExt; + status + .code() + .unwrap_or_else(|| 128 + status.signal().unwrap_or(1)) +} + +/// Keep the parent alive long enough to finish accounting when its child exits. +pub fn wait(command: Command) -> Result { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(async { + let mut child = tokio::process::Command::from(command).spawn()?; + wait_child(&mut child, false).await + }) +} + +async fn wait_child(child: &mut tokio::process::Child, group: bool) -> Result { + use tokio::signal::unix::{SignalKind, signal}; + let mut interrupt = signal(SignalKind::interrupt())?; + let mut terminate = signal(SignalKind::terminate())?; + loop { + let signum = tokio::select! {status=child.wait()=>return Ok(status?),_=interrupt.recv()=>nix::sys::signal::Signal::SIGINT,_=terminate.recv()=>nix::sys::signal::Signal::SIGTERM}; + if let Some(pid) = child.id() { + let pid = pid as i32; + let _ = nix::sys::signal::kill( + nix::unistd::Pid::from_raw(if group { -pid } else { pid }), + signum, + ); + } + } +} + +async fn capture( + mut reader: impl tokio::io::AsyncRead + Unpin, + stderr: bool, +) -> io::Result<(Vec, u64, bool)> { + let mut bytes = Vec::new(); + let mut count = 0u64; + let mut streamed = false; + let mut chunk = [0u8; 8192]; + loop { + let n = reader.read(&mut chunk).await?; + if n == 0 { + break; + } + count = count.saturating_add(n as u64); + if !streamed && bytes.len() + n <= MAX_CAPTURE { + bytes.extend_from_slice(&chunk[..n]); + } else { + if stderr { + let mut out = io::stderr().lock(); + if !streamed { + out.write_all(&bytes)?; + } + out.write_all(&chunk[..n])?; + } else { + let mut out = io::stdout().lock(); + if !streamed { + out.write_all(&bytes)?; + } + out.write_all(&chunk[..n])?; + } + bytes.clear(); + streamed = true; + } + } + Ok((bytes, count, streamed)) +} + +fn execute(mut command: Command, args: &[OsString]) -> Result { + use std::os::unix::process::CommandExt; + let args = args + .iter() + .map(|v| v.to_string_lossy().into_owned()) + .collect::>(); + let filter = sparkplane_rtk::select(&args); + let metric_id = uuid::Uuid::new_v4().to_string(); + // An interrupted wrapper or failed final write leaves an incomplete record. + metric_named( + &metric_id, + &Metric { + category: "command".into(), + raw_bytes: 0, + filtered_bytes: 0, + error: true, + }, + ); + command + .stdin(Stdio::inherit()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(async { + let mut child = match tokio::process::Command::from(command).spawn() { + Ok(child) => child, + Err(error) => { + eprintln!("could not execute command: {error}"); + return Ok(if error.kind() == io::ErrorKind::NotFound { + 127 + } else { + 126 + }); + } + }; + let stdout = child.stdout.take().expect("stdout pipe"); + let stderr = child.stderr.take().expect("stderr pipe"); + let (status, out, err) = tokio::join!( + wait_child(&mut child, true), + capture(stdout, false), + capture(stderr, true) + ); + let status = exit_code(status?); + let (out, out_count, out_streamed) = out?; + let (err, err_count, err_streamed) = err?; + let filtered_out = filter + .as_ref() + .map(|f| sparkplane_rtk::filter(f, &out, status)) + .unwrap_or_else(|| out.clone()); + let filtered_err = match &filter { + Some(sparkplane_rtk::Filter::Cargo(label)) => sparkplane_rtk::filter( + &sparkplane_rtk::Filter::Cargo(if *label == "test" { "build" } else { label }), + &err, + status, + ), + _ => err.clone(), + }; + if !out_streamed { + io::stdout().write_all(&filtered_out)?; + } + if !err_streamed { + io::stderr().write_all(&filtered_err)?; + } + let filtered_count = if out_streamed { + out_count + } else { + filtered_out.len() as u64 + } + if err_streamed { + err_count + } else { + filtered_err.len() as u64 + }; + metric_named( + &metric_id, + &Metric { + category: "command".into(), + raw_bytes: out_count.saturating_add(err_count), + filtered_bytes: filtered_count, + error: false, + }, + ); + Ok(status) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hooks_preserve_permissions_and_stay_invocation_local() { + let session = EcoSession::create(&ulid::Ulid::new().to_string(), true, "auto").unwrap(); + let mut claude = Command::new("claude"); + claude + .args([ + "--permission-mode", + "bypassPermissions", + "--settings", + r#"{"sandbox":{"enabled":false},"skipDangerousModePermissionPrompt":true}"#, + ]) + .env("FIXTURE", "kept"); + session + .configure(&mut claude, LaunchIntegration::Claude) + .unwrap(); + let args = claude + .get_args() + .map(|s| s.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + args.iter().filter(|s| s.as_str() == "--settings").count(), + 1 + ); + let settings: Value = serde_json::from_str(&args[3]).unwrap(); + assert_eq!(settings["sandbox"]["enabled"], false); + assert_eq!(settings["skipDangerousModePermissionPrompt"], true); + assert!(settings["hooks"]["PreToolUse"].is_array()); + assert!(claude.get_envs().any(|(key,value)|key=="FIXTURE"&&value==Some(std::ffi::OsStr::new("kept")))); + let mut codex = Command::new("codex"); + session + .configure(&mut codex, LaunchIntegration::Codex) + .unwrap(); + let args = codex + .get_args() + .map(|s| s.to_string_lossy().into_owned()) + .collect::>(); + for arg in args.iter().skip(1).step_by(2) { + let _: toml::Value = toml::from_str(arg).unwrap(); + } + assert!(!args.iter().any(|s| s.contains("bypass-hook-trust"))); + let inherited = + EcoSession::create(&ulid::Ulid::new().to_string(), true, "inherit").unwrap(); + let mut native = Command::new("codex"); + native.args(["-c", "approval_policy=\"on-request\""]); + inherited + .configure(&mut native, LaunchIntegration::Codex) + .unwrap(); + assert_eq!(native.get_args().count(), 2); + assert!(inherited.summary().status.contains("inherited")); + } +} diff --git a/src/spark/economics.rs b/src/spark/economics.rs new file mode 100644 index 0000000..f2cb813 --- /dev/null +++ b/src/spark/economics.rs @@ -0,0 +1,507 @@ +//! Reproducible public-price comparisons and private aggregate report storage. +use super::wire::LaunchSessionDocument; +use anyhow::{Result, ensure}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + fs, + io::{self, Read, Write}, + os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt}, + path::{Path, PathBuf}, +}; + +pub const SCHEMA: &str = "sparkplane.session-economics/v1"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Price { + pub model: String, + pub source: String, + pub verified_at: String, + pub effective_from: String, + pub input_microusd_per_million: u64, + pub output_microusd_per_million: u64, + pub long_context_threshold: Option, + pub long_input_microusd_per_million: Option, + pub long_output_microusd_per_million: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PriceCatalog { + pub version: String, + pub currency: String, + pub tariff: String, + pub models: Vec, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct Compression { + pub status: String, + pub commands: u64, + pub raw_bytes: u64, + pub filtered_bytes: u64, + pub metrics_errors: u64, +} +impl Compression { + pub fn saved_tokens(&self) -> u64 { + self.raw_bytes + .div_ceil(4) + .saturating_sub(self.filtered_bytes.div_ceil(4)) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Comparison { + pub model: String, + pub inference_usd_nanos: Option, + pub estimated_rtk_usd_nanos: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Report { + pub schema: String, + pub session_id: String, + pub host: String, + pub model: String, + pub integration: String, + pub eco_mode: String, + pub started_at: String, + pub finished_at: Option, + pub exit_code: Option, + pub inference: Option, + pub compression: Compression, + pub catalog: PriceCatalog, + pub comparisons: Vec, +} +impl Report { + pub fn calculate(&mut self) { + self.comparisons = self + .catalog + .models + .iter() + .map(|price| { + let inference = self.inference.as_ref().and_then(|session| { + let usage = &session.usage; + if (session.finished_at.is_none() && usage.requests == 0) + || (usage.requests > 0 && usage.unknown_usage_requests == usage.requests) + { + return None; + } + let short_input = usage + .input_tokens + .checked_sub(usage.long_context_input_tokens)?; + let short_output = usage + .output_tokens + .checked_sub(usage.long_context_output_tokens)?; + let long_in = price + .long_input_microusd_per_million + .unwrap_or(price.input_microusd_per_million); + let long_out = price + .long_output_microusd_per_million + .unwrap_or(price.output_microusd_per_million); + let value = u128::from(short_input) + * u128::from(price.input_microusd_per_million) + + u128::from(short_output) * u128::from(price.output_microusd_per_million) + + u128::from(usage.long_context_input_tokens) * u128::from(long_in) + + u128::from(usage.long_context_output_tokens) * u128::from(long_out); + u64::try_from(value / 1000).ok() + }); + let rtk = (self.compression.metrics_errors == 0 + && self.compression.status == "active") + .then(|| { + u64::try_from( + u128::from(self.compression.saved_tokens()) + * u128::from(price.input_microusd_per_million) + / 1000, + ) + .ok() + }) + .flatten(); + Comparison { + model: price.model.clone(), + inference_usd_nanos: inference, + estimated_rtk_usd_nanos: rtk, + } + }) + .collect(); + } +} + +pub fn catalog() -> Result { + Ok(serde_json::from_str(include_str!( + "../../data/economics-prices.json" + ))?) +} + +pub(crate) fn private_dir(path: &Path) -> Result<()> { + if !path.exists() { + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(path)?; + } + let meta = fs::symlink_metadata(path)?; + ensure!( + meta.is_dir() + && meta.uid() == rustix::process::geteuid().as_raw() + && meta.mode() & 0o077 == 0, + "economics directory has unsafe ownership or permissions" + ); + Ok(()) +} + +pub(crate) fn write_record(path: &Path, value: &impl Serialize) -> Result<()> { + let parent = path + .parent() + .ok_or_else(|| anyhow::anyhow!("record has no parent"))?; + private_dir(parent)?; + let bytes = serde_json::to_vec_pretty(value)?; + let temporary = parent.join(format!(".staging-{}", uuid::Uuid::new_v4())); + let mut file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(&temporary)?; + file.write_all(&bytes)?; + file.sync_all()?; + fs::rename(&temporary, path)?; + fs::File::open(parent)?.sync_all()?; + Ok(()) +} + +pub(crate) fn read_record(path: &Path) -> Result { + let file = fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path)?; + let meta = file.metadata()?; + ensure!( + meta.is_file() + && meta.uid() == rustix::process::geteuid().as_raw() + && meta.mode() & 0o077 == 0 + && meta.len() <= 1024 * 1024, + "invalid economics record" + ); + let mut bytes = Vec::new(); + file.take(1024 * 1024 + 1).read_to_end(&mut bytes)?; + ensure!(bytes.len() <= 1024 * 1024, "economics record too large"); + Ok(serde_json::from_slice(&bytes)?) +} + +fn directory(config: &Path, host: &str) -> PathBuf { + config + .join("economics") + .join(format!("{:x}", Sha256::digest(host.as_bytes()))) +} + +pub fn save(config: &Path, report: &Report) -> Result<()> { + ensure!( + report.session_id.parse::().is_ok(), + "invalid session id" + ); + let dir = directory(config, &report.host); + private_dir(&dir)?; + write_record(&dir.join(format!("{}.json", report.session_id)), report)?; + prune(&dir) +} + +fn prune(dir: &Path) -> Result<()> { + let cutoff = chrono::Utc::now() - chrono::Duration::days(90); + for entry in fs::read_dir(dir)? { + let path = entry?.path(); + if path.extension().is_some_and(|s| s == "json") + && let Ok(old) = read_record::(&path) + && chrono::DateTime::parse_from_rfc3339(&old.started_at).is_ok_and(|t| t < cutoff) + { + fs::remove_file(path)?; + } + } + Ok(()) +} + +pub fn load(config: &Path, host: &str, id: Option<&str>) -> Result { + let dir = directory(config, host); + // Enforce retention even when no further launch writes a report. + ensure!( + fs::symlink_metadata(&dir)?.is_dir(), + "invalid economics directory" + ); + private_dir(&dir)?; + prune(&dir)?; + let path = if let Some(id) = id { + ensure!(id.parse::().is_ok(), "invalid session id"); + dir.join(format!("{id}.json")) + } else { + let mut paths = fs::read_dir(&dir)? + .filter_map(Result::ok) + .map(|e| e.path()) + .filter(|p| { + p.extension().is_some_and(|s| s == "json") + && p.file_stem() + .and_then(|s| s.to_str()) + .is_some_and(|s| s.parse::().is_ok()) + }) + .collect::>(); + paths.sort(); + paths + .pop() + .ok_or_else(|| anyhow::anyhow!("no retained economics reports"))? + }; + let report: Report = read_record(&path)?; + ensure!( + report.schema == SCHEMA && report.host == host, + "economics report identity mismatch" + ); + Ok(report) +} + +fn money(value: Option) -> String { + match value { + None => "unavailable".into(), + Some(0) => "$0.00".into(), + Some(v) if v < 10_000_000 => "<$0.01".into(), + Some(v) => { + let cents = (u128::from(v) + 5_000_000) / 10_000_000; + format!("${}.{:02}", cents / 100, cents % 100) + } + } +} + +pub fn render(report: &Report, fancy: bool, width: usize) -> String { + let width = width.clamp(40, 100); + let title = if fancy { + "\x1b[1;32mSession economics\x1b[0m" + } else { + "Session economics" + }; + let mut text = format!("\n{}\n{title}\n", "─".repeat(width)); + text.push_str(&format!( + "{} · {} · exit {}\n", + report.integration, + report.model, + report + .exit_code + .map(|v| v.to_string()) + .unwrap_or_else(|| "interrupted".into()) + )); + if let (Ok(start), Some(end)) = ( + chrono::DateTime::parse_from_rfc3339(&report.started_at), + report + .finished_at + .as_ref() + .and_then(|t| chrono::DateTime::parse_from_rfc3339(t).ok()), + ) { + let seconds = (end - start).num_seconds().max(0); + text.push_str(&format!("Duration: {}m {}s\n", seconds / 60, seconds % 60)); + } + if let Some(session) = &report.inference { + let usage = &session.usage; + let incomplete = usage.unknown_usage_requests > 0 + || usage.pending_requests > 0 + || usage.accounting_errors > 0 + || session.finished_at.is_none(); + text.push_str(&format!( + "Tokens burned{}: {} input + {} output = {} total · {} requests\n", + if incomplete { " (known)" } else { "" }, + usage.input_tokens, + usage.output_tokens, + usage.input_tokens.saturating_add(usage.output_tokens), + usage.requests + )); + if usage.unknown_usage_requests > 0 + || usage.pending_requests > 0 + || usage.accounting_errors > 0 + || session.finished_at.is_none() + { + text.push_str("Inference accounting incomplete; known usage is a lower bound.\n"); + } + } else { + text.push_str("Inference usage unavailable.\n"); + } + let rtk = &report.compression; + text.push_str(&format!( + "RTK: {} · {} commands\n", + rtk.status, rtk.commands + )); + if rtk.metrics_errors > 0 { + text.push_str("RTK metrics incomplete; dollar savings unavailable.\n"); + } + if rtk.commands > 0 { + let pct = if rtk.raw_bytes > 0 { + 100.0 * (1.0 - rtk.filtered_bytes as f64 / rtk.raw_bytes as f64) + } else { + 0.0 + }; + text.push_str(&format!( + "Tool output: ~{} → ~{} tokens · {:.1}% smaller · ~{} tokens saved\n", + rtk.raw_bytes.div_ceil(4), + rtk.filtered_bytes.div_ceil(4), + pct, + rtk.saved_tokens() + )); + if fancy { + let kept = rtk + .filtered_bytes + .saturating_mul(20) + .checked_div(rtk.raw_bytes) + .unwrap_or(0) + .min(20) as usize; + text.push_str(&format!( + "[{}{}]\n", + "█".repeat(kept), + "░".repeat(20 - kept) + )); + } + } + text.push_str("\nCloud equivalent / estimated RTK savings (USD):\n"); + for row in &report.comparisons { + text.push_str(&format!( + " {:<20} {:>11} / {:>11}\n", + row.model, + money(row.inference_usd_nanos), + money(row.estimated_rtk_usd_nanos) + )); + } + text.push_str(&format!( + "Public standard uncached rates · verified {}\n", + report + .catalog + .models + .first() + .map(|p| p.verified_at.as_str()) + .unwrap_or("unknown") + )); + text.push_str("Estimates use local token counts; RTK is valued once at input rates.\nCloud comparisons exclude electricity/hardware and are not added together.\n"); + text +} + +pub fn display(report: &Report) { + use std::io::IsTerminal; + eprint!( + "{}", + render( + report, + io::stderr().is_terminal() && std::env::var_os("NO_COLOR").is_none(), + std::env::var("COLUMNS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(80) + ) + ); +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn money_rounds_and_tiny_values_are_visible() { + assert_eq!(money(Some(1)), "<$0.01"); + assert_eq!(money(None), "unavailable"); + assert_eq!(money(Some(999_000_000)), "$1.00"); + } + #[test] + fn catalog_has_official_sources_and_known_context_tier() { + let prices = catalog().unwrap(); + assert_eq!(prices.models.len(), 3); + assert_eq!(prices.models[0].long_context_threshold, Some(272_000)); + } + + fn report() -> Report { + let now = chrono::Utc::now().to_rfc3339(); + Report { + schema: SCHEMA.into(), + session_id: ulid::Ulid::new().to_string(), + host: "fixture".into(), + model: "local-model".into(), + integration: "codex".into(), + eco_mode: "max".into(), + started_at: now.clone(), + finished_at: Some(now.clone()), + exit_code: Some(0), + inference: Some(LaunchSessionDocument { + schema: "sparkplane.launch-session/v1".into(), + id: ulid::Ulid::new().to_string(), + instance: "fixture".into(), + model: "local-model".into(), + integration: "codex".into(), + eco_mode: "max".into(), + started_at: now.clone(), + finished_at: Some(now), + usage: super::super::wire::SessionUsage { + requests: 1, + input_tokens: 272001, + output_tokens: 4, + long_context_input_tokens: 272001, + long_context_output_tokens: 4, + ..Default::default() + }, + }), + compression: Compression { + status: "active".into(), + commands: 1, + raw_bytes: 400, + filtered_bytes: 100, + metrics_errors: 0, + }, + catalog: catalog().unwrap(), + comparisons: Vec::new(), + } + } + + #[test] + fn costs_use_each_requests_context_tier_and_keep_rtk_separate() { + let mut report = report(); + report.calculate(); + assert_eq!( + report.comparisons[0].inference_usd_nanos, + Some(1_088_064_000) + ); + assert_eq!(report.comparisons[0].estimated_rtk_usd_nanos, Some(150_000)); + assert_eq!(report.comparisons[1].inference_usd_nanos, Some(816_063_000)); + let usage = &mut report.inference.as_mut().unwrap().usage; + usage.input_tokens = 272000; + usage.long_context_input_tokens = 0; + usage.long_context_output_tokens = 0; + report.calculate(); + assert_eq!(report.comparisons[0].inference_usd_nanos, Some(544_040_000)); + report + .inference + .as_mut() + .unwrap() + .usage + .unknown_usage_requests = 1; + report.compression.metrics_errors = 1; + report.calculate(); + assert_eq!(report.comparisons[0].estimated_rtk_usd_nanos, None); + let text = render(&report, false, 40); + assert!(text.contains("lower bound")); + assert!(!text.contains('\x1b')); + } + + #[test] + fn reports_are_private_scoped_and_expire_on_read() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let root = tempfile::tempdir().unwrap(); + let mut report = report(); + report.calculate(); + save(root.path(), &report).unwrap(); + assert_eq!( + load(root.path(), "fixture", None).unwrap().session_id, + report.session_id + ); + assert!(load(root.path(), "other", Some(&report.session_id)).is_err()); + assert!(load(root.path(), "fixture", Some("../escape")).is_err()); + let path = directory(root.path(), "fixture").join(format!("{}.json", report.session_id)); + assert_eq!(fs::metadata(&path).unwrap().mode() & 0o777, 0o600); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + assert!(load(root.path(), "fixture", None).is_err()); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + let link = path.with_extension("link"); + symlink(&path, &link).unwrap(); + assert!(read_record::(&link).is_err()); + report.started_at = (chrono::Utc::now() - chrono::Duration::days(91)).to_rfc3339(); + write_record(&path, &report).unwrap(); + assert!(load(root.path(), "fixture", None).is_err()); + assert!(!path.exists()); + } +} diff --git a/src/spark/gateway.rs b/src/spark/gateway.rs index 5a8f027..1d9baed 100644 --- a/src/spark/gateway.rs +++ b/src/spark/gateway.rs @@ -17,7 +17,14 @@ use super::upstream::{ decode_embedding_response, }; -pub const MAX_COMPLETION_BODY_BYTES: usize = 1024 * 1024; +pub const MAX_COMPLETION_BODY_BYTES: usize = 8 * 1024 * 1024; + +#[path = "gateway_images.rs"] +mod images; +// Protocol harnesses also include this file as a private module. +#[allow(unused_imports)] +pub use images::{HARD_REQUEST_BYTES, ImageProtocol, IngressLimits, normalize_image_request}; + pub const RETRY_AFTER_SECONDS: &str = "1"; pub const MAX_OUTPUT_TOKENS: u64 = 32_768; const MAX_INSTANCE_CONCURRENCY: usize = 4; @@ -276,17 +283,20 @@ struct AnthropicImageSource { #[serde(untagged)] enum AnthropicToolResultContent { Text(String), - Blocks(Vec), + Blocks(Vec), } #[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct AnthropicToolResultText { - #[serde(rename = "type")] - kind: String, - text: String, - #[serde(default)] - cache_control: Option, +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +enum AnthropicToolResultBlock { + Text { + text: String, + #[serde(default)] + cache_control: Option, + }, + Image { + source: AnthropicImageSource, + }, } #[derive(Debug, Deserialize)] @@ -741,6 +751,7 @@ fn omit_empty_tool_controls(object: &mut serde_json::Map) { struct ImageBudget { count: usize, bytes: usize, + source_pixels: u64, } fn parse_data_uri(value: &str) -> Result<(&str, &str), OpenAiError> { @@ -1549,18 +1560,36 @@ pub fn rewrite_chat_request_with_profile( .get("messages") .and_then(Value::as_array) .ok_or_else(|| invalid_request("messages must be an array"))?; - if messages.iter().any(|message| { - message["content"].as_array().is_some_and(|parts| { - parts.iter().any(|part| { - matches!(part["type"].as_str(), Some("image_url" | "input_image")) - || part.get("image_url").is_some() - || part.get("file_id").is_some() - }) - }) - }) { - return Err(invalid_request( - "chat image inputs are unsupported; use the Responses route", - )); + let mut images = ImageBudget::default(); + for message in messages { + for part in message["content"].as_array().into_iter().flatten() { + if part["type"] == "image_url" { + if !matches!(message["role"].as_str(), Some("user" | "tool")) + || part.as_object().is_none_or(|o| { + o.keys() + .any(|key| !matches!(key.as_str(), "type" | "image_url")) + }) + || part["image_url"].as_object().is_none_or(|o| { + o.keys() + .any(|key| !matches!(key.as_str(), "url" | "detail")) + }) + { + return Err(invalid_request( + "chat images must be inline user or tool-result content", + )); + } + let url = part["image_url"]["url"] + .as_str() + .ok_or_else(|| invalid_request("image_url must be an inline data URI"))?; + let (media, data) = parse_data_uri(url)?; + decode_image(media, data, profile.vision.as_ref(), &mut images)?; + } else if part.get("image_url").is_some() + || part.get("file_id").is_some() + || part["type"] == "input_image" + { + return Err(invalid_request("chat image content type is unsupported")); + } + } } let stream = object .get("stream") @@ -1991,9 +2020,17 @@ fn anthropic_blocks( "tool_result does not match a prior tool_use", )); } - let content = anthropic_tool_result_text(content)?; + let content = anthropic_tool_result_content(content, vision, images)?; let content = if is_error { - format!("Tool error: {content}") + match content { + Value::String(text) => Value::String(format!("Tool error: {text}")), + Value::Array(mut parts) => { + parts + .insert(0, serde_json::json!({"type":"text","text":"Tool error:"})); + Value::Array(parts) + } + value => value, + } } else { content }; @@ -2044,22 +2081,45 @@ fn anthropic_blocks( Ok(()) } -fn anthropic_tool_result_text( +fn anthropic_tool_result_content( content: AnthropicToolResultContent, -) -> Result { + vision: Option<&VisionPolicy>, + images: &mut ImageBudget, +) -> Result { match content { - AnthropicToolResultContent::Text(text) => Ok(text), + AnthropicToolResultContent::Text(text) => Ok(text.into()), AnthropicToolResultContent::Blocks(blocks) => { - blocks - .into_iter() - .try_fold(String::new(), |mut text, block| { - validate_cache_control(block.cache_control.as_ref())?; - if block.kind != "text" { - return Err(anthropic_invalid("tool_result content type is unsupported")); + let mut content = Vec::new(); + let mut text = String::new(); + let mut multimodal = false; + for block in blocks { + match block { + AnthropicToolResultBlock::Text { + text: part, + cache_control, + } => { + validate_cache_control(cache_control.as_ref())?; + text.push_str(&part); + content.push(serde_json::json!({"type":"text","text":part})); } - text.push_str(&block.text); - Ok(text) - }) + AnthropicToolResultBlock::Image { source } => { + if source.kind != "base64" { + return Err(anthropic_invalid( + "image source must be inline base64 tool-result content", + )); + } + let image = decode_image(&source.media_type, &source.data, vision, images) + .map_err(|error| anthropic_invalid(error.message))?; + content.push(canonical_image(&image)); + multimodal = true; + } + } + } + Ok(if multimodal { + Value::Array(content) + } else { + text.into() + }) } } } @@ -2537,7 +2597,9 @@ fn response_input_item( Some("message") => response_message(item, vision, images), Some("function_call") => response_tool_call(item, false), Some("custom_tool_call") => response_tool_call(item, true), - Some("function_call_output" | "custom_tool_call_output") => response_tool_output(item), + Some("function_call_output" | "custom_tool_call_output") => { + response_tool_output(item, vision, images) + } Some("reasoning") => response_reasoning(item), Some("computer_call" | "computer_call_output") => Err(OpenAiError { code: "unsupported_tool", @@ -2602,10 +2664,10 @@ fn response_content( "text":part["text"].as_str().ok_or_else(|| invalid_request("text content is invalid"))? })), Some("input_image") => { - if role != "user" || part.as_object().is_none_or(|object| { + if !matches!(role, "user" | "tool") || part.as_object().is_none_or(|object| { object.keys().any(|key| !matches!(key.as_str(), "type" | "image_url" | "detail")) }) || part.get("file_id").is_some() { - return Err(invalid_request("image input must be bounded inline user content")); + return Err(invalid_request("image input must be bounded inline user or tool-result content")); } let url = part["image_url"] .as_str() @@ -2645,17 +2707,36 @@ fn response_tool_call(item: &Value, custom: bool) -> Result ) } -fn response_tool_output(item: &Value) -> Result { +fn response_tool_output( + item: &Value, + vision: Option<&VisionPolicy>, + images: &mut ImageBudget, +) -> Result { let call_id = item["call_id"] .as_str() .ok_or_else(|| invalid_request("tool output call_id is required"))?; let output = item .get("output") .ok_or_else(|| invalid_request("tool output is required"))?; - let content = output - .as_str() - .map(str::to_owned) - .unwrap_or_else(|| output.to_string()); + // Codex view_image returns typed output blocks. Serializing their base64 as + // text both loses the image and can consume the entire model context. + let content = if output.as_array().is_some_and(|parts| { + parts.iter().any(|part| { + matches!( + part["type"].as_str(), + Some("input_text" | "output_text" | "input_image" | "input_file") + ) + }) + }) { + response_content(output, "tool", vision, images)? + } else { + Value::String( + output + .as_str() + .map(str::to_owned) + .unwrap_or_else(|| output.to_string()), + ) + }; Ok(serde_json::json!({"role": "tool", "tool_call_id": call_id, "content": content})) } diff --git a/src/spark/gateway_images.rs b/src/spark/gateway_images.rs new file mode 100644 index 0000000..3a51bd7 --- /dev/null +++ b/src/spark/gateway_images.rs @@ -0,0 +1,394 @@ +//! Public image uploads have separate limits from qualified engine inputs. +use std::io::{self, Cursor, Write}; + +use base64::Engine as _; +use image::{ + DynamicImage, ImageDecoder, ImageEncoder, ImageFormat, ImageReader, imageops::FilterType, +}; +use serde_json::Value; + +use super::{ + BASE64, GatewayProfile, ImageBudget, MAX_COMPLETION_BODY_BYTES, OpenAiError, VisionPolicy, + decode_image, invalid_request, parse_data_uri, +}; + +pub const HARD_REQUEST_BYTES: usize = 64 * 1024 * 1024; +const DECODE_MEMORY_BYTES: u64 = 128 * 1024 * 1024; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct IngressLimits { + pub request_body_bytes: usize, + pub image_bytes: usize, + pub image_pixels: u64, + pub image_dimension: u32, + pub max_parallel_requests: usize, +} + +impl Default for IngressLimits { + fn default() -> Self { + Self { + request_body_bytes: 32 * 1024 * 1024, + image_bytes: 16 * 1024 * 1024, + image_pixels: 32_000_000, + image_dimension: 16384, + max_parallel_requests: 2, + } + } +} + +impl IngressLimits { + pub fn validate(&self) -> Result<(), &'static str> { + if self.request_body_bytes == 0 + || self.request_body_bytes > HARD_REQUEST_BYTES + || self.image_bytes == 0 + || self.image_bytes > 32 * 1024 * 1024 + || self.image_pixels == 0 + || self.image_pixels > 32_000_000 + || self.image_dimension == 0 + || self.image_dimension > 16384 + || !(1..=4).contains(&self.max_parallel_requests) + { + return Err("gateway ingress limits exceed supported resource bounds"); + } + Ok(()) + } +} + +#[derive(Clone, Copy)] +pub enum ImageProtocol { + Responses, + Anthropic, + Chat, +} + +fn too_large(message: &'static str) -> OpenAiError { + OpenAiError { + code: "request_too_large", + message, + } +} + +pub fn normalize_image_request( + bytes: &[u8], + protocol: ImageProtocol, + profile: &GatewayProfile, + limits: &IngressLimits, +) -> Result, OpenAiError> { + limits.validate().map_err(invalid_request)?; + if bytes.len() > limits.request_body_bytes { + return Err(too_large("request exceeds the gateway upload byte limit")); + } + let mut request: Value = + serde_json::from_slice(bytes).map_err(|_| invalid_request("request JSON is invalid"))?; + let history_protocol = match protocol { + ImageProtocol::Responses => super::super::image_history::Protocol::Responses, + ImageProtocol::Anthropic => super::super::image_history::Protocol::Anthropic, + ImageProtocol::Chat => super::super::image_history::Protocol::Chat, + }; + if let Some(vision) = &profile.vision { + super::super::image_history::project( + &mut request, + history_protocol, + vision.max_count, + None, + ) + .map_err(|_| invalid_request("image count exceeds the model profile limit"))?; + } + let field = match protocol { + ImageProtocol::Responses => "input", + ImageProtocol::Anthropic | ImageProtocol::Chat => "messages", + }; + let mut image_count = 0; + super::super::image_history::visit_parts(&mut request[field], history_protocol, &mut |_| { + image_count += 1 + }); + let mut image_policy = profile.vision.clone(); + if image_count > 0 { + let policy = image_policy + .as_mut() + .ok_or_else(|| invalid_request("image input requires a vision profile"))?; + if image_count > policy.max_count { + return Err(invalid_request( + "image count exceeds the model profile limit", + )); + } + // Reserve an equal share for each frame, including later tool results. + // The strict rewriter still checks the original shared engine budget. + policy.max_bytes = policy.max_bytes.min(policy.max_total_bytes / image_count); + } + let mut budget = ImageBudget::default(); + if let Some(messages) = request.get_mut(field).and_then(Value::as_array_mut) { + for message in messages { + let user = message["role"] == "user"; + let tool = (matches!(protocol, ImageProtocol::Responses) + && matches!( + message["type"].as_str(), + Some("function_call_output" | "custom_tool_call_output") + )) + || (matches!(protocol, ImageProtocol::Chat) && message["role"] == "tool"); + let mut error = None; + super::super::image_history::visit_parts(message, history_protocol, &mut |part| { + if error.is_some() { + return; + } + if !user && !tool { + error = Some(invalid_request( + "image input must be inline user or tool-result content", + )); + return; + } + let prepared = (|| { + let (media, data) = match protocol { + ImageProtocol::Responses | ImageProtocol::Chat => { + let uri = if matches!(protocol, ImageProtocol::Responses) { + part["image_url"].as_str() + } else { + part["image_url"]["url"].as_str() + } + .ok_or_else(|| { + invalid_request("image_url must be an inline data URI") + })?; + parse_data_uri(uri)? + } + ImageProtocol::Anthropic => { + let source = &part["source"]; + if source["type"] != "base64" { + return Err(invalid_request( + "image source must be inline base64 user content", + )); + } + ( + source["media_type"].as_str().ok_or_else(|| { + invalid_request("image media type is required") + })?, + source["data"] + .as_str() + .ok_or_else(|| invalid_request("image base64 is required"))?, + ) + } + }; + prepare_image(media, data, image_policy.as_ref(), &mut budget, limits) + })(); + match prepared { + Ok(image) => match protocol { + ImageProtocol::Responses => { + part["image_url"] = format!( + "data:{};base64,{}", + image.media_type, + BASE64.encode(&image.bytes) + ) + .into() + } + ImageProtocol::Chat => { + part["image_url"]["url"] = format!( + "data:{};base64,{}", + image.media_type, + BASE64.encode(&image.bytes) + ) + .into() + } + ImageProtocol::Anthropic => { + part["source"]["media_type"] = image.media_type.into(); + part["source"]["data"] = BASE64.encode(&image.bytes).into(); + } + }, + Err(problem) => error = Some(problem), + } + }); + if let Some(error) = error { + return Err(error); + } + } + } + let bytes = + serde_json::to_vec(&request).map_err(|_| invalid_request("request cannot be encoded"))?; + if bytes.len() > MAX_COMPLETION_BODY_BYTES { + return Err(too_large( + "normalized request exceeds the 8 MiB inference body limit", + )); + } + Ok(bytes) +} + +fn prepare_image( + media: &str, + encoded: &str, + policy: Option<&VisionPolicy>, + budget: &mut ImageBudget, + limits: &IngressLimits, +) -> Result { + let policy = policy.ok_or_else(|| invalid_request("image input requires a vision profile"))?; + if budget.count >= policy.max_count { + return Err(invalid_request( + "image count exceeds the model profile limit", + )); + } + if !policy.media_types.contains(media) { + return Err(invalid_request( + "image format is unsupported by the model profile", + )); + } + if encoded.len() > limits.image_bytes.div_ceil(3) * 4 { + return Err(too_large("image exceeds the gateway upload byte limit")); + } + let bytes = BASE64 + .decode(encoded) + .map_err(|_| invalid_request("image base64 is invalid"))?; + if bytes.is_empty() { + return Err(invalid_request("image data is empty")); + } + if bytes.len() > limits.image_bytes { + return Err(too_large("image exceeds the gateway upload byte limit")); + } + let format = match media { + "image/png" if bytes.starts_with(b"\x89PNG\r\n\x1a\n") => ImageFormat::Png, + "image/jpeg" if bytes.starts_with(b"\xff\xd8\xff") => ImageFormat::Jpeg, + "image/webp" + if bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP".as_slice()) => + { + ImageFormat::WebP + } + _ => { + return Err(invalid_request( + "image media type does not match its magic bytes", + )); + } + }; + let mut decode_limits = image::Limits::default(); + decode_limits.max_image_width = Some(limits.image_dimension); + decode_limits.max_image_height = Some(limits.image_dimension); + decode_limits.max_alloc = Some(DECODE_MEMORY_BYTES); + let mut reader = ImageReader::with_format(Cursor::new(&bytes), format); + reader.limits(decode_limits); + let mut decoder = reader.into_decoder().map_err(|error| match error { + image::ImageError::Limits(_) => { + too_large("image exceeds the gateway dimension or decode memory limit") + } + _ => invalid_request("image header is invalid"), + })?; + let (width, height) = decoder.dimensions(); + let pixels = u64::from(width) * u64::from(height); + if width == 0 + || height == 0 + || pixels > limits.image_pixels.saturating_sub(budget.source_pixels) + || decoder.total_bytes() > DECODE_MEMORY_BYTES + { + return Err(too_large( + "images exceed the gateway total pixel or decode memory limit", + )); + } + budget.source_pixels += pixels; + let available = policy + .max_total_bytes + .saturating_sub(budget.bytes) + .min(policy.max_bytes); + if available == 0 || policy.max_width == 0 || policy.max_height == 0 { + return Err(invalid_request("image cannot fit the model profile limits")); + } + if bytes.len() <= available && width <= policy.max_width && height <= policy.max_height { + return decode_image(media, encoded, Some(policy), budget); + } + let orientation = decoder + .orientation() + .map_err(|_| invalid_request("image orientation metadata is invalid"))?; + let mut decoded = DynamicImage::from_decoder(decoder) + .map_err(|_| invalid_request("image pixels are invalid"))?; + decoded.apply_orientation(orientation); + let alpha = decoded.color().has_alpha(); + let mut image = decoded.resize( + policy.max_width.min(decoded.width()), + policy.max_height.min(decoded.height()), + FilterType::Lanczos3, + ); + drop(decoded); + image = if alpha { + DynamicImage::ImageRgba8(image.to_rgba8()) + } else { + DynamicImage::ImageRgb8(image.to_rgb8()) + }; + for _ in 0..16 { + let mut formats = Vec::new(); + if alpha || format == ImageFormat::Png { + formats.push(("image/png", ImageFormat::Png, 0)); + } + if !alpha { + formats.extend([ + ("image/jpeg", ImageFormat::Jpeg, 90), + ("image/jpeg", ImageFormat::Jpeg, 80), + ]); + } + formats.push(("image/webp", ImageFormat::WebP, 0)); + for (media, format, quality) in formats { + if !policy.media_types.contains(media) { + continue; + } + if let Some(bytes) = encode(&image, format, quality, available)? { + return decode_image(media, &BASE64.encode(&bytes), Some(policy), budget); + } + } + if image.width() == 1 && image.height() == 1 { + break; + } + let width = (image.width() * 3 / 4).max(1); + let height = (image.height() * 3 / 4).max(1); + image = image.resize(width, height, FilterType::Lanczos3); + } + Err(invalid_request("image cannot fit the model profile limits")) +} + +struct CappedWriter { + bytes: Vec, + limit: usize, + exceeded: bool, +} +impl Write for CappedWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if bytes.len() > self.limit.saturating_sub(self.bytes.len()) { + self.exceeded = true; + return Err(io::Error::other("image encoding byte limit")); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn encode( + image: &DynamicImage, + format: ImageFormat, + quality: u8, + limit: usize, +) -> Result>, OpenAiError> { + let mut output = CappedWriter { + bytes: Vec::new(), + limit, + exceeded: false, + }; + let result = match format { + ImageFormat::Png => image::codecs::png::PngEncoder::new(&mut output).write_image( + image.as_bytes(), + image.width(), + image.height(), + image.color().into(), + ), + ImageFormat::Jpeg => { + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut output, quality) + .encode_image(image) + } + ImageFormat::WebP => image::codecs::webp::WebPEncoder::new_lossless(&mut output).encode( + image.as_bytes(), + image.width(), + image.height(), + image.color().into(), + ), + _ => return Err(invalid_request("image output format is unsupported")), + }; + if output.exceeded { + return Ok(None); + } + result.map_err(|_| invalid_request("image cannot be encoded"))?; + Ok(Some(output.bytes)) +} diff --git a/src/spark/image_history.rs b/src/spark/image_history.rs new file mode 100644 index 0000000..2c071d3 --- /dev/null +++ b/src/spark/image_history.rs @@ -0,0 +1,435 @@ +//! Bounded model-visible image history, shared by the launch adapter and gateway. +//! Original tool calls, observations and instructions are never summarized here. +use std::{cell::Cell, collections::BTreeMap, io::Read, path::PathBuf, rc::Rc}; + +use serde::de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +pub const REQUEST_BYTES: usize = 32 * 1024 * 1024; +const TEXT_BYTES: usize = 8 * 1024 * 1024; +const RECENT: usize = 6; +const HIGH_WATER: usize = 12; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Protocol { + Responses, + Anthropic, + Chat, +} +impl Protocol { + fn field(self) -> &'static str { + if self == Self::Responses { + "input" + } else { + "messages" + } + } + fn image(self, part: &Value) -> bool { + part["type"] + == match self { + Self::Responses => "input_image", + Self::Anthropic => "image", + Self::Chat => "image_url", + } + } + fn text(self, text: String) -> Value { + json!({"type": if self == Self::Responses { "input_text" } else { "text" }, "text":text}) + } +} + +#[derive(Debug, Default, Clone, Copy)] +pub struct Statistics { + pub evicted: usize, + pub removed_bytes: usize, +} +pub type Archive<'a> = dyn FnMut(&str, &Value, Protocol) -> Option + 'a; + +pub fn visit_parts(value: &mut Value, protocol: Protocol, visit: &mut impl FnMut(&mut Value)) { + match value { + Value::Array(parts) => { + for part in parts { + visit_parts(part, protocol, visit); + } + } + Value::Object(_) if protocol.image(value) => visit(value), + Value::Object(object) => { + // Restrict traversal to protocol content; tool arguments and metadata are data. + for field in ["content", "output"] { + if let Some(parts) = object.get_mut(field) { + visit_parts(parts, protocol, visit); + } + } + } + _ => {} + } +} +fn acknowledged(item: &Value, protocol: Protocol) -> bool { + item["role"] == "assistant" + || (protocol == Protocol::Responses + && matches!( + item["type"].as_str(), + Some("function_call" | "custom_tool_call") + )) +} +fn image_payload(part: &Value, protocol: Protocol) -> Option<&str> { + match protocol { + Protocol::Responses => part["image_url"].as_str(), + Protocol::Anthropic => part["source"]["data"].as_str(), + Protocol::Chat => part["image_url"]["url"].as_str(), + } +} +pub fn image_id(part: &Value, protocol: Protocol) -> Option { + image_payload(part, protocol).map(|payload| format!("{:x}", Sha256::digest(payload.as_bytes()))) +} + +fn evictable(part: &Value, protocol: Protocol) -> bool { + let Some(object) = part.as_object() else { + return false; + }; + match protocol { + Protocol::Responses | Protocol::Chat => { + if object + .keys() + .any(|key| !matches!(key.as_str(), "type" | "image_url" | "detail")) + { + return false; + } + let Some(payload) = image_payload(part, protocol) else { + return false; + }; + payload.starts_with("data:image/png;base64,") + || payload.starts_with("data:image/jpeg;base64,") + || payload.starts_with("data:image/webp;base64,") + } + Protocol::Anthropic => { + object + .keys() + .all(|key| matches!(key.as_str(), "type" | "source")) + && part["source"]["type"] == "base64" + && matches!( + part["source"]["media_type"].as_str(), + Some("image/png" | "image/jpeg" | "image/webp") + ) + } + } +} + +fn prune_items( + items: &mut [Value], + protocol: Protocol, + max_count: usize, + archive: &mut Option<&mut Archive<'_>>, + enforce: bool, +) -> Result { + let ack = items.iter().rposition(|item| acknowledged(item, protocol)); + let mut total = 0; + let mut unseen = 0; + let mut payload_bytes = 0; + for (index, item) in items.iter_mut().enumerate() { + visit_parts(item, protocol, &mut |part| { + total += 1; + payload_bytes += image_payload(part, protocol).map(str::len).unwrap_or(0); + if ack.is_none_or(|ack| index >= ack) { + unseen += 1; + } + }); + } + if enforce && unseen > max_count { + return Err("new image batch exceeds the model limit; split the batch"); + } + let high = HIGH_WATER.min(max_count); + if total < high && payload_bytes <= REQUEST_BYTES / 2 { + return Ok(Statistics::default()); + } + let keep = RECENT.min(max_count).max(unseen); + let mut remove = total.saturating_sub(keep); + let mut stats = Statistics::default(); + let mut paths = BTreeMap::new(); + for (index, item) in items.iter_mut().enumerate() { + if matches!( + item["type"].as_str(), + Some("function_call" | "custom_tool_call") + ) && let (Some(id), Some(arguments)) = + (item["call_id"].as_str(), item["arguments"].as_str()) + && let Ok(arguments) = serde_json::from_str::(arguments) + && let Some(path) = arguments["path"].as_str().filter(|path| path.len() <= 4096) + { + paths.insert(id.to_owned(), path.to_owned()); + } + let path = item["call_id"] + .as_str() + .and_then(|id| paths.get(id)) + .cloned(); + if ack.is_none_or(|ack| index >= ack) { + continue; + } + if item["role"] != "user" + && !(protocol == Protocol::Responses + && matches!( + item["type"].as_str(), + Some("function_call_output" | "custom_tool_call_output") + )) + && !(protocol == Protocol::Chat && item["role"] == "tool") + { + continue; + } + visit_parts(item, protocol, &mut |part| { + if (remove == 0 && payload_bytes <= REQUEST_BYTES / 2) || !evictable(part, protocol) { + return; + } + let Some(id) = image_id(part, protocol) else { + return; + }; + let saved = archive + .as_deref_mut() + .and_then(|archive| archive(&id, part, protocol)); + let original = saved + .map(|path| path.to_string_lossy().into_owned()) + .or_else(|| path.clone()); + let reference = match original { + Some(path) => format!("Reopen with view_image using path {}. A source path may have changed; prefer the archived copy when present.", json!(path)), + None => "Original is not available through this gateway; request the image again if details are needed.".into(), + }; + let bytes = image_payload(part, protocol).map(str::len).unwrap_or(0); + stats.removed_bytes += bytes; + payload_bytes = payload_bytes.saturating_sub(bytes); + stats.evicted += 1; + remove = remove.saturating_sub(1); + *part = protocol.text(format!("[Older image {id} omitted from active history. {reference} Written observations are retained.]")); + }); + } + Ok(stats) +} + +/// Project an already buffered gateway request. Never evict images awaiting inspection. +pub fn project( + value: &mut Value, + protocol: Protocol, + max_count: usize, + mut archive: Option<&mut Archive<'_>>, +) -> Result { + let Some(items) = value + .get_mut(protocol.field()) + .and_then(Value::as_array_mut) + else { + return Ok(Statistics::default()); + }; + let ack = items.iter().rposition(|item| acknowledged(item, protocol)); + let mut unseen = 0; + for (index, item) in items.iter_mut().enumerate() { + if ack.is_none_or(|ack| index >= ack) { + visit_parts(item, protocol, &mut |_| unseen += 1); + } + } + if unseen > max_count { + return Err("new image batch exceeds the model limit; split the batch"); + } + let original = std::mem::take(items); + let mut stats = Statistics::default(); + for item in original { + items.push(item); + let next = prune_items(items, protocol, max_count, &mut archive, false)?; + stats.evicted += next.evicted; + stats.removed_bytes += next.removed_bytes; + } + let next = prune_items(items, protocol, max_count, &mut archive, true)?; + stats.evicted += next.evicted; + stats.removed_bytes += next.removed_bytes; + Ok(stats) +} + +struct CountWriter(usize); +impl std::io::Write for CountWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 = self.0.saturating_add(bytes.len()); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} +fn size(value: &Value) -> usize { + let mut writer = CountWriter(0); + let _ = serde_json::to_writer(&mut writer, value); + writer.0 +} +struct ItemReader { + reader: R, + count: Rc>, + deadline: std::time::Instant, +} +impl Read for ItemReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if std::time::Instant::now() >= self.deadline { + return Err(std::io::Error::other("history processing timed out")); + } + let count = self.reader.read(out)?; + let total = self.count.get().saturating_add(count); + self.count.set(total); + if total > REQUEST_BYTES { + return Err(std::io::Error::other( + "individual history item exceeds 32 MiB", + )); + } + Ok(count) + } +} +struct RequestSeed<'a, 'b> { + protocol: Protocol, + max_count: usize, + archive: Option<&'a mut Archive<'b>>, + count: Rc>, + stats: Statistics, +} +impl<'de> DeserializeSeed<'de> for &mut RequestSeed<'_, '_> { + type Value = Value; + fn deserialize>(self, deserializer: D) -> Result { + deserializer.deserialize_map(self) + } +} +impl<'de> Visitor<'de> for &mut RequestSeed<'_, '_> { + type Value = Value; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("an inference request object") + } + fn visit_map>(self, mut map: M) -> Result { + let mut object = serde_json::Map::new(); + let mut other_bytes = 0; + while let Some(key) = map.next_key::()? { + if object.contains_key(&key) { + return Err(de::Error::custom("duplicate request field")); + } + self.count.set(0); + let value = if key == self.protocol.field() { + map.next_value_seed(HistorySeed(&mut *self))? + } else { + map.next_value()? + }; + if key != self.protocol.field() { + other_bytes += key.len() + size(&value); + if other_bytes > TEXT_BYTES { + return Err(de::Error::custom("request text exceeds 8 MiB")); + } + } + object.insert(key, value); + } + Ok(Value::Object(object)) + } + fn visit_seq>(self, mut sequence: S) -> Result { + let mut items = Vec::new(); + let mut text_bytes = 0; + loop { + self.count.set(0); + let Some(mut item) = sequence.next_element::()? else { + break; + }; + let mut image_bytes = 0; + visit_parts(&mut item, self.protocol, &mut |part| { + image_bytes += image_payload(part, self.protocol) + .map(str::len) + .unwrap_or(0); + }); + text_bytes += size(&item).saturating_sub(image_bytes); + if text_bytes > TEXT_BYTES { + return Err(de::Error::custom( + "history text exceeds 8 MiB; compact the conversation", + )); + } + items.push(item); + let stats = prune_items( + &mut items, + self.protocol, + self.max_count, + &mut self.archive, + false, + ) + .map_err(de::Error::custom)?; + self.stats.evicted += stats.evicted; + self.stats.removed_bytes += stats.removed_bytes; + // Bound retained memory independently of the number of discarded wire bytes. + let mut retained = 0; + for item in &mut items { + visit_parts(item, self.protocol, &mut |part| { + retained += image_payload(part, self.protocol) + .map(str::len) + .unwrap_or(0); + }); + } + if retained > REQUEST_BYTES { + return Err(de::Error::custom("active image batch exceeds 32 MiB")); + } + } + let stats = prune_items( + &mut items, + self.protocol, + self.max_count, + &mut self.archive, + true, + ) + .map_err(de::Error::custom)?; + self.stats.evicted += stats.evicted; + self.stats.removed_bytes += stats.removed_bytes; + Ok(Value::Array(items)) + } + fn visit_str(self, text: &str) -> Result { + Ok(text.into()) + } +} +struct HistorySeed<'a, 'b, 'c>(&'a mut RequestSeed<'b, 'c>); +impl<'de> DeserializeSeed<'de> for HistorySeed<'_, '_, '_> { + type Value = Value; + fn deserialize>(self, deserializer: D) -> Result { + deserializer.deserialize_any(&mut *self.0) + } +} + +/// Stream old inline payloads through a bounded item buffer; the full source body +/// is never retained. The adapter also imposes a finite upload/processing deadline. +pub fn project_reader( + reader: R, + protocol: Protocol, + max_count: usize, + archive: Option<&mut Archive<'_>>, +) -> Result<(Value, Statistics), &'static str> { + let count = Rc::new(Cell::new(0)); + let reader = ItemReader { + reader, + count: count.clone(), + deadline: std::time::Instant::now() + std::time::Duration::from_secs(120), + }; + let mut deserializer = serde_json::Deserializer::from_reader(reader); + let mut seed = RequestSeed { + protocol, + max_count, + archive, + count, + stats: Statistics::default(), + }; + let mut value = (&mut seed) + .deserialize(&mut deserializer) + .map_err(|error| { + let diagnostic = error.to_string(); + if diagnostic.starts_with("new image batch exceeds") { + "new image batch exceeds the model limit; split the batch" + } else if diagnostic.starts_with("history text exceeds") + || diagnostic.starts_with("request text exceeds") + { + "history text exceeds the adapter budget; compact the conversation" + } else if error.is_io() || diagnostic.starts_with("active image batch exceeds") { + "image or request exceeds the adapter byte budget" + } else { + "request JSON is invalid" + } + })?; + deserializer + .end() + .map_err(|_| "request has trailing data")?; + let stats = project(&mut value, protocol, max_count, None)?; + seed.stats.evicted += stats.evicted; + seed.stats.removed_bytes += stats.removed_bytes; + if size(&value) > REQUEST_BYTES { + return Err("projected request exceeds 32 MiB; compact the conversation"); + } + Ok((value, seed.stats)) +} diff --git a/src/spark/inference_adapter.rs b/src/spark/inference_adapter.rs new file mode 100644 index 0000000..8dbd367 --- /dev/null +++ b/src/spark/inference_adapter.rs @@ -0,0 +1,670 @@ +//! Invocation-local authenticated loopback adapter. Only fixed inference routes +//! reach the pinned appliance; images are projected before the network upload. +use super::{ + client::{self, ClientError}, + image_history::{self, Protocol}, +}; +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use bytes::Bytes; +use http_body_util::{BodyExt, Full, StreamBody, combinators::UnsyncBoxBody}; +use hyper::{ + Request, Response, + body::{Frame, Incoming}, + server::conn::http1, + service::service_fn, +}; +use hyper_util::rt::TokioIo; +use serde_json::{Value, json}; +use std::{ + fs, + io::{self, Read, Write}, + net::TcpListener, + os::unix::fs::{MetadataExt, OpenOptionsExt}, + path::{Path, PathBuf}, + sync::{Arc, Mutex, mpsc}, + thread, + time::Duration, +}; +use tokio::sync::{Semaphore, oneshot}; + +type Body = UnsyncBoxBody; +const ARCHIVE_BYTES: u64 = 512 * 1024 * 1024; +const IMAGE_BYTES: usize = 16 * 1024 * 1024; + +struct Archive { + root: PathBuf, +} +impl Archive { + fn save(&self, id: &str, part: &Value, protocol: Protocol) -> Option { + let (media, data) = if protocol == Protocol::Anthropic { + if part["source"]["type"] != "base64" { + return None; + } + ( + part["source"]["media_type"].as_str()?, + part["source"]["data"].as_str()?, + ) + } else { + let uri = if protocol == Protocol::Responses { + part["image_url"].as_str()? + } else { + part["image_url"]["url"].as_str()? + }; + uri.strip_prefix("data:")?.split_once(";base64,")? + }; + let extension = match media { + "image/png" => "png", + "image/jpeg" => "jpg", + "image/webp" => "webp", + _ => return None, + }; + if data.len() > IMAGE_BYTES.div_ceil(3) * 4 { + return None; + } + let path = self.root.join(format!("{id}.{extension}")); + let lock = fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(self.root.join("archive.lock")) + .ok()?; + let _lock = + nix::fcntl::Flock::lock(lock, nix::fcntl::FlockArg::LockExclusiveNonblock).ok()?; + if let Ok(meta) = fs::symlink_metadata(&path) { + let expected_bytes = (data.len() / 4 * 3) + .checked_sub(data.bytes().rev().take_while(|byte| *byte == b'=').count())?; + if !meta.is_file() + || meta.mode() & 0o077 != 0 + || meta.uid() != rustix::process::geteuid().as_raw() + || meta.len() != expected_bytes as u64 + { + return None; + } + let file = fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(&path) + .ok()?; + let _ = file.set_modified(std::time::SystemTime::now()); + return Some(path); + } + let bytes = BASE64.decode(data).ok()?; + if bytes.is_empty() || bytes.len() > IMAGE_BYTES { + return None; + } + if !match media { + "image/png" => bytes.starts_with(b"\x89PNG\r\n\x1a\n"), + "image/jpeg" => bytes.starts_with(b"\xff\xd8\xff"), + "image/webp" => bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP"), + _ => false, + } { + return None; + } + let mut entries = Vec::new(); + let mut total = 0; + for (index, entry) in fs::read_dir(&self.root).ok()?.take(4098).enumerate() { + if index > 4096 { + return None; + } + let entry = entry.ok()?; + let name = entry.file_name(); + let name = name.to_str()?; + if name.starts_with(".staging-") && name[9..].parse::().is_ok() { + fs::remove_file(entry.path()).ok()?; + continue; + } + let Some((stem, ext)) = name.split_once('.') else { + continue; + }; + if stem.len() != 64 + || !stem.bytes().all(|b| b.is_ascii_hexdigit()) + || !matches!(ext, "png" | "jpg" | "webp") + { + continue; + } + let meta = fs::symlink_metadata(entry.path()).ok()?; + if !meta.is_file() { + return None; + } + total += meta.len(); + entries.push((meta.modified().ok()?, meta.len(), entry.path())); + } + entries.sort_by_key(|entry| entry.0); + let mut remaining = entries.len(); + for (_, len, old) in &entries { + let expired = fs::symlink_metadata(old) + .ok()? + .modified() + .ok()? + .elapsed() + .ok()? + .as_secs() + > 90 * 86400; + if !expired && total + bytes.len() as u64 <= ARCHIVE_BYTES && remaining < 4096 { + break; + } + fs::remove_file(old).ok()?; + total = total.saturating_sub(*len); + remaining -= 1; + } + if total + bytes.len() as u64 > ARCHIVE_BYTES { + return None; + } + let staging = self.root.join(format!(".staging-{}", uuid::Uuid::new_v4())); + let mut file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(&staging) + .ok()?; + if file + .write_all(&bytes) + .and_then(|_| file.sync_data()) + .and_then(|_| fs::rename(&staging, &path)) + .is_err() + { + let _ = fs::remove_file(&staging); + return None; + } + Some(path) + } +} +struct State { + http: reqwest::Client, + base: reqwest::Url, + instance: String, + remote_token: String, + local_token: String, + archive: Arc>, + slots: Arc, +} +pub struct Adapter { + pub base_url: String, + token: String, + stop: Option>, + worker: Option>, +} +impl Adapter { + pub fn start( + config_dir: &Path, + host: &str, + instance: &str, + remote_token: &str, + ) -> Result { + let (http, base) = client::inference_transport(config_dir, host)?; + let root = config_dir.join("spark").join("image-archive"); + super::economics::private_dir(&root) + .map_err(|_| failure("private image archive is unavailable"))?; + Self::start_with(http, base, instance, remote_token, root) + } + fn start_with( + http: reqwest::Client, + base: reqwest::Url, + instance: &str, + remote_token: &str, + root: PathBuf, + ) -> Result { + client::validate_instance_reference(instance)?; + let listener = TcpListener::bind("127.0.0.1:0") + .map_err(|_| failure("cannot bind the private inference adapter"))?; + listener + .set_nonblocking(true) + .map_err(|_| failure("cannot configure inference adapter"))?; + let address = listener + .local_addr() + .map_err(|_| failure("cannot identify inference adapter"))?; + let token = format!( + "sparkplane_local_{}{}", + uuid::Uuid::new_v4().simple(), + uuid::Uuid::new_v4().simple() + ); + let state = Arc::new(State { + http, + base, + instance: instance.into(), + remote_token: remote_token.into(), + local_token: token.clone(), + archive: Arc::new(Mutex::new(Archive { root })), + slots: Arc::new(Semaphore::new(2)), + }); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .map_err(|_| failure("cannot start inference adapter runtime"))?; + let (stop, mut stopping) = oneshot::channel(); + let worker = thread::spawn(move || { + runtime.block_on(async move { + let Ok(listener) = tokio::net::TcpListener::from_std(listener) else { return; }; + let mut connections = tokio::task::JoinSet::new(); + loop { + tokio::select! { + _ = &mut stopping => break, + Some(_) = connections.join_next(), if !connections.is_empty() => {}, + accepted = listener.accept() => { + let Ok((stream, _)) = accepted else { break; }; + if connections.len() >= 32 { continue; } + let state = state.clone(); + connections.spawn(async move { + let service = service_fn(move |request| { + let state = state.clone(); + async move { Ok::<_, std::convert::Infallible>(handle(request, state).await) } + }); + let _ = http1::Builder::new().max_buf_size(16 * 1024).serve_connection(TokioIo::new(stream), service).await; + }); + } + } + } + connections.abort_all(); + }) + }); + Ok(Self { + base_url: format!("http://{address}"), + token, + stop: Some(stop), + worker: Some(worker), + }) + } + pub fn token(&self) -> &str { + &self.token + } +} +impl Drop for Adapter { + fn drop(&mut self) { + if let Some(stop) = self.stop.take() { + let _ = stop.send(()); + } + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} +fn failure(message: &'static str) -> ClientError { + ClientError { + code: super::EXIT_INTERNAL, + message: message.into(), + } +} +fn error(status: u16, message: &'static str) -> Response { + let bytes = serde_json::to_vec(&json!({"type":"error","error":{"type":"invalid_request_error","code":if status == 413 { "request_too_large" } else if message.starts_with("history text exceeds") { "context_length_exceeded" } else { "invalid_request_error" },"message":message}})).expect("static error"); + let mut builder = Response::builder() + .status(status) + .header("content-type", "application/json"); + if status == 429 { + builder = builder.header("retry-after", "1"); + } + builder + .body( + Full::new(Bytes::from(bytes)) + .map_err(|never| match never {}) + .boxed_unsync(), + ) + .expect("static response") +} +struct ChunkReader { + receiver: mpsc::Receiver, + current: std::io::Cursor, +} +impl Read for ChunkReader { + fn read(&mut self, out: &mut [u8]) -> io::Result { + if out.is_empty() { + return Ok(0); + } + loop { + let n = self.current.read(out)?; + if n != 0 { + return Ok(n); + } + match self.receiver.recv_timeout(Duration::from_secs(30)) { + Ok(bytes) => self.current = std::io::Cursor::new(bytes), + Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(0), + Err(_) => return Err(io::Error::other("upload timed out")), + } + } + } +} +async fn handle(request: Request, state: Arc) -> Response { + if request + .headers() + .get("authorization") + .and_then(|v| v.to_str().ok()) + != Some(&format!("Bearer {}", state.local_token)) + { + return error(401, "private adapter authentication failed"); + } + let path = request.uri().path(); + if (request.uri().query().is_some() + && !(path.starts_with("/anthropic/") && request.uri().query() == Some("beta=true"))) + || request.headers().contains_key("origin") + { + return error(400, "adapter route is unsupported"); + } + let (route, protocol) = match (request.method().as_str(), path) { + ("POST", "/openai/v1/responses") => ( + format!("openai/{}/v1/responses", state.instance), + Some(Protocol::Responses), + ), + ("POST", "/openai/v1/chat/completions") => ( + format!("openai/{}/v1/chat/completions", state.instance), + Some(Protocol::Chat), + ), + ("POST", "/anthropic/v1/messages") => ( + format!("anthropic/{}/v1/messages", state.instance), + Some(Protocol::Anthropic), + ), + ("POST", "/anthropic/v1/messages/count_tokens") => ( + format!("anthropic/{}/v1/messages/count_tokens", state.instance), + Some(Protocol::Anthropic), + ), + ("GET", "/openai/v1/models") => (format!("openai/{}/v1/models", state.instance), None), + _ => return error(404, "adapter route is unsupported"), + }; + if request + .headers() + .get("content-encoding") + .is_some_and(|v| v != "identity") + { + return error(415, "compressed adapter uploads are unsupported"); + } + let permit = match state.slots.clone().try_acquire_owned() { + Ok(p) => p, + Err(_) => return error(429, "private adapter is busy; retry shortly"), + }; + let (parts, mut body) = request.into_parts(); + let payload = if let Some(protocol) = protocol { + let (sender, receiver) = mpsc::sync_channel(2); + let archive = state.archive.clone(); + let parsing = tokio::task::spawn_blocking(move || { + // The permit remains held until blocking parsing exits, including cancellation. + let _permit = permit; + let mut save = + |id: &str, part: &Value, protocol| archive.lock().ok()?.save(id, part, protocol); + image_history::project_reader( + std::io::BufReader::new(ChunkReader { + receiver, + current: std::io::Cursor::new(Bytes::new()), + }), + protocol, + 16, + Some(&mut save), + ) + }); + let feeding = async { + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|_| ())?; + if let Ok(data) = frame.into_data() { + // Backpressure without blocking the asynchronous transport workers. + let sender = sender.clone(); + tokio::task::spawn_blocking(move || sender.send(data)) + .await + .map_err(|_| ())? + .map_err(|_| ())?; + } + } + drop(sender); + Ok::<_, ()>(()) + }; + let fed = tokio::time::timeout(Duration::from_secs(120), feeding).await; + if fed.is_err() { + return error(408, "private adapter upload timed out"); + } + match tokio::time::timeout(Duration::from_secs(120), parsing).await { + Ok(Ok(Ok((value, _)))) if matches!(fed, Ok(Ok(()))) => match serde_json::to_vec(&value) + { + Ok(bytes) => Some(bytes), + Err(_) => return error(400, "request cannot be encoded"), + }, + Ok(Ok(Err(message))) => { + return error( + if message.contains("byte budget") || message.contains("32 MiB") { + 413 + } else { + 400 + }, + message, + ); + } + _ => { + return error( + 413, + "history cannot fit the adapter budgets; compact text or split the new image batch", + ); + } + } + } else { + drop(permit); + None + }; + let Ok(mut url) = state.base.join(&route) else { + return error(502, "invalid pinned upstream route"); + }; + if let Some(query) = parts.uri.query() { + url.set_query(Some(query)); + } + let mut upstream = state + .http + .request(parts.method, url) + .bearer_auth(&state.remote_token); + for name in ["anthropic-version", "anthropic-beta", "accept"] { + if let Some(value) = parts.headers.get(name) { + upstream = upstream.header(name, value); + } + } + if let Some(payload) = payload { + upstream = upstream + .header("content-type", "application/json") + .body(payload); + } + let response = match upstream.send().await { + Ok(r) => r, + Err(_) => return error(502, "pinned Spark inference transport is unavailable"), + }; + let mut builder = Response::builder().status(response.status()); + for name in ["content-type", "retry-after"] { + if let Some(value) = response.headers().get(name) { + builder = builder.header(name, value); + } + } + let stream = futures_util::stream::try_unfold(response, |mut response| async move { + match response.chunk().await { + Ok(Some(chunk)) => Ok(Some((Frame::data(chunk), response))), + Ok(None) => Ok(None), + Err(_) => Err(io::Error::other("inference response stream disconnected")), + } + }); + builder + .body(StreamBody::new(stream).boxed_unsync()) + .expect("upstream status and headers") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct HistoryReader { + next: usize, + current: io::Cursor>, + image: String, + read: Arc, + } + impl Read for HistoryReader { + fn read(&mut self, out: &mut [u8]) -> io::Result { + loop { + let n = self.current.read(out)?; + if n > 0 { + self.read.fetch_add(n, Ordering::Relaxed); + return Ok(n); + } + let bytes = match self.next { + 0 => br#"{"input":["#.to_vec(), + 1..=200 => { + let i = self.next - 1; + format!("{}{}", if i == 0 { "" } else { "," }, + json!({"type":"message","role":"user","content":[{"type":"input_text","text":format!("frame {i}")},{"type":"input_image","image_url":self.image}]})).into_bytes() + } + 201 => b"],\"stream\":true}".to_vec(), + _ => return Ok(0), + }; + // Each inspected frame is followed by an assistant observation. + let bytes = if (1..200).contains(&self.next) { + let mut bytes = bytes; + bytes.extend_from_slice(format!(",{}", json!({"type":"message","role":"assistant","content":"Button remains visible."})).as_bytes()); + bytes + } else { + bytes + }; + self.next += 1; + self.current = io::Cursor::new(bytes); + } + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn streamed_history_exceeds_gateway_upload_limit_but_forwarded_body_stays_small() { + let sent = Arc::new(Mutex::new(Vec::new())); + let observed = sent.clone(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let observed = observed.clone(); + tokio::spawn(async move { + let service = service_fn(move |request: Request| { + let observed = observed.clone(); + async move { + assert_eq!(request.uri().path(), "/openai/fixture/v1/responses"); + assert_eq!(request.headers()["authorization"], "Bearer REMOTE_FIXTURE"); + let bytes = request.into_body().collect().await.unwrap().to_bytes(); + assert!(bytes.len() < image_history::REQUEST_BYTES); + observed + .lock() + .unwrap() + .push(serde_json::from_slice::(&bytes).unwrap()); + Ok::<_, std::convert::Infallible>(Response::new(Full::new( + Bytes::from_static(b"data: {\"type\":\"response.completed\"}\n\n"), + ))) + } + }); + let _ = http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + }); + let root = tempfile::tempdir().unwrap(); + let archive = root.path().join("archive"); + super::super::economics::private_dir(&archive).unwrap(); + let adapter = Adapter::start_with( + reqwest::Client::new(), + format!("http://{address}").parse().unwrap(), + "fixture", + "REMOTE_FIXTURE", + archive.clone(), + ) + .unwrap(); + let http = reqwest::Client::new(); + let unauthorized = http + .post(format!("{}/openai/v1/responses", adapter.base_url)) + .body("bad") + .send() + .await + .unwrap(); + assert_eq!(unauthorized.status(), 401); + let forbidden = http + .post(format!("{}/api/sparkplane/v1/tokens", adapter.base_url)) + .bearer_auth(adapter.token()) + .body("{}") + .send() + .await + .unwrap(); + assert_eq!(forbidden.status(), 404); + assert!(sent.lock().unwrap().is_empty()); + let malformed = http + .post(format!("{}/openai/v1/responses", adapter.base_url)) + .bearer_auth(adapter.token()) + .body("{") + .send() + .await + .unwrap(); + assert_eq!(malformed.status(), 400); + let excessive = json!({"input":[{"type":"message","role":"user","content":(0..17).map(|_| json!({"type":"input_image","image_url":"data:image/png;base64,aW1hZ2U="})).collect::>()}]}); + let excessive = http + .post(format!("{}/openai/v1/responses", adapter.base_url)) + .bearer_auth(adapter.token()) + .json(&excessive) + .send() + .await + .unwrap(); + assert_eq!(excessive.status(), 400); + assert!(excessive.text().await.unwrap().contains("split the batch")); + assert!(sent.lock().unwrap().is_empty()); + let config: toml::Value = toml::from_str(include_str!( + "../../configs/sparkplane/engines/vllm-qwen38-mmap.toml" + )) + .unwrap(); + let mut original = BASE64 + .decode( + config["profiles"][0]["vision"]["health_image_base64"] + .as_str() + .unwrap(), + ) + .unwrap(); + original.resize(200_000, 0); + let image = format!("data:image/png;base64,{}", BASE64.encode(&original)); + let bytes_read = Arc::new(AtomicUsize::new(0)); + let reader = HistoryReader { + next: 0, + current: io::Cursor::new(Vec::new()), + image, + read: bytes_read.clone(), + }; + let endpoint = format!("{}/openai/v1/responses", adapter.base_url); + let token = adapter.token().to_owned(); + let response = tokio::task::spawn_blocking(move || { + reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(120)) + .build() + .unwrap() + .post(endpoint) + .bearer_auth(token) + .body(reqwest::blocking::Body::new(reader)) + .send() + .unwrap() + .text() + .unwrap() + }) + .await + .unwrap(); + assert!(response.contains("response.completed"), "{response}"); + assert!(bytes_read.load(Ordering::Relaxed) > image_history::REQUEST_BYTES); + let value = sent.lock().unwrap().pop().unwrap(); + let mut projected = value.clone(); + let mut images = 0; + image_history::visit_parts(&mut projected["input"], Protocol::Responses, &mut |_| { + images += 1 + }); + assert!(images <= 12); + assert_eq!(value["input"].as_array().unwrap().len(), 399); + assert_eq!(value["input"][1]["content"], "Button remains visible."); + let marker = value["input"][0]["content"][1]["text"].as_str().unwrap(); + let files = fs::read_dir(&archive) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "png")) + .collect::>(); + assert_eq!(files.len(), 1); + assert!(marker.contains(&files[0].path().to_string_lossy().to_string())); + assert_eq!(fs::read(files[0].path()).unwrap(), original); + assert_eq!(fs::metadata(files[0].path()).unwrap().mode() & 0o777, 0o600); + assert_eq!(fs::metadata(&archive).unwrap().mode() & 0o777, 0o700); + let local_address = adapter.base_url.strip_prefix("http://").unwrap().to_owned(); + drop(adapter); + assert!(tokio::net::TcpStream::connect(local_address).await.is_err()); + server.abort(); + } +} diff --git a/src/spark/launch.rs b/src/spark/launch.rs index dfcae32..0f79081 100644 --- a/src/spark/launch.rs +++ b/src/spark/launch.rs @@ -45,6 +45,8 @@ struct LaunchPlan { extra_argument_count: usize, allow_network: bool, mode: String, + eco_mode: String, + eco_capability: &'static str, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -182,6 +184,8 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client extra_argument_count: args.extra_args.len(), allow_network: args.allow_network, mode: args.mode.clone(), + eco_mode: args.eco_mode.clone(), + eco_capability: eco_capability(&args), }, args.json, ); @@ -257,6 +261,8 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client extra_argument_count: args.extra_args.len(), allow_network: args.allow_network, mode: args.mode.clone(), + eco_mode: args.eco_mode.clone(), + eco_capability: eco_capability(&args), }; if args.configure { return render_plan(&plan, args.json); @@ -275,7 +281,51 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client "Network access requested for this invocation; filesystem and approval policies are unchanged." ); } - launch_child( + let session_id = ulid::Ulid::new().to_string(); + let started_at = chrono::Utc::now().to_rfc3339(); + let created = match client.create_launch_session(&super::wire::LaunchSessionRequest { + id: session_id.clone(), + instance: instance.name.clone(), + integration: args.integration.as_str().into(), + eco_mode: args.eco_mode.clone(), + }) { + Ok(value) => value, + Err(_) => { + eprintln!("Session accounting unavailable; inference usage will be marked unknown."); + // Creation may have committed before its response was lost. + let _ = client.finish_launch_session(&session_id); + None + } + }; + let mut inference = created.as_ref().map(|value| value.session.clone()); + let session_token = created + .and_then(|value| value.bearer_token) + .map(LaunchSecret); + if inference.is_some() && session_token.is_none() { + // A lost one-time credential cannot be recovered. Revoke it rather than + // attributing the shared fallback credential to this session. + let _ = client.finish_launch_session(&session_id); + inference = None; + } + let eco = match super::eco::EcoSession::create(&session_id, args.eco_mode == "max", &args.mode) + { + Ok(value) => Some(value), + Err(_) => { + eprintln!("Eco integration unavailable; launch continues without compression."); + None + } + }; + if args.eco_mode == "max" { + eprintln!( + "Eco mode: embedded RTK; {}.", + if args.mode == "inherit" { + "compression inactive to preserve inherited permissions" + } else { + "agent hook trust still applies" + } + ); + } + let result = launch_child( args.integration, args.yes, args.allow_network, @@ -285,10 +335,59 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client host, instance: &instance, model: &model, - token: &token, + token: session_token.as_ref().unwrap_or(&token), }, &args.extra_args, - ) + eco.as_ref(), + ); + if inference.is_some() { + match client.finish_launch_session(&session_id) { + Ok(value) => inference = Some(value), + Err(_) => { + inference = None; + eprintln!("Session finalization unavailable; inference usage is unknown."); + } + } + } + let compression = eco + .as_ref() + .map(super::eco::EcoSession::summary) + .unwrap_or_else(|| super::economics::Compression { + status: "unavailable".into(), + metrics_errors: 1, + ..Default::default() + }); + let mut report = super::economics::Report { + schema: super::economics::SCHEMA.into(), + session_id, + host: host.into(), + model: model.canonical.clone(), + integration: args.integration.as_str().into(), + eco_mode: args.eco_mode.clone(), + started_at, + finished_at: Some(chrono::Utc::now().to_rfc3339()), + exit_code: Some(result.as_ref().err().map(|error| error.code).unwrap_or(0)), + inference, + compression, + catalog: super::economics::catalog().expect("bundled price catalog"), + comparisons: Vec::new(), + }; + report.calculate(); + if super::economics::save(config_dir, &report).is_err() { + eprintln!("Could not retain the private economics report."); + } + super::economics::display(&report); + result +} + +fn eco_capability(args: &LaunchArgs) -> &'static str { + if args.eco_mode == "none" { + "disabled" + } else if args.mode == "inherit" { + "inactive_policy" + } else { + "embedded_hooks_require_agent_support_and_trust" + } } fn validate_args(args: &LaunchArgs) -> Result<(), ClientError> { @@ -732,11 +831,17 @@ fn launch_child( mode: &str, launch: ReadyLaunch<'_>, extra_args: &[String], + eco: Option<&super::eco::EcoSession>, ) -> Result<(), ClientError> { let executable = ensure_executable(integration, yes)?; let mut command = Command::new(executable); if integration != LaunchIntegration::Opencode { configure_launch_permissions(&mut command, integration, mode, allow_network)?; + if let Some(eco) = eco + && eco.configure(&mut command, integration).is_err() + { + eprintln!("Eco hook setup unavailable; launch continues without compression."); + } } command .stdin(Stdio::inherit()) @@ -774,12 +879,88 @@ fn launch_child( if integration == LaunchIntegration::Opencode { configure_launch_permissions(&mut command, integration, mode, allow_network)?; } - let status = command - .status() + if integration == LaunchIntegration::Opencode + && let Some(eco) = eco + && eco.configure(&mut command, integration).is_err() + { + eprintln!("Eco hook setup unavailable; launch continues without compression."); + } + let adapter = if launch + .instance + .input_modalities + .iter() + .any(|modality| modality == "image") + { + let adapter = super::inference_adapter::Adapter::start( + launch.config_dir, + launch.host, + &launch.instance.name, + launch.token.expose(), + )?; + configure_image_adapter(&mut command, integration, &launch.instance.name, &adapter)?; + eprintln!("Image history: recent frames retained; older originals can be reopened."); + Some(adapter) + } else { + None + }; + let status = super::eco::wait(command) .map_err(|_| usage(format!("could not start {}", integration.as_str())))?; + drop(adapter); child_result(integration, status) } +fn configure_image_adapter( + command: &mut Command, + integration: LaunchIntegration, + instance: &str, + adapter: &super::inference_adapter::Adapter, +) -> Result<(), ClientError> { + match integration { + LaunchIntegration::Codex => { + command + .arg("-c") + .arg(format!( + "model_providers.{}.base_url={:?}", + provider_name(instance), + format!("{}/openai/v1", adapter.base_url) + )) + .env("SPARKPLANE_INFERENCE_TOKEN", adapter.token()) + .env("OPENAI_API_KEY", adapter.token()); + } + LaunchIntegration::Claude => { + command + .env( + "ANTHROPIC_BASE_URL", + format!("{}/anthropic", adapter.base_url), + ) + .env("ANTHROPIC_AUTH_TOKEN", adapter.token()); + } + LaunchIntegration::Opencode => { + let content = command + .get_envs() + .find(|(key, _)| *key == "OPENCODE_CONFIG_CONTENT") + .and_then(|(_, value)| value) + .ok_or_else(|| failure(EXIT_INTERNAL, "missing OpenCode session configuration"))?; + let mut value: Value = serde_json::from_str(&content.to_string_lossy()) + .map_err(|_| failure(EXIT_INTERNAL, "invalid OpenCode session configuration"))?; + value["provider"]["sparkplane"]["options"]["baseURL"] = + format!("{}/openai/v1", adapter.base_url).into(); + command + .env( + "OPENCODE_CONFIG_CONTENT", + serde_json::to_string(&value).map_err(|_| { + failure( + EXIT_INTERNAL, + "cannot encode OpenCode session configuration", + ) + })?, + ) + .env("SPARKPLANE_INFERENCE_TOKEN", adapter.token()); + } + } + Ok(()) +} + fn configure_claude_command( command: &mut Command, config_dir: &Path, @@ -849,6 +1030,11 @@ fn configure_codex_command( "responses" ), format!("model_catalog_json={:?}", catalog), + // Leave room for new tool results and a compaction request on local models. + format!( + "model_auto_compact_token_limit={}", + instance.context_window * 3 / 4 + ), ]; if let Some(effort) = &instance.default_reasoning_effort { overrides.push(format!("model_reasoning_effort={effort:?}")); @@ -1072,10 +1258,7 @@ fn child_result(integration: LaunchIntegration, status: ExitStatus) -> Result<() if status.success() { return Ok(()); } - let code = status - .code() - .filter(|code| (1..=125).contains(code)) - .unwrap_or(1); + let code = super::eco::exit_code(status); Err(failure( code, format!("{} exited with status {code}", integration.as_str()), @@ -1838,6 +2021,7 @@ mod tests { let args = LaunchArgs { integration: LaunchIntegration::Codex, mode: "auto".into(), + eco_mode: "max".into(), allow_network: false, model: Some("ornith".into()), configure: false, @@ -1861,4 +2045,18 @@ mod tests { 7 ); } + + #[test] + fn child_signal_exit_status_is_preserved() { + let status = Command::new("sh") + .args(["-c", "kill -TERM $$"]) + .status() + .unwrap(); + assert_eq!( + child_result(LaunchIntegration::Codex, status) + .unwrap_err() + .code, + 143 + ); + } } diff --git a/src/spark/mod.rs b/src/spark/mod.rs index b7c7075..3ee65e9 100644 --- a/src/spark/mod.rs +++ b/src/spark/mod.rs @@ -2,13 +2,19 @@ pub mod cli; pub mod client; +pub mod eco; +pub mod economics; #[cfg(feature = "appliance")] pub mod engine; +pub mod image_history; +pub mod inference_adapter; pub mod install; pub mod launch; pub mod qualification; #[cfg(feature = "appliance")] pub mod reconcile; +#[cfg(feature = "appliance")] +pub mod sessions; pub mod wire; #[cfg(feature = "appliance")] diff --git a/src/spark/sessions.rs b/src/spark/sessions.rs new file mode 100644 index 0000000..ef98676 --- /dev/null +++ b/src/spark/sessions.rs @@ -0,0 +1,460 @@ +//! Private per-launch inference ledger; all SQL runs on the existing database actor. +use std::sync::{Arc, Mutex}; + +use rusqlite::{Connection, OptionalExtension, params}; +use secrecy::SecretString; +use tokio::sync::oneshot; + +use super::{ + state::{DbActor, StateError}, + wire::{LaunchSessionCreated, LaunchSessionDocument, LaunchSessionRequest, SessionUsage}, +}; + +pub(crate) const MIGRATION: &str = r#" +CREATE TABLE launch_sessions ( + id TEXT PRIMARY KEY, token_id TEXT NOT NULL UNIQUE REFERENCES token_metadata(id), + request_hash TEXT NOT NULL, instance TEXT NOT NULL, model TEXT NOT NULL, + integration TEXT NOT NULL, eco_mode TEXT NOT NULL, started_at TEXT NOT NULL, + finished_at TEXT +); +CREATE TABLE launch_requests ( + id TEXT PRIMARY KEY, session_id TEXT NOT NULL REFERENCES launch_sessions(id) ON DELETE CASCADE, + instance TEXT NOT NULL, started_at TEXT NOT NULL, finished_at TEXT, + input_tokens INTEGER, output_tokens INTEGER, outcome TEXT NOT NULL DEFAULT 'pending', accounting_error INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX launch_requests_session_idx ON launch_requests(session_id); +CREATE INDEX launch_sessions_retention_idx ON launch_sessions(started_at); +"#; + +pub(crate) fn migration_needed(db: &Connection) -> Result { + db.query_row("SELECT NOT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='launch_session_migrations')",[],|row|row.get(0)).map_err(sql) +} + +pub(crate) fn migrate(db: &mut Connection) -> Result<(), StateError> { + use sha2::{Digest, Sha256}; + let checksum = format!("{:x}", Sha256::digest(MIGRATION.as_bytes())); + if !migration_needed(db)? { + let versions = db + .prepare("SELECT version,checksum FROM launch_session_migrations ORDER BY version") + .map_err(sql)? + .query_map([], |row| { + Ok((row.get::<_, u32>(0)?, row.get::<_, String>(1)?)) + }) + .map_err(sql)? + .collect::, _>>() + .map_err(sql)?; + if versions != [(1, checksum)] { + return Err(StateError::Unavailable( + "unsupported or modified session ledger migration".into(), + )); + } + return Ok(()); + } + let tx = db.transaction().map_err(sql)?; + tx.execute_batch("CREATE TABLE launch_session_migrations(version INTEGER PRIMARY KEY,checksum TEXT NOT NULL);").map_err(sql)?; + tx.execute_batch(MIGRATION).map_err(sql)?; + tx.execute( + "INSERT INTO launch_session_migrations(version,checksum) VALUES(1,?1)", + [checksum], + ) + .map_err(sql)?; + tx.commit().map_err(sql) +} + +pub(crate) enum Command { + Create { + actor: String, + request: LaunchSessionRequest, + reply: oneshot::Sender>, + }, + Get { + id: String, + finish: bool, + actor: String, + reply: oneshot::Sender>, + }, + Begin { + token: String, + id: String, + instance: String, + reply: oneshot::Sender>, + }, + Record { + id: String, + usage: Option<(u64, u64)>, + outcome: String, + incomplete: bool, + reply: oneshot::Sender>, + }, +} + +impl DbActor { + pub async fn create_launch_session( + &self, + actor: &str, + request: LaunchSessionRequest, + ) -> Result { + let (reply, receive) = oneshot::channel(); + self.session_command(Command::Create { + actor: actor.into(), + request, + reply, + })?; + receive + .await + .map_err(|_| StateError::Unavailable("session database unavailable".into()))? + } + pub async fn launch_session( + &self, + id: &str, + finish: bool, + actor: &str, + ) -> Result { + let (reply, receive) = oneshot::channel(); + self.session_command(Command::Get { + id: id.into(), + finish, + actor: actor.into(), + reply, + })?; + receive + .await + .map_err(|_| StateError::Unavailable("session database unavailable".into()))? + } + async fn begin_usage(&self, token: &str, id: &str, instance: &str) -> Result { + let (reply, receive) = oneshot::channel(); + self.session_command(Command::Begin { + token: token.into(), + id: id.into(), + instance: instance.into(), + reply, + })?; + receive + .await + .map_err(|_| StateError::Unavailable("session database unavailable".into()))? + } + async fn record_usage( + &self, + id: &str, + usage: Option<(u64, u64)>, + outcome: &str, + incomplete: bool, + ) -> Result<(), StateError> { + let (reply, receive) = oneshot::channel(); + self.session_command(Command::Record { + id: id.into(), + usage, + outcome: outcome.into(), + incomplete, + reply, + })?; + receive + .await + .map_err(|_| StateError::Unavailable("session database unavailable".into()))? + } +} + +fn sql(error: rusqlite::Error) -> StateError { + StateError::Unavailable(format!("session database: {error}")) +} + +pub(crate) fn dispatch(db: &mut Connection, pepper: &SecretString, command: Command) { + match command { + Command::Create { + actor, + request, + reply, + } => { + let _ = reply.send(create(db, pepper, &actor, request)); + } + Command::Get { + id, + finish, + actor, + reply, + } => { + let result = if finish { + finish_session(db, &id, &actor).and_then(|()| read(db, &id)) + } else { + read(db, &id) + }; + let _ = reply.send(result); + } + Command::Begin { + token, + id, + instance, + reply, + } => { + let result = (|| { + let expected = db + .query_row( + "SELECT instance,finished_at FROM launch_sessions WHERE token_id=?1", + [&token], + |r| Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?)), + ) + .optional() + .map_err(sql)?; + let Some((expected, finished)) = expected else { + return Ok(false); + }; + if expected != instance || finished.is_some() { + return Err(StateError::Invalid( + "launch session is not active for this instance".into(), + )); + } + db.execute("INSERT INTO launch_requests(id,session_id,instance,started_at) SELECT ?1,id,?2,?3 FROM launch_sessions WHERE token_id=?4 AND finished_at IS NULL", params![id,instance,super::state::now(),token]).map(|n| n == 1).map_err(sql) + })(); + let _ = reply.send(result); + } + Command::Record { + id, + usage, + outcome, + incomplete, + reply, + } => { + let result = usage.map(|(a,b)| (i64::try_from(a),i64::try_from(b))).map(|(a,b)| Ok((a.map_err(|_| StateError::Invalid("usage overflow".into()))?,b.map_err(|_| StateError::Invalid("usage overflow".into()))?))).transpose().and_then(|counts| { + db.execute("UPDATE launch_requests SET input_tokens=?2,output_tokens=?3,outcome=?4,finished_at=?5,accounting_error=?6 WHERE id=?1 AND finished_at IS NULL",params![id,counts.map(|c|c.0),counts.map(|c|c.1),outcome,super::state::now(),incomplete]).map(|_|()).map_err(sql) + }); + let _ = reply.send(result); + } + } +} + +fn create( + db: &mut Connection, + pepper: &SecretString, + actor: &str, + request: LaunchSessionRequest, +) -> Result { + if request.id.parse::().is_err() + || !["codex", "claude", "opencode"].contains(&request.integration.as_str()) + || !["max", "none"].contains(&request.eco_mode.as_str()) + { + return Err(StateError::Invalid("invalid launch session".into())); + } + let hash = super::wire::canonical_request_sha256(&request) + .map_err(|_| StateError::Invalid("invalid session request".into()))?; + if let Some(existing) = db + .query_row( + "SELECT request_hash FROM launch_sessions WHERE id=?1", + [&request.id], + |r| r.get::<_, String>(0), + ) + .optional() + .map_err(sql)? + { + if existing != hash { + return Err(StateError::Conflict("session identity conflict".into())); + } + return Ok(LaunchSessionCreated { + session: read(db, &request.id)?, + bearer_token: None, + }); + } + let instance = super::state::read_instance(db, &request.instance)?; + let active: i64 = db.query_row("SELECT COUNT(*) FROM token_metadata WHERE revoked_at IS NULL AND (expires_at IS NULL OR julianday(expires_at)>julianday('now'))",[],|r|r.get(0)).map_err(sql)?; + if active >= 1024 { + return Err(StateError::Overloaded); + } + let token_id = ulid::Ulid::new().to_string(); + let secret = super::state::random_secret()?; + let verifier = super::state::token_hmac(pepper, &token_id, &secret)?; + let expires = (chrono::Utc::now() + chrono::Duration::days(7)).to_rfc3339(); + let now = super::state::now(); + let tx = db.transaction().map_err(sql)?; + tx.execute( + "DELETE FROM launch_sessions WHERE julianday(started_at) Result<(), StateError> { + let token: String = db + .query_row( + "SELECT token_id FROM launch_sessions WHERE id=?1", + [id], + |r| r.get(0), + ) + .optional() + .map_err(sql)? + .ok_or(StateError::NotFound)?; + let tx = db.transaction().map_err(sql)?; + let now = super::state::now(); + let changed = tx + .execute( + "UPDATE launch_sessions SET finished_at=?2 WHERE id=?1 AND finished_at IS NULL", + params![id, now], + ) + .map_err(sql)?; + tx.execute( + "UPDATE token_metadata SET revoked_at=COALESCE(revoked_at,?2) WHERE id=?1", + params![token, now], + ) + .map_err(sql)?; + if changed > 0 { + tx.execute("INSERT INTO audit(occurred_at,actor_token_id,action,target,outcome,metadata_json) VALUES(?1,?2,'launch-session.finish',?3,'succeeded','{}')",params![now,actor,id]).map_err(sql)?; + } + tx.commit().map_err(sql) +} + +fn read(db: &Connection, id: &str) -> Result { + let mut session = db.query_row("SELECT id,instance,model,integration,eco_mode,started_at,finished_at FROM launch_sessions WHERE id=?1",[id],|r|Ok(LaunchSessionDocument { schema:"sparkplane.launch-session/v1".into(),id:r.get(0)?,instance:r.get(1)?,model:r.get(2)?,integration:r.get(3)?,eco_mode:r.get(4)?,started_at:r.get(5)?,finished_at:r.get(6)?,usage:SessionUsage::default() })).optional().map_err(sql)?.ok_or(StateError::NotFound)?; + session.usage = db.query_row("SELECT COUNT(*),COALESCE(SUM(input_tokens),0),COALESCE(SUM(output_tokens),0),COALESCE(SUM(input_tokens IS NULL),0),COALESCE(SUM(finished_at IS NULL),0),COALESCE(SUM(outcome!='succeeded' AND outcome!='pending'),0),COALESCE(SUM(CASE WHEN input_tokens>272000 THEN input_tokens ELSE 0 END),0),COALESCE(SUM(CASE WHEN input_tokens>272000 THEN output_tokens ELSE 0 END),0),COALESCE(SUM(accounting_error),0) FROM launch_requests WHERE session_id=?1",[id],|r|Ok(SessionUsage {requests:r.get::<_,i64>(0)? as u64,input_tokens:r.get::<_,i64>(1)? as u64,output_tokens:r.get::<_,i64>(2)? as u64,unknown_usage_requests:r.get::<_,i64>(3)? as u64,pending_requests:r.get::<_,i64>(4)? as u64,failed_requests:r.get::<_,i64>(5)? as u64,long_context_input_tokens:r.get::<_,i64>(6)? as u64,long_context_output_tokens:r.get::<_,i64>(7)? as u64,accounting_errors:r.get::<_,i64>(8)? as u64})).map_err(sql)?; + Ok(session) +} + +#[derive(Clone, Default)] +pub(crate) struct UsageMeter(Option>); +impl std::fmt::Debug for UsageMeter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("UsageMeter") + } +} +#[derive(Clone, Copy)] +struct MeterValues { + usage: Option<(u64, u64)>, + outcome: &'static str, + incomplete: bool, +} +struct MeterInner { + db: DbActor, + id: String, + values: Mutex, +} +impl UsageMeter { + pub async fn begin( + db: Option<&DbActor>, + token: &str, + instance: &str, + ) -> Result { + let Some(db) = db else { + return Ok(Self::default()); + }; + let id = ulid::Ulid::new().to_string(); + match db.begin_usage(token, &id, instance).await { + Ok(true) => Ok(Self(Some(Arc::new(MeterInner { + db: db.clone(), + id, + values: Mutex::new(MeterValues { + usage: None, + outcome: "interrupted", + incomplete: false, + }), + })))), + Ok(false) => Ok(Self::default()), + Err(error) => Err(error), + } + } + pub fn observe(&self, event: &super::upstream::GenerationEvent) { + if let Some(inner) = &self.0 + && let Ok(mut values) = inner.values.lock() + { + match event { + super::upstream::GenerationEvent::Usage { + prompt_tokens, + completion_tokens, + } => values.usage = Some((*prompt_tokens, *completion_tokens)), + super::upstream::GenerationEvent::Done => values.outcome = "succeeded", + _ => (), + } + } + } + pub fn json(&self, bytes: &[u8], success: bool) { + if let Some(inner) = &self.0 + && let Ok(mut values) = inner.values.lock() + { + values.outcome = if success { "succeeded" } else { "failed" }; + if let Ok(document) = serde_json::from_slice::(bytes) + && let Some(usage) = document.get("usage") + { + values.usage = usage + .get("prompt_tokens") + .or_else(|| usage.get("input_tokens")) + .and_then(|v| v.as_u64()) + .zip( + usage + .get("completion_tokens") + .or_else(|| usage.get("output_tokens")) + .and_then(|v| v.as_u64()), + ); + } + } + } + pub fn raw(&self, bytes: &[u8]) { + // RawResponseStream frames events; SSE permits data: with no space and + // JSON spread across several data lines, with LF or CRLF separators. + let data = bytes + .split(|b| *b == b'\n') + .filter_map(|line| { + line.strip_suffix(b"\r") + .unwrap_or(line) + .strip_prefix(b"data:") + }) + .map(|line| line.strip_prefix(b" ").unwrap_or(line)) + .collect::>() + .join(&b'\n'); + if let Ok(document) = serde_json::from_slice::(&data) { + let response = document.get("response").unwrap_or(&document); + let terminal = matches!( + document.get("type").and_then(|v| v.as_str()), + Some("response.completed" | "response.incomplete" | "response.failed") + ); + let success = + document.get("type").and_then(|v| v.as_str()) == Some("response.completed"); + if terminal || response.get("usage").is_some() { + self.json(&serde_json::to_vec(response).unwrap_or_default(), success); + } + } + } +} +impl Drop for MeterInner { + fn drop(&mut self) { + let mut values = self.values.lock().map(|v| *v).unwrap_or(MeterValues { + usage: None, + outcome: "interrupted", + incomplete: true, + }); + values.incomplete |= values.outcome != "succeeded"; + let db = self.db.clone(); + let id = self.id.clone(); + tokio::spawn(async move { + if db + .record_usage(&id, values.usage, values.outcome, values.incomplete) + .await + .is_err() + { + tracing::warn!("session usage could not be recorded"); + } + }); + } +} + +impl super::upstream::UsageObserver for UsageMeter { + fn observe(&self, event: &super::upstream::GenerationEvent) { + UsageMeter::observe(self, event); + } + fn json(&self, bytes: &[u8], success: bool) { + UsageMeter::json(self, bytes, success); + } + fn raw(&self, bytes: &[u8]) { + UsageMeter::raw(self, bytes); + } + fn incomplete(&self) { + if let Some(inner) = &self.0 + && let Ok(mut values) = inner.values.lock() + { + values.incomplete = true; + } + } +} diff --git a/src/spark/state.rs b/src/spark/state.rs index daf8e2a..9aefe4d 100644 --- a/src/spark/state.rs +++ b/src/spark/state.rs @@ -297,6 +297,7 @@ enum Command { reply: oneshot::Sender, StateError>>, }, Snapshot(oneshot::Sender>), + Sessions(super::sessions::Command), Shutdown(std_mpsc::SyncSender<()>), } @@ -308,6 +309,15 @@ pub struct DbActor { } impl DbActor { + pub(crate) fn session_command( + &self, + command: super::sessions::Command, + ) -> Result<(), StateError> { + self.sender + .try_send(Command::Sessions(command)) + .map_err(|_| StateError::Overloaded) + } + pub fn open( path: impl AsRef, backup_dir: impl AsRef, @@ -690,7 +700,7 @@ fn open_connection( let version: u32 = connection .pragma_query_value(None, "user_version", |row| row.get(0)) .map_err(state_sql)?; - if version > 0 && version < 4 { + if version > 0 && (version < 4 || super::sessions::migration_needed(&connection)?) { verified_backup(&connection, backup_dir, max_backups)?; } let migrations = Migrations::new(vec![ @@ -705,6 +715,9 @@ fn open_connection( migrations .to_latest(&mut connection) .map_err(state_migration)?; + // Additive launch tables have a separate version ledger. Older releases + // still read the qualified engine state under the unchanged main schema. + super::sessions::migrate(&mut connection)?; verified_backup(&connection, backup_dir, max_backups)?; Ok(connection) } @@ -777,6 +790,9 @@ fn actor_loop( ) { while let Some(command) = receiver.blocking_recv() { match command { + Command::Sessions(command) => { + super::sessions::dispatch(&mut connection, &pepper, command) + } Command::Health(reply) => { let _ = reply.send(database_health(&connection)); } @@ -1588,7 +1604,10 @@ fn list_instances(connection: &Connection) -> Result, Stat .collect() } -fn read_instance(connection: &Connection, reference: &str) -> Result { +pub(crate) fn read_instance( + connection: &Connection, + reference: &str, +) -> Result { let values = { let mut statement = connection .prepare("SELECT metadata_json FROM instances WHERE id=?1 OR name=?1 LIMIT 2") @@ -2341,7 +2360,11 @@ fn valid_cidr(value: &str) -> bool { prefix <= if address.is_ipv4() { 32 } else { 128 } } -fn token_hmac(pepper: &SecretString, id: &str, secret: &str) -> Result<[u8; 32], StateError> { +pub(crate) fn token_hmac( + pepper: &SecretString, + id: &str, + secret: &str, +) -> Result<[u8; 32], StateError> { let mut mac = HmacSha256::new_from_slice(pepper.expose_secret().as_bytes()) .map_err(|_| StateError::Unavailable("token pepper is invalid".into()))?; mac.update(id.as_bytes()); @@ -2349,7 +2372,7 @@ fn token_hmac(pepper: &SecretString, id: &str, secret: &str) -> Result<[u8; 32], Ok(mac.finalize().into_bytes().into()) } -fn random_secret() -> Result { +pub(crate) fn random_secret() -> Result { use std::io::Read; let mut bytes = [0_u8; 32]; fs::File::open("/dev/urandom") @@ -2381,7 +2404,7 @@ fn parse_state(value: &str) -> Result { } } -fn now() -> String { +pub(crate) fn now() -> String { chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true) } fn json(value: &T) -> Result { @@ -2501,6 +2524,123 @@ mod tests { actor.shutdown().unwrap(); } + #[tokio::test] + async fn launch_sessions_isolate_usage_and_finish_idempotently() { + use crate::spark::{ + sessions::UsageMeter, upstream::GenerationEvent, wire::LaunchSessionRequest, + }; + let root = tempfile::tempdir().unwrap(); + let db = actor(root.path()); + let model = model_document("m_0123456789abcdef0123456789abcdef", "ornith:9b"); + db.promote_model(model.clone(), false).await.unwrap(); + db.begin_serve(instance_document(&model, "ornith", "spark-vllm")) + .await + .unwrap(); + let request = LaunchSessionRequest { + id: ulid::Ulid::new().to_string(), + instance: "ornith".into(), + integration: "codex".into(), + eco_mode: "max".into(), + }; + let created = db + .create_launch_session("admin", request.clone()) + .await + .unwrap(); + assert!(created.bearer_token.is_some()); + assert!( + db.create_launch_session("admin", request.clone()) + .await + .unwrap() + .bearer_token + .is_none() + ); + let mut conflict = request.clone(); + conflict.eco_mode = "none".into(); + assert!(matches!( + db.create_launch_session("admin", conflict).await, + Err(StateError::Conflict(_)) + )); + let token = created.bearer_token.unwrap(); + let token_id = token + .strip_prefix("sparkplane_") + .unwrap() + .split('_') + .next() + .unwrap(); + let meter = UsageMeter::begin(Some(&db), token_id, "ornith") + .await + .unwrap(); + meter.observe(&GenerationEvent::Usage { + prompt_tokens: 272001, + completion_tokens: 4, + }); + meter.observe(&GenerationEvent::Usage { + prompt_tokens: 272001, + completion_tokens: 4, + }); + meter.observe(&GenerationEvent::Done); + drop(meter); + let missing = UsageMeter::begin(Some(&db), token_id, "ornith") + .await + .unwrap(); + drop(missing); + for _ in 0..100 { + if db + .launch_session(&request.id, false, "") + .await + .unwrap() + .usage + .pending_requests + == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + } + let finished = db.launch_session(&request.id, true, "admin").await.unwrap(); + assert_eq!(finished.usage.requests, 2); + assert_eq!(finished.usage.input_tokens, 272001); + assert_eq!(finished.usage.output_tokens, 4); + assert_eq!(finished.usage.long_context_input_tokens, 272001); + assert_eq!(finished.usage.unknown_usage_requests, 1); + assert_eq!(finished.usage.pending_requests, 0); + assert_eq!( + finished.finished_at, + db.launch_session(&request.id, true, "admin") + .await + .unwrap() + .finished_at + ); + assert!( + !db.auth_snapshot() + .await + .unwrap() + .tokens + .contains_key(token_id) + ); + db.shutdown().unwrap(); + // The previous release can still open its schema and read the exact + // qualified instance after the additive feature migration. + let mut legacy = Connection::open(root.path().join("state.sqlite3")).unwrap(); + Migrations::new(vec![ + M::up(MIGRATION_1), + M::up(MIGRATION_2), + M::up(MIGRATION_3), + M::up(MIGRATION_4), + M::up(MIGRATION_5), + M::up(MIGRATION_6), + ]) + .to_latest(&mut legacy) + .unwrap(); + let retained = read_instance(&legacy, "ornith").unwrap(); + assert_eq!(retained.context_window, 65536); + assert_eq!(retained.model, model.canonical); + assert_eq!( + retained.engine_fingerprint, + format!("sha256:{}", "b".repeat(64)) + ); + } + #[test] fn migrations_are_valid_checksummed_and_n_minus_one_readable() { assert_eq!( diff --git a/src/spark/upstream.rs b/src/spark/upstream.rs index 0c3bf84..1bbeb5a 100644 --- a/src/spark/upstream.rs +++ b/src/spark/upstream.rs @@ -8,7 +8,7 @@ use http_body_util::{BodyExt, Full, Limited}; use hyper::{Request, client::conn::http1, header}; use hyper_util::rt::TokioIo; -pub const MAX_BODY_BYTES: usize = 1024 * 1024; +pub const MAX_BODY_BYTES: usize = 8 * 1024 * 1024; const MAX_RESPONSE_BYTES: usize = 1024 * 1024; const MAX_ROUTE_COUNT: usize = 32; const MAX_PATH_BYTES: usize = 256; @@ -22,23 +22,107 @@ pub const DEFAULT_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(120); const PROTOCOL_TOOL_MAX_TOKENS: u32 = 256; #[derive(Debug, Clone, PartialEq, Eq)] -pub struct UpstreamError(&'static str); +pub struct UpstreamError { + message: &'static str, + rejection: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RequestRejection { + InvalidRequest, + ContextLength, + PayloadTooLarge, + RateLimited, +} + +impl RequestRejection { + pub const fn status(self) -> u16 { + match self { + Self::InvalidRequest | Self::ContextLength => 400, + Self::PayloadTooLarge => 413, + Self::RateLimited => 429, + } + } + pub const fn code(self) -> &'static str { + match self { + Self::InvalidRequest => "invalid_request_error", + Self::ContextLength => "context_length_exceeded", + Self::PayloadTooLarge => "request_too_large", + Self::RateLimited => "rate_limit_error", + } + } + pub const fn message(self) -> &'static str { + match self { + Self::InvalidRequest => "engine rejected the inference request", + Self::ContextLength => { + "request exceeds the model context window; compact the conversation or reduce requested output" + } + Self::PayloadTooLarge => "request exceeds the engine body limit", + Self::RateLimited => "engine request capacity is exhausted; retry shortly", + } + } + pub fn from_response(status: u16, bytes: &[u8]) -> Option { + match status { + 400 | 422 => { + let value = (bytes.len() <= 16 * 1024) + .then(|| serde_json::from_slice::(bytes).ok()) + .flatten(); + let error = value + .as_ref() + .and_then(|value| value.get("error")) + .or(value.as_ref()); + let context = error.is_some_and(|error| { + error["code"] == "context_length_exceeded" + || error["type"] == "context_length_exceeded" + || error["message"].as_str().is_some_and(|message| { + let message = message.to_ascii_lowercase(); + message.contains("maximum context length") + || message.contains("max_model_len") + }) + }); + Some(if context { + Self::ContextLength + } else { + Self::InvalidRequest + }) + } + 413 => Some(Self::PayloadTooLarge), + 429 => Some(Self::RateLimited), + _ => None, + } + } +} impl std::fmt::Display for UpstreamError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(self.0) + formatter.write_str(self.message) } } impl std::error::Error for UpstreamError {} impl UpstreamError { + const fn new(message: &'static str) -> Self { + Self { + message, + rejection: None, + } + } + fn rejected(rejection: RequestRejection) -> Self { + Self { + message: rejection.message(), + rejection: Some(rejection), + } + } + pub const fn request_rejection(&self) -> Option { + self.rejection + } pub const fn identity_mismatch() -> Self { - Self("engine route identity changed") + Self::new("engine route identity changed") } pub const fn diagnostic(&self) -> &'static str { - self.0 + self.message } } @@ -95,23 +179,23 @@ pub fn decode_embedding_response( tolerance_ppm: u32, ) -> Result { if bytes.len() > MAX_RESPONSE_BYTES { - return Err(UpstreamError("engine embedding response is too large")); + return Err(UpstreamError::new("engine embedding response is too large")); } let value: serde_json::Value = serde_json::from_slice(bytes) - .map_err(|_| UpstreamError("engine embedding response is invalid"))?; + .map_err(|_| UpstreamError::new("engine embedding response is invalid"))?; let data = value["data"] .as_array() .filter(|data| data.len() == expected_count) - .ok_or(UpstreamError("engine embedding count changed"))?; + .ok_or(UpstreamError::new("engine embedding count changed"))?; if value["object"] != "list" || value["model"] != served_model { - return Err(UpstreamError("engine embedding identity changed")); + return Err(UpstreamError::new("engine embedding identity changed")); } let mut vectors = Vec::with_capacity(data.len()); for (expected_index, item) in data.iter().enumerate() { let values = item["embedding"] .as_array() .filter(|values| values.len() == dimensions) - .ok_or(UpstreamError("engine embedding dimension changed"))? + .ok_or(UpstreamError::new("engine embedding dimension changed"))? .iter() .map(|value| { value @@ -119,13 +203,13 @@ pub fn decode_embedding_response( .filter(|value| value.is_finite()) .map(|value| value as f32) .filter(|value| value.is_finite()) - .ok_or(UpstreamError( + .ok_or(UpstreamError::new( "engine embedding contains a non-finite value", )) }) .collect::, _>>()?; if item["object"] != "embedding" || item["index"].as_u64() != Some(expected_index as u64) { - return Err(UpstreamError("engine embedding order changed")); + return Err(UpstreamError::new("engine embedding order changed")); } let norm = values .iter() @@ -134,7 +218,7 @@ pub fn decode_embedding_response( .sqrt(); let tolerance = f64::from(tolerance_ppm) / 1_000_000.0; if normalized && (norm - 1.0).abs() > tolerance { - return Err(UpstreamError("engine embedding normalization changed")); + return Err(UpstreamError::new("engine embedding normalization changed")); } vectors.push(EmbeddingVector { index: expected_index, @@ -143,11 +227,11 @@ pub fn decode_embedding_response( } let prompt_tokens = value["usage"]["prompt_tokens"] .as_u64() - .ok_or(UpstreamError("engine embedding usage is invalid"))?; + .ok_or(UpstreamError::new("engine embedding usage is invalid"))?; let total_tokens = value["usage"]["total_tokens"] .as_u64() .filter(|total| *total == prompt_tokens) - .ok_or(UpstreamError("engine embedding usage is invalid"))?; + .ok_or(UpstreamError::new("engine embedding usage is invalid"))?; Ok(EmbeddingBatch { vectors, prompt_tokens, @@ -182,7 +266,15 @@ pub enum GenerationEvent { Done, } +pub trait UsageObserver: std::fmt::Debug + Send + Sync { + fn observe(&self, event: &GenerationEvent); + fn json(&self, bytes: &[u8], success: bool); + fn raw(&self, bytes: &[u8]); + fn incomplete(&self) {} +} + pub struct CompletionStream { + meter: Option>, receiver: tokio::sync::mpsc::Receiver>, task: tokio::task::JoinHandle<()>, connection_task: tokio::task::JoinHandle<()>, @@ -192,8 +284,22 @@ pub struct CompletionStream { impl CompletionStream { pub async fn next(&mut self) -> Option> { match tokio::time::timeout(self.idle_timeout, self.receiver.recv()).await { - Ok(event) => event, - Err(_) => Some(Err(UpstreamError("engine stream idle timeout"))), + Ok(event) => { + if let Some(meter) = &self.meter { + match &event { + Some(Ok(event)) => meter.observe(event), + Some(Err(_)) => meter.incomplete(), + None => (), + } + } + event + } + Err(_) => { + if let Some(meter) = &self.meter { + meter.incomplete(); + } + Some(Err(UpstreamError::new("engine stream idle timeout"))) + } } } } @@ -206,6 +312,9 @@ impl Drop for CompletionStream { } pub struct RawResponseStream { + meter_failed: bool, + pending_meter: Vec, + meter: Option>, receiver: tokio::sync::mpsc::Receiver>, task: tokio::task::JoinHandle<()>, connection_task: tokio::task::JoinHandle<()>, @@ -214,8 +323,49 @@ pub struct RawResponseStream { impl RawResponseStream { pub async fn next(&mut self) -> Option> { match tokio::time::timeout(DEFAULT_STREAM_IDLE_TIMEOUT, self.receiver.recv()).await { - Ok(chunk) => chunk, - Err(_) => Some(Err(UpstreamError("engine stream idle timeout"))), + Ok(chunk) => { + if let Some(meter) = &self.meter { + match &chunk { + Some(Ok(bytes)) if !self.meter_failed => { + if self.pending_meter.len().saturating_add(bytes.len()) + > MAX_RESPONSE_BYTES + { + self.pending_meter.clear(); + self.meter_failed = true; + meter.incomplete(); + } else { + self.pending_meter.extend_from_slice(bytes); + while let Some((end, size)) = self + .pending_meter + .windows(2) + .position(|v| v == b"\n\n") + .map(|i| (i, 2)) + .or_else(|| { + self.pending_meter + .windows(4) + .position(|v| v == b"\r\n\r\n") + .map(|i| (i, 4)) + }) + { + let event = + self.pending_meter.drain(..end + size).collect::>(); + meter.raw(&event); + } + } + } + Some(Err(_)) => meter.incomplete(), + None if !self.pending_meter.is_empty() => meter.incomplete(), + _ => (), + } + } + chunk + } + Err(_) => { + if let Some(meter) = &self.meter { + meter.incomplete(); + } + Some(Err(UpstreamError::new("engine stream idle timeout"))) + } } } } @@ -234,6 +384,7 @@ impl Drop for RawResponseStream { /// route after another executor inspection. #[derive(Debug, Clone)] pub struct ObservedRoute { + meter: Option>, instance_id: String, generation: u64, process_identity: Option<(u32, u64)>, @@ -244,6 +395,10 @@ pub struct ObservedRoute { } impl ObservedRoute { + pub fn with_meter(mut self, meter: Arc) -> Self { + self.meter = Some(meter); + self + } pub fn new<'a>( instance_id: &str, generation: u64, @@ -252,14 +407,16 @@ impl ObservedRoute { allowed: impl IntoIterator, ) -> Result { if instance_id.is_empty() || instance_id.len() > 96 || generation == 0 || port == 0 { - return Err(UpstreamError("invalid observed engine identity")); + return Err(UpstreamError::new("invalid observed engine identity")); } let address_allowed = match address { IpAddr::V4(value) => value.is_private(), IpAddr::V6(value) => value.is_unique_local(), } || cfg!(test) && address.is_loopback(); if !address_allowed { - return Err(UpstreamError("engine address is outside a private bridge")); + return Err(UpstreamError::new( + "engine address is outside a private bridge", + )); } let allowed = allowed .into_iter() @@ -271,7 +428,7 @@ impl ObservedRoute { .iter() .any(|(method, path)| !valid_method(method) || !valid_path(path)) { - return Err(UpstreamError("recipe route allowlist is invalid")); + return Err(UpstreamError::new("recipe route allowlist is invalid")); } Ok(Self { instance_id: instance_id.into(), @@ -281,6 +438,7 @@ impl ObservedRoute { port, allowed, connections: Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT_CONNECTIONS)), + meter: None, }) } @@ -300,11 +458,11 @@ impl ObservedRoute { body_bytes: usize, ) -> Result { if body_bytes > MAX_BODY_BYTES { - return Err(UpstreamError("engine request body is too large")); + return Err(UpstreamError::rejected(RequestRejection::PayloadTooLarge)); } let key = (method.to_owned(), path.to_owned()); if !self.allowed.contains(&key) { - return Err(UpstreamError("engine method or route is not allowed")); + return Err(UpstreamError::new("engine method or route is not allowed")); } Ok(UpstreamRequest { method: key.0, @@ -327,9 +485,15 @@ impl ObservedRoute { timeout: Duration, ) -> Result { self.request(&request.method, &request.path, body.len())?; - tokio::time::timeout(timeout, self.send_inner(request, body)) + let response = tokio::time::timeout(timeout, self.send_inner(request, body)) .await - .map_err(|_| UpstreamError("engine request timed out"))? + .map_err(|_| UpstreamError::new("engine request timed out"))?; + if let Ok(response) = &response + && let Some(meter) = &self.meter + { + meter.json(&response.bytes, (200..300).contains(&response.status)); + } + response } pub async fn semantic_probe( @@ -343,14 +507,14 @@ impl ObservedRoute { let models = self.send(&models, &[]).await?; let models_status = models.status; let models: serde_json::Value = serde_json::from_slice(&models.bytes) - .map_err(|_| UpstreamError("engine model identity response is invalid"))?; + .map_err(|_| UpstreamError::new("engine model identity response is invalid"))?; let identity_matches = models["data"].as_array().is_some_and(|models| { models .iter() .any(|model| model["id"].as_str() == Some(served_model)) }); if !(200..300).contains(&models_status) || !identity_matches { - return Err(UpstreamError("engine served-model identity mismatch")); + return Err(UpstreamError::new("engine served-model identity mismatch")); } let body = serde_json::to_vec(&serde_json::json!({ "model": served_model, @@ -360,16 +524,16 @@ impl ObservedRoute { "ignore_eos": true, "stream": false })) - .map_err(|_| UpstreamError("semantic probe cannot be encoded"))?; + .map_err(|_| UpstreamError::new("semantic probe cannot be encoded"))?; let completion = self.request("POST", "/v1/completions", body.len())?; let completion = self.send_with_timeout(&completion, &body, timeout).await?; let completion_status = completion.status; let completion: serde_json::Value = serde_json::from_slice(&completion.bytes) - .map_err(|_| UpstreamError("semantic probe response is invalid"))?; + .map_err(|_| UpstreamError::new("semantic probe response is invalid"))?; if !(200..300).contains(&completion_status) || !valid_semantic_completion(&completion, served_model, max_tokens) { - return Err(UpstreamError("semantic completion contract rejected")); + return Err(UpstreamError::new("semantic completion contract rejected")); } Ok(()) } @@ -386,7 +550,7 @@ impl ObservedRoute { let models = self.request("GET", "/v1/models", 0)?; let models = self.send(&models, &[]).await?; let identity: serde_json::Value = serde_json::from_slice(&models.bytes) - .map_err(|_| UpstreamError("engine model identity response is invalid"))?; + .map_err(|_| UpstreamError::new("engine model identity response is invalid"))?; if !(200..300).contains(&models.status) || !identity["data"].as_array().is_some_and(|models| { models @@ -394,16 +558,16 @@ impl ObservedRoute { .any(|model| model["id"].as_str() == Some(served_model)) }) { - return Err(UpstreamError("engine served-model identity mismatch")); + return Err(UpstreamError::new("engine served-model identity mismatch")); } let body = serde_json::to_vec(&serde_json::json!({ "model":served_model,"input":input,"encoding_format":"float" })) - .map_err(|_| UpstreamError("embedding probe cannot be encoded"))?; + .map_err(|_| UpstreamError::new("embedding probe cannot be encoded"))?; let request = self.request("POST", "/v1/embeddings", body.len())?; let response = self.send_with_timeout(&request, &body, timeout).await?; if !(200..300).contains(&response.status) { - return Err(UpstreamError("embedding probe was rejected")); + return Err(UpstreamError::new("embedding probe was rejected")); } decode_embedding_response( &response.bytes, @@ -432,7 +596,7 @@ impl ObservedRoute { let models = self.request("GET", "/v1/models", 0)?; let models = self.send(&models, &[]).await?; let identity: serde_json::Value = serde_json::from_slice(&models.bytes) - .map_err(|_| UpstreamError("engine model identity response is invalid"))?; + .map_err(|_| UpstreamError::new("engine model identity response is invalid"))?; if !(200..300).contains(&models.status) || !identity["data"].as_array().is_some_and(|models| { models @@ -440,7 +604,7 @@ impl ObservedRoute { .any(|model| model["id"].as_str() == Some(served_model)) }) { - return Err(UpstreamError("engine served-model identity mismatch")); + return Err(UpstreamError::new("engine served-model identity mismatch")); } let body = serde_json::to_vec(&serde_json::json!({ "model":served_model, @@ -453,41 +617,43 @@ impl ObservedRoute { "max_tokens":max_tokens,"thinking_budget_tokens":if disable_thinking { 0 } else { -1 },"temperature":0,"stream":false, "chat_template_kwargs":{"enable_thinking":!disable_thinking,"reasoning_strength":if disable_thinking { "low" } else { "high" }} })) - .map_err(|_| UpstreamError("vision probe cannot be encoded"))?; + .map_err(|_| UpstreamError::new("vision probe cannot be encoded"))?; let request = self.request("POST", "/v1/chat/completions", body.len())?; let response = self.send_with_timeout(&request, &body, timeout).await?; if !(200..300).contains(&response.status) { - return Err(UpstreamError(match response.status { + return Err(UpstreamError::new(match response.status { 400..=499 => "vision probe rejected with HTTP 4xx", 500..=599 => "vision probe rejected with HTTP 5xx", _ => "vision probe returned a non-success HTTP status", })); } let value: serde_json::Value = serde_json::from_slice(&response.bytes) - .map_err(|_| UpstreamError("vision probe response is invalid"))?; + .map_err(|_| UpstreamError::new("vision probe response is invalid"))?; if value["model"] != served_model { - return Err(UpstreamError("vision probe model identity changed")); + return Err(UpstreamError::new("vision probe model identity changed")); } let choice = value["choices"] .as_array() .filter(|choices| choices.len() == 1) .map(|choices| &choices[0]) - .ok_or(UpstreamError("vision probe choice shape is invalid"))?; + .ok_or(UpstreamError::new("vision probe choice shape is invalid"))?; let content = choice["message"]["content"] .as_str() - .ok_or(UpstreamError("vision probe final content is missing"))?; + .ok_or(UpstreamError::new("vision probe final content is missing"))?; if !content .to_lowercase() .contains(&expected_text.to_lowercase()) { - return Err(UpstreamError("vision probe expected answer is missing")); + return Err(UpstreamError::new( + "vision probe expected answer is missing", + )); } let prompt_tokens = value["usage"]["prompt_tokens"].as_u64(); let completion_tokens = value["usage"]["completion_tokens"].as_u64(); if prompt_tokens.is_none_or(|tokens| tokens == 0) || completion_tokens.is_none_or(|tokens| tokens == 0 || tokens > u64::from(max_tokens)) { - return Err(UpstreamError("vision probe usage contract is invalid")); + return Err(UpstreamError::new("vision probe usage contract is invalid")); } Ok(()) } @@ -503,7 +669,7 @@ impl ObservedRoute { self.protocol_probe_inner(served_model, require_tools), ) .await - .map_err(|_| UpstreamError("engine protocol probe timed out"))? + .map_err(|_| UpstreamError::new("engine protocol probe timed out"))? } async fn protocol_probe_inner( @@ -516,7 +682,7 @@ impl ObservedRoute { "max_tokens":PROTOCOL_TOOL_MAX_TOKENS,"temperature":0,"stream":true, "stream_options":{"include_usage":true},"chat_template_kwargs":{"enable_thinking":false,"reasoning_strength":"low"} })) - .map_err(|_| UpstreamError("protocol probe cannot be encoded"))?; + .map_err(|_| UpstreamError::new("protocol probe cannot be encoded"))?; validate_protocol_stream(self.chat_stream(&body).await?, false).await?; if require_tools { let body = serde_json::to_vec(&serde_json::json!({ @@ -529,7 +695,7 @@ impl ObservedRoute { "parallel_tool_calls":true,"max_tokens":PROTOCOL_TOOL_MAX_TOKENS, "temperature":0,"stream":true,"stream_options":{"include_usage":true},"chat_template_kwargs":{"enable_thinking":false,"reasoning_strength":"low"} })) - .map_err(|_| UpstreamError("tool protocol probe cannot be encoded"))?; + .map_err(|_| UpstreamError::new("tool protocol probe cannot be encoded"))?; validate_protocol_stream(self.chat_stream(&body).await?, true).await?; } Ok(()) @@ -563,13 +729,13 @@ impl ObservedRoute { let permit = Arc::clone(&self.connections) .acquire_owned() .await - .map_err(|_| UpstreamError("engine connection pool is closed"))?; + .map_err(|_| UpstreamError::new("engine connection pool is closed"))?; let stream = tokio::net::TcpStream::connect((self.address, self.port)) .await - .map_err(|_| UpstreamError("engine endpoint is unreachable"))?; + .map_err(|_| UpstreamError::new("engine endpoint is unreachable"))?; let (mut sender, connection) = http1::handshake(TokioIo::new(stream)) .await - .map_err(|_| UpstreamError("engine HTTP handshake failed"))?; + .map_err(|_| UpstreamError::new("engine HTTP handshake failed"))?; let connection_task = tokio::spawn(async move { let _ = connection.await; }); @@ -579,14 +745,15 @@ impl ObservedRoute { .header(header::HOST, "engine.internal") .header(header::CONTENT_TYPE, "application/json") .body(Full::new(Bytes::copy_from_slice(body))) - .map_err(|_| UpstreamError("engine request is malformed"))?; + .map_err(|_| UpstreamError::new("engine request is malformed"))?; let response = sender .send_request(outbound) .await - .map_err(|_| UpstreamError("engine response failed"))?; + .map_err(|_| UpstreamError::new("engine response failed"))?; if !(200..300).contains(&response.status().as_u16()) { + let error = rejected_stream_response(response).await; connection_task.abort(); - return Err(UpstreamError("engine rejected response stream")); + return Err(error); } let (send, receiver) = tokio::sync::mpsc::channel(STREAM_CHANNEL_CAPACITY); let mut incoming = response.into_body(); @@ -594,7 +761,9 @@ impl ObservedRoute { let _permit = permit; while let Some(frame) = incoming.frame().await { let Ok(frame) = frame else { - let _ = send.send(Err(UpstreamError("engine stream failed"))).await; + let _ = send + .send(Err(UpstreamError::new("engine stream failed"))) + .await; break; }; let Ok(data) = frame.into_data() else { @@ -608,6 +777,9 @@ impl ObservedRoute { } }); Ok(RawResponseStream { + meter_failed: false, + pending_meter: Vec::new(), + meter: self.meter.clone(), receiver, task, connection_task, @@ -624,13 +796,13 @@ impl ObservedRoute { let permit = Arc::clone(&self.connections) .acquire_owned() .await - .map_err(|_| UpstreamError("engine connection pool is closed"))?; + .map_err(|_| UpstreamError::new("engine connection pool is closed"))?; let stream = tokio::net::TcpStream::connect((self.address, self.port)) .await - .map_err(|_| UpstreamError("engine endpoint is unreachable"))?; + .map_err(|_| UpstreamError::new("engine endpoint is unreachable"))?; let (mut sender, connection) = http1::handshake(TokioIo::new(stream)) .await - .map_err(|_| UpstreamError("engine HTTP handshake failed"))?; + .map_err(|_| UpstreamError::new("engine HTTP handshake failed"))?; let connection_task = tokio::spawn(async move { let _ = connection.await; }); @@ -640,13 +812,15 @@ impl ObservedRoute { .header(header::HOST, "engine.internal") .header(header::CONTENT_TYPE, "application/json") .body(Full::new(Bytes::copy_from_slice(body))) - .map_err(|_| UpstreamError("engine request is malformed"))?; + .map_err(|_| UpstreamError::new("engine request is malformed"))?; let response = sender .send_request(outbound) .await - .map_err(|_| UpstreamError("engine response failed"))?; + .map_err(|_| UpstreamError::new("engine response failed"))?; if !(200..300).contains(&response.status().as_u16()) { - return Err(UpstreamError("engine rejected completion stream")); + let error = rejected_stream_response(response).await; + connection_task.abort(); + return Err(error); } let (send, receiver) = tokio::sync::mpsc::channel(STREAM_CHANNEL_CAPACITY); let mut incoming = response.into_body(); @@ -661,7 +835,7 @@ impl ObservedRoute { pending.extend_from_slice(&data); if pending.len() > MAX_STREAM_EVENT_BYTES { let _ = send - .send(Err(UpstreamError("engine stream event is too large"))) + .send(Err(UpstreamError::new("engine stream event is too large"))) .await; break; } @@ -681,6 +855,7 @@ impl ObservedRoute { task, connection_task, idle_timeout, + meter: self.meter.clone(), }) } @@ -693,13 +868,13 @@ impl ObservedRoute { .connections .acquire() .await - .map_err(|_| UpstreamError("engine connection pool is closed"))?; + .map_err(|_| UpstreamError::new("engine connection pool is closed"))?; let stream = tokio::net::TcpStream::connect((self.address, self.port)) .await - .map_err(|_| UpstreamError("engine endpoint is unreachable"))?; + .map_err(|_| UpstreamError::new("engine endpoint is unreachable"))?; let (mut sender, connection) = http1::handshake(TokioIo::new(stream)) .await - .map_err(|_| UpstreamError("engine HTTP handshake failed"))?; + .map_err(|_| UpstreamError::new("engine HTTP handshake failed"))?; tokio::spawn(async move { let _ = connection.await; }); @@ -709,16 +884,16 @@ impl ObservedRoute { .header(header::HOST, "engine.internal") .header(header::CONTENT_TYPE, "application/json") .body(Full::new(Bytes::copy_from_slice(body))) - .map_err(|_| UpstreamError("engine request is malformed"))?; + .map_err(|_| UpstreamError::new("engine request is malformed"))?; let response = sender .send_request(outbound) .await - .map_err(|_| UpstreamError("engine response failed"))?; + .map_err(|_| UpstreamError::new("engine response failed"))?; let status = response.status().as_u16(); let bytes = Limited::new(response.into_body(), MAX_RESPONSE_BYTES) .collect() .await - .map_err(|_| UpstreamError("engine response is too large"))? + .map_err(|_| UpstreamError::new("engine response is too large"))? .to_bytes() .to_vec(); Ok(UpstreamResponse { status, bytes }) @@ -729,6 +904,28 @@ impl ObservedRoute { } } +async fn rejected_stream_response( + response: hyper::Response, +) -> UpstreamError { + let status = response.status().as_u16(); + let bytes = if matches!(status, 400 | 422) { + match tokio::time::timeout( + Duration::from_secs(2), + Limited::new(response.into_body(), 16 * 1024).collect(), + ) + .await + { + Ok(Ok(body)) => body.to_bytes(), + _ => Bytes::new(), + } + } else { + Bytes::new() + }; + RequestRejection::from_response(status, &bytes) + .map(UpstreamError::rejected) + .unwrap_or_else(|| UpstreamError::new("engine rejected response stream")) +} + async fn validate_protocol_stream( mut stream: CompletionStream, require_tool: bool, @@ -741,7 +938,7 @@ async fn validate_protocol_stream( match event? { GenerationEvent::ReasoningDelta { text } | GenerationEvent::TextDelta { text } => { if finish_reason.is_some() || text.is_empty() { - return Err(UpstreamError("engine protocol event order is invalid")); + return Err(UpstreamError::new("engine protocol event order is invalid")); } output = true; } @@ -752,7 +949,7 @@ async fn validate_protocol_stream( arguments, } => { if finish_reason.is_some() || !require_tool { - return Err(UpstreamError("engine protocol tool event is invalid")); + return Err(UpstreamError::new("engine protocol tool event is invalid")); } let tool = tools.entry(index).or_default(); if let Some(call_id) = call_id { @@ -767,7 +964,9 @@ async fn validate_protocol_stream( finish_reason: reason, } => { if finish_reason.is_some() || reason.is_none() { - return Err(UpstreamError("engine protocol finish event is invalid")); + return Err(UpstreamError::new( + "engine protocol finish event is invalid", + )); } finish_reason = reason; } @@ -777,7 +976,7 @@ async fn validate_protocol_stream( } => { if finish_reason.is_none() || usage || prompt_tokens == 0 || completion_tokens == 0 { - return Err(UpstreamError("engine protocol usage event is invalid")); + return Err(UpstreamError::new("engine protocol usage event is invalid")); } usage = true; } @@ -798,11 +997,13 @@ async fn validate_protocol_stream( { return Ok(()); } - return Err(UpstreamError("engine protocol stream is incomplete")); + return Err(UpstreamError::new("engine protocol stream is incomplete")); } } } - Err(UpstreamError("engine protocol stream ended before done")) + Err(UpstreamError::new( + "engine protocol stream ended before done", + )) } fn valid_semantic_completion( @@ -855,10 +1056,10 @@ fn decode_sse_events(bytes: &[u8]) -> Vec } let value: serde_json::Value = match serde_json::from_str(data) { Ok(value) => value, - Err(_) => return vec![Err(UpstreamError("engine stream event is invalid"))], + Err(_) => return vec![Err(UpstreamError::new("engine stream event is invalid"))], }; if value.get("error").is_some() { - return vec![Err(UpstreamError("engine stream returned an error"))]; + return vec![Err(UpstreamError::new("engine stream returned an error"))]; } let mut events = Vec::new(); let choice = value["choices"] @@ -886,7 +1087,7 @@ fn decode_sse_events(bytes: &[u8]) -> Vec .as_u64() .and_then(|value| value.try_into().ok()) else { - return vec![Err(UpstreamError("engine tool call index is invalid"))]; + return vec![Err(UpstreamError::new("engine tool call index is invalid"))]; }; events.push(Ok(GenerationEvent::ToolCallDelta { index, @@ -907,10 +1108,10 @@ fn decode_sse_events(bytes: &[u8]) -> Vec } if let Some(usage) = value.get("usage").filter(|usage| !usage.is_null()) { let Some(prompt_tokens) = usage["prompt_tokens"].as_u64() else { - return vec![Err(UpstreamError("engine stream usage is invalid"))]; + return vec![Err(UpstreamError::new("engine stream usage is invalid"))]; }; let Some(completion_tokens) = usage["completion_tokens"].as_u64() else { - return vec![Err(UpstreamError("engine stream usage is invalid"))]; + return vec![Err(UpstreamError::new("engine stream usage is invalid"))]; }; events.push(Ok(GenerationEvent::Usage { prompt_tokens, @@ -1264,6 +1465,7 @@ mod tests { }); let connection_task = tokio::spawn(std::future::pending()); let stream = CompletionStream { + meter: None, receiver, task, connection_task, @@ -1287,6 +1489,7 @@ mod tests { }); let connection_task = tokio::spawn(std::future::pending()); let mut stream = CompletionStream { + meter: None, receiver, task, connection_task, diff --git a/src/spark/wire.rs b/src/spark/wire.rs index 911aebc..2cf1b91 100644 --- a/src/spark/wire.rs +++ b/src/spark/wire.rs @@ -1356,3 +1356,50 @@ mod tests { ); } } +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] +pub struct LaunchSessionRequest { + pub id: String, + pub instance: String, + pub integration: String, + pub eco_mode: String, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] +pub struct SessionUsage { + pub requests: u64, + pub input_tokens: u64, + pub output_tokens: u64, + pub unknown_usage_requests: u64, + pub pending_requests: u64, + pub failed_requests: u64, + #[serde(default)] + pub long_context_input_tokens: u64, + #[serde(default)] + pub long_context_output_tokens: u64, + #[serde(default)] + pub accounting_errors: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] +pub struct LaunchSessionDocument { + pub schema: String, + pub id: String, + pub instance: String, + pub model: String, + pub integration: String, + pub eco_mode: String, + pub started_at: String, + pub finished_at: Option, + pub usage: SessionUsage, +} + +#[derive(Serialize, Deserialize)] +#[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] +pub struct LaunchSessionCreated { + pub session: LaunchSessionDocument, + pub bearer_token: Option, +} diff --git a/tests/eco_launch.rs b/tests/eco_launch.rs new file mode 100644 index 0000000..b65c32d --- /dev/null +++ b/tests/eco_launch.rs @@ -0,0 +1,205 @@ +use std::{ + io::Write, + os::unix::fs::PermissionsExt, + process::{Command, Stdio}, +}; + +#[test] +fn eco_mode_is_default_and_can_be_disabled() { + let help = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["fixture", "launch", "codex", "--help"]) + .output() + .unwrap(); + let help = String::from_utf8(help.stdout).unwrap(); + assert!(help.contains("--eco-mode")); + assert!(help.contains("[default: max]")); + let rejected = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["fixture", "launch", "codex", "--eco-mode=invalid"]) + .output() + .unwrap(); + assert_eq!(rejected.status.code(), Some(2)); +} + +#[test] +fn embedded_rewrite_preserves_shell_and_structured_output() { + for command in [ + "git status --porcelain", + "git status > result", + "git status | cat", + "echo $(git status)", + "cargo test --message-format=json", + "printf '%s' '$secret'", + ] { + let output = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["__eco", "rewrite", command]) + .output() + .unwrap(); + assert!(output.status.success()); + assert!(output.stdout.is_empty(), "{command}"); + } + let output = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["__eco", "rewrite", "git status"]) + .output() + .unwrap(); + assert!(output.status.success()); + assert!( + String::from_utf8(output.stdout) + .unwrap() + .contains("__eco exec") + ); +} + +#[test] +fn hooks_compress_only_supported_calls_and_collect_private_metrics() { + let root = tempfile::tempdir().unwrap(); + let bin = root.path().join("bin"); + std::fs::create_dir(&bin).unwrap(); + let cargo = bin.join("cargo"); + std::fs::write(&cargo,"#!/bin/sh\nprintf 'running 2 tests\\ntest first ... ok\\ntest second ... ok\\ntest result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s\\n'\nexit 0\n").unwrap(); + std::fs::set_permissions(&cargo, std::fs::Permissions::from_mode(0o700)).unwrap(); + let metrics = root.path().join("metrics"); + let binary = env!("CARGO_BIN_EXE_sparkplane"); + let mut hook = Command::new(binary) + .args(["__eco", "hook", "codex"]) + .env("SPARKPLANE_ECO_REWRITE_ENABLED", "1") + .env("SPARKPLANE_ECO_SESSION_DIR", &metrics) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + hook.stdin.take().unwrap().write_all(br#"{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"cargo test"}}"#).unwrap(); + let output = hook.wait_with_output().unwrap(); + let output: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let rewritten = output["hookSpecificOutput"]["updatedInput"]["command"] + .as_str() + .unwrap(); + let filtered = Command::new("/bin/sh") + .args(["-c", rewritten]) + .env("PATH", &bin) + .env("SPARKPLANE_ECO_SESSION_DIR", &metrics) + .output() + .unwrap(); + assert!(filtered.status.success()); + let raw = Command::new(binary) + .args(["__eco", "raw", "--", "cargo", "test"]) + .env("PATH", &bin) + .output() + .unwrap(); + assert!(raw.status.success()); + assert!(filtered.stdout.len() < raw.stdout.len()); + assert!(String::from_utf8_lossy(&filtered.stdout).contains("2 passed")); + assert_eq!( + std::fs::metadata(&metrics).unwrap().permissions().mode() & 0o777, + 0o700 + ); + for file in std::fs::read_dir(&metrics).unwrap() { + let path = file.unwrap().path(); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let record = std::fs::read_to_string(path).unwrap(); + assert!(!record.contains("first")); + assert!(!record.contains("cargo test")); + } + std::fs::write(&cargo,"#!/bin/sh\nprintf 'full stdout diagnostic\\n'\nprintf 'full stderr diagnostic\\n' >&2\nexit 7\n").unwrap(); + let failure = Command::new(binary) + .args(["__eco", "exec", "--", "cargo", "test"]) + .env("PATH", &bin) + .output() + .unwrap(); + assert_eq!(failure.status.code(), Some(7)); + assert_eq!(failure.stdout, b"full stdout diagnostic\n"); + assert_eq!(failure.stderr, b"full stderr diagnostic\n"); +} + +#[test] +fn disabled_hooks_emit_no_rewrite() { + let output = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["__eco", "rewrite", "git status"]) + .env("SPARKPLANE_ECO_REWRITE_ENABLED", "0") + .output() + .unwrap(); + assert!(output.status.success()); + assert!(output.stdout.is_empty()); +} + +#[test] +fn economics_json_is_a_retained_offline_report() { + use sparkplane::spark::economics::{self, Compression, Report}; + let root = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now().to_rfc3339(); + let mut report = Report { + schema: economics::SCHEMA.into(), + session_id: ulid::Ulid::new().to_string(), + host: "offline".into(), + model: "fixture".into(), + integration: "codex".into(), + eco_mode: "none".into(), + started_at: now.clone(), + finished_at: Some(now), + exit_code: Some(7), + inference: None, + compression: Compression { + status: "disabled".into(), + ..Default::default() + }, + catalog: economics::catalog().unwrap(), + comparisons: Vec::new(), + }; + report.calculate(); + economics::save(root.path(), &report).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["offline", "economics", "--json", "--config-dir"]) + .arg(root.path()) + .output() + .unwrap(); + assert!(output.status.success()); + assert!(output.stderr.is_empty()); + let value: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(value["session_id"], report.session_id); + assert_eq!(value["exit_code"], 7); + assert!(value["inference"].is_null()); + assert!(value["comparisons"][0]["inference_usd_nanos"].is_null()); +} + +#[test] +fn wrapper_forwards_termination_and_preserves_signal_exit_status() { + let root = tempfile::tempdir().unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args([ + "__eco", + "exec", + "--", + "sh", + "-c", + "printf ready > ready; exec sleep 30", + ]) + .current_dir(root.path()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !root.path().join("ready").exists() { + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + nix::sys::signal::kill( + nix::unistd::Pid::from_raw(child.id() as i32), + nix::sys::signal::Signal::SIGTERM, + ) + .unwrap(); + loop { + if let Some(status) = child.try_wait().unwrap() { + assert_eq!(status.code(), Some(143)); + break; + } + if std::time::Instant::now() >= deadline { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("eco child did not terminate"); + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } +} diff --git a/tests/gateway_image_upload.rs b/tests/gateway_image_upload.rs new file mode 100644 index 0000000..301260e --- /dev/null +++ b/tests/gateway_image_upload.rs @@ -0,0 +1,545 @@ +#![cfg(feature = "appliance")] + +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use image::{DynamicImage, ImageFormat, ImageReader, RgbImage}; +use sparkplane::spark::gateway::{self, GatewayProfile, ImageProtocol, IngressLimits}; + +fn profile() -> GatewayProfile { + let config: toml::Value = toml::from_str(include_str!( + "../configs/sparkplane/engines/vllm-qwen38-mmap.toml" + )) + .unwrap(); + let mut profile = GatewayProfile::text(); + profile.capabilities.insert("vision".into()); + profile.vision = Some(config["profiles"][0]["vision"].clone().try_into().unwrap()); + let vision = profile.vision.as_mut().unwrap(); + vision.max_width = 64; + vision.max_height = 32; + vision.max_bytes = 4096; + vision.max_total_bytes = 4096; + vision.max_count = 1; + profile +} + +fn png() -> Vec { + let mut seed = 0x12345678_u32; + let pixels = (0..1024 * 512 * 3) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed as u8 + }) + .collect(); + let image = DynamicImage::ImageRgb8(RgbImage::from_raw(1024, 512, pixels).unwrap()); + let mut out = std::io::Cursor::new(Vec::new()); + image.write_to(&mut out, ImageFormat::Png).unwrap(); + out.into_inner() +} + +fn responses(image: &[u8]) -> Vec { + serde_json::to_vec(&serde_json::json!({"input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"describe"},{"type":"input_image","image_url":format!("data:image/png;base64,{}",BASE64.encode(image)),"detail":"high"}]}],"stream":true,"store":false})).unwrap() +} + +fn check_image(url: &str, profile: &GatewayProfile) { + let (media, data) = url + .strip_prefix("data:") + .unwrap() + .split_once(";base64,") + .unwrap(); + let bytes = BASE64.decode(data).unwrap(); + let vision = profile.vision.as_ref().unwrap(); + assert!(vision.media_types.contains(media)); + assert!(bytes.len() <= vision.max_bytes); + let decoded = ImageReader::new(std::io::Cursor::new(&bytes)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + assert!(decoded.width() <= vision.max_width); + assert!(decoded.height() <= vision.max_height); + assert_eq!(decoded.width(), decoded.height() * 2); +} + +#[test] +fn large_uploads_fit_the_model_in_both_public_protocols() { + let image = png(); + let request = responses(&image); + assert!(request.len() > 1024 * 1024); + let profile = profile(); + let limits = IngressLimits::default(); + assert!(gateway::rewrite_responses_request_with_profile(&request, "served", &profile).is_err()); + let normalized = + gateway::normalize_image_request(&request, ImageProtocol::Responses, &profile, &limits) + .unwrap(); + let request = + gateway::rewrite_responses_request_with_profile(&normalized, "served", &profile).unwrap(); + let request: serde_json::Value = serde_json::from_slice(&request.body).unwrap(); + check_image( + request["messages"][0]["content"][1]["image_url"]["url"] + .as_str() + .unwrap(), + &profile, + ); + let request=serde_json::to_vec(&serde_json::json!({"model":"public","max_tokens":8,"messages":[{"role":"user","content":[{"type":"text","text":"describe"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":BASE64.encode(image)}}]}]})).unwrap(); + let normalized = + gateway::normalize_image_request(&request, ImageProtocol::Anthropic, &profile, &limits) + .unwrap(); + let request = + gateway::rewrite_anthropic_request_with_profile(&normalized, "served", &profile).unwrap(); + let request: serde_json::Value = serde_json::from_slice(&request.body).unwrap(); + check_image( + request["messages"][0]["content"][1]["image_url"]["url"] + .as_str() + .unwrap(), + &profile, + ); +} + +#[test] +fn native_responses_receives_normalized_images_and_preserves_other_fields() { + let mut profile = profile(); + profile.native_responses = true; + let normalized = gateway::normalize_image_request( + &responses(&png()), + ImageProtocol::Responses, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let request = + gateway::rewrite_native_responses_request_with_profile(&normalized, "public", &profile) + .unwrap(); + let request: serde_json::Value = serde_json::from_slice(&request.body).unwrap(); + assert_eq!(request["store"], false); + assert_eq!(request["input"][0]["content"][0]["text"], "describe"); + assert_eq!(request["input"][0]["content"][1]["detail"], "high"); + check_image( + request["input"][0]["content"][1]["image_url"] + .as_str() + .unwrap(), + &profile, + ); +} + +#[test] +fn upload_byte_pixel_and_model_image_count_limits_stay_finite() { + let request = responses(&png()); + let profile = profile(); + for limits in [ + IngressLimits { + request_body_bytes: 1024, + ..IngressLimits::default() + }, + IngressLimits { + image_bytes: 1024, + ..IngressLimits::default() + }, + IngressLimits { + image_pixels: 1024, + ..IngressLimits::default() + }, + IngressLimits { + image_dimension: 512, + ..IngressLimits::default() + }, + ] { + let error = + gateway::normalize_image_request(&request, ImageProtocol::Responses, &profile, &limits) + .unwrap_err(); + assert_eq!(error.code, "request_too_large"); + } + let mut request: serde_json::Value = serde_json::from_slice(&request).unwrap(); + let image = request["input"][0]["content"][1].clone(); + request["input"][0]["content"] + .as_array_mut() + .unwrap() + .push(image); + assert!( + gateway::normalize_image_request( + &serde_json::to_vec(&request).unwrap(), + ImageProtocol::Responses, + &profile, + &IngressLimits::default() + ) + .is_err() + ); +} + +#[test] +fn compatible_images_are_unchanged_and_resizing_preserves_transparency() { + let profile = profile(); + let image = DynamicImage::ImageRgb8(RgbImage::from_pixel(2, 1, image::Rgb([10, 20, 30]))); + let mut original = std::io::Cursor::new(Vec::new()); + image.write_to(&mut original, ImageFormat::Png).unwrap(); + let original = original.into_inner(); + let normalized = gateway::normalize_image_request( + &responses(&original), + ImageProtocol::Responses, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let normalized: serde_json::Value = serde_json::from_slice(&normalized).unwrap(); + assert_eq!( + normalized["input"][0]["content"][1]["image_url"], + format!("data:image/png;base64,{}", BASE64.encode(&original)) + ); + + let image = image::DynamicImage::ImageRgba8(image::RgbaImage::from_fn(128, 64, |_, _| { + image::Rgba([200, 30, 80, 0]) + })); + let mut png = std::io::Cursor::new(Vec::new()); + image.write_to(&mut png, ImageFormat::Png).unwrap(); + let normalized = gateway::normalize_image_request( + &responses(&png.into_inner()), + ImageProtocol::Responses, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let normalized: serde_json::Value = serde_json::from_slice(&normalized).unwrap(); + let url = normalized["input"][0]["content"][1]["image_url"] + .as_str() + .unwrap(); + check_image(url, &profile); + let bytes = BASE64 + .decode(url.split_once(";base64,").unwrap().1) + .unwrap(); + let decoded = ImageReader::new(std::io::Cursor::new(bytes)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + assert!(decoded.color().has_alpha()); + assert!(decoded.to_rgba8().pixels().all(|pixel| pixel[3] == 0)); +} + +#[test] +fn normalization_keeps_inline_only_and_verified_vision_requirements() { + let mut profile = profile(); + let mut request: serde_json::Value = serde_json::from_slice(&responses(&png())).unwrap(); + for invalid in [ + "https://example.com/picture.png", + "file:///tmp/picture.png", + "data:image/jpeg;base64,iVBORw0KGgo=", + ] { + request["input"][0]["content"][1]["image_url"] = invalid.into(); + assert!( + gateway::normalize_image_request( + &serde_json::to_vec(&request).unwrap(), + ImageProtocol::Responses, + &profile, + &IngressLimits::default() + ) + .is_err() + ); + } + profile.vision = None; + assert!( + gateway::normalize_image_request( + &responses(&png()), + ImageProtocol::Responses, + &profile, + &IngressLimits::default() + ) + .is_err() + ); +} + +#[test] +fn codex_view_image_tool_results_are_images_and_share_the_model_budget() { + let profile = profile(); + for (call, output) in [ + ("function_call", "function_call_output"), + ("custom_tool_call", "custom_tool_call_output"), + ] { + let image = serde_json::json!({"type":"input_image","image_url":format!("data:image/png;base64,{}",BASE64.encode(png())),"detail":"high"}); + let request = serde_json::json!({"input":[ + {"type":"message","role":"user","content":"inspect the screenshot"}, + {"type":call,"call_id":"call_view","name":"view_image","arguments":"{}","input":"image"}, + {"type":output,"call_id":"call_view","output":[{"type":"input_text","text":"Screenshot from the tool."},image]} + ],"stream":true}); + let normalized = gateway::normalize_image_request( + &serde_json::to_vec(&request).unwrap(), + ImageProtocol::Responses, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let rewritten = + gateway::rewrite_responses_request_with_profile(&normalized, "served", &profile) + .unwrap(); + let rewritten: serde_json::Value = serde_json::from_slice(&rewritten.body).unwrap(); + let tool = &rewritten["messages"][2]; + assert_eq!(tool["role"], "tool"); + assert_eq!(tool["tool_call_id"], "call_view"); + assert!( + tool["content"].is_array(), + "view_image output must not become base64 text" + ); + assert_eq!( + tool["content"][0], + serde_json::json!({"type":"text","text":"Screenshot from the tool."}) + ); + check_image( + tool["content"][1]["image_url"]["url"].as_str().unwrap(), + &profile, + ); + let mut native = profile.clone(); + native.native_responses = true; + let rewritten = + gateway::rewrite_native_responses_request_with_profile(&normalized, "public", &native) + .unwrap(); + let rewritten: serde_json::Value = serde_json::from_slice(&rewritten.body).unwrap(); + check_image( + rewritten["input"][2]["output"][1]["image_url"] + .as_str() + .unwrap(), + &native, + ); + let mut two_images = request.clone(); + two_images["input"][2]["output"] + .as_array_mut() + .unwrap() + .push(image); + assert!( + gateway::normalize_image_request( + &serde_json::to_vec(&two_images).unwrap(), + ImageProtocol::Responses, + &profile, + &IngressLimits::default() + ) + .is_err() + ); + assert!( + gateway::normalize_image_request( + &serde_json::to_vec(&request).unwrap(), + ImageProtocol::Responses, + &GatewayProfile::text(), + &IngressLimits::default() + ) + .is_err() + ); + } +} + +#[test] +fn successive_image_tool_results_reserve_bytes_for_every_frame() { + let config: toml::Value = toml::from_str(include_str!( + "../configs/sparkplane/engines/vllm-qwen38-mmap.toml" + )) + .unwrap(); + let mut profile = profile(); + profile.vision.as_mut().unwrap().max_count = config["profiles"][0]["vision"]["max_count"] + .as_integer() + .unwrap() as usize; + assert_eq!(profile.vision.as_ref().unwrap().max_count, 16); + let source = DynamicImage::ImageRgb8(RgbImage::from_fn(128, 64, |x, y| { + image::Rgb([ + (x * 31 + y * 17) as u8, + (x * 71 + y * 43) as u8, + (x * 19 + y * 83) as u8, + ]) + })); + let mut encoded = std::io::Cursor::new(Vec::new()); + source.write_to(&mut encoded, ImageFormat::Png).unwrap(); + let image = serde_json::json!({"type":"input_image","image_url":format!("data:image/png;base64,{}",BASE64.encode(encoded.into_inner())),"detail":"high"}); + for count in [2, 3, 16] { + let mut input = vec![ + serde_json::json!({"type":"message","role":"user","content":"compare the screenshots"}), + ]; + for index in 0..count { + let id = format!("view_{index}"); + input.push(serde_json::json!({"type":"function_call","name":"view_image","call_id":id,"arguments":"{}"})); + input.push(serde_json::json!({"type":"function_call_output","call_id":id,"output":[{"type":"input_text","text":format!("Frame {index}")},image]})); + if index + 1 < count { + input.push(serde_json::json!({"type":"message","role":"assistant","content":"Now inspect the next frame."})); + } + } + let request = + serde_json::to_vec(&serde_json::json!({"input":input,"stream":true})).unwrap(); + let normalized = gateway::normalize_image_request( + &request, + ImageProtocol::Responses, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let normalized: serde_json::Value = serde_json::from_slice(&normalized).unwrap(); + let mut total_bytes = 0; + let mut frames = 0; + let retained = if count == 16 { 10 } else { count }; + for (index, item) in normalized["input"].as_array().unwrap().iter().enumerate() { + if item["type"] != "function_call_output" { + continue; + } + assert_eq!( + item["output"][0]["text"], + format!("Frame {}", (index - 1) / 3) + ); + let Some(url) = item["output"][1]["image_url"].as_str() else { + continue; + }; + let bytes = BASE64 + .decode(url.split_once(";base64,").unwrap().1) + .unwrap(); + assert!(bytes.len() <= profile.vision.as_ref().unwrap().max_total_bytes / retained); + total_bytes += bytes.len(); + frames += 1; + } + assert_eq!(frames, retained); + assert!(total_bytes <= profile.vision.as_ref().unwrap().max_total_bytes); + let bytes = serde_json::to_vec(&normalized).unwrap(); + assert!( + gateway::rewrite_responses_request_with_profile(&bytes, "served", &profile).is_ok() + ); + let mut native = profile.clone(); + native.native_responses = true; + assert!( + gateway::rewrite_native_responses_request_with_profile(&bytes, "served", &native) + .is_ok() + ); + } +} + +#[test] +fn multiple_images_share_the_gateway_pixel_limit_and_keep_a_finite_count() { + let mut profile = profile(); + profile.vision.as_mut().unwrap().max_count = 16; + let source = DynamicImage::ImageRgb8(RgbImage::from_pixel(128, 64, image::Rgb([10, 20, 30]))); + let mut encoded = std::io::Cursor::new(Vec::new()); + source.write_to(&mut encoded, ImageFormat::Png).unwrap(); + let image = serde_json::json!({"type":"input_image","image_url":format!("data:image/png;base64,{}",BASE64.encode(encoded.into_inner()))}); + let make_request = |count| { + serde_json::to_vec(&serde_json::json!({"input":[{"role":"user","type":"message","content":vec![image.clone();count]}]})).unwrap() + }; + let limits = IngressLimits { + image_pixels: 128 * 64 + 1, + ..IngressLimits::default() + }; + assert!( + gateway::normalize_image_request( + &make_request(1), + ImageProtocol::Responses, + &profile, + &limits + ) + .is_ok() + ); + assert_eq!( + gateway::normalize_image_request( + &make_request(2), + ImageProtocol::Responses, + &profile, + &limits + ) + .unwrap_err() + .code, + "request_too_large" + ); + assert_eq!( + gateway::normalize_image_request( + &make_request(17), + ImageProtocol::Responses, + &profile, + &IngressLimits::default() + ) + .unwrap_err() + .message, + "image count exceeds the model profile limit" + ); +} + +#[test] +fn anthropic_nested_image_results_keep_calls_and_recent_frames() { + let mut profile = profile(); + profile.vision.as_mut().unwrap().max_count = 16; + let image = BASE64.encode(png()); + let mut messages = Vec::new(); + for index in 0..20 { + messages.push(serde_json::json!({"role":"assistant","content":[{"type":"text","text":"Inspect another frame."},{"type":"tool_use","id":format!("t{index}"),"name":"view_image","input":{"path":format!("/tmp/{index}.png")}}]})); + messages.push(serde_json::json!({"role":"user","content":[{"type":"tool_result","tool_use_id":format!("t{index}"),"content":[{"type":"text","text":format!("Frame {index}")},{"type":"image","source":{"type":"base64","media_type":"image/png","data":image}}]}]})); + } + let bytes = serde_json::to_vec( + &serde_json::json!({"model":"public","max_tokens":8,"messages":messages}), + ) + .unwrap(); + // Use small originals so the source body fits normal direct gateway ingress. + let source = gateway::normalize_image_request( + &bytes, + ImageProtocol::Anthropic, + &profile, + &IngressLimits::default(), + ) + .unwrap_err(); + assert_eq!(source.code, "request_too_large"); + let image = &profile.vision.as_ref().unwrap().health_image_base64; + let mut value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + for message in value["messages"].as_array_mut().unwrap() { + if message["role"] == "user" { + message["content"][0]["content"][1]["source"]["data"] = image.clone().into(); + } + } + let normalized = gateway::normalize_image_request( + &serde_json::to_vec(&value).unwrap(), + ImageProtocol::Anthropic, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let rewritten = + gateway::rewrite_anthropic_request_with_profile(&normalized, "served", &profile).unwrap(); + let rewritten: serde_json::Value = serde_json::from_slice(&rewritten.body).unwrap(); + let tools = rewritten["messages"] + .as_array() + .unwrap() + .iter() + .filter(|message| message["role"] == "tool") + .collect::>(); + assert_eq!(tools.len(), 20); + assert_eq!(tools.last().unwrap()["tool_call_id"], "t19"); + assert!( + tools[0]["content"] + .as_str() + .unwrap() + .contains("Older image") + ); + assert_eq!(tools[19]["content"][0]["text"], "Frame 19"); + assert_eq!( + tools + .iter() + .filter(|tool| tool["content"].is_array()) + .count(), + 8 + ); +} + +#[test] +fn chat_vision_uses_the_same_retention_and_finite_profile_budget() { + let mut profile = profile(); + profile.vision.as_mut().unwrap().max_count = 16; + let url = format!( + "data:image/png;base64,{}", + profile.vision.as_ref().unwrap().health_image_base64 + ); + let mut messages = Vec::new(); + for index in 0..30 { + messages.push(serde_json::json!({"role":"user","content":[{"type":"text","text":format!("Frame {index}")},{"type":"image_url","image_url":{"url":url}}]})); + if index < 29 { + messages.push(serde_json::json!({"role":"assistant","content":"Observed frame."})); + } + } + let normalized = gateway::normalize_image_request( + &serde_json::to_vec(&serde_json::json!({"messages":messages})).unwrap(), + ImageProtocol::Chat, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + assert!(gateway::rewrite_chat_request_with_profile(&normalized, "served", &profile).is_ok()); + assert!( + gateway::rewrite_chat_request_with_profile(&normalized, "served", &GatewayProfile::text()) + .is_err() + ); +} diff --git a/tests/image_history.rs b/tests/image_history.rs new file mode 100644 index 0000000..1275d6a --- /dev/null +++ b/tests/image_history.rs @@ -0,0 +1,104 @@ +use serde_json::{Value, json}; +use sparkplane::spark::image_history::{Protocol, project, project_reader}; + +fn frame(i: usize) -> Value { + json!({"type":"function_call_output","call_id":format!("call_{i}"),"output":[ + {"type":"input_text","text":format!("screenshot {i}")}, + {"type":"input_image","image_url":format!("data:image/png;base64,aW1hZ2U{i:04}")} + ]}) +} +fn images(value: &Value) -> usize { + match value { + Value::Array(a) => a.iter().map(images).sum(), + Value::Object(o) => { + usize::from(value["type"] == "input_image" || value["type"] == "image") + + o.values().map(images).sum::() + } + _ => 0, + } +} + +#[test] +fn five_hundred_views_keep_new_images_and_tool_pairing() { + let mut input = vec![]; + for i in 0..500 { + input.push(json!({"type":"function_call","call_id":format!("call_{i}"),"name":"view_image","arguments":format!("{{\"path\":\"/tmp/frame-{i}.png\"}}") })); + input.push(frame(i)); + if i != 499 { + input.push(json!({"type":"message","role":"assistant","content":format!("Observation {i}: button visible")})); + } + } + let original = json!({"model":"model","input":input,"store":false}); + let mut value = original.clone(); + let stats = project(&mut value, Protocol::Responses, 16, None).unwrap(); + assert!(images(&value) <= 12); + assert!(stats.evicted > 480); + assert_eq!(value["input"].as_array().unwrap().len(), 1499); + assert_eq!(value["input"][1]["call_id"], "call_0"); + assert_eq!(value["input"][1]["output"][0]["text"], "screenshot 0"); + assert!( + value["input"][1]["output"][1]["text"] + .as_str() + .unwrap() + .contains("/tmp/frame-0.png") + ); + assert_eq!(value["input"][1498], original["input"][1498]); + assert_eq!(value["input"][2], original["input"][2]); + let projected = project_reader( + std::io::Cursor::new(serde_json::to_vec(&original).unwrap()), + Protocol::Responses, + 16, + None, + ) + .unwrap(); + assert_eq!(projected.0, value); + let repeated = project(&mut value, Protocol::Responses, 16, None).unwrap(); + assert_eq!(repeated.evicted, 0); +} + +#[test] +fn unseen_batch_is_never_silently_discarded() { + let mut value = json!({"input":(0..17).map(frame).collect::>()}); + let before = value.clone(); + assert!(project(&mut value, Protocol::Responses, 16, None).is_err()); + assert_eq!(value, before); + let mut value = json!({"input":(0..16).map(frame).collect::>()}); + project(&mut value, Protocol::Responses, 16, None).unwrap(); + assert_eq!(images(&value), 16); +} + +#[test] +fn anthropic_nested_results_and_current_comparison_survive() { + let mut messages = vec![]; + for i in 0..20 { + messages.push(json!({"role":"assistant","content":[{"type":"tool_use","id":format!("t{i}"),"name":"screenshot","input":{}}]})); + messages.push(json!({"role":"user","content":[{"type":"tool_result","tool_use_id":format!("t{i}"),"content":[{"type":"text","text":"observed"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"aW1hZ2U="}}]}]})); + } + messages.push(json!({"role":"user","content":[{"type":"image","source":{"type":"base64","media_type":"image/png","data":"Y29tcGFyZTE="}},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"Y29tcGFyZTI="}}]})); + let mut value = json!({"messages":messages,"system":"preserve instructions"}); + let last = value["messages"][40].clone(); + project(&mut value, Protocol::Anthropic, 16, None).unwrap(); + assert!(images(&value) <= 12); + assert_eq!(value["messages"][40], last); + assert_eq!(value["messages"][1]["content"][0]["tool_use_id"], "t0"); + assert_eq!(value["system"], "preserve instructions"); +} + +#[test] +fn byte_pressure_evicts_processed_images_before_the_count_threshold() { + let image = format!("data:image/png;base64,{}", "a".repeat(6 * 1024 * 1024)); + let mut input = Vec::new(); + for index in 0..6 { + input.push(json!({"type":"message","role":"user","content":[{"type":"input_text","text":format!("Frame {index}")},{"type":"input_image","image_url":image}]})); + if index < 5 { + input.push(json!({"type":"message","role":"assistant","content":"Observed frame."})); + } + } + let mut request = json!({"input":input}); + let newest = request["input"][10].clone(); + let stats = project(&mut request, Protocol::Responses, 16, None).unwrap(); + assert_eq!(stats.evicted, 4); + assert_eq!(images(&request), 2); + assert_eq!(request["input"][10], newest); + assert!(serde_json::to_vec(&request).unwrap().len() < 16 * 1024 * 1024); +} diff --git a/tests/spark_anthropic_e2e.rs b/tests/spark_anthropic_e2e.rs index 8551b8d..e69fa1e 100644 --- a/tests/spark_anthropic_e2e.rs +++ b/tests/spark_anthropic_e2e.rs @@ -1,5 +1,7 @@ #![cfg(feature = "appliance")] +use sparkplane::spark::image_history; + #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] mod gateway; diff --git a/tests/spark_modalities_e2e.rs b/tests/spark_modalities_e2e.rs index 2bea587..518f5b3 100644 --- a/tests/spark_modalities_e2e.rs +++ b/tests/spark_modalities_e2e.rs @@ -1,5 +1,7 @@ #![cfg(feature = "appliance")] +use sparkplane::spark::image_history; + #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] mod gateway; diff --git a/tests/spark_openai_e2e.rs b/tests/spark_openai_e2e.rs index 8574e55..02892c4 100644 --- a/tests/spark_openai_e2e.rs +++ b/tests/spark_openai_e2e.rs @@ -1,5 +1,7 @@ #![cfg(feature = "appliance")] +use sparkplane::spark::image_history; + #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] mod gateway; diff --git a/tests/spark_qwen_vllm_optimization.rs b/tests/spark_qwen_vllm_optimization.rs index 3d28d01..3b148c5 100644 --- a/tests/spark_qwen_vllm_optimization.rs +++ b/tests/spark_qwen_vllm_optimization.rs @@ -32,6 +32,8 @@ fn qwen38_checkpoint_and_selected_engine_accept_inline_images() { vision.processor_sha256, "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516" ); + assert_eq!(vision.max_count, 16); + assert_eq!(vision.max_total_bytes, 524288); let image = serde_json::json!({"type":"input_image", "image_url":format!("data:image/png;base64,{}", vision.health_image_base64)}); let mut request = serde_json::json!({"input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"Name the color."},image]}]}); let rewrite = gateway::rewrite_responses_request_with_profile( @@ -59,6 +61,15 @@ fn qwen38_checkpoint_and_selected_engine_accept_inline_images() { ) .is_err() ); + request["input"][0]["content"] = serde_json::json!(vec![image.clone(); vision.max_count]); + assert!( + gateway::rewrite_responses_request_with_profile( + &serde_json::to_vec(&request).unwrap(), + "served-model", + &profile + ) + .is_ok() + ); request["input"][0]["content"] .as_array_mut() .unwrap() diff --git a/tests/spark_vllm_e2e.rs b/tests/spark_vllm_e2e.rs index 5ca3133..a6683d2 100644 --- a/tests/spark_vllm_e2e.rs +++ b/tests/spark_vllm_e2e.rs @@ -1,5 +1,7 @@ #![cfg(feature = "appliance")] +use sparkplane::spark::image_history; + #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] mod gateway; diff --git a/tests/standalone_contract.py b/tests/standalone_contract.py index d68548a..507deae 100644 --- a/tests/standalone_contract.py +++ b/tests/standalone_contract.py @@ -1,5 +1,7 @@ """Distribution boundary: this checkout must stand on its own.""" import pathlib +import hashlib +import json import re import tomllib import unittest @@ -8,6 +10,13 @@ class StandaloneContract(unittest.TestCase): + def test_vendored_rtk_sources_match_pinned_provenance(self): + root = ROOT / "crates/sparkplane-rtk" + provenance = json.loads((root / "PROVENANCE.json").read_text()) + for name, digest in provenance["files"].items(): + self.assertEqual(hashlib.sha256((root / "upstream" / name).read_bytes()).hexdigest(), digest, name) + self.assertEqual((root / "LICENSE").read_bytes(), (root / "upstream/LICENSE").read_bytes()) + def test_documentation_links_resolve_inside_this_checkout(self): for document in [ROOT / "README.md", *ROOT.glob("docs/**/*.md")]: for destination in re.findall(r"\]\(([^)]+)\)", document.read_text()):