diff --git a/Cargo.lock b/Cargo.lock index 48a851e..472cd96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3097,8 +3097,10 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "shlex", "sparkplane-core", "sparkplane-ipc", + "sparkplane-rtk", "tempfile", "tokio", "tokio-util", @@ -3150,6 +3152,16 @@ dependencies = [ "ulid", ] +[[package]] +name = "sparkplane-rtk" +version = "0.49.0" +dependencies = [ + "regex", + "serde", + "serde_json", + "toml", +] + [[package]] name = "spinning_top" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 6dfc6b5..f4a9902 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "crates/sparkplane-core", "crates/sparkplane-ipc"] +members = [".", "crates/sparkplane-core", "crates/sparkplane-ipc", "crates/sparkplane-rtk"] resolver = "3" [workspace.package] @@ -37,7 +37,7 @@ publish = false [features] default = [] -appliance = ["dep:axum", "dep:axum-server", "dep:hmac", "dep:governor", "dep:rcgen", "dep:rusqlite", "dep:rusqlite_migration", "dep:rustls", "dep:secrecy", "dep:tower", "dep:utoipa", "dep:x509-parser", "dep:bollard", "dep:http-body-util", "dep:hyper", "dep:hyper-util", "dep:hf-hub", "dep:image", "dep:arc-swap", "dep:sparkplane-ipc", "dep:tempfile"] +appliance = ["dep:axum", "dep:axum-server", "dep:hmac", "dep:governor", "dep:rcgen", "dep:rusqlite", "dep:rusqlite_migration", "dep:rustls", "dep:secrecy", "dep:tower", "dep:utoipa", "dep:x509-parser", "dep:bollard", "dep:hf-hub", "dep:image", "dep:arc-swap", "dep:sparkplane-ipc", "dep:tempfile"] [dependencies] anyhow.workspace = true @@ -52,6 +52,8 @@ futures-util.workspace = true tracing.workspace = true tracing-subscriber.workspace = true sparkplane-core.workspace = true +sparkplane-rtk = { path = "crates/sparkplane-rtk", version = "=0.49.0" } +shlex = "1" sparkplane-ipc = { workspace = true, optional = true } tempfile = { workspace = true, optional = true } clap = { version = "4", features = ["derive", "env"] } @@ -62,7 +64,7 @@ reqwest = { version = "0.12", default-features = false, features = ["blocking", sha2 = "0.10" base64 = "0.22" minisign-verify = "0.2" -nix = { version = "0.29", features = ["fs"] } +nix = { version = "0.29", features = ["fs", "signal", "process"] } uuid = { version = "1", features = ["v4"] } zstd = "0.13" walkdir = "2" @@ -81,9 +83,9 @@ rusqlite = { version = "0.40.0", default-features = false, features = ["backup", rusqlite_migration = { version = "2.6.0", default-features = false, optional = true } governor = { version = "0.10.4", default-features = false, features = ["dashmap", "std"], optional = true } bollard = { version = "0.21.1", default-features = false, features = ["pipe"], optional = true } -http-body-util = { version = "0.1", optional = true } -hyper = { version = "1", default-features = false, features = ["client", "http1"], optional = true } -hyper-util = { version = "0.1", default-features = false, features = ["tokio"], optional = true } +http-body-util = { version = "0.1" } +hyper = { version = "1", default-features = false, features = ["client", "server", "http1"] } +hyper-util = { version = "0.1", default-features = false, features = ["tokio"] } hf-hub = { version = "1.0.0", default-features = false, features = ["rustls-tls"], optional = true } image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"], optional = true } arc-swap = { version = "1", optional = true } diff --git a/README.md b/README.md index 060614d..e1f3d55 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,17 @@ without action approval prompts. Use `--mode inherit` to keep the agent's own permission settings. In inherit mode, `--allow-network` grants network access within a supported sandbox. See [launch permissions](docs/reference/spark.md#internet-access-for-development). +Launches also default to `--eco-mode=max`, using RTK filters embedded in the +Sparkplane executable. Use `--eco-mode=none` to disable compression. When the +coding session exits, Sparkplane prints token usage and public cloud price +comparisons; `sparkplane economics --json` reads the latest retained report. +See [session economics](docs/reference/spark.md#eco-mode-and-session-economics). + +Vision launches also retain recent images automatically and archive older +originals for reopening. A private local adapter prunes historical image payloads +before upload, while the gateway enforces the qualified model limits. See +[image history and gateway limits](docs/reference/spark.md#image-history-and-upload-limits). + ## Architecture The unprivileged HTTPS agent owns state and protocol translation. A separate, diff --git a/configs/sparkplane/agent.toml b/configs/sparkplane/agent.toml index d6d5766..3081d6b 100644 --- a/configs/sparkplane/agent.toml +++ b/configs/sparkplane/agent.toml @@ -6,6 +6,14 @@ allowed_client_cidrs = ["10.1.30.0/24"] plain_http_loopback_only = true executor_socket = "/run/sparkplane/executor.sock" +[gateway] +# Public uploads are normalized to the selected engine's stricter image policy. +request_body_bytes = 33554432 +image_bytes = 16777216 +image_pixels = 32000000 # Total source pixels across the request. +image_dimension = 16384 +max_parallel_requests = 2 + [models] cache_root = "/var/lib/sparkplane/huggingface" endpoint = "https://huggingface.co" diff --git a/configs/sparkplane/engines/vllm-qwen38-mmap.toml b/configs/sparkplane/engines/vllm-qwen38-mmap.toml index a9ae53a..dc7d7c0 100644 --- a/configs/sparkplane/engines/vllm-qwen38-mmap.toml +++ b/configs/sparkplane/engines/vllm-qwen38-mmap.toml @@ -110,6 +110,7 @@ arguments = [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", ] capabilities = ["text_generation", "tool_calling", "vision"] -vision = { processor_sha256 = "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", media_types = ["image/jpeg", "image/png", "image/webp"], max_bytes = 524288, max_total_bytes = 524288, max_count = 1, max_width = 4096, max_height = 4096, health_media_type = "image/png", health_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAOAAAADgAQAAAAAyVldMAAAAHUlEQVRYw+3BMQEAAADCoPVPbQZ/oAAAAAAAAHgMGWAAAct2tnAAAAAASUVORK5CYII=", health_image_sha256 = "1b8ab2918e9fb346d8ff5b7579372510b0d8b1566d590dac31aea311fff15fc6", health_prompt = "Name the single dominant color using one lowercase word.", health_expected_text = "black", health_max_tokens = 32, health_disable_thinking = true } +# Multi-frame requests retain the same total byte and dimension budgets. +vision = { processor_sha256 = "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", media_types = ["image/jpeg", "image/png", "image/webp"], max_bytes = 524288, max_total_bytes = 524288, max_count = 16, max_width = 4096, max_height = 4096, health_media_type = "image/png", health_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAOAAAADgAQAAAAAyVldMAAAAHUlEQVRYw+3BMQEAAADCoPVPbQZ/oAAAAAAAAHgMGWAAAct2tnAAAAAASUVORK5CYII=", health_image_sha256 = "1b8ab2918e9fb346d8ff5b7579372510b0d8b1566d590dac31aea311fff15fc6", health_prompt = "Name the single dominant color using one lowercase word.", health_expected_text = "black", health_max_tokens = 32, health_disable_thinking = true } startup_protocol_probe = true sampling = { temperature = 1.0, top_p = 0.95, top_k = 20, min_p = 0.0, presence_penalty = 0.0, repetition_penalty = 1.0, thinking_token_budget = 8192, reasoning_effort_map = { low = "low", medium = "medium", high = "xhigh", max = "xhigh" } } diff --git a/crates/sparkplane-rtk/ADAPTATION.patch b/crates/sparkplane-rtk/ADAPTATION.patch new file mode 100644 index 0000000..e7c5b5a --- /dev/null +++ b/crates/sparkplane-rtk/ADAPTATION.patch @@ -0,0 +1,280 @@ +--- upstream/src/core/toml_filter.rs ++++ src/core/toml_filter.rs +@@ -29,7 +29,7 @@ + use std::sync::LazyLock; + + // Built-in filters: concatenated from src/filters/*.toml by build.rs at compile time. +-const BUILTIN_TOML: &str = include_str!(concat!(env!("OUT_DIR"), "/builtin_filters.toml")); ++const BUILTIN_TOML: &str = include_str!("builtin_filters.toml"); + + // --------------------------------------------------------------------------- + // Deserialization types (TOML schema) +@@ -188,38 +188,12 @@ + fn load() -> Self { + let mut filters = Vec::new(); + +- for path in crate::hooks::trust::gated_filter_paths() { +- Self::extend_with_trusted(&mut filters, &path); +- } +- + match Self::parse_and_compile(BUILTIN_TOML, "builtin") { + Ok(f) => filters.extend(f), + Err(e) => eprintln!("[rtk] warning: builtin filters: {}", e), + } + + TomlFilterRegistry { filters } +- } +- +- fn extend_with_trusted(filters: &mut Vec, path: &std::path::Path) { +- if !path.exists() { +- return; +- } +- let label = path.display().to_string(); +- let (status, content) = crate::hooks::trust::check_trust_with_content(path) +- .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); +- match status { +- crate::hooks::trust::TrustStatus::Trusted +- | crate::hooks::trust::TrustStatus::EnvOverride => { +- if let Some(content) = content { +- match Self::parse_and_compile(&content, &label) { +- Ok(f) => filters.extend(f), +- Err(e) => eprintln!("[rtk] warning: {}: {}", label, e), +- } +- } +- } +- crate::hooks::trust::TrustStatus::Untrusted +- | crate::hooks::trust::TrustStatus::ContentChanged { .. } => {} +- } + } + + fn parse_and_compile(content: &str, source: &str) -> Result, String> { +@@ -447,24 +421,7 @@ + } + + fn collect_match_patterns() -> Vec { +- let mut patterns: Vec = Vec::new(); +- for path in crate::hooks::trust::gated_filter_paths() { +- if !path.exists() { +- continue; +- } +- let (status, content) = crate::hooks::trust::check_trust_with_content(&path) +- .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); +- if matches!( +- status, +- crate::hooks::trust::TrustStatus::Trusted +- | crate::hooks::trust::TrustStatus::EnvOverride +- ) && let Some(content) = content +- { +- patterns.extend(match_patterns_in(&content)); +- } +- } +- patterns.extend(match_patterns_in(BUILTIN_TOML)); +- patterns ++ match_patterns_in(BUILTIN_TOML) + } + + /// Strip a leading inline-flag group such as `(?i)` so the anchor check sees the +@@ -722,131 +679,6 @@ + /// + /// - `filter_name_opt`: if `Some`, only run tests for that filter name. + /// - Returns `VerifyResults` with all outcomes and filters that have no tests. +-pub fn run_filter_tests(filter_name_opt: Option<&str>) -> VerifyResults { +- let mut outcomes = Vec::new(); +- let mut all_filter_names: Vec = Vec::new(); +- let mut tested_filter_names: std::collections::HashSet = +- std::collections::HashSet::new(); +- +- let builtin = BUILTIN_TOML; +- collect_test_outcomes( +- builtin, +- filter_name_opt, +- &mut outcomes, +- &mut all_filter_names, +- &mut tested_filter_names, +- ); +- +- // Trust-gated: only verify project-local filters if trusted (SA-2025-RTK-002) +- let project_path = std::path::Path::new(".rtk/filters.toml"); +- if project_path.exists() { +- let (trust_status, verified_content) = +- crate::hooks::trust::check_trust_with_content(project_path) +- .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); +- match trust_status { +- crate::hooks::trust::TrustStatus::Trusted +- | crate::hooks::trust::TrustStatus::EnvOverride => { +- if let Some(content) = verified_content { +- collect_test_outcomes( +- &content, +- filter_name_opt, +- &mut outcomes, +- &mut all_filter_names, +- &mut tested_filter_names, +- ); +- } +- } +- _ => { +- eprintln!("[rtk] WARNING: untrusted project filters skipped in verify"); +- } +- } +- } +- +- let filters_without_tests = all_filter_names +- .into_iter() +- .filter(|name| { +- // When a specific filter is requested, only report that one as missing tests +- filter_name_opt.is_none_or(|f| name == f) +- }) +- .filter(|name| !tested_filter_names.contains(name)) +- .collect(); +- +- VerifyResults { +- outcomes, +- filters_without_tests, +- } +-} +- +-fn collect_test_outcomes( +- content: &str, +- filter_name_opt: Option<&str>, +- outcomes: &mut Vec, +- all_filter_names: &mut Vec, +- tested_filter_names: &mut std::collections::HashSet, +-) { +- let file: TomlFilterFile = match toml::from_str(content) { +- Ok(f) => f, +- Err(e) => { +- eprintln!("[rtk] warning: TOML parse error during verify: {}", e); +- return; +- } +- }; +- +- // Compile all filters and track their names +- let mut compiled_filters: BTreeMap = BTreeMap::new(); +- for (name, def) in file.filters { +- all_filter_names.push(name.clone()); +- match compile_filter(name.clone(), def) { +- Ok(f) => { +- compiled_filters.insert(name, f); +- } +- Err(e) => eprintln!("[rtk] warning: filter '{}' compilation error: {}", name, e), +- } +- } +- +- // Run tests +- for (filter_name, tests) in file.tests { +- if let Some(name) = filter_name_opt +- && filter_name != name +- { +- continue; +- } +- +- tested_filter_names.insert(filter_name.clone()); +- +- let compiled = match compiled_filters.get(&filter_name) { +- Some(f) => f, +- None => { +- eprintln!( +- "[rtk] warning: [[tests.{}]] references unknown filter", +- filter_name +- ); +- continue; +- } +- }; +- +- for test in tests { +- let actual = apply_filter(compiled, &test.input); +- // Trim trailing newlines: TOML multiline strings end with a newline +- let actual_cmp = actual.trim_end_matches('\n').to_string(); +- let expected_cmp = test.expected.trim_end_matches('\n').to_string(); +- outcomes.push(TestOutcome { +- filter_name: filter_name.clone(), +- test_name: test.name, +- passed: actual_cmp == expected_cmp, +- actual: actual_cmp, +- expected: expected_cmp, +- }); +- } +- } +-} +- +-// --------------------------------------------------------------------------- +-// Convenience wrapper (uses singleton — for run_fallback) +-// --------------------------------------------------------------------------- +- +-/// Find a matching filter from the global registry. Initialises the registry +-/// lazily on first call. Returns `None` if no filter matches. + pub fn find_matching_filter(command: &str) -> Option<&'static CompiledFilter> { + if std::env::var("RTK_TOML_DEBUG").is_ok() { + eprintln!( +@@ -2227,3 +2059,71 @@ + assert_eq!(found.unwrap().name, "my-new-tool"); + } + } ++ ++pub fn registry() -> &'static TomlFilterRegistry { ++ ®ISTRY ++} ++ ++fn collect_test_outcomes( ++ content: &str, ++ filter_name_opt: Option<&str>, ++ outcomes: &mut Vec, ++ all_filter_names: &mut Vec, ++ tested_filter_names: &mut std::collections::HashSet, ++) { ++ let file: TomlFilterFile = match toml::from_str(content) { ++ Ok(f) => f, ++ Err(e) => { ++ eprintln!("[rtk] warning: TOML parse error during verify: {}", e); ++ return; ++ } ++ }; ++ ++ // Compile all filters and track their names ++ let mut compiled_filters: BTreeMap = BTreeMap::new(); ++ for (name, def) in file.filters { ++ all_filter_names.push(name.clone()); ++ match compile_filter(name.clone(), def) { ++ Ok(f) => { ++ compiled_filters.insert(name, f); ++ } ++ Err(e) => eprintln!("[rtk] warning: filter '{}' compilation error: {}", name, e), ++ } ++ } ++ ++ // Run tests ++ for (filter_name, tests) in file.tests { ++ if let Some(name) = filter_name_opt ++ && filter_name != name ++ { ++ continue; ++ } ++ ++ tested_filter_names.insert(filter_name.clone()); ++ ++ let compiled = match compiled_filters.get(&filter_name) { ++ Some(f) => f, ++ None => { ++ eprintln!( ++ "[rtk] warning: [[tests.{}]] references unknown filter", ++ filter_name ++ ); ++ continue; ++ } ++ }; ++ ++ for test in tests { ++ let actual = apply_filter(compiled, &test.input); ++ // Trim trailing newlines: TOML multiline strings end with a newline ++ let actual_cmp = actual.trim_end_matches('\n').to_string(); ++ let expected_cmp = test.expected.trim_end_matches('\n').to_string(); ++ outcomes.push(TestOutcome { ++ filter_name: filter_name.clone(), ++ test_name: test.name, ++ passed: actual_cmp == expected_cmp, ++ actual: actual_cmp, ++ expected: expected_cmp, ++ }); ++ } ++ } ++} diff --git a/crates/sparkplane-rtk/Cargo.toml b/crates/sparkplane-rtk/Cargo.toml new file mode 100644 index 0000000..210deed --- /dev/null +++ b/crates/sparkplane-rtk/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "sparkplane-rtk" +version = "0.49.0" +edition = "2024" +license = "Apache-2.0" +publish = false + +[dependencies] +regex = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +toml = "0.8" diff --git a/crates/sparkplane-rtk/LICENSE b/crates/sparkplane-rtk/LICENSE new file mode 100644 index 0000000..0afaf4b --- /dev/null +++ b/crates/sparkplane-rtk/LICENSE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 rtk-ai and rtk-ai Labs + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/sparkplane-rtk/PROVENANCE.json b/crates/sparkplane-rtk/PROVENANCE.json new file mode 100644 index 0000000..d30a693 --- /dev/null +++ b/crates/sparkplane-rtk/PROVENANCE.json @@ -0,0 +1,96 @@ +{ + "repository": "https://github.com/rtk-ai/rtk", + "revision": "6d4b77eadee1c66dc1f68466ad77e96d1b6e4989", + "archive_sha256": "400abfa429aa974604ba9b0223871d82aad59443cb2ec6af7d37fad8f0cb8de0", + "files": { + "Cargo.toml": "681d327949ac960367c13fc14f6df8ce3fcc2500acde370c37b8f1612ecdf967", + "LICENSE": "4044ade9c21d8b084d3d16a03375cf3b7e166b946a327bb37a3fbbdb53287cfd", + "src/cmds/git/git_cmd.rs": "7ae499835723e3715c48d91d8fa5121e3c20d74054719b9b5f535c102984dbaa", + "src/cmds/python/pytest_cmd.rs": "497e9a12174a4fcf28fad2bb62a470c0c2431e7c6be2431cc76087f3db76ec53", + "src/cmds/rust/cargo_cmd.rs": "87aac95895e4418fd6abcee0a4f99a2504e830e0e5da02b9e92a65bb570c559c", + "src/core/constants.rs": "1b65b0fc6ef9e221fe10ca30092fe239e9c1237f8f38c55ff80dc2422f3c37d1", + "src/core/guard.rs": "64cbff90d714e97de74767fc26770779ebeb25705cabb5dc28e98af01b287dce", + "src/core/stream.rs": "3953c190b2fe586374355c622adc57f3c7312836443f56a846e68f855c13fc61", + "src/core/toml_filter.rs": "da4313afd8e33715760b7512c17b5991c81d64ace500cfa33710debed097d17d", + "src/core/truncate.rs": "6c3ea9994eb0dbdd2f5de231f19cb375e23dbcb0695a9c046f4849e70173ff7d", + "src/core/utils.rs": "0ed65369a95bde40d717445bfc64ecbdabbc3755696a9bba110de02e5eefccff", + "src/filters/README.md": "92e3c23bfa488dc03a72632e57a3c868e18845657b152a38c99d3b60624152fc", + "src/filters/ansible-playbook.toml": "5a6c880f1baf60399b4afece9eb2f0761e599bb1cb050dbf8c4cc4a57527c7e2", + "src/filters/basedpyright.toml": "f6df2dfffb311a193194d74f8a536dd006913c038b97af20d1aa301400eeb410", + "src/filters/biome.toml": "b476a13b4cf253c4aa4968cf5192ef3e61b352ee2db0a4f054bdc09ec5d4b04b", + "src/filters/brew-install.toml": "9ce35afac31248db32e2033a88bfb32ae9d64ce4700a4c9faa0effb490bbafdf", + "src/filters/bundle-install.toml": "f9bdea56d32f4cb94388226ec53a49ffec458ac3f887895ca2aebd2da81d37a3", + "src/filters/composer-install.toml": "e3c53ea5d13c8eb06bfcec2952f55346dce4a41732bb85bd5665aa24c2a25d31", + "src/filters/df.toml": "6c496a1559cd22ce8227f6ba5e88c48ba9e06e92fccdbb3ca5c128bd3dd1d532", + "src/filters/dotnet-build.toml": "32941a4c4b21e6f54f4f2fbe012b61ac491f69e5cbba142fe77579851f52a4aa", + "src/filters/du.toml": "cb363ec3e163c6fed016339d3adeb9b3c36bedb3b7d1ae3501a78b049e8ef595", + "src/filters/fail2ban-client.toml": "66a46d62819254bc9e26db95dc51a403e54fc6e089168a6c5ef0bb514e559fd9", + "src/filters/gcc.toml": "1d0d719ee01fa0f5e23ddfcbd3b02b47ece743cf61af335e26f313b3faca617c", + "src/filters/gcloud.toml": "30bb8ccb7b5a642a17c68545612707ddec801e9c0cb980e190c40fcf0bbaff91", + "src/filters/hadolint.toml": "51575715ef665ea55e57219d784e38336d2c6924829a0abb52062f2a78aa8a22", + "src/filters/helm.toml": "a29171e30f1e2a41f9f82616bf00a3430b2c66052dbb5c897342406777bf76c0", + "src/filters/iptables.toml": "0b2802aab90ccd03e9c1d674e8cd49f04861bb5b7bc34aef4a92ea659222563e", + "src/filters/jira.toml": "9557fa46444e0308c36ec1cb5e735ddb043cc841330ba0ac0cb58f6ef2d1e454", + "src/filters/jj.toml": "da5cc860cd564a38169b404a4ab3d55c4e91154c90083da364cacc7549f48a8a", + "src/filters/jq.toml": "fec22286464e3c66229d784febc3ba51d0d6f98f907af18f28930fa997980b98", + "src/filters/just.toml": "3ab46b21148ebf4af210200d408515c12ebb5f6590b14048db8af31ba5b2923e", + "src/filters/liquibase.toml": "5fac9073de0e1e9c897eed522ffe487bba66afa5d37fbfd2b06e35ebed75bf94", + "src/filters/make.toml": "0d2f67f2e2fd4820853e998f7ca2bf9445ffa6632642d10b15a681e9f5c36614", + "src/filters/markdownlint.toml": "827cf11614d4dca4d64563060e6665ce135b01d61ebb86621ba8af12ea04dbd9", + "src/filters/mise.toml": "5a416be11bdb8ea5ea1d06c47e9c96920ecea1e57d8fb17956f6a2cbc1e0d192", + "src/filters/mix-compile.toml": "4167205058fecc76f43d26bf71a7b904b5d553ed81216732fa4dcac51763cd6c", + "src/filters/mix-format.toml": "f5be87d80dd5d8a0276a101bc5c8bad16662d4b58c4c691f6e3561e1db427cc6", + "src/filters/nx.toml": "dde44afaf6026c8b4ce912e7c8f61a0f2890eb52b8245169a2766306910ae562", + "src/filters/ollama.toml": "bf2898bffa2fbbf8a7945c344abf52d3a760dce67f28113bd26bba031836088c", + "src/filters/oxlint.toml": "4cdd71f51128e588f0abb100e4e61c8ff4f2a7906383b90cf3b821d9c0532488", + "src/filters/ping.toml": "a22fd6e62016c3846eed832bfb0213d4a91f4ae976b204ac03817c99acc6c970", + "src/filters/pio-run.toml": "ccc98f26a35f76e7cf10befa13d95179cd96bec7184d10fc2468c255620c1c57", + "src/filters/poetry-install.toml": "ebff2dac219dc7b86b8ddeeb9dd37d12fa91c9b81368e58ba06aa25242019b3b", + "src/filters/pre-commit.toml": "02446f5d004d999f762f791c468dc8b2808e3bdcb5bfdd03dd7e07bc6c695305", + "src/filters/ps.toml": "325d04ccee15075c02feb2ee63ecc08763bc253a2fb55b48bb08432bd527bcb5", + "src/filters/pulumi-destroy.toml": "115a118816320f93fec2f976a277e47f731f618f5faa6a2ba44f92a3040a7b7b", + "src/filters/pulumi-preview.toml": "b72c409df7103acac6a8cb881c96820a1370bc2fc048cfc8b3aefa85b0b025ed", + "src/filters/pulumi-refresh.toml": "47d215de95ea56d14e21028cbbe582da6d65a1a0e7464e69f45d0d705b2b7e13", + "src/filters/pulumi-stack.toml": "4e0034b23a74b9660f11c307fca35de21b306b6a8af498867ca8905eca641993", + "src/filters/pulumi-up.toml": "09afaf8881c6cd44b6d09f36897c9eba90fbff075242db00b0d1824a3cc51a2a", + "src/filters/quarto-render.toml": "1894954417a930b25b7f8463b6301d178680dd17fe6a7c56e933e8961f170dcf", + "src/filters/rsync.toml": "abf8979dcef021d69b6ff2fc49df55127b80232bc93ee1c5bda862f4aa851d39", + "src/filters/shellcheck.toml": "aacca530bb613df62fe73cf01a428b1a0e7828c7098939ee6aec1428bea7f4e4", + "src/filters/shopify-theme.toml": "b2193351d19ccefe433b4dedca792fa3e4e9208a6ffce57790adffaba497ae9f", + "src/filters/skopeo.toml": "a9347ce2a4d0f43aa2b19eaae43f93a3d69a951c7657210e1a0547c14847f032", + "src/filters/sops.toml": "5c400d57fda6db0f7a69d567aa0aaa28be3a680c2266da79b9f7e6f46b60905a", + "src/filters/spring-boot.toml": "41c8ef73ae1336d79e317219f8c6c1d6b271be17f9523e85f28c97271e489eca", + "src/filters/ssh.toml": "f80964f50ff7345e622060c34ee1772dd3f8d468335a0f4a964a02a4067b7e05", + "src/filters/stat.toml": "f3a9814c2bf2eded442bd6e66fedde4ae60dc0d3139bb96dd36ce360d5d1f189", + "src/filters/swift-build.toml": "1c397ddd29292ceef5a4615c0b744c10be77febcbb270767a9a9d9d7dd76eeab", + "src/filters/systemctl-status.toml": "6299897c067c39972712421d30c8faaa965673543360570c52f67bb2a0bbfe7a", + "src/filters/task.toml": "2151fdb559927059edd19c6fbc4311739faf4493f2b27817de9b38150e306e2f", + "src/filters/terraform-plan.toml": "05b7014a38de8781a41778a775ef19ad0474d1f304018483b682b9c0c0dc5a50", + "src/filters/tofu-fmt.toml": "3c8731e0c1f764cc2fed3b4d0a4c2101813ec64d03ff8c68f8728ea5ebb976a1", + "src/filters/tofu-init.toml": "127abb8fbedd19f2b0f860c9733d44788a8ed11f423d0077cda447976f14df8f", + "src/filters/tofu-plan.toml": "e444466da52ce184f3ff351ff9acdf71b4bbd1e230aea94e010c05dfb5c196a7", + "src/filters/tofu-validate.toml": "419c8acf53ee7160f43960ef4e7be10f1ecac72ad3f7490bab7743d923d4f909", + "src/filters/trunk-build.toml": "0d587fc0403985326813b6a3daffc4a4b9889e9488d37b319c599bfc7575d17b", + "src/filters/turbo.toml": "1a3c521e4b9d3ec68c5d5da746453de7d64ab671827101f510f00f7adc2a4353", + "src/filters/ty.toml": "233607e246349ca4ef50b73b081bf2138936381a6394f72eaa495cc9885b00c9", + "src/filters/uv-sync.toml": "1b78722bdca34d821dd7d721ffdeaf67c9d2e407d01dc78f48ab88e09c245d04", + "src/filters/xcodebuild.toml": "0fc3dc2d11ff554cdf2e58cefed56c4d9f2428b271dad328ecdd69d985412eec", + "src/filters/yadm.toml": "1dc264e305d5ddac99aa803d34a5074702c154b02f5cf178ba57f8e3c10919ed", + "src/filters/yamllint.toml": "3230ef732858c5a6d830d4f689501695ba5056f37e238c89096b19e4715705df" + }, + "adaptation": { + "scope": "Pure output reducers and bundled declarative filters; Sparkplane owns execution, hooks and per-session metrics. No RTK installer, binary, SQLite telemetry, project filters, trust override or API-key access.", + "modules": { + "cargo.rs": "src/cmds/rust/cargo_cmd.rs: CargoBuildHandler, CargoTestHandler, diagnostics, filter_cargo_build_labeled, filter_cargo_test", + "pytest.rs": "src/cmds/python/pytest_cmd.rs: filter_pytest_output and ParseState", + "git.rs": "src/cmds/git/git_cmd.rs: filter_status_with_args", + "core/utils.rs": "src/core/utils.rs: truncate, strip_ansi, join_with_overflow", + "core/stream.rs": "src/core/stream.rs: BlockHandler only", + "core/toml_filter.rs": "src/core/toml_filter.rs: bundled registry only; filesystem trust/custom rules and CLI verify removed", + "core/constants.rs": "src/core/constants.rs: unchanged", + "core/guard.rs": "src/core/guard.rs: unchanged", + "core/truncate.rs": "src/core/truncate.rs: unchanged" + }, + "filter_bundle": "schema_version = 1 plus upstream/src/filters/*.toml in lexical filename order, with each schema_version header removed" + } +} diff --git a/crates/sparkplane-rtk/src/cargo.rs b/crates/sparkplane-rtk/src/cargo.rs new file mode 100644 index 0000000..cdf7981 --- /dev/null +++ b/crates/sparkplane-rtk/src/cargo.rs @@ -0,0 +1,666 @@ +use crate::core::stream::BlockHandler; +use crate::core::truncate::{CAP_ERRORS, CAP_WARNINGS}; +use crate::core::utils::{join_with_overflow, truncate}; +use serde::Deserialize; +use std::sync::LazyLock; +struct CargoBuildHandler { + compiled: usize, + warnings: usize, + error_count: usize, + finished_line: Option, + label: &'static str, +} + +impl CargoBuildHandler { + fn with_label(label: &'static str) -> Self { + Self { + compiled: 0, + warnings: 0, + error_count: 0, + finished_line: None, + label, + } + } +} + +impl BlockHandler for CargoBuildHandler { + fn should_skip(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if trimmed.starts_with("Compiling") || trimmed.starts_with("Checking") { + self.compiled += 1; + return true; + } + if trimmed.starts_with("Downloading") || trimmed.starts_with("Downloaded") { + return true; + } + if trimmed.starts_with("Finished") { + self.finished_line = Some(trimmed.to_string()); + return true; + } + if line.starts_with("warning:") && line.contains("generated") && line.contains("warning") { + return true; + } + if (line.starts_with("error:") || line.starts_with("error[")) + && (line.contains("aborting due to") || line.contains("could not compile")) + { + return true; + } + false + } + + fn is_block_start(&mut self, line: &str) -> bool { + if line.starts_with("error[") || line.starts_with("error:") { + self.error_count += 1; + return true; + } + if line.starts_with("warning:") || line.starts_with("warning[") { + self.warnings += 1; + return true; + } + false + } + + fn is_block_continuation(&mut self, line: &str, block: &[String]) -> bool { + !(line.trim().is_empty() && block.len() > 3) + } + + fn format_summary(&self, exit_code: i32, raw: &str) -> Option { + if self.error_count == 0 && self.warnings == 0 && exit_code == 0 { + let summary = + cargo_build_success_line(self.compiled, self.finished_line.as_deref(), self.label); + return Some(crate::core::guard::never_worse(raw, &summary).to_string()); + } + // The streamed path only runs for non-json build/check; error blocks are + // emitted live, so the summary carries no rendered diagnostics. + let empty = JsonDiagnostics { + errors: Vec::new(), + warnings: Vec::new(), + }; + Some(cargo_build_failure_summary( + self.compiled, + self.error_count, + self.warnings, + &empty, + self.label, + exit_code, + )) + } +} + +struct CargoTestHandler { + in_failure_section: bool, + in_failure_names: bool, + summary_lines: Vec, + has_compile_errors: bool, +} + +impl CargoTestHandler { + fn new() -> Self { + Self { + in_failure_section: false, + in_failure_names: false, + summary_lines: Vec::new(), + has_compile_errors: false, + } + } + + /// Compacted `cargo test` summary, before the never-worse guard. + fn compute_test_summary(&self, raw: &str) -> Option { + if self.summary_lines.is_empty() { + let json = extract_json_diagnostics(raw); + if self.has_compile_errors || !json.errors.is_empty() { + // Content-based (exit 0): a real compile error yields "cargo test: N + // errors"; a bare "could not compile" leaves the raw tail fallback. + let build_filtered = filter_cargo_build_labeled(raw, "test", 0); + if build_filtered.contains("cargo test:") { + return Some(format!("{}\n", build_filtered)); + } + // Fallback: last 5 meaningful lines + let meaningful: Vec<&str> = raw + .lines() + .filter(|l| !l.trim().is_empty() && !l.trim_start().starts_with("Compiling")) + .collect(); + let last5: Vec<&str> = meaningful.iter().rev().take(5).rev().copied().collect(); + return Some(format!("{}\n", last5.join("\n"))); + } + } + + // No failures emitted — aggregate pass results + let mut aggregated: Option = None; + let mut all_parsed = true; + + for line in &self.summary_lines { + if let Some(parsed) = AggregatedTestResult::parse_line(line) { + if let Some(ref mut agg) = aggregated { + agg.merge(&parsed); + } else { + aggregated = Some(parsed); + } + } else { + all_parsed = false; + break; + } + } + + if all_parsed + && let Some(agg) = aggregated + && agg.suites > 0 + { + return Some(format!("{}\n", agg.format_compact())); + } + + // Fallback: show raw summary lines + if !self.summary_lines.is_empty() { + let mut s = String::new(); + for line in &self.summary_lines { + s.push_str(line); + s.push('\n'); + } + return Some(s); + } + + None + } +} + +impl BlockHandler for CargoTestHandler { + fn should_skip(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if trimmed.starts_with("Compiling") + || trimmed.starts_with("Downloading") + || trimmed.starts_with("Downloaded") + || trimmed.starts_with("Finished") + { + return true; + } + if line.starts_with("running ") { + return true; + } + if line.starts_with("test ") && line.ends_with("... ok") { + return true; + } + // Track compile errors for fallback + if trimmed.starts_with("error[") || trimmed.starts_with("error:") { + self.has_compile_errors = true; + } + // "failures:" toggles section state + if line == "failures:" { + if self.in_failure_section { + // Second "failures:" = list of failure names — skip them + self.in_failure_names = true; + } + self.in_failure_section = true; + return true; + } + // Skip the failure name listing section + if self.in_failure_names { + if line.starts_with("test result:") { + self.in_failure_names = false; + self.in_failure_section = false; + self.summary_lines.push(line.to_string()); + return true; + } + return true; + } + if line.starts_with("test result:") { + self.summary_lines.push(line.to_string()); + self.in_failure_section = false; + return true; + } + false + } + + fn is_block_start(&mut self, line: &str) -> bool { + self.in_failure_section && line.starts_with("---- ") + } + + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + self.in_failure_section && !line.starts_with("---- ") + } + + fn format_summary(&self, _exit_code: i32, raw: &str) -> Option { + // Same never-worse guard as CargoBuildHandler (#3430 review): if the + // compacted summary ends up larger than the raw output (e.g. a tiny + // `cargo test` run), keep the raw output instead of "compacting" it + // into something bigger. + let summary = self.compute_test_summary(raw)?; + Some(crate::core::guard::never_worse(raw, &summary).to_string()) + } +} + +/// Generic cargo command runner with filtering. +/// Builds the Command with restored `--` separator, then delegates to shared runner. + +struct JsonDiagnostics { + errors: Vec, + warnings: Vec, +} + +#[derive(Deserialize)] +struct CargoJsonLine { + reason: String, + message: Option, +} + +#[derive(Deserialize)] +struct CargoDiagnostic { + level: String, + #[serde(default)] + message: String, + rendered: Option, +} + +fn extract_json_diagnostics(raw: &str) -> JsonDiagnostics { + let mut errors = Vec::new(); + let mut warnings = Vec::new(); + for line in raw.lines() { + let line = line.trim_start(); + if !line.starts_with('{') { + continue; + } + let Ok(entry) = serde_json::from_str::(line) else { + continue; + }; + if entry.reason != "compiler-message" { + continue; + } + let Some(msg) = entry.message else { + continue; + }; + let bucket = match msg.level.as_str() { + "error" | "error: internal compiler error" => &mut errors, + "warning" => &mut warnings, + _ => continue, + }; + if msg.message.starts_with("aborting due to") + || msg.message.starts_with("could not compile") + || (msg.message.contains("warning") && msg.message.contains("generated")) + { + continue; + } + if let Some(rendered) = msg.rendered { + bucket.push( + crate::core::utils::strip_ansi(&rendered) + .trim_end() + .to_string(), + ); + } else if !msg.message.is_empty() { + bucket.push(msg.message); + } + } + JsonDiagnostics { errors, warnings } +} + +fn merge_diag_counts( + error_count: usize, + warnings: usize, + json: &JsonDiagnostics, +) -> (usize, usize) { + ( + error_count.max(json.errors.len()), + warnings.max(json.warnings.len()), + ) +} + +fn cargo_build_success_line(compiled: usize, finished: Option<&str>, label: &str) -> String { + match finished { + Some(f) => format!("cargo {} ({} crates compiled)\n{}\n", label, compiled, f), + None => format!("cargo {} ({} crates compiled)\n", label, compiled), + } +} + +fn cargo_build_failure_summary( + compiled: usize, + errors: usize, + warnings: usize, + json: &JsonDiagnostics, + label: &str, + exit_code: i32, +) -> String { + let mut out = if errors == 0 && warnings == 0 { + format!("cargo {}: failed (exit {})\n", label, exit_code) + } else { + format!( + "cargo {}: {} errors, {} warnings ({} crates)\n", + label, errors, warnings, compiled + ) + }; + if !json.errors.is_empty() { + let shown: Vec = json.errors.iter().take(CAP_ERRORS).cloned().collect(); + out.push_str(&join_with_overflow( + &shown, + json.errors.len(), + CAP_ERRORS, + "errors", + )); + out.push('\n'); + } + if !json.warnings.is_empty() { + let shown: Vec = json.warnings.iter().take(CAP_WARNINGS).cloned().collect(); + out.push_str(&join_with_overflow( + &shown, + json.warnings.len(), + CAP_WARNINGS, + "warnings", + )); + out.push('\n'); + } + if json.errors.len() > CAP_ERRORS || json.warnings.len() > CAP_WARNINGS { + let full = json + .errors + .iter() + .chain(json.warnings.iter()) + .map(String::as_str) + .collect::>() + .join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&full, "cargo-json-issues") { + out.push_str(&format!(" {}\n", hint)); + } + } + out +} + +fn filter_cargo_build(output: &str) -> String { + filter_cargo_build_labeled(output, "build", 0) +} + +pub fn filter_cargo_build_labeled(output: &str, label: &'static str, exit_code: i32) -> String { + let mut handler = CargoBuildHandler::with_label(label); + let mut blocks: Vec> = Vec::new(); + let mut current_block: Vec = Vec::new(); + let mut in_block = false; + + for line in output.lines() { + if handler.should_skip(line) { + continue; + } + if handler.is_block_start(line) { + if in_block && !current_block.is_empty() { + blocks.push(std::mem::take(&mut current_block)); + } + in_block = true; + current_block.push(line.to_string()); + } else if in_block { + if handler.is_block_continuation(line, ¤t_block) { + current_block.push(line.to_string()); + } else { + blocks.push(std::mem::take(&mut current_block)); + in_block = false; + } + } + } + if !current_block.is_empty() { + blocks.push(current_block); + } + + let json = extract_json_diagnostics(output); + let (errors, warnings) = merge_diag_counts(handler.error_count, handler.warnings, &json); + + if errors == 0 && warnings == 0 && exit_code == 0 { + let summary = + cargo_build_success_line(handler.compiled, handler.finished_line.as_deref(), label); + return crate::core::guard::never_worse(output, &summary).to_string(); + } + + let mut result = + cargo_build_failure_summary(handler.compiled, errors, warnings, &json, label, exit_code); + const MAX_CHECK_BLOCKS: usize = CAP_ERRORS; + for (i, blk) in blocks.iter().enumerate().take(MAX_CHECK_BLOCKS) { + result.push_str(&blk.join("\n")); + result.push('\n'); + if i < blocks.len() - 1 { + result.push('\n'); + } + } + if blocks.len() > MAX_CHECK_BLOCKS { + result.push_str(&format!( + "\n… +{} more issues\n", + blocks.len() - MAX_CHECK_BLOCKS + )); + let all_blocks: String = blocks + .iter() + .map(|b| b.join("\n")) + .collect::>() + .join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_blocks, "cargo-check-issues") { + result.push_str(&format!(" {}\n", hint)); + } + } + result.trim().to_string() +} + +/// Aggregated test results for compact display +#[derive(Debug, Default, Clone)] +struct AggregatedTestResult { + passed: usize, + failed: usize, + ignored: usize, + measured: usize, + filtered_out: usize, + suites: usize, + duration_secs: f64, + has_duration: bool, +} + +impl AggregatedTestResult { + /// Parse a test result summary line + /// Format: "test result: ok. 15 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s" + fn parse_line(line: &str) -> Option { + static RE: LazyLock = LazyLock::new(|| { + regex::Regex::new( + r"test result: (\w+)\.\s+(\d+) passed;\s+(\d+) failed;\s+(\d+) ignored;\s+(\d+) measured;\s+(\d+) filtered out(?:;\s+finished in ([\d.]+)s)?" + ).unwrap() + }); + + let caps = RE.captures(line)?; + let status = caps.get(1)?.as_str(); + + // Only aggregate if status is "ok" (all tests passed) + if status != "ok" { + return None; + } + + let passed = caps.get(2)?.as_str().parse().ok()?; + let failed = caps.get(3)?.as_str().parse().ok()?; + let ignored = caps.get(4)?.as_str().parse().ok()?; + let measured = caps.get(5)?.as_str().parse().ok()?; + let filtered_out = caps.get(6)?.as_str().parse().ok()?; + + let (duration_secs, has_duration) = if let Some(duration_match) = caps.get(7) { + (duration_match.as_str().parse().unwrap_or(0.0), true) + } else { + (0.0, false) + }; + + Some(Self { + passed, + failed, + ignored, + measured, + filtered_out, + suites: 1, + duration_secs, + has_duration, + }) + } + + /// Merge another test result into this one + fn merge(&mut self, other: &Self) { + self.passed += other.passed; + self.failed += other.failed; + self.ignored += other.ignored; + self.measured += other.measured; + self.filtered_out += other.filtered_out; + self.suites += other.suites; + self.duration_secs += other.duration_secs; + self.has_duration = self.has_duration && other.has_duration; + } + + /// Format as compact single line + fn format_compact(&self) -> String { + let mut parts = vec![format!("{} passed", self.passed)]; + + if self.ignored > 0 { + parts.push(format!("{} ignored", self.ignored)); + } + if self.filtered_out > 0 { + parts.push(format!("{} filtered out", self.filtered_out)); + } + + let counts = parts.join(", "); + + let suite_text = if self.suites == 1 { + "1 suite".to_string() + } else { + format!("{} suites", self.suites) + }; + + if self.has_duration { + format!( + "cargo test: {} ({}, {:.2}s)", + counts, suite_text, self.duration_secs + ) + } else { + format!("cargo test: {} ({})", counts, suite_text) + } + } +} + +pub(crate) fn filter_cargo_test(output: &str) -> String { + let mut failures: Vec = Vec::new(); + let mut summary_lines: Vec = Vec::new(); + let mut in_failure_section = false; + let mut current_failure = Vec::new(); + + for line in output.lines() { + // Skip compilation lines + if line.trim_start().starts_with("Compiling") + || line.trim_start().starts_with("Downloading") + || line.trim_start().starts_with("Downloaded") + || line.trim_start().starts_with("Finished") + { + continue; + } + + // Skip "running N tests" and individual "test ... ok" lines + if line.starts_with("running ") || (line.starts_with("test ") && line.ends_with("... ok")) { + continue; + } + + // Detect failures section + if line == "failures:" { + in_failure_section = true; + continue; + } + + if in_failure_section { + if line.starts_with("test result:") { + in_failure_section = false; + summary_lines.push(line.to_string()); + } else if line.starts_with(" ") || line.starts_with("---- ") { + current_failure.push(line.to_string()); + } else if line.trim().is_empty() && !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } else if !line.trim().is_empty() { + current_failure.push(line.to_string()); + } + } + + // Capture test result summary + if !in_failure_section && line.starts_with("test result:") { + summary_lines.push(line.to_string()); + } + } + + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + } + + let mut result = String::new(); + + if failures.is_empty() && !summary_lines.is_empty() { + // All passed - try to aggregate + let mut aggregated: Option = None; + let mut all_parsed = true; + + for line in &summary_lines { + if let Some(parsed) = AggregatedTestResult::parse_line(line) { + if let Some(ref mut agg) = aggregated { + agg.merge(&parsed); + } else { + aggregated = Some(parsed); + } + } else { + all_parsed = false; + break; + } + } + + // If all lines parsed successfully and we have at least one suite, return compact format + if all_parsed + && let Some(agg) = aggregated + && agg.suites > 0 + { + return agg.format_compact(); + } + + // Fallback: use original behavior if regex failed + for line in &summary_lines { + result.push_str(&format!("{}\n", line)); + } + return result.trim().to_string(); + } + + if !failures.is_empty() { + result.push_str(&format!("FAILURES ({}):\n", failures.len())); + const MAX_FAILURES: usize = CAP_WARNINGS; + for (i, failure) in failures.iter().enumerate().take(MAX_FAILURES) { + result.push_str(&format!("{}. {}\n", i + 1, truncate(failure, 200))); + } + if failures.len() > MAX_FAILURES { + result.push_str(&format!( + "\n… +{} more failures\n", + failures.len() - MAX_FAILURES + )); + let all_failures = failures.join("\n\n"); + if let Some(hint) = + crate::core::tee::force_tee_hint(&all_failures, "cargo-test-failures") + { + result.push_str(&format!(" {}\n", hint)); + } + } + result.push('\n'); + } + + for line in &summary_lines { + result.push_str(&format!("{}\n", line)); + } + + if result.trim().is_empty() { + let json = extract_json_diagnostics(output); + let has_compile_errors = !json.errors.is_empty() + || output.lines().any(|line| { + let trimmed = line.trim_start(); + trimmed.starts_with("error[") || trimmed.starts_with("error:") + }); + + if has_compile_errors { + let build_filtered = filter_cargo_build_labeled(output, "test", 0); + if build_filtered.contains("cargo test:") { + return build_filtered; + } + } + + // Fallback: show last meaningful lines + let meaningful: Vec<&str> = output + .lines() + .filter(|l| !l.trim().is_empty() && !l.trim_start().starts_with("Compiling")) + .collect(); + for line in meaningful.iter().rev().take(5).rev() { + result.push_str(&format!("{}\n", line)); + } + } + + result.trim().to_string() +} diff --git a/crates/sparkplane-rtk/src/core/builtin_filters.toml b/crates/sparkplane-rtk/src/core/builtin_filters.toml new file mode 100644 index 0000000..b381b4e --- /dev/null +++ b/crates/sparkplane-rtk/src/core/builtin_filters.toml @@ -0,0 +1,2493 @@ +schema_version = 1 + +[filters.ansible-playbook] +description = "Compact ansible-playbook output" +match_command = "^ansible-playbook\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ok: \\[", + "^skipping: \\[", +] +max_lines = 60 + +[[tests.ansible-playbook]] +name = "strips ok and skipping lines, keeps changed and failures" +input = """ +PLAY [all] ********************************************************************* + +TASK [Gathering Facts] ********************************************************* +ok: [web01] +ok: [web02] + +TASK [Install nginx] *********************************************************** +changed: [web01] +skipping: [web02] + +PLAY RECAP ********************************************************************* +web01 : ok=2 changed=1 unreachable=0 failed=0 +web02 : ok=1 changed=0 unreachable=0 failed=0 +""" +expected = "PLAY [all] *********************************************************************\nTASK [Gathering Facts] *********************************************************\nTASK [Install nginx] ***********************************************************\nchanged: [web01]\nPLAY RECAP *********************************************************************\nweb01 : ok=2 changed=1 unreachable=0 failed=0\nweb02 : ok=1 changed=0 unreachable=0 failed=0" + +[[tests.ansible-playbook]] +name = "failed task preserved" +input = "TASK [Start service] ***\nfailed: [web01] => {\"msg\": \"Service not found\"}\nPLAY RECAP ***\nweb01 : ok=1 failed=1" +expected = "TASK [Start service] ***\nfailed: [web01] => {\"msg\": \"Service not found\"}\nPLAY RECAP ***\nweb01 : ok=1 failed=1" + + +[filters.basedpyright] +description = "Compact basedpyright type checker output — strip blank lines, keep errors" +match_command = "^basedpyright\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Searching for source files", + "^Found \\d+ source file", + "^Pyright \\d+\\.\\d+", + "^basedpyright \\d+\\.\\d+", +] +max_lines = 50 +on_empty = "basedpyright: ok" + +[[tests.basedpyright]] +name = "strips noise, keeps errors and summary" +input = """ +basedpyright 1.22.0 +Searching for source files +Found 42 source files + +/home/user/app/main.py + /home/user/app/main.py:10:5 - error: "foo" is not defined (reportUndefinedVariable) + /home/user/app/main.py:25:1 - error: Type "str" is not assignable to type "int" (reportAssignmentType) + +/home/user/app/utils.py + /home/user/app/utils.py:8:9 - warning: Variable "x" is not accessed (reportUnusedVariable) + +3 errors, 1 warning, 0 informations +""" +expected = "/home/user/app/main.py\n /home/user/app/main.py:10:5 - error: \"foo\" is not defined (reportUndefinedVariable)\n /home/user/app/main.py:25:1 - error: Type \"str\" is not assignable to type \"int\" (reportAssignmentType)\n/home/user/app/utils.py\n /home/user/app/utils.py:8:9 - warning: Variable \"x\" is not accessed (reportUnusedVariable)\n3 errors, 1 warning, 0 informations" + +[[tests.basedpyright]] +name = "clean output" +input = """ +basedpyright 1.22.0 +Searching for source files +Found 10 source files + +0 errors, 0 warnings, 0 informations +""" +expected = "0 errors, 0 warnings, 0 informations" + +[[tests.basedpyright]] +name = "empty input returns on_empty message" +input = "" +expected = "basedpyright: ok" + + +[filters.biome] +description = "Compact Biome lint/format output — strip blank lines, keep diagnostics" +match_command = "^biome\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Checked \\d+ file", + "^Fixed \\d+ file", + "^The following command", + "^Run it with", +] +max_lines = 50 +on_empty = "biome: ok" + +[[tests.biome]] +name = "lint strips noise, keeps diagnostics" +input = """ +Checked 42 files in 0.5s + +src/app.tsx:5:3 lint/suspicious/noExplicitAny ━━━━━━━━━━━━━━━━━━━━ + × Unexpected any. Specify a different type. + 3 │ interface Props { + 4 │ data: any; + 5 │ ^^^ + +src/utils.ts:12:1 lint/complexity/noForEach ━━━━━━━━━━━━━━━━━━━━ + × Prefer for...of instead of forEach. + 12 │ items.forEach(item => process(item)); + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Found 2 errors. +""" +expected = "src/app.tsx:5:3 lint/suspicious/noExplicitAny ━━━━━━━━━━━━━━━━━━━━\n × Unexpected any. Specify a different type.\n 3 │ interface Props {\n 4 │ data: any;\n 5 │ ^^^\nsrc/utils.ts:12:1 lint/complexity/noForEach ━━━━━━━━━━━━━━━━━━━━\n × Prefer for...of instead of forEach.\n 12 │ items.forEach(item => process(item));\n │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nFound 2 errors." + +[[tests.biome]] +name = "clean check" +input = """ +Checked 42 files in 0.3s +""" +expected = "biome: ok" + +[[tests.biome]] +name = "empty input returns on_empty message" +input = "" +expected = "biome: ok" + + +[filters.brew-install] +description = "Compact brew install/upgrade output — strip downloads, short-circuit when already installed" +match_command = "^brew\\s+(install|upgrade)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^==> Downloading", + "^==> Pouring", + "^Already downloaded:", + "^###", + "^==> Fetching", +] +match_output = [ + { pattern = "already installed", message = "ok (already installed)" }, +] +max_lines = 20 + +[[tests.brew-install]] +name = "already installed short-circuits" +input = """ +Warning: rtk 0.27.1 is already installed and up-to-date. +To reinstall 0.27.1, run: + brew reinstall rtk +""" +expected = "ok (already installed)" + +[[tests.brew-install]] +name = "install strips download lines" +input = """ +==> Fetching jq +==> Downloading https://homebrew.bintray.com/bottles/jq-1.7.1.arm64_sonoma.bottle.tar.gz +######################################################################## 100.0% +==> Pouring jq-1.7.1.arm64_sonoma.bottle.tar.gz +==> Summary +/opt/homebrew/Cellar/jq/1.7.1: 18 files, 1.2MB +""" +expected = "==> Summary\n/opt/homebrew/Cellar/jq/1.7.1: 18 files, 1.2MB" + + +[filters.bundle-install] +description = "Compact bundle install/update — strip 'Using' lines, keep installs and errors" +match_command = "^bundle\\s+(install|update)\\b" +strip_ansi = true +strip_lines_matching = [ + "^Using ", + "^\\s*$", + "^Fetching gem metadata", + "^Resolving dependencies", +] +match_output = [ + { pattern = "Bundle complete!", message = "ok bundle: complete" }, + { pattern = "Bundle updated!", message = "ok bundle: updated" }, +] +max_lines = 30 + +[[tests.bundle-install]] +name = "all cached short-circuits" +input = """ +Using bundler 2.5.6 +Using rake 13.1.0 +Using ast 2.4.2 +Using base64 0.2.0 +Using minitest 5.22.2 +Bundle complete! 85 Gemfile dependencies, 200 gems now installed. +Use `bundle info [gemname]` to see where a bundled gem is installed. +""" +expected = "ok bundle: complete" + +[[tests.bundle-install]] +name = "mixed install keeps Fetching and Installing lines" +input = """ +Fetching gem metadata from https://rubygems.org/......... +Resolving dependencies... +Using rake 13.1.0 +Using ast 2.4.2 +Fetching rspec 3.13.0 +Installing rspec 3.13.0 +Using rubocop 1.62.0 +Fetching simplecov 0.22.0 +Installing simplecov 0.22.0 +Bundle complete! 85 Gemfile dependencies, 202 gems now installed. +""" +expected = "ok bundle: complete" + +[[tests.bundle-install]] +name = "update output" +input = """ +Fetching gem metadata from https://rubygems.org/......... +Resolving dependencies... +Using rake 13.1.0 +Fetching rspec 3.14.0 (was 3.13.0) +Installing rspec 3.14.0 (was 3.13.0) +Bundle updated! +""" +expected = "ok bundle: updated" + +[[tests.bundle-install]] +name = "empty output" +input = "" +expected = "" + + +[filters.composer-install] +description = "Compact composer install/update/require output — strip downloads, short-circuit when up-to-date" +match_command = "^composer\\s+(install|update|require)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ - Downloading ", + "^ - Installing ", + "^Loading composer", + "^Updating dependencies", +] +match_output = [ + { pattern = "Nothing to install, update or remove", message = "ok (up to date)" }, +] +max_lines = 30 + +[[tests.composer-install]] +name = "nothing to do short-circuits" +input = """ +Loading composer repositories with package information +Updating dependencies +Lock file operations: 0 installs, 0 updates, 0 removals +Nothing to install, update or remove +Generating autoload files +""" +expected = "ok (up to date)" + +[[tests.composer-install]] +name = "install strips download lines" +input = """ +Loading composer repositories with package information +Updating dependencies + - Downloading symfony/console (v6.4.0) + - Installing symfony/console (v6.4.0): Extracting archive + - Downloading psr/log (3.0.0) + - Installing psr/log (3.0.0): Extracting archive +Writing lock file +Generating autoload files +""" +expected = "Writing lock file\nGenerating autoload files" + + +[filters.df] +description = "Compact df output — truncate wide columns, limit rows" +match_command = "^df(\\s|$)" +strip_ansi = true +truncate_lines_at = 80 +max_lines = 20 + +[[tests.df]] +name = "short output passes through unchanged" +input = "Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/sda1 4096000 123456 3972544 4% /" +expected = "Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/sda1 4096000 123456 3972544 4% /" + +[[tests.df]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.dotnet-build] +description = "Compact dotnet build output — short-circuit on success, strip banners" +match_command = "^dotnet\\s+build\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Microsoft \\(R\\)", + "^Copyright \\(C\\)", + "^ Determining projects", +] +match_output = [ + { pattern = "0 Warning\\(s\\)\\n\\s+0 Error\\(s\\)", message = "ok (build succeeded)" }, +] +max_lines = 40 + +[[tests.dotnet-build]] +name = "successful build short-circuits to ok" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... + All projects are up-to-date for restore. + MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll + +Build succeeded. + 0 Warning(s) + 0 Error(s) + +Time Elapsed 00:00:02.34 +""" +expected = "ok (build succeeded)" + +[[tests.dotnet-build]] +name = "build with warnings not short-circuited" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... + MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll + +Build succeeded. + 3 Warning(s) + 0 Error(s) + +Time Elapsed 00:00:01.87 +""" +expected = " MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll\nBuild succeeded.\n 3 Warning(s)\n 0 Error(s)\nTime Elapsed 00:00:01.87" + +[[tests.dotnet-build]] +name = "build errors pass through" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... +src/Program.cs(10,5): error CS1002: ; expected [/home/user/MyApp/MyApp.csproj] + +Build FAILED. + 0 Warning(s) + 1 Error(s) +""" +expected = "src/Program.cs(10,5): error CS1002: ; expected [/home/user/MyApp/MyApp.csproj]\nBuild FAILED.\n 0 Warning(s)\n 1 Error(s)" + + +[filters.du] +description = "Compact du output" +match_command = "^du\\b" +strip_lines_matching = ["^\\s*$"] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.du]] +name = "preserves sizes, strips blank lines" +input = "4.0K\t./src\n\n8.0K\t./tests\n16K\t." +expected = "4.0K\t./src\n8.0K\t./tests\n16K\t." + +[[tests.du]] +name = "single line passthrough" +input = "128K\t." +expected = "128K\t." + + +[filters.fail2ban-client] +description = "Compact fail2ban-client output" +match_command = "^fail2ban-client\\b" +strip_lines_matching = ["^\\s*$"] +max_lines = 30 + +[[tests.fail2ban-client]] +name = "strips blank lines" +input = "Status for the jail: sshd\n|- Filter\n| |- Currently failed: 3\n\n|- Actions\n `- Total banned: 42" +expected = "Status for the jail: sshd\n|- Filter\n| |- Currently failed: 3\n|- Actions\n `- Total banned: 42" + +[[tests.fail2ban-client]] +name = "single line passthrough" +input = "Shutdown successful" +expected = "Shutdown successful" + + +[filters.gcc] +description = "Compact gcc/g++ compiler output — strip notes, keep errors and warnings" +match_command = "^g(cc|\\+\\+)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s+\\|\\s*$", + "^In file included from", + "^\\s+from\\s", + "^\\d+ warnings? generated", + "^\\d+ errors? generated", +] +max_lines = 50 +on_empty = "gcc: ok" + +[[tests.gcc]] +name = "strips include chain, keeps errors and warnings" +input = """ +In file included from /usr/include/stdio.h:42: + from main.c:1: +main.c:10:5: error: use of undeclared identifier 'foo' + foo(); + ^ +main.c:15:12: warning: unused variable 'x' [-Wunused-variable] + int x = 42; + ^ +2 warnings generated. +1 error generated. +""" +expected = "main.c:10:5: error: use of undeclared identifier 'foo'\n foo();\n ^\nmain.c:15:12: warning: unused variable 'x' [-Wunused-variable]\n int x = 42;\n ^" + +[[tests.gcc]] +name = "clean compilation" +input = """ +""" +expected = "gcc: ok" + +[[tests.gcc]] +name = "linker error kept" +input = """ +/usr/bin/ld: /tmp/main.o: undefined reference to 'missing_func' +collect2: error: ld returned 1 exit status +""" +expected = "/usr/bin/ld: /tmp/main.o: undefined reference to 'missing_func'\ncollect2: error: ld returned 1 exit status" + +[[tests.gcc]] +name = "empty input returns on_empty message" +input = "" +expected = "gcc: ok" + + +[filters.gcloud] +description = "Compact gcloud output" +match_command = "^gcloud\\b" +strip_ansi = true +strip_lines_matching = ["^\\s*$"] +truncate_lines_at = 120 +max_lines = 30 + +[[tests.gcloud]] +name = "strips blank lines, preserves output" +input = """ +Updated property [core/project]. + +NAME REGION STATUS +my-cluster us-central1 RUNNING +""" +expected = "Updated property [core/project].\nNAME REGION STATUS\nmy-cluster us-central1 RUNNING" + +[[tests.gcloud]] +name = "single line passthrough" +input = "Listed 0 items." +expected = "Listed 0 items." + + +[filters.hadolint] +description = "Compact hadolint Dockerfile linting output" +match_command = "^hadolint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.hadolint]] +name = "Dockerfile warnings kept, blank lines stripped" +input = """ +Dockerfile:3 DL3008 warning: Pin versions in apt-get install +Dockerfile:5 DL3009 info: Delete apt-get lists after installing + +Dockerfile:8 DL4006 warning: Set SHELL option -o pipefail before RUN with pipe +""" +expected = "Dockerfile:3 DL3008 warning: Pin versions in apt-get install\nDockerfile:5 DL3009 info: Delete apt-get lists after installing\nDockerfile:8 DL4006 warning: Set SHELL option -o pipefail before RUN with pipe" + +[[tests.hadolint]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.helm] +description = "Compact helm output" +match_command = "^helm\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^W\\d{4}", +] +truncate_lines_at = 120 + +[[tests.helm]] +name = "strips blank lines, preserves release info" +input = """ +NAME: my-release +LAST DEPLOYED: Mon Jan 15 10:30:00 2024 +NAMESPACE: default +STATUS: deployed +REVISION: 3 + +NOTES: +Application is running. +""" +expected = "NAME: my-release\nLAST DEPLOYED: Mon Jan 15 10:30:00 2024\nNAMESPACE: default\nSTATUS: deployed\nREVISION: 3\nNOTES:\nApplication is running." + +[[tests.helm]] +name = "strips glog W-prefix warnings" +input = "W0115 10:30:00 warning message from internal\nNAME: my-chart\nSTATUS: deployed" +expected = "NAME: my-chart\nSTATUS: deployed" + +[[tests.helm]] +name = "helm template output is not truncated" +input = """ +# Source: my-chart/templates/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: my-ns +--- +# Source: my-chart/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-secret +--- +# Source: my-chart/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-cm +--- +# Source: my-chart/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc +--- +# Source: my-chart/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: my-svc +--- +# Source: my-chart/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-deploy +""" +expected = """# Source: my-chart/templates/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: my-ns +--- +# Source: my-chart/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-secret +--- +# Source: my-chart/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-cm +--- +# Source: my-chart/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc +--- +# Source: my-chart/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: my-svc +--- +# Source: my-chart/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-deploy""" + + +[filters.iptables] +description = "Compact iptables output" +match_command = "^iptables\\b" +strip_lines_matching = [ + "^\\s*$", + "^Chain DOCKER", + "^Chain BR-", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.iptables]] +name = "strips Docker chains, preserves real rules" +input = """ +Chain INPUT (policy ACCEPT) +num target prot opt source destination +1 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 +Chain DOCKER (1 references) + DOCKER all -- 0.0.0.0/0 0.0.0.0/0 +Chain BR-abcdef (0 references) +""" +expected = "Chain INPUT (policy ACCEPT)\nnum target prot opt source destination\n1 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0\n DOCKER all -- 0.0.0.0/0 0.0.0.0/0" + +[[tests.iptables]] +name = "preserves FORWARD and OUTPUT chains" +input = "Chain FORWARD (policy DROP)\n1 ACCEPT tcp\nChain OUTPUT (policy ACCEPT)\n1 ACCEPT all" +expected = "Chain FORWARD (policy DROP)\n1 ACCEPT tcp\nChain OUTPUT (policy ACCEPT)\n1 ACCEPT all" + + +[filters.jira] +description = "Compact Jira CLI output — strip verbose metadata, keep essentials" +match_command = "^jira\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*--", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.jira]] +name = "strips blank lines from issue list" +input = "TYPE\tKEY\tSUMMARY\tSTATUS\n\nStory\tPROJ-123\tAdd login feature\tIn Progress\n\nBug\tPROJ-456\tFix crash on startup\tOpen" +expected = "TYPE\tKEY\tSUMMARY\tSTATUS\nStory\tPROJ-123\tAdd login feature\tIn Progress\nBug\tPROJ-456\tFix crash on startup\tOpen" + +[[tests.jira]] +name = "single issue view" +input = "KEY: PROJ-123\nSummary: Add login feature\nStatus: In Progress\nAssignee: john@example.com" +expected = "KEY: PROJ-123\nSummary: Add login feature\nStatus: In Progress\nAssignee: john@example.com" + + +[filters.jj] +description = "Compact Jujutsu (jj) output — strip blank lines, truncate" +match_command = "^jj\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Hint:", + "^Working copy now at:", +] +max_lines = 30 +truncate_lines_at = 120 + +[[tests.jj]] +name = "log output stripped of hints" +input = """ +@ qpvuntsm patrick@example.com 2026-03-10 12:00 abc123 +│ feat: add new feature +◉ zzzzzzzz root() + +Working copy now at: qpvuntsm abc123 feat: add new feature +Hint: use `jj log` to see the full history +""" +expected = "@ qpvuntsm patrick@example.com 2026-03-10 12:00 abc123\n│ feat: add new feature\n◉ zzzzzzzz root()" + +[[tests.jj]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.jq] +description = "Compact jq output — truncate large JSON results" +match_command = "^jq\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 40 +truncate_lines_at = 120 + +[[tests.jq]] +name = "short output passes through" +input = """ +{ + "name": "test", + "version": "1.0" +} +""" +expected = "{\n \"name\": \"test\",\n \"version\": \"1.0\"\n}" + +[[tests.jq]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.just] +description = "Compact just task runner output — strip recipe headers, keep command output" +match_command = "^just\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Available recipes:", + "^\\s*just --list", +] +truncate_lines_at = 150 +max_lines = 50 + +[[tests.just]] +name = "preserves command output" +input = "cargo test\n\ntest result: ok. 42 passed; 0 failed\n" +expected = "cargo test\ntest result: ok. 42 passed; 0 failed" + +[[tests.just]] +name = "preserves error output" +input = "error: Compilation failed\nsrc/main.rs:10: expected `;`" +expected = "error: Compilation failed\nsrc/main.rs:10: expected `;`" + +[[tests.just]] +name = "empty input" +input = "" +expected = "" + + +[filters.liquibase] +description = "Compact liquibase output — strip headers and generic info" +match_command = "^liquibase(?:\\s|$)" +strip_ansi = true +filter_stderr = true +strip_lines_matching = [ + "^\\s*$", + "^Starting Liquibase at", + "^Liquibase (?:Community|Open Source)", + "^Liquibase Home:", + "^Java Home", + "^Libraries:", + "^\\s*-\\s+\\S+\\.jar", + "^INFO \\[liquibase\\.integration\\]", + "^INFO \\[liquibase\\.core\\] Reading resource", + "^INFO \\[liquibase\\.core\\] Parsing", + "^(?:\\[?INFO\\]?\\s*)?#+$", + "^\\s*##" +] +on_empty = "liquibase: ok" +max_lines = 200 + +[[tests.liquibase]] +name = "strip ascii banner and info logs from subcommand" +input = ''' +#################################################### +## _ _ _ _ ## +## | | (_) (_) | ## +#################################################### +Starting Liquibase at 10:12:11 (version 4.29.1) +Liquibase Version: 4.29.1 +Liquibase Open Source 4.29.1 by Liquibase +INFO [liquibase.integration] Starting command +INFO [liquibase.core] Reading resource db/changelog.xml +INFO [liquibase.core] Parsing db/changelog.xml +Running Changeset: filepath::id::author +Changeset filepath::id::author ran successfully +''' +expected = ''' +Liquibase Version: 4.29.1 +Running Changeset: filepath::id::author +Changeset filepath::id::author ran successfully''' + +[[tests.liquibase]] +name = "strip --version noise, keep only version line" +input = ''' +#################################################### +## _ _ _ _ ## +#################################################### +Starting Liquibase at 13:45:24 using Java 17.0.15 (version 4.30.0 #4943 built at 2024-10-31 17:00+0000) +Liquibase Home: D:\mcp\bash\lbr\third-party +Java Home C:\Program Files\Java\jdk-17.0.15 (Version 17.0.15) +Libraries: + - internal\lib\commons-io.jar: Apache Commons IO 2.17.0 By The Apache Software Foundation + - internal\lib\picocli.jar: picocli 4.7.6 By Remko Popma + - lib\ojdbc10-19.30.0.0.jar: JDBC 19.30.0.0.0 By Oracle Corporation + +Liquibase Version: 4.30.0 +Liquibase Open Source 4.30.0 by Liquibase +''' +expected = ''' +Liquibase Version: 4.30.0''' + +[[tests.liquibase]] +name = "keep status and error lines" +input = ''' +#################################################### +## _ _ _ _ ## +#################################################### +Starting Liquibase at 10:00:00 (version 4.30.0) +Liquibase Version: 4.30.0 +Liquibase Open Source 4.30.0 by Liquibase +HR@jdbc:oracle:thin:@localhost:1523:XE is up to date +Liquibase command 'status' was executed successfully. +''' +expected = ''' +Liquibase Version: 4.30.0 +HR@jdbc:oracle:thin:@localhost:1523:XE is up to date +Liquibase command 'status' was executed successfully.''' + +[[tests.liquibase]] +name = "empty input" +input = "" +expected = "liquibase: ok" + + +[filters.make] +description = "Compact make output" +match_command = "^make\\b" +strip_lines_matching = [ + "^make\\[\\d+\\]:", + "^\\s*$", + "^Nothing to be done", +] +max_lines = 50 +on_empty = "make: ok" + +[[tests.make]] +name = "strips entering/leaving lines" +input = """ +make[1]: Entering directory '/home/user' +gcc -O2 foo.c +make[1]: Leaving directory '/home/user' +""" +expected = """ +gcc -O2 foo.c +""" + +[[tests.make]] +name = "strips blank lines" +input = """ +gcc -O2 foo.c + +gcc -O2 bar.c +""" +expected = """ +gcc -O2 foo.c +gcc -O2 bar.c +""" + +[[tests.make]] +name = "on_empty when all stripped" +input = """ +make[1]: Entering directory '/home/user' +make[1]: Leaving directory '/home/user' +""" +expected = "make: ok" + + +[filters.markdownlint] +description = "Compact markdownlint output — strip blank lines, limit rows" +match_command = "^markdownlint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.markdownlint]] +name = "linting errors stripped of blank lines" +input = """ +README.md:1:1 MD041/first-line-heading/first-line-h1 First line in file should be a top level heading +README.md:10:1 MD022/blanks-around-headings Headings should be surrounded by blank lines + +README.md:15:80 MD013/line-length Line length [Expected: 80; Actual: 95] +""" +expected = "README.md:1:1 MD041/first-line-heading/first-line-h1 First line in file should be a top level heading\nREADME.md:10:1 MD022/blanks-around-headings Headings should be surrounded by blank lines\nREADME.md:15:80 MD013/line-length Line length [Expected: 80; Actual: 95]" + +[[tests.markdownlint]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.mise] +description = "Compact mise task runner output — strip status lines, keep task results" +match_command = "^mise\\s+(run|exec|install|upgrade)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^mise\\s+(trust|install|upgrade).*✓", + "^mise\\s+Installing\\s", + "^mise\\s+Downloading\\s", + "^mise\\s+Extracting\\s", + "^mise\\s+\\w+@[\\d.]+ installed", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "mise: ok" + +[[tests.mise]] +name = "strips install noise, keeps task output" +input = "mise Installing node@20.0.0\nmise Downloading node@20.0.0\nmise Extracting node@20.0.0\nmise node@20.0.0 installed\n\nlint check passed\n2 warnings found" +expected = "lint check passed\n2 warnings found" + +[[tests.mise]] +name = "preserves error output" +input = "mise run lint\nError: biome check failed\nsrc/index.ts:5 — unused variable" +expected = "mise run lint\nError: biome check failed\nsrc/index.ts:5 — unused variable" + +[[tests.mise]] +name = "empty after stripping" +input = "mise trust ~/dev/.mise.toml ✓\nmise install node@20 ✓\n" +expected = "mise: ok" + + +[filters.mix-compile] +description = "Compact mix compile output" +match_command = "^mix\\s+compile(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^Compiling \\d+ file", + "^\\s*$", + "^Generated\\s", +] +max_lines = 40 +on_empty = "mix compile: ok" + +[[tests.mix-compile]] +name = "strips compile noise, preserves warnings" +input = """ +Compiling 12 files (.ex) +Generated my_app app + +warning: variable "conn" is unused + lib/router.ex:42 +""" +expected = "warning: variable \"conn\" is unused\n lib/router.ex:42" + +[[tests.mix-compile]] +name = "on_empty when only noise" +input = "Compiling 3 files (.ex)\nGenerated my_app app\n" +expected = "mix compile: ok" + + +[filters.mix-format] +description = "Compact mix format output" +match_command = "^mix\\s+format(\\s|$)" +on_empty = "mix format: ok" +max_lines = 20 + +[[tests.mix-format]] +name = "empty output returns ok" +input = "" +expected = "mix format: ok" + +[[tests.mix-format]] +name = "changed files pass through" +input = "lib/my_app.ex\ntest/my_app_test.exs" +expected = "lib/my_app.ex\ntest/my_app_test.exs" + + +[filters.nx] +description = "Compact Nx monorepo output — strip task graph noise, keep results" +match_command = "^(pnpm\\s+)?nx\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*>\\s*NX\\s+Running target", + "^\\s*>\\s*NX\\s+Nx read the output", + "^\\s*>\\s*NX\\s+View logs", + "^———————", + "^—————————", + "^\\s+Nx \\(powered by", +] +truncate_lines_at = 150 +max_lines = 60 + +[[tests.nx]] +name = "strips Nx noise, keeps build output" +input = "\n > NX Running target build for project myapp\n\n———————————————————————————————————————\nCompiled successfully.\nOutput: dist/apps/myapp\n\n > NX View logs at /tmp/.nx/runs/abc123\n\n Nx (powered by computation caching)\n" +expected = "Compiled successfully.\nOutput: dist/apps/myapp" + +[[tests.nx]] +name = "preserves error output" +input = "ERROR: Cannot find module '@myapp/shared'\n\n > NX Running target build for project myapp\n\nFailed at step: build" +expected = "ERROR: Cannot find module '@myapp/shared'\nFailed at step: build" + + +[filters.ollama] +description = "Strip ANSI spinners and cursor control from ollama output, keep final text" +match_command = "^ollama\\s+run\\b" +strip_ansi = true +strip_lines_matching = [ + "^[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏\\s]*$", + "^\\s*$", +] + +[[tests.ollama]] +name = "strips spinner lines, keeps response" +input = "⠋ \n⠙ \n⠹ \nHello! How can I help you today?" +expected = "Hello! How can I help you today?" + +[[tests.ollama]] +name = "preserves multi-line response" +input = "⠋ \n⠙ \nLine one of the response.\nLine two of the response." +expected = "Line one of the response.\nLine two of the response." + +[[tests.ollama]] +name = "empty input" +input = "" +expected = "" + + +[filters.oxlint] +description = "Compact oxlint output — strip blank lines, keep diagnostics" +match_command = "^oxlint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Finished in \\d+", + "^Found \\d+ warning", +] +max_lines = 50 +on_empty = "oxlint: ok" + +[[tests.oxlint]] +name = "strips noise, keeps diagnostics" +input = """ + × eslint(no-console): Unexpected console statement. + ╭─[src/app.ts:5:3] + 5 │ console.log("debug"); + │ ^^^^^^^^^^^ + ╰──── + + × eslint(no-unused-vars): 'x' is defined but never used. + ╭─[src/utils.ts:2:7] + 2 │ let x = 42; + │ ^ + ╰──── + +Found 2 warnings on 2 files. +Finished in 12ms on 100 files. +""" +expected = " × eslint(no-console): Unexpected console statement.\n ╭─[src/app.ts:5:3]\n 5 │ console.log(\"debug\");\n │ ^^^^^^^^^^^\n ╰────\n × eslint(no-unused-vars): 'x' is defined but never used.\n ╭─[src/utils.ts:2:7]\n 2 │ let x = 42;\n │ ^\n ╰────" + +[[tests.oxlint]] +name = "clean output" +input = """ +Finished in 5ms on 100 files. +""" +expected = "oxlint: ok" + +[[tests.oxlint]] +name = "empty input returns on_empty message" +input = "" +expected = "oxlint: ok" + + +[filters.ping] +description = "Compact ping output — strip per-packet lines, keep summary" +match_command = "^ping\\b" +strip_ansi = true +strip_lines_matching = [ + "^PING ", + "^Pinging ", + "^\\d+ bytes from ", + "^Reply from .+: bytes=", + "^\\s*$", +] +tail_lines = 4 + +[[tests.ping]] +name = "success keeps summary only" +input = """ +PING example.com (93.184.216.34): 56 data bytes +64 bytes from 93.184.216.34: icmp_seq=0 ttl=56 time=14.2 ms +64 bytes from 93.184.216.34: icmp_seq=1 ttl=56 time=13.8 ms +64 bytes from 93.184.216.34: icmp_seq=2 ttl=56 time=14.1 ms +64 bytes from 93.184.216.34: icmp_seq=3 ttl=56 time=13.9 ms + +--- example.com ping statistics --- +4 packets transmitted, 4 packets received, 0.0% packet loss +round-trip min/avg/max/stddev = 13.8/14.0/14.2/0.2 ms +""" +expected = """--- example.com ping statistics --- +4 packets transmitted, 4 packets received, 0.0% packet loss +round-trip min/avg/max/stddev = 13.8/14.0/14.2/0.2 ms""" + +[[tests.ping]] +name = "windows format keeps stats block only" +input = """ +Pinging 192.0.2.1 with 32 bytes of data: +Reply from 192.0.2.1: bytes=32 time=14ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=13ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=14ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=13ms TTL=56 + +Ping statistics for 192.0.2.1: + Packets: Sent = 4, Received = 4, Lost = 0 (0% loss), +Approximate round trip times in milli-seconds: + Minimum = 13ms, Maximum = 14ms, Average = 13ms +""" +expected = """Ping statistics for 192.0.2.1: + Packets: Sent = 4, Received = 4, Lost = 0 (0% loss), +Approximate round trip times in milli-seconds: + Minimum = 13ms, Maximum = 14ms, Average = 13ms""" + +[[tests.ping]] +name = "unreachable host passes error through" +input = """ +PING unreachable.example.com (192.0.2.1): 56 data bytes +Request timeout for icmp_seq 0 +Request timeout for icmp_seq 1 + +--- unreachable.example.com ping statistics --- +2 packets transmitted, 0 packets received, 100.0% packet loss +""" +expected = """Request timeout for icmp_seq 0 +Request timeout for icmp_seq 1 +--- unreachable.example.com ping statistics --- +2 packets transmitted, 0 packets received, 100.0% packet loss""" + + +[filters.pio-run] +description = "Compact PlatformIO build output" +match_command = "^pio\\s+run(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Verbose mode", + "^CONFIGURATION:", + "^LDF:", + "^Library Manager:", + "^Compiling\\s", + "^Linking\\s", + "^Building\\s", + "^Checking size", +] +max_lines = 30 +on_empty = "pio run: ok" + +[[tests.pio-run]] +name = "strips build noise, preserves errors" +input = """ +Verbose mode can be enabled via `-v, --verbose` option +CONFIGURATION: https://docs.platformio.org/page/boards/espressif32/esp32dev.html +LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf +Compiling .pio/build/esp32dev/src/main.cpp.o +Building .pio/build/esp32dev/firmware.elf +Linking .pio/build/esp32dev/firmware.elf +Checking size .pio/build/esp32dev/firmware.elf +src/main.cpp:10:3: error: 'LED_BUILTINN' was not declared +""" +expected = "src/main.cpp:10:3: error: 'LED_BUILTINN' was not declared" + +[[tests.pio-run]] +name = "on_empty when clean build with only noise" +input = """ +Verbose mode can be enabled via `-v, --verbose` option +Compiling .pio/build/esp32dev/src/main.cpp.o +Linking .pio/build/esp32dev/firmware.elf +""" +expected = "pio run: ok" + + +[filters.poetry-install] +description = "Compact poetry install/lock/update output — strip downloads, short-circuit when up-to-date" +match_command = "^poetry\\s+(install|lock|update)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ [-•] Downloading ", + "^ [-•] Installing .* \\(", + "^Creating virtualenv", + "^Using virtualenv", +] +match_output = [ + { pattern = "No dependencies to install or update|No changes\\.", message = "ok (up to date)" }, +] +max_lines = 30 + +[[tests.poetry-install]] +name = "up to date short-circuits" +input = """ +Installing dependencies from lock file + +No dependencies to install or update +""" +expected = "ok (up to date)" + +[[tests.poetry-install]] +name = "poetry 2.x bullet syntax short-circuits to ok" +input = """ +• Installing requests (2.31.0) +• Installing certifi (2023.11.17) + +No changes. +""" +expected = "ok (up to date)" + +[[tests.poetry-install]] +name = "install strips download lines" +input = """ +Installing dependencies from lock file + + - Downloading requests-2.31.0-py3-none-any.whl (62.6 kB) + - Installing certifi (2023.11.17) + - Installing charset-normalizer (3.3.2) + - Installing idna (3.6) + - Installing urllib3 (2.1.0) + - Installing requests (2.31.0) + +Writing lock file +""" +expected = "Installing dependencies from lock file\nWriting lock file" + + +[filters.pre-commit] +description = "Compact pre-commit output" +match_command = "^pre-commit\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\[INFO\\] Installing environment", + "^\\[INFO\\] Once installed this environment will be reused", + "^\\[INFO\\] This may take a few minutes", + "^\\s*$", +] +max_lines = 40 + +[[tests.pre-commit]] +name = "strips INFO install noise, keeps hook results" +input = """ +[INFO] Installing environment for https://github.com/psf/black. +[INFO] Once installed this environment will be reused. +[INFO] This may take a few minutes... +Trim Trailing Whitespace.................................................Passed +Fix End of Files.........................................................Passed +Check Yaml...............................................................Failed +- hook id: check-yaml +- exit code: 1 +""" +expected = "Trim Trailing Whitespace.................................................Passed\nFix End of Files.........................................................Passed\nCheck Yaml...............................................................Failed\n- hook id: check-yaml\n- exit code: 1" + +[[tests.pre-commit]] +name = "all passed — no INFO noise" +input = """ +[INFO] Installing environment for https://github.com/pre-commit/mirrors-isort. +[INFO] Once installed this environment will be reused. +isort....................................................................Passed +black....................................................................Passed +""" +expected = "isort....................................................................Passed\nblack....................................................................Passed" + + +[filters.ps] +description = "Compact ps output — truncate wide lines, limit rows" +match_command = "^ps(\\s|$)" +strip_ansi = true +truncate_lines_at = 120 +max_lines = 30 + +[[tests.ps]] +name = "short process list passes through unchanged" +input = "USER PID %CPU %MEM COMMAND\nroot 1 0.0 0.0 /sbin/launchd\nflorian 42 0.1 0.2 bash" +expected = "USER PID %CPU %MEM COMMAND\nroot 1 0.0 0.0 /sbin/launchd\nflorian 42 0.1 0.2 bash" + +[[tests.ps]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.pulumi-destroy] +description = "Compact Pulumi destroy output" +match_command = "^pulumi\\s+destroy(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Destroying \\(", + "^Previewing (update|destroy)", + "^View in Browser", + "^View Live:", + "^Duration:", + "^Permalink:", + "^\\s*Type\\s+Name\\s+", + "^Loading policy packs", + "^@ (Previewing (update|destroy)|Destroying)", + "^More information at:", + "^Use `pulumi ", + "^The resources in the stack have been deleted", + "^If you want to remove the stack completely", + "^\\s+-\\s+.*\\bdeleting\\s+\\(", + "^\\s{4,}at\\s+\\S+\\s*\\(", + "^\\s{4,}at\\s+/", + "^\\s+at\\s+processTicksAndRejections", + "^\\s+promise:\\s+Promise", + "^\\s+\\[Circular", + "^\\s*-.jar name, and jars under a path containing spaces, are deliberately left to full passthrough rather than risk misclassifying an unrelated tool." +match_command = "^(mvn\\s+spring-boot:run|java\\s+-jar\\s+(?:\\S*[/\\\\])?[^/\\\\\\s]*(?i:spring)[^/\\\\\\s]*\\.jar|gradle\\s+.*bootRun)" +strip_ansi = true +keep_lines_matching = [ + "Started\\s.*\\sin\\s", + "Tomcat started on port", + "ERROR", + "WARN", + "Exception", + "Caused by:", + "Application run failed", + "BUILD\\s", + "Tests run:", + "FAILURE", + "listening on port", +] +max_lines = 30 + +[[tests.spring-boot]] +name = "keeps startup summary and errors" +input = " . ____ _ \n /\\\\ / ___'_ __ _ _(_)_ __ \n( ( )\\___ | '_ | '_| | '_ \\ \n \\/ ___)| |_)| | | | | || )\n ' |____| .__|_| |_|_| |_\\__|\n :: Spring Boot :: (v3.2.0)\n2024-01-01 INFO Initializing Spring\n2024-01-01 INFO Bean 'dataSource' created\n2024-01-01 INFO Tomcat started on port 8080\n2024-01-01 INFO Started MyApp in 3.2 seconds" +expected = "2024-01-01 INFO Tomcat started on port 8080\n2024-01-01 INFO Started MyApp in 3.2 seconds" + +[[tests.spring-boot]] +name = "preserves errors" +input = " :: Spring Boot :: (v3.2.0)\n2024-01-01 INFO Initializing Spring\n2024-01-01 ERROR Application run failed\nCaused by: java.lang.NullPointerException" +expected = "2024-01-01 ERROR Application run failed\nCaused by: java.lang.NullPointerException" + + +[filters.ssh] +description = "Compact ssh output — strip connection banners, keep command output" +match_command = "^ssh(?:\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Warning: Permanently added", + "^Connection to .+ closed", + "^Authenticated to", + "^debug1:", + "^OpenSSH_", + "^Pseudo-terminal", +] +max_lines = 200 +truncate_lines_at = 120 + +[[tests.ssh]] +name = "strips connection banners, keeps command output" +input = """ +Warning: Permanently added '192.168.1.10' (ED25519) to the list of known hosts. + +total 32 +drwxr-xr-x 4 user user 4096 Mar 10 12:00 app +-rw-r--r-- 1 user user 1234 Mar 10 11:00 config.yaml + +Connection to 192.168.1.10 closed. +""" +expected = "total 32\ndrwxr-xr-x 4 user user 4096 Mar 10 12:00 app\n-rw-r--r-- 1 user user 1234 Mar 10 11:00 config.yaml" + +[[tests.ssh]] +name = "verbose debug lines stripped" +input = """ +debug1: Connecting to host.example.com port 22. +debug1: Connection established. +Authenticated to host.example.com ([1.2.3.4]:22). +uptime: 12:00:00 up 42 days, load average: 0.10, 0.15, 0.12 +Connection to host.example.com closed. +""" +expected = "uptime: 12:00:00 up 42 days, load average: 0.10, 0.15, 0.12" + +[[tests.ssh]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.stat] +description = "Compact stat output — strip device/inode/birth noise" +match_command = "^stat\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Device:", + "^\\s*Birth:", +] +truncate_lines_at = 120 +max_lines = 20 + +[[tests.stat]] +name = "linux stat output strips device and birth" +input = """ + File: main.rs + Size: 12345 Blocks: 24 IO Block: 4096 regular file +Device: 801h/2049d Inode: 1234567 Links: 1 +Access: (0644/-rw-r--r--) Uid: ( 1000/ patrick) Gid: ( 1000/ patrick) +Access: 2026-03-10 12:00:00.000000000 +0100 +Modify: 2026-03-10 11:00:00.000000000 +0100 +Change: 2026-03-10 11:00:00.000000000 +0100 + Birth: 2026-03-09 10:00:00.000000000 +0100 +""" +expected = " File: main.rs\n Size: 12345 Blocks: 24 IO Block: 4096 regular file\nAccess: (0644/-rw-r--r--) Uid: ( 1000/ patrick) Gid: ( 1000/ patrick)\nAccess: 2026-03-10 12:00:00.000000000 +0100\nModify: 2026-03-10 11:00:00.000000000 +0100\nChange: 2026-03-10 11:00:00.000000000 +0100" + +[[tests.stat]] +name = "macOS stat -x strips device and birth" +input = """ + File: "main.rs" + Size: 82848 FileType: Regular File + Mode: (0644/-rw-r--r--) Uid: ( 501/ patrick) Gid: ( 20/ staff) +Device: 1,15 Inode: 66302332 Links: 1 +Access: Wed Mar 18 21:21:15 2026 +Modify: Wed Mar 18 20:56:11 2026 +Change: Wed Mar 18 20:56:11 2026 + Birth: Wed Mar 18 20:56:11 2026 +""" +expected = " File: \"main.rs\"\n Size: 82848 FileType: Regular File\n Mode: (0644/-rw-r--r--) Uid: ( 501/ patrick) Gid: ( 20/ staff)\nAccess: Wed Mar 18 21:21:15 2026\nModify: Wed Mar 18 20:56:11 2026\nChange: Wed Mar 18 20:56:11 2026" + +[[tests.stat]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.swift-build] +description = "Compact swift build output — short-circuit on success, strip Compiling/Linking" +match_command = "^swift\\s+build\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Compiling ", + "^Linking ", +] +match_output = [ + { pattern = "Build complete!", message = "ok (build complete)", unless = "warning:|error:" }, +] +max_lines = 40 + +[[tests.swift-build]] +name = "successful build short-circuits to ok" +input = """ +Build complete! +""" +expected = "ok (build complete)" + +[[tests.swift-build]] +name = "build errors pass through after stripping noise" +input = """ +Compiling MyApp MyApp.swift +/home/user/MyApp/Sources/MyApp/main.swift:5:1: error: use of unresolved identifier 'foo' +foo() +^~~ +Linking MyApp +error: build had 1 command failure +""" +expected = "/home/user/MyApp/Sources/MyApp/main.swift:5:1: error: use of unresolved identifier 'foo'\nfoo()\n^~~\nerror: build had 1 command failure" + +[[tests.swift-build]] +name = "warnings not swallowed when Build complete present" +input = """ +CompileSwift normal x86_64 MyFile.swift +/path/to/MyFile.swift:42:10: warning: unused variable 'x' +Build complete! (with warnings) +""" +expected = "CompileSwift normal x86_64 MyFile.swift\n/path/to/MyFile.swift:42:10: warning: unused variable 'x'\nBuild complete! (with warnings)" + + +[filters.systemctl-status] +description = "Compact systemctl status output — strip blank lines, limit to 20 lines" +match_command = "^systemctl\\s+status\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 20 + +[[tests.systemctl-status]] +name = "verbose unit status stripped of blank lines" +input = """ +● nginx.service - A high performance web server + Loaded: loaded (/lib/systemd/system/nginx.service; enabled) + Active: active (running) since Mon 2024-01-15 10:30:00 UTC; 2h ago + Docs: man:nginx(8) + Main PID: 1234 (nginx) + Tasks: 3 (limit: 4915) + Memory: 8.5M + + CGroup: /system.slice/nginx.service + ├─1234 nginx: master process /usr/sbin/nginx + └─1235 nginx: worker process + +Jan 15 10:30:00 host nginx[1234]: nginx/1.24.0 +Jan 15 10:30:00 host systemd[1]: Started nginx.service +""" +expected = "● nginx.service - A high performance web server\n Loaded: loaded (/lib/systemd/system/nginx.service; enabled)\n Active: active (running) since Mon 2024-01-15 10:30:00 UTC; 2h ago\n Docs: man:nginx(8)\n Main PID: 1234 (nginx)\n Tasks: 3 (limit: 4915)\n Memory: 8.5M\n CGroup: /system.slice/nginx.service\n ├─1234 nginx: master process /usr/sbin/nginx\n └─1235 nginx: worker process\nJan 15 10:30:00 host nginx[1234]: nginx/1.24.0\nJan 15 10:30:00 host systemd[1]: Started nginx.service" + +[[tests.systemctl-status]] +name = "empty input passes through" +input = "" +expected = "" + + +[filters.task] +description = "Compact go-task output — strip task headers, keep command results" +match_command = "^task\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^task: \\[.*\\] ", + "^task: Task .* is up to date", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "task: ok" + +[[tests.task]] +name = "strips task headers, keeps output" +input = "task: [build] go build ./...\n\ntask: [test] go test ./...\nok myapp 0.5s\n\ntask: Task \"lint\" is up to date" +expected = "ok myapp 0.5s" + +[[tests.task]] +name = "preserves error output" +input = "task: [build] go build ./...\n./main.go:10: undefined: foo\ntask: Failed to run task \"build\": exit status 1" +expected = "./main.go:10: undefined: foo\ntask: Failed to run task \"build\": exit status 1" + +[[tests.task]] +name = "all up to date" +input = "task: Task \"build\" is up to date\ntask: Task \"lint\" is up to date\n" +expected = "task: ok" + + +[filters.terraform-plan] +description = "Compact Terraform plan output" +match_command = "^terraform\\s+plan(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^Refreshing state", + "^\\s*#.*unchanged", + "^\\s*$", + "^Acquiring state lock", + "^Releasing state lock", +] +max_lines = 80 +on_empty = "terraform plan: no changes detected" + +[[tests.terraform-plan]] +name = "strips Refreshing state lines and blank lines" +input = """ +Acquiring state lock. This may take a few moments... +Refreshing state... [id=vpc-abc] +Refreshing state... [id=sg-123] +Releasing state lock. This may take a few moments... + +Terraform will perform the following actions: + + # aws_instance.web will be created + + resource "aws_instance" "web" {} + +Plan: 1 to add, 0 to change, 0 to destroy. +""" +expected = "Terraform will perform the following actions:\n # aws_instance.web will be created\n + resource \"aws_instance\" \"web\" {}\nPlan: 1 to add, 0 to change, 0 to destroy." + +[[tests.terraform-plan]] +name = "strips noise, preserves non-blank content" +input = "Refreshing state... [id=vpc-abc]\nNo changes. Your infrastructure matches the configuration." +expected = "No changes. Your infrastructure matches the configuration." + + +[filters.tofu-fmt] +description = "Compact OpenTofu fmt output" +match_command = "^tofu\\s+fmt(\\s|$)" +strip_ansi = true +on_empty = "tofu fmt: ok (no changes)" +max_lines = 30 + +[[tests.tofu-fmt]] +name = "empty output returns on_empty message" +input = "" +expected = "tofu fmt: ok (no changes)" + +[[tests.tofu-fmt]] +name = "changed files pass through" +input = "main.tf\nvariables.tf" +expected = "main.tf\nvariables.tf" + + +[filters.tofu-init] +description = "Compact OpenTofu init output" +match_command = "^tofu\\s+init(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^- Downloading", + "^- Installing", + "^- Using previously-installed", + "^\\s*$", + "^Initializing provider", + "^Initializing the backend", + "^Initializing modules", +] +max_lines = 20 +on_empty = "tofu init: ok" + +[[tests.tofu-init]] +name = "strips downloading/installing lines" +input = """ +Initializing the backend... +Initializing provider plugins... +- Downloading hashicorp/aws 5.0.0... +- Installing hashicorp/aws 5.0.0... +- Using previously-installed hashicorp/random 3.5.1 + +OpenTofu has been successfully initialized! +""" +expected = "OpenTofu has been successfully initialized!" + +[[tests.tofu-init]] +name = "on_empty when all noise stripped" +input = """ +Initializing the backend... +Initializing provider plugins... +- Using previously-installed hashicorp/aws 5.0.0 + +""" +expected = "tofu init: ok" + + +[filters.tofu-plan] +description = "Compact OpenTofu plan output" +match_command = "^tofu\\s+plan(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^Refreshing state", + "^\\s*#.*unchanged", + "^\\s*$", + "^Acquiring state lock", + "^Releasing state lock", +] +max_lines = 80 +on_empty = "tofu plan: no changes detected" + +[[tests.tofu-plan]] +name = "strips Refreshing state and lock lines" +input = """ +Acquiring state lock. This may take a few moments... +Refreshing state... [id=vpc-abc123] +Refreshing state... [id=sg-def456] +Releasing state lock. This may take a few moments... + +OpenTofu will perform the following actions: + + # aws_instance.web will be created + + resource "aws_instance" "web" {} + +Plan: 1 to add, 0 to change, 0 to destroy. +""" +expected = "OpenTofu will perform the following actions:\n # aws_instance.web will be created\n + resource \"aws_instance\" \"web\" {}\nPlan: 1 to add, 0 to change, 0 to destroy." + +[[tests.tofu-plan]] +name = "on_empty when all noise stripped" +input = "Refreshing state... [id=vpc-abc]\nAcquiring state lock. This may take a few moments...\nReleasing state lock. This may take a few moments..." +expected = "tofu plan: no changes detected" + + +[filters.tofu-validate] +description = "Compact OpenTofu validate output" +match_command = "^tofu\\s+validate(\\s|$)" +strip_ansi = true +match_output = [ + { pattern = "Success! The configuration is valid", message = "ok (valid)" }, +] + +[[tests.tofu-validate]] +name = "success short-circuits to ok" +input = "Success! The configuration is valid." +expected = "ok (valid)" + +[[tests.tofu-validate]] +name = "error passes through unchanged" +input = "Error: Invalid resource type\n on main.tf line 3: resource \"aws_instancee\" \"web\"" +expected = "Error: Invalid resource type\n on main.tf line 3: resource \"aws_instancee\" \"web\"" + + +[filters.trunk-build] +description = "Compact trunk build output" +match_command = "^trunk\\s+build(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Compiling\\s", + "^\\s*Downloading\\s", + "^\\s*Fetching\\s", + "^\\s*Fresh\\s", + "^\\s*Checking\\s", +] +tail_lines = 10 +max_lines = 30 +on_empty = "trunk build: ok" + +[[tests.trunk-build]] +name = "strips compile noise, keeps tail summary" +input = """ + Compiling tokio v1.35.0 + Compiling hyper v0.14.28 + Compiling my-crate v0.1.0 + Downloading serde v1.0.195 + Fresh regex v1.10.2 + + Finished release [optimized] target(s) in 45.23s + Binary: target/release/my-crate (5.2MB) +""" +expected = " Finished release [optimized] target(s) in 45.23s\n Binary: target/release/my-crate (5.2MB)" + +[[tests.trunk-build]] +name = "on_empty when all noise stripped" +input = """ + Compiling my-crate v0.1.0 + Fresh serde v1.0 + Checking tokio v1.35.0 + +""" +expected = "trunk build: ok" + + +[filters.turbo] +description = "Compact Turborepo output — strip cache status noise, keep task results" +match_command = "^turbo\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*cache (hit|miss|bypass)", + "^\\s*\\d+ packages in scope", + "^\\s*Tasks:\\s+\\d+", + "^\\s*Duration:\\s+", + "^\\s*Remote caching (enabled|disabled)", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "turbo: ok" + +[[tests.turbo]] +name = "strips cache noise, keeps task output" +input = " cache hit, replaying logs abc123\n cache miss, executing abc456\n\n3 packages in scope\n\n> myapp:build\n\nCompiled successfully.\n\nTasks: 2 successful, 2 total (1 cached)\nDuration: 3.2s" +expected = "> myapp:build\nCompiled successfully." + +[[tests.turbo]] +name = "preserves error output" +input = "> myapp:lint\n\nError: src/index.ts(5,1): error TS2304\n\nTasks: 0 successful, 1 total\nDuration: 1.1s" +expected = "> myapp:lint\nError: src/index.ts(5,1): error TS2304" + +[[tests.turbo]] +name = "empty after stripping" +input = " cache hit, replaying logs abc\n\n" +expected = "turbo: ok" + + +[filters.ty] +description = "Compact ty type checker output — strip blank lines, keep errors" +match_command = "^ty\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Checking \\d+ file", + "^ty \\d+\\.\\d+", +] +max_lines = 50 +on_empty = "ty: ok" + +[[tests.ty]] +name = "strips noise, keeps diagnostics" +input = """ +ty 0.1.0 +Checking 15 files + +error[unresolved-reference]: Name `foo` used when not defined + --> app/main.py:10:5 + | +10 | foo() + | ^^^ + | + +warning[unused-variable]: Variable `x` is not used + --> app/utils.py:8:9 + | + 8 | x = 42 + | ^ + | + +Found 1 error, 1 warning +""" +expected = "error[unresolved-reference]: Name `foo` used when not defined\n --> app/main.py:10:5\n |\n10 | foo()\n | ^^^\n |\nwarning[unused-variable]: Variable `x` is not used\n --> app/utils.py:8:9\n |\n 8 | x = 42\n | ^\n |\nFound 1 error, 1 warning" + +[[tests.ty]] +name = "clean output" +input = """ +ty 0.1.0 +Checking 10 files + +All checks passed! +""" +expected = "All checks passed!" + +[[tests.ty]] +name = "empty input returns on_empty message" +input = "" +expected = "ty: ok" + + +[filters.uv-sync] +description = "Compact uv sync/pip install output — strip downloads, short-circuit when up-to-date" +match_command = "^uv\\s+(sync|pip\\s+install)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s+Downloading ", + "^\\s+Using cached ", + "^\\s+Preparing ", +] +match_output = [ + { pattern = "Audited \\d+ package", message = "ok (up to date)" }, +] +max_lines = 20 + +[[tests.uv-sync]] +name = "audited packages short-circuits to ok" +input = """ +Resolved 42 packages in 123ms +Audited 42 packages in 0.05ms +""" +expected = "ok (up to date)" + +[[tests.uv-sync]] +name = "install strips download and cached lines" +input = """ + Downloading requests-2.31.0-py3-none-any.whl (62.6 kB) + Using cached certifi-2023.11.17-py3-none-any.whl (162 kB) + Preparing packages... +Installed 5 packages in 23ms + + certifi==2023.11.17 + + charset-normalizer==3.3.2 + + idna==3.6 + + requests==2.31.0 + + urllib3==2.1.0 +""" +expected = "Installed 5 packages in 23ms\n + certifi==2023.11.17\n + charset-normalizer==3.3.2\n + idna==3.6\n + requests==2.31.0\n + urllib3==2.1.0" + + +[filters.xcodebuild] +description = "Compact xcodebuild output — strip build phases, keep errors/warnings/summary" +match_command = "^xcodebuild\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^CompileC\\s", + "^CompileSwift\\s", + "^Ld\\s", + "^CreateBuildDirectory\\s", + "^MkDir\\s", + "^ProcessInfoPlistFile\\s", + "^CopySwiftLibs\\s", + "^CodeSign\\s", + "^Signing Identity:", + "^RegisterWithLaunchServices", + "^Validate\\s", + "^ProcessProductPackaging", + "^Touch\\s", + "^LinkStoryboards", + "^CompileStoryboard", + "^CompileAssetCatalog", + "^GenerateDSYMFile", + "^PhaseScriptExecution", + "^PBXCp\\s", + "^SetMode\\s", + "^SetOwnerAndGroup\\s", + "^Ditto\\s", + "^CpResource\\s", + "^CpHeader\\s", + "^\\s+cd\\s+/", + "^\\s+export\\s", + "^\\s+/Applications/Xcode", + "^\\s+/usr/bin/", + "^\\s+builtin-", + "^note: Using new build system", +] +max_lines = 60 +on_empty = "xcodebuild: ok" + +[[tests.xcodebuild]] +name = "strips build phases, keeps errors and summary" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/App/ViewController.swift + cd /Users/dev/App + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-frontend -c +CompileSwift normal arm64 /Users/dev/App/AppDelegate.swift + cd /Users/dev/App + export DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer +Ld /Users/dev/Build/Products/Debug/App normal arm64 + cd /Users/dev/App + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang +CodeSign /Users/dev/Build/Products/Debug/App.app + cd /Users/dev/App + builtin-codesign --force --sign + +/Users/dev/App/ViewController.swift:42:9: error: use of unresolved identifier 'foo' +/Users/dev/App/Model.swift:18:5: warning: variable 'x' was never used + +** BUILD FAILED ** +""" +expected = "/Users/dev/App/ViewController.swift:42:9: error: use of unresolved identifier 'foo'\n/Users/dev/App/Model.swift:18:5: warning: variable 'x' was never used\n** BUILD FAILED **" + +[[tests.xcodebuild]] +name = "clean build success" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/App/Main.swift + cd /Users/dev/App +Ld /Users/dev/Build/Products/Debug/App normal arm64 + cd /Users/dev/App +CodeSign /Users/dev/Build/Products/Debug/App.app + cd /Users/dev/App + builtin-codesign --force --sign + +** BUILD SUCCEEDED ** +""" +expected = "** BUILD SUCCEEDED **" + +[[tests.xcodebuild]] +name = "test output keeps test results" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/AppTests/Tests.swift + cd /Users/dev/App +Test Suite 'All tests' started at 2026-03-10 12:00:00 +Test Suite 'AppTests' started at 2026-03-10 12:00:00 +Test Case '-[AppTests testExample]' passed (0.001 seconds). +Test Case '-[AppTests testFailing]' failed (0.002 seconds). +Test Suite 'AppTests' passed at 2026-03-10 12:00:01 +Executed 2 tests, with 1 failure in 0.003 seconds +""" +expected = "Test Suite 'All tests' started at 2026-03-10 12:00:00\nTest Suite 'AppTests' started at 2026-03-10 12:00:00\nTest Case '-[AppTests testExample]' passed (0.001 seconds).\nTest Case '-[AppTests testFailing]' failed (0.002 seconds).\nTest Suite 'AppTests' passed at 2026-03-10 12:00:01\nExecuted 2 tests, with 1 failure in 0.003 seconds" + +[[tests.xcodebuild]] +name = "empty input returns on_empty message" +input = "" +expected = "xcodebuild: ok" + + +[filters.yadm] +description = "Compact yadm (git wrapper) output — same filtering as git" +match_command = "^yadm\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*\\(use \"git ", + "^\\s*\\(use \"yadm ", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.yadm]] +name = "strips hint lines" +input = "On branch main\nYour branch is up to date with 'origin/main'.\n\n (use \"yadm add\" to update what will be committed)\n\nChanges not staged for commit:\n modified: .bashrc" +expected = "On branch main\nYour branch is up to date with 'origin/main'.\nChanges not staged for commit:\n modified: .bashrc" + +[[tests.yadm]] +name = "short output preserved" +input = "Already up to date." +expected = "Already up to date." + + +[filters.yamllint] +description = "Compact yamllint output — strip blank lines, limit rows" +match_command = "^yamllint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.yamllint]] +name = "multi-warning output stripped of blank lines" +input = """ +config.yml + 3:1 warning missing document start "---" (document-start) + 5:12 error too many spaces inside braces (braces) + + 8:1 error wrong indentation: expected 2 but found 4 (indentation) +""" +expected = "config.yml\n 3:1 warning missing document start \"---\" (document-start)\n 5:12 error too many spaces inside braces (braces)\n 8:1 error wrong indentation: expected 2 but found 4 (indentation)" + +[[tests.yamllint]] +name = "empty input passes through" +input = "" +expected = "" + + diff --git a/crates/sparkplane-rtk/src/core/constants.rs b/crates/sparkplane-rtk/src/core/constants.rs new file mode 100644 index 0000000..e7cc757 --- /dev/null +++ b/crates/sparkplane-rtk/src/core/constants.rs @@ -0,0 +1,38 @@ +pub const RTK_DATA_DIR: &str = "rtk"; +pub const HISTORY_DB: &str = "history.db"; +pub const RECALL_DB: &str = "recall.db"; +pub const CONFIG_TOML: &str = "config.toml"; +pub const FILTERS_TOML: &str = "filters.toml"; +pub const TRUSTED_FILTERS_JSON: &str = "trusted_filters.json"; +pub const DEFAULT_HISTORY_DAYS: i64 = 90; + +/// RTK-only subcommands that should never fall back to raw execution. +/// When adding a new RTK-only subcommand to `Commands`, add its clap name here. +pub const RTK_META_COMMANDS: &[&str] = &[ + "gain", + "discover", + "learn", + "init", + "config", + "proxy", + "recall", + "run", + "hook", + "hook-audit", + "pipe", + "cc-economics", + "verify", + "trust", + "untrust", + "session", + "rewrite", + "telemetry", + "smart", + "deps", + "json", + // `err` and `summary` name no real binary, and since they grew `--shell` + // they have a flag to get wrong. Falling through would try to exec a + // program called `err` instead of reporting the flag error (#4125 review). + "err", + "summary", +]; diff --git a/crates/sparkplane-rtk/src/core/guard.rs b/crates/sparkplane-rtk/src/core/guard.rs new file mode 100644 index 0000000..3824d44 --- /dev/null +++ b/crates/sparkplane-rtk/src/core/guard.rs @@ -0,0 +1,71 @@ +//! Never-worse output guard: RTK never emits more tokens than the raw command. +//! +//! Two callers are allowed past it. `rtk diff` prints a one-line message for a +//! difference `str::lines()` cannot render — CRLF against LF, or a missing +//! final newline — where the raw fallback is two blobs that look identical and +//! answer the question worse at any size. The exception is bounded by the case +//! rather than by a token count: it fires only when the bytes differ and the +//! line vectors do not, so the message never competes with a change list. A +//! fixed allowance above raw was tried and could not be met by construction: +//! the shortest form of the message is ~20 tokens and a one-line pair is ~2, so +//! the ceiling sat under the message's own floor and dropped it on 90% of +//! one-line pairs. +//! +//! The other is a clean run of a command whose output format rtk injected +//! (`runner::RunOptions::clean_outputs`): ruff's `[]` answers a question the +//! user never asked, so the filter's summary is shown although it is longer. + +use crate::core::tracking::estimate_tokens; + +/// Returns `filtered`, or `raw` when `filtered` would emit more tokens. +pub fn never_worse<'a>(raw: &'a str, filtered: &'a str) -> &'a str { + if estimate_tokens(filtered) > estimate_tokens(raw) { + raw + } else { + filtered + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_filtered_when_smaller() { + let raw = "a".repeat(400); + assert_eq!(never_worse(&raw, "ok"), "ok"); + } + + #[test] + fn falls_back_to_raw_when_filtered_bigger() { + let raw = "{}"; + let filtered = "{\n \"pretty\": true\n}"; + assert_eq!(never_worse(raw, filtered), raw); + } + + #[test] + fn tie_keeps_filtered() { + assert_eq!(never_worse("abcd", "wxyz"), "wxyz"); + } + + #[test] + fn token_boundary_follows_estimate_tokens() { + assert_eq!(never_worse("abcd", "abcde"), "abcd"); + assert_eq!(never_worse("abcdefgh", "ijklmnop"), "ijklmnop"); + } + + #[test] + fn empty_raw_returns_raw() { + assert_eq!(never_worse("", "0 matches"), ""); + } + + #[test] + fn empty_filtered_returns_filtered() { + assert_eq!(never_worse("data", ""), ""); + } + + #[test] + fn both_empty_returns_filtered() { + assert_eq!(never_worse("", ""), ""); + } +} diff --git a/crates/sparkplane-rtk/src/core/stream.rs b/crates/sparkplane-rtk/src/core/stream.rs new file mode 100644 index 0000000..52f938f --- /dev/null +++ b/crates/sparkplane-rtk/src/core/stream.rs @@ -0,0 +1,13 @@ +use std::borrow::Cow; +pub trait BlockHandler { + /// Rewrite a raw line before it is matched or emitted — the place to + /// strip ANSI escapes for tools that colour by default. Identity unless + /// a handler opts in, so the rest of the pipeline sees raw bytes. + fn normalize_line<'a>(&self, line: &'a str) -> Cow<'a, str> { + Cow::Borrowed(line) + } + fn should_skip(&mut self, line: &str) -> bool; + fn is_block_start(&mut self, line: &str) -> bool; + fn is_block_continuation(&mut self, line: &str, block: &[String]) -> bool; + fn format_summary(&self, exit_code: i32, raw: &str) -> Option; +} diff --git a/crates/sparkplane-rtk/src/core/toml_filter.rs b/crates/sparkplane-rtk/src/core/toml_filter.rs new file mode 100644 index 0000000..fa15516 --- /dev/null +++ b/crates/sparkplane-rtk/src/core/toml_filter.rs @@ -0,0 +1,2129 @@ +//! Applies TOML-defined filter rules to command output. +/// +/// Provides a declarative pipeline of 8 stages that can be configured +/// via TOML files. Lookup priority (first match wins): +/// 1. `.rtk/filters.toml` — project-local, committable with the repo +/// 2. `~/.config/rtk/filters.toml` — user-global, applies to all projects +/// 3. Built-in TOML — `src/filters/*.toml`, concatenated by build.rs and embedded at compile time +/// 4. Passthrough — no match, handled by caller +/// +/// `rtk init` generates a commented template for both levels (project or global). +/// +/// Environment variables: +/// - `RTK_NO_TOML=1` — bypass TOML engine entirely +/// - `RTK_TOML_DEBUG=1` — print which filter matched and line counts to stderr +/// +/// Pipeline stages (applied in order): +/// 1. strip_ansi — remove ANSI escape codes +/// 2. replace — regex substitutions, line-by-line, chainable +/// 3. match_output — short-circuit: if blob matches a pattern, return message immediately +/// 4. strip/keep_lines — filter lines by regex +/// 5. truncate_lines_at — truncate each line to N chars +/// 6. head/tail_lines — keep first/last N lines +/// 7. max_lines — absolute line cap +/// 8. on_empty — message if result is empty +use super::constants::RTK_META_COMMANDS; +use regex::{Regex, RegexSet}; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::sync::LazyLock; + +// Built-in filters: concatenated from src/filters/*.toml by build.rs at compile time. +const BUILTIN_TOML: &str = include_str!("builtin_filters.toml"); + +// --------------------------------------------------------------------------- +// Deserialization types (TOML schema) +// --------------------------------------------------------------------------- + +/// A match-output rule: if `pattern` matches anywhere in the full output blob, +/// the filter short-circuits and returns `message` immediately. +/// First matching rule wins; remaining rules are not evaluated. +/// Optional `unless`: if this regex also matches the blob, the rule is skipped +/// (prevents short-circuiting when errors or warnings are present). +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct MatchOutputRule { + pattern: String, + message: String, + #[serde(default)] + unless: Option, +} + +/// A regex substitution applied line-by-line. Rules are chained sequentially: +/// rule N+1 operates on the output of rule N. +/// Backreferences (`$1`, `$2`, ...) are supported via the `regex` crate. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ReplaceRule { + pattern: String, + replacement: String, +} + +/// An inline test case attached to a filter in the TOML. +/// Lives in `[[tests.]]` sections, separate from `[filters.*]`. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TomlFilterTestDef { + pub name: String, + pub input: String, + pub expected: String, +} + +#[derive(Deserialize)] +struct TomlFilterFile { + schema_version: u32, + #[serde(default)] + filters: BTreeMap, + /// Inline tests keyed by filter name. Kept separate from `filters` so that + /// `TomlFilterDef` can keep `deny_unknown_fields` without touching test data. + #[serde(default)] + tests: BTreeMap>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TomlFilterDef { + description: Option, + match_command: String, + #[serde(default)] + strip_ansi: bool, + /// Regex substitutions, applied line-by-line before match_output (stage 2). + #[serde(default)] + replace: Vec, + /// Short-circuit rules: if the full output blob matches, return the message (stage 3). + #[serde(default)] + match_output: Vec, + #[serde(default)] + strip_lines_matching: Vec, + #[serde(default)] + keep_lines_matching: Vec, + truncate_lines_at: Option, + head_lines: Option, + tail_lines: Option, + max_lines: Option, + on_empty: Option, + /// When true, stderr is captured and merged with stdout before filtering. + /// Use for tools like liquibase that emit banners/logs to stderr. + #[serde(default)] + filter_stderr: bool, +} + +// --------------------------------------------------------------------------- +// Compiled types (post-validation, ready to use) +// --------------------------------------------------------------------------- + +#[derive(Debug)] +struct CompiledMatchOutputRule { + pattern: Regex, + message: String, + /// If set and matches the blob, this rule is skipped (prevents swallowing errors). + unless: Option, +} + +#[derive(Debug)] +struct CompiledReplaceRule { + pattern: Regex, + replacement: String, +} + +#[derive(Debug)] +enum LineFilter { + None, + Strip(RegexSet), + Keep(RegexSet), +} + +/// A filter that has been parsed and compiled — all regexes are ready. +#[derive(Debug)] +pub struct CompiledFilter { + pub name: String, + #[allow(dead_code)] + pub description: Option, + match_regex: Regex, + strip_ansi: bool, + replace: Vec, + match_output: Vec, + line_filter: LineFilter, + truncate_lines_at: Option, + head_lines: Option, + tail_lines: Option, + pub max_lines: Option, + on_empty: Option, + /// When true, the runner should capture stderr and merge it with stdout. + pub filter_stderr: bool, +} + +// --------------------------------------------------------------------------- +// Results for `rtk verify` +// --------------------------------------------------------------------------- + +/// Outcome of running a single inline test. +pub struct TestOutcome { + pub filter_name: String, + pub test_name: String, + pub passed: bool, + pub actual: String, + pub expected: String, +} + +/// Aggregated results from `run_filter_tests`. +pub struct VerifyResults { + /// Individual test outcomes (all filters, or just the requested one). + pub outcomes: Vec, + /// Filter names that have no inline tests (used by `--require-all`). + pub filters_without_tests: Vec, +} + +// --------------------------------------------------------------------------- +// Registry +// --------------------------------------------------------------------------- + +pub struct TomlFilterRegistry { + pub filters: Vec, +} + +impl TomlFilterRegistry { + /// Load registry from disk + built-in. Emits warnings to stderr on parse + /// errors but never panics — bad files are silently ignored. + fn load() -> Self { + let mut filters = Vec::new(); + + match Self::parse_and_compile(BUILTIN_TOML, "builtin") { + Ok(f) => filters.extend(f), + Err(e) => eprintln!("[rtk] warning: builtin filters: {}", e), + } + + TomlFilterRegistry { filters } + } + + fn parse_and_compile(content: &str, source: &str) -> Result, String> { + let file: TomlFilterFile = toml::from_str(content) + .map_err(|e| format!("TOML parse error in {}: {}", source, e))?; + + if file.schema_version != 1 { + return Err(format!( + "unsupported schema_version {} in {} (expected 1)", + file.schema_version, source + )); + } + + let mut compiled = Vec::new(); + for (name, def) in file.filters { + if !is_fully_anchored(&def.match_command) { + eprintln!( + "[rtk] warning: filter '{}' in {}: match_command '{}' has a top-level branch \ + that does not start with '^'; it would match a path component mid-command. \ + Filter ignored.", + name, source, def.match_command + ); + continue; + } + match compile_filter(name.clone(), def) { + Ok(f) => compiled.push(f), + Err(e) => eprintln!("[rtk] warning: filter '{}' in {}: {}", name, source, e), + } + } + Ok(compiled) + } +} + +/// Commands already handled by dedicated Rust modules (routed by Clap before TOML). +/// A TOML filter whose match_command matches one of these will never activate — +/// Clap routes the command before `run_fallback()` is reached. +const RUST_HANDLED_COMMANDS: &[&str] = &[ + "ls", + "tree", + "read", + "smart", + "git", + "gh", + "aws", + "psql", + "pnpm", + "err", + "test", + "json", + "deps", + "env", + "find", + "diff", + "log", + "docker", + "kubectl", + "summary", + "grep", + "init", + "wget", + "wc", + "gain", + "config", + "ctest", + "vitest", + "prisma", + "tsc", + "next", + "lint", + "prettier", + "format", + "playwright", + "cargo", + "npm", + "npx", + "curl", + "discover", + "ruff", + "pytest", + "mypy", + "pip", + "sqlfluff", + "go", + "golangci-lint", + "rewrite", + "proxy", + "verify", + "learn", +]; + +pub fn is_rtk_reserved_command(name: &str) -> bool { + RUST_HANDLED_COMMANDS.contains(&name) || RTK_META_COMMANDS.contains(&name) +} + +fn compile_filter(name: String, def: TomlFilterDef) -> Result { + // Mutual exclusion: strip and keep cannot both be set + if !def.strip_lines_matching.is_empty() && !def.keep_lines_matching.is_empty() { + return Err("strip_lines_matching and keep_lines_matching are mutually exclusive".into()); + } + + let match_regex = Regex::new(&def.match_command) + .map_err(|e| format!("invalid match_command regex: {}", e))?; + + // Shadow warning: if match_command matches a Rust-handled command, this filter + // will never activate (Clap routes before run_fallback). Warn the author. + for cmd in RUST_HANDLED_COMMANDS { + if match_regex.is_match(cmd) { + eprintln!( + "[rtk] warning: filter '{}' match_command matches '{}' which is already \ + handled by a Rust module — this filter will never activate for that command", + name, cmd + ); + break; + } + } + + let replace = def + .replace + .into_iter() + .map(|r| { + let pat = r.pattern.clone(); + Regex::new(&r.pattern) + .map(|pattern| CompiledReplaceRule { + pattern, + replacement: r.replacement, + }) + .map_err(|e| format!("invalid replace pattern '{}': {}", pat, e)) + }) + .collect::, _>>()?; + + let match_output = def + .match_output + .into_iter() + .map(|r| -> Result { + let pat = r.pattern.clone(); + let pattern = Regex::new(&r.pattern) + .map_err(|e| format!("invalid match_output pattern '{}': {}", pat, e))?; + let unless = r + .unless + .as_deref() + .map(|u| { + Regex::new(u) + .map_err(|e| format!("invalid match_output unless pattern '{}': {}", u, e)) + }) + .transpose()?; + Ok(CompiledMatchOutputRule { + pattern, + message: r.message, + unless, + }) + }) + .collect::, _>>()?; + + let line_filter = if !def.strip_lines_matching.is_empty() { + let set = RegexSet::new(&def.strip_lines_matching) + .map_err(|e| format!("invalid strip_lines_matching regex: {}", e))?; + LineFilter::Strip(set) + } else if !def.keep_lines_matching.is_empty() { + let set = RegexSet::new(&def.keep_lines_matching) + .map_err(|e| format!("invalid keep_lines_matching regex: {}", e))?; + LineFilter::Keep(set) + } else { + LineFilter::None + }; + + Ok(CompiledFilter { + name, + description: def.description, + match_regex, + strip_ansi: def.strip_ansi, + replace, + match_output, + line_filter, + truncate_lines_at: def.truncate_lines_at, + head_lines: def.head_lines, + tail_lines: def.tail_lines, + max_lines: def.max_lines, + on_empty: def.on_empty, + filter_stderr: def.filter_stderr, + }) +} + +// --------------------------------------------------------------------------- +// Singleton (lazy-loaded, one-time cost) +// --------------------------------------------------------------------------- + +static REGISTRY: LazyLock = LazyLock::new(TomlFilterRegistry::load); + +pub fn toml_disabled() -> bool { + std::env::var("RTK_NO_TOML").ok().as_deref() == Some("1") +} + +pub fn active_filter_summaries(content: &str) -> Vec<(String, String)> { + toml::from_str::(content) + .map(|f| { + f.filters + .into_iter() + .map(|(name, def)| (name, def.match_command)) + .collect() + }) + .unwrap_or_default() +} + +pub fn filter_parse_error(content: &str) -> Option { + toml::from_str::(content) + .err() + .map(|e| e.to_string()) +} + +static MATCH_SET: LazyLock = LazyLock::new(build_match_set); + +pub fn command_matches_filter(command: &str) -> bool { + MATCH_SET.is_match(command) +} + +fn build_match_set() -> RegexSet { + let patterns = collect_match_patterns(); + RegexSet::new(&patterns).unwrap_or_else(|_| { + let valid: Vec = patterns + .into_iter() + .filter(|p| Regex::new(p).is_ok()) + .collect(); + RegexSet::new(&valid).unwrap_or_else(|_| RegexSet::empty()) + }) +} + +fn collect_match_patterns() -> Vec { + match_patterns_in(BUILTIN_TOML) +} + +/// Strip a leading inline-flag group such as `(?i)` so the anchor check sees the +/// pattern body. `(?:` opens a non-capturing group and is left alone. +fn strip_inline_flags(branch: &str) -> &str { + let Some(rest) = branch.strip_prefix("(?") else { + return branch; + }; + let Some(end) = rest.find(')') else { + return branch; + }; + if rest[..end].chars().all(|c| "imsuxU-".contains(c)) { + &rest[end + 1..] + } else { + branch + } +} + +/// Split `pattern` on top-level `|`, ignoring alternations nested inside groups +/// or character classes. +fn top_level_branches(pattern: &str) -> Vec<&str> { + let mut branches = Vec::new(); + let mut depth = 0usize; + let mut escaped = false; + let mut in_class = false; + let mut start = 0usize; + + for (idx, character) in pattern.char_indices() { + if escaped { + escaped = false; + continue; + } + match character { + '\\' => escaped = true, + '[' if !in_class => in_class = true, + ']' if in_class => in_class = false, + '(' if !in_class => depth += 1, + ')' if !in_class => depth = depth.saturating_sub(1), + '|' if !in_class && depth == 0 => { + branches.push(&pattern[start..idx]); + start = idx + 1; + } + _ => {} + } + } + branches.push(&pattern[start..]); + branches +} + +/// A filter selects on argv[0], but its regex runs against the whole command +/// line. A top-level branch that does not start with `^` therefore matches a +/// path component or wrapper argument mid-line, so `timeout 5 /usr/bin/liquibase +/// update` activates the liquibase filter and rewrites the wrapper instead. +fn is_fully_anchored(pattern: &str) -> bool { + top_level_branches(pattern) + .iter() + .all(|branch| strip_inline_flags(branch).starts_with('^')) +} + +fn match_patterns_in(content: &str) -> Vec { + match toml::from_str::(content) { + Ok(file) if file.schema_version == 1 => file + .filters + .into_values() + .map(|def| def.match_command) + .filter(|pattern| is_fully_anchored(pattern)) + .collect(), + _ => Vec::new(), + } +} + +// --------------------------------------------------------------------------- +// Public API — pure functions (testable without global state) +// --------------------------------------------------------------------------- + +/// Find the first matching filter in a slice. O(N) on the number of filters. +/// Tests should call this directly with a local filter list. +pub fn find_filter_in<'a>( + command: &str, + filters: &'a [CompiledFilter], +) -> Option<&'a CompiledFilter> { + filters.iter().find(|f| f.match_regex.is_match(command)) +} + +/// Apply a compiled filter pipeline to raw stdout. Pure String -> String. +/// +/// Pipeline stages (in order): +/// 1. strip_ansi — remove ANSI escape codes +/// 2. replace — regex substitutions, line-by-line, chainable +/// 3. match_output — short-circuit if blob matches a pattern +/// 4. strip/keep_lines — filter lines by regex +/// 5. truncate_lines_at — truncate each line to N chars +/// 6. head/tail_lines — keep first/last N lines +/// 7. max_lines — absolute line cap +/// 8. on_empty — message if result is empty +pub fn apply_filter(filter: &CompiledFilter, stdout: &str) -> String { + apply_filter_with_info(filter, stdout).0 +} + +#[derive(Debug, PartialEq)] +pub enum Lossiness { + None, + /// `tail -n +{tail_offset}` over `tee_payload` reproduces the dropped lines, + /// up to the tee `max_file_size` cap (larger payloads are truncated in the tee file). + Tail { + tee_payload: String, + tail_offset: usize, + }, + Whole, +} + +pub fn apply_filter_with_info(filter: &CompiledFilter, stdout: &str) -> (String, Lossiness) { + let mut lines: Vec = stdout.lines().map(String::from).collect(); + + // 1. strip_ansi + if filter.strip_ansi { + lines = lines + .into_iter() + .map(|l| crate::core::utils::strip_ansi(&l)) + .collect(); + } + + // 2. replace — line-by-line, rules chained sequentially + if !filter.replace.is_empty() { + lines = lines + .into_iter() + .map(|mut line| { + for rule in &filter.replace { + line = rule + .pattern + .replace_all(&line, rule.replacement.as_str()) + .into_owned(); + } + line + }) + .collect(); + } + + // 3. match_output — short-circuit on full blob match (first rule wins) + // If `unless` is set and also matches the blob, the rule is skipped. + if !filter.match_output.is_empty() { + let blob = lines.join("\n"); + for rule in &filter.match_output { + if rule.pattern.is_match(&blob) { + if let Some(ref unless_re) = rule.unless + && unless_re.is_match(&blob) + { + continue; // errors/warnings present — skip this rule + } + return (rule.message.clone(), Lossiness::Whole); + } + } + } + + // 4. strip OR keep (mutually exclusive) + match &filter.line_filter { + LineFilter::Strip(set) => lines.retain(|l| !set.is_match(l)), + LineFilter::Keep(set) => lines.retain(|l| set.is_match(l)), + LineFilter::None => {} + } + + // 5. truncate_lines_at — uses utils::truncate (unicode-safe) + let mut intra_line_loss = false; + if let Some(max_chars) = filter.truncate_lines_at { + lines = lines + .into_iter() + .map(|line| { + let truncated = crate::core::utils::truncate(&line, max_chars); + if truncated != line { + intra_line_loss = true; + } + truncated + }) + .collect(); + } + + let snapshot_for_tail = !intra_line_loss + && filter.tail_lines.is_none() + && (filter.head_lines.is_some() || filter.max_lines.is_some()); + let pre_cut = snapshot_for_tail.then(|| lines.clone()); + + // 6. head + tail + let total = lines.len(); + let mut noncontiguous_drop = false; + let mut head_cut: Option = None; + if let (Some(head), Some(tail)) = (filter.head_lines, filter.tail_lines) { + if total > head + tail { + let mut result = lines[..head].to_vec(); + result.push(format!("... ({} lines omitted)", total - head - tail)); + result.extend_from_slice(&lines[total - tail..]); + lines = result; + noncontiguous_drop = true; + } + } else if let Some(head) = filter.head_lines { + if total > head { + lines.truncate(head); + lines.push(format!("... ({} lines omitted)", total - head)); + head_cut = Some(head); + } + } else if let Some(tail) = filter.tail_lines + && total > tail + { + let omitted = total - tail; + lines = lines[omitted..].to_vec(); + lines.insert(0, format!("... ({} lines omitted)", omitted)); + noncontiguous_drop = true; + } + + // 7. max_lines — absolute cap applied after head/tail (includes omit messages) + let mut max_cut: Option = None; + if let Some(max) = filter.max_lines + && lines.len() > max + { + let dropped = lines.len() - max; + lines.truncate(max); + lines.push(format!("... ({} lines truncated)", dropped)); + max_cut = Some(max); + } + + // 8. on_empty + let result = lines.join("\n"); + if result.trim().is_empty() + && let Some(ref msg) = filter.on_empty + { + return (msg.clone(), Lossiness::None); + } + + let loss = if let Some(snapshot) = pre_cut { + match (head_cut, max_cut) { + (Some(_), Some(_)) => Lossiness::Whole, + (Some(head), None) => Lossiness::Tail { + tee_payload: snapshot.join("\n"), + tail_offset: head + 1, + }, + (None, Some(max)) => Lossiness::Tail { + tee_payload: snapshot.join("\n"), + tail_offset: max + 1, + }, + (None, None) => Lossiness::None, + } + } else if noncontiguous_drop || intra_line_loss || head_cut.is_some() || max_cut.is_some() { + Lossiness::Whole + } else { + Lossiness::None + }; + + (result, loss) +} + +// --------------------------------------------------------------------------- +// rtk verify — inline test execution +// --------------------------------------------------------------------------- + +/// Run inline tests from loaded TOML files (builtin + project-local). +/// +/// - `filter_name_opt`: if `Some`, only run tests for that filter name. +/// - Returns `VerifyResults` with all outcomes and filters that have no tests. +pub fn find_matching_filter(command: &str) -> Option<&'static CompiledFilter> { + if std::env::var("RTK_TOML_DEBUG").is_ok() { + eprintln!( + "[rtk:toml] looking up filter for: {:?} ({} filters loaded)", + command, + REGISTRY.filters.len() + ); + } + let result = find_filter_in(command, ®ISTRY.filters); + if std::env::var("RTK_TOML_DEBUG").is_ok() { + match result { + Some(f) => eprintln!("[rtk:toml] matched filter: '{}'", f.name), + None => eprintln!("[rtk:toml] no filter matched — passthrough"), + } + } + result +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // Helper: build a CompiledFilter from inline TOML for tests. + // Never touches the lazy registry. + fn make_filters(toml: &str) -> Vec { + TomlFilterRegistry::parse_and_compile(toml, "test").expect("test TOML should be valid") + } + + fn first_filter(toml: &str) -> CompiledFilter { + make_filters(toml) + .into_iter() + .next() + .expect("expected at least one filter") + } + + #[test] + fn command_matches_filter_agrees_with_find_matching_filter() { + for cmd in ["jj log", "jq .", "frobnicate xyz", "cd /tmp"] { + assert_eq!( + command_matches_filter(cmd), + find_matching_filter(cmd).is_some(), + "match-set disagreed with registry for {cmd:?}" + ); + } + } + + #[test] + fn test_loss_tail_configured_unfired_max_fires_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntail_lines = 100\nmax_lines = 2\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc\nd\ne"); + assert!(out.contains("lines truncated")); + assert_ne!(loss, Lossiness::None); + } + + #[test] + fn test_match_patterns_in_honors_schema_version() { + let v1 = "schema_version = 1\n[filters.a]\nmatch_command = \"^aaa\"\n"; + assert_eq!(match_patterns_in(v1), vec!["^aaa".to_string()]); + + let v2 = "schema_version = 2\n[filters.a]\nmatch_command = \"^aaa\"\n"; + assert!(match_patterns_in(v2).is_empty()); + + assert!(match_patterns_in("not valid toml {{{").is_empty()); + } + + #[test] + fn test_filter_parse_error() { + assert!( + filter_parse_error("schema_version = 1\n[filters.a]\nmatch_command = \"^a\"\n") + .is_none() + ); + assert!(filter_parse_error("[filters.a]\nmatch_command = \"^a\"\n").is_some()); + assert!(filter_parse_error("this is { not toml").is_some()); + } + + #[test] + fn test_toml_parse_tolerates_utf8_bom() { + // Hand-edited filter files on Windows often carry a BOM. The toml + // crate tolerates it natively — this pin is why the TOML file-read + // sites (config.rs, toml_filter.rs) deliberately skip + // strip_leading_bom; if a crate upgrade regresses this, add it there. + let bom = "\u{feff}schema_version = 1\n[filters.a]\nmatch_command = \"^a\"\n"; + assert!(filter_parse_error(bom).is_none()); + assert_eq!(match_patterns_in(bom), vec!["^a".to_string()]); + } + + #[test] + fn test_is_rtk_reserved_command() { + assert!(is_rtk_reserved_command("git")); + assert!(is_rtk_reserved_command("cargo")); + assert!(is_rtk_reserved_command("json")); + assert!(is_rtk_reserved_command("rewrite")); + assert!(!is_rtk_reserved_command("jj")); + assert!(!is_rtk_reserved_command("jq")); + assert!(!is_rtk_reserved_command("just")); + assert!(!is_rtk_reserved_command("frobnicate")); + } + + fn loss_of(toml: &str, input: &str) -> Lossiness { + apply_filter_with_info(&first_filter(toml), input).1 + } + + #[test] + fn test_loss_head_lines_is_tail() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc\nd\ne"); + assert!(out.starts_with("a\nb\n")); + match loss { + Lossiness::Tail { + tee_payload, + tail_offset, + } => { + assert_eq!(tail_offset, 3); + let recovered: Vec<&str> = tee_payload.lines().skip(tail_offset - 1).collect(); + assert_eq!(recovered, vec!["c", "d", "e"]); + } + other => panic!("expected Tail, got {:?}", other), + } + } + + #[test] + fn test_loss_max_lines_is_tail() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nmax_lines = 2\n"; + match loss_of(toml, "a\nb\nc\nd\ne") { + Lossiness::Tail { + tee_payload, + tail_offset, + } => { + assert_eq!(tail_offset, 3); + let recovered: Vec<&str> = tee_payload.lines().skip(tail_offset - 1).collect(); + assert_eq!(recovered, vec!["c", "d", "e"]); + } + other => panic!("expected Tail, got {:?}", other), + } + } + + #[test] + fn test_loss_tail_lines_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntail_lines = 2\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_head_then_max_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\nmax_lines = 2\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_head_plus_tail_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 1\ntail_lines = 1\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_truncate_lines_at_is_whole() { + let toml = + "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntruncate_lines_at = 3\n"; + assert_eq!(loss_of(toml, "abcdefgh\nshort"), Lossiness::Whole); + } + + #[test] + fn test_loss_match_output_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\n[[filters.f.match_output]]\npattern = \"ok\"\nmessage = \"all good\"\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "everything ok here\nmore"); + assert_eq!(out, "all good"); + assert_eq!(loss, Lossiness::Whole); + } + + #[test] + fn test_loss_strip_only_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nstrip_lines_matching = [\"^noise\"]\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "keep\nnoise line\nkeep2"); + assert_eq!(out, "keep\nkeep2"); + assert_eq!(loss, Lossiness::None); + } + + #[test] + fn test_loss_on_empty_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nstrip_lines_matching = [\".\"]\non_empty = \"nothing\"\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc"); + assert_eq!(out, "nothing"); + assert_eq!(loss, Lossiness::None); + } + + #[test] + fn test_loss_no_truncation_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 10\n"; + assert_eq!(loss_of(toml, "a\nb\nc"), Lossiness::None); + } + + #[test] + fn test_apply_filter_wrapper_matches_with_info() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\n"; + let f = first_filter(toml); + let input = "a\nb\nc\nd"; + assert_eq!(apply_filter(&f, input), apply_filter_with_info(&f, input).0); + } + + // --- Pipeline primitives (existing) --- + + #[test] + fn test_strip_ansi_removes_codes() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +"#, + ); + let out = apply_filter(&f, "\x1b[31mError\x1b[0m\nnormal"); + assert_eq!(out, "Error\nnormal"); + } + + #[test] + fn test_strip_lines_matching_basic() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["^noise", "^verbose"] +"#, + ); + let input = "noise line\nkeep this\nverbose stuff\nalso keep"; + let out = apply_filter(&f, input); + assert_eq!(out, "keep this\nalso keep"); + } + + #[test] + fn test_keep_lines_matching_basic() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +keep_lines_matching = ["^PASS", "^FAIL"] +"#, + ); + let input = "PASS test_a\nsome noise\nFAIL test_b\nmore noise"; + let out = apply_filter(&f, input); + assert_eq!(out, "PASS test_a\nFAIL test_b"); + } + + #[test] + fn test_truncate_lines_at_unicode_safe() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +truncate_lines_at = 5 +"#, + ); + // utils::truncate(s, 5) takes 2 chars + "..." when len > 5 + // "hello" = 5 chars exactly, stays unchanged + // "日本語xyz" = 6 chars, truncated to "日本..." (take 2 + "...") + let out = apply_filter(&f, "hello\n日本語xyz"); + assert_eq!(out, "hello\n日本..."); + } + + #[test] + fn test_head_lines() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +head_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + assert!(out.starts_with("a\nb\n")); + assert!(out.contains("3 lines omitted")); + } + + #[test] + fn test_tail_lines() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +tail_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + assert!(out.contains("3 lines omitted")); + assert!(out.ends_with("d\ne")); + } + + #[test] + fn test_head_and_tail_combined() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +head_lines = 2 +tail_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne\nf"; + let out = apply_filter(&f, input); + assert!(out.starts_with("a\nb\n")); + assert!(out.contains("2 lines omitted")); + assert!(out.ends_with("e\nf")); + } + + #[test] + fn test_max_lines_counts_omit_message() { + // max_lines applied AFTER head — the "omitted" message counts as a line + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +max_lines = 3 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + let line_count = out.lines().count(); + // 3 content lines + 1 truncated message = 4 lines output + assert_eq!(line_count, 4); + assert!(out.contains("lines truncated")); + } + + #[test] + fn test_on_empty_when_all_filtered() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = [".*"] +on_empty = "nothing left" +"#, + ); + let out = apply_filter(&f, "line1\nline2"); + assert_eq!(out, "nothing left"); + } + + #[test] + fn test_on_empty_not_triggered_when_output_remains() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +keep_lines_matching = ["keep"] +on_empty = "nothing left" +"#, + ); + let out = apply_filter(&f, "keep this\nnoise"); + assert_eq!(out, "keep this"); + } + + #[test] + fn test_full_pipeline_order() { + // Verify all 8 stages fire in order on a single input + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +strip_lines_matching = ["^noise"] +truncate_lines_at = 10 +head_lines = 3 +max_lines = 4 +on_empty = "empty" +"#, + ); + let input = + "\x1b[31mred line\x1b[0m\nnoise skip\nkeep one\nkeep two\nkeep three\nkeep four"; + let out = apply_filter(&f, input); + // After strip_ansi: "red line", strip noise: removed, head 3 from remaining 4 lines + assert!(out.contains("red line")); + assert!(!out.contains("noise skip")); + assert!(out.contains("lines omitted") || out.contains("lines truncated")); + } + + // --- Validation --- + + #[test] + fn test_mutual_exclusion_strip_keep_errors() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["a"] +keep_lines_matching = ["b"] +"#, + ); + // The filter should be skipped (warning emitted), resulting in empty list + assert!(result.is_empty()); + } + + #[test] + fn test_invalid_regex_returns_err() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "[" +"#, + ); + assert!(result.is_empty()); + } + + #[test] + fn test_schema_version_mismatch_errors() { + let result = TomlFilterRegistry::parse_and_compile( + r#"schema_version = 99 +[filters.f] +match_command = "^cmd" +"#, + "test", + ); + assert!(result.is_err()); + } + + #[test] + fn test_unknown_field_typo_errors() { + // deny_unknown_fields should catch this + let result = TomlFilterRegistry::parse_and_compile( + r#"schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi_typo = true +"#, + "test", + ); + assert!(result.is_err()); + } + + #[test] + fn test_empty_filter_passthrough() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +"#, + ); + let input = "line1\nline2\nline3"; + let out = apply_filter(&f, input); + assert_eq!(out, input); + } + + // --- Registry / find --- + + #[test] + fn test_builtin_filters_compile() { + // Compile-time safety: panics if any src/filters/*.toml is broken + let builtin = BUILTIN_TOML; + let result = TomlFilterRegistry::parse_and_compile(builtin, "builtin"); + assert!( + result.is_ok(), + "builtin filters failed to compile: {:?}", + result + ); + assert!(!result.unwrap().is_empty()); + } + + #[test] + fn test_find_filter_matches_terraform() { + let filters = make_filters( + r#" +schema_version = 1 +[filters.terraform-plan] +match_command = "^terraform\\s+plan" +strip_ansi = true +"#, + ); + let found = find_filter_in("terraform plan -out=tfplan", &filters); + assert!(found.is_some()); + assert_eq!(found.unwrap().name, "terraform-plan"); + } + + #[test] + fn test_find_filter_no_match_returns_none() { + let filters = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^terraform" +"#, + ); + let found = find_filter_in("kubectl get pods", &filters); + assert!(found.is_none()); + } + + #[test] + fn test_project_filters_priority_over_builtin() { + // Project filter has same name but different max_lines — project wins + let project = make_filters( + r#" +schema_version = 1 +[filters.make] +match_command = "^make\\b" +max_lines = 999 +"#, + ); + let builtin = make_filters(BUILTIN_TOML); + + // Simulate the registry: project first + let mut all = project; + all.extend(builtin); + + let found = find_filter_in("make all", &all).expect("should match"); + assert_eq!(found.name, "make"); + // The first (project) match has max_lines=999 + assert_eq!(found.max_lines, Some(999)); + } + + // --- Token savings --- + + #[test] + fn test_terraform_savings_above_60pct() { + let filters = make_filters(BUILTIN_TOML); + let filter = find_filter_in("terraform plan", &filters).expect("terraform-plan built-in"); + + // Inline fixture: realistic terraform plan with many Refreshing state lines (noise). + // Real infra refreshes 30+ resources; the plan section is small. + // All Refreshing/lock/blank/unchanged lines are stripped -> >60% savings. + let input = concat!( + "Acquiring state lock. This may take a few moments...\n", + "Refreshing state... [id=vpc-0a1b2c3d]\n", + "Refreshing state... [id=subnet-11111111]\n", + "Refreshing state... [id=subnet-22222222]\n", + "Refreshing state... [id=subnet-33333333]\n", + "Refreshing state... [id=subnet-44444444]\n", + "Refreshing state... [id=igw-aabbccdd]\n", + "Refreshing state... [id=rtb-aabbccdd]\n", + "Refreshing state... [id=rtb-11223344]\n", + "Refreshing state... [id=sg-00112233]\n", + "Refreshing state... [id=sg-44556677]\n", + "Refreshing state... [id=sg-88990011]\n", + "Refreshing state... [id=nacl-00aabbcc]\n", + "Refreshing state... [id=acm-arn:us-east-1:cert/abc]\n", + "Refreshing state... [id=Z1234567890ABC]\n", + "Refreshing state... [id=alb-arn:my-alb]\n", + "Refreshing state... [id=tg-arn:my-tg]\n", + "Refreshing state... [id=db-ABCDEFGHIJKLMNO]\n", + "Refreshing state... [id=rds-cluster:my-aurora]\n", + "Refreshing state... [id=elasticache:my-cluster]\n", + "Refreshing state... [id=lambda:my-api-function]\n", + "Refreshing state... [id=lambda:my-worker]\n", + "Refreshing state... [id=iam-role:my-lambda-role]\n", + "Refreshing state... [id=iam-role:my-ecs-role]\n", + "Refreshing state... [id=s3:::my-app-assets]\n", + "Refreshing state... [id=s3:::my-app-logs]\n", + "Refreshing state... [id=cloudfront:ABCDEFGHIJK]\n", + "Refreshing state... [id=ssm:/my/app/db-url]\n", + "Refreshing state... [id=ssm:/my/app/api-key]\n", + "Refreshing state... [id=secretsmanager:my-secret]\n", + "Releasing state lock. This may take a few moments...\n", + "\n", + "Terraform will perform the following actions:\n", + "\n", + " # aws_instance.web will be created\n", + " + resource \"aws_instance\" \"web\" {\n", + " + ami = \"ami-0c55b159cbfafe1f0\"\n", + " + instance_type = \"t3.micro\"\n", + " }\n", + "\n", + "Plan: 1 to add, 0 to change, 0 to destroy.\n", + ); + let out = apply_filter(filter, input); + let input_words = input.split_whitespace().count(); + let out_words = out.split_whitespace().count(); + let savings = 100.0 - (out_words as f64 / input_words as f64 * 100.0); + assert!( + savings >= 60.0, + "terraform-plan filter: expected >=60% savings, got {:.1}% (in={} out={})", + savings, + input_words, + out_words + ); + } + + #[test] + fn test_make_savings_above_60pct() { + let filters = make_filters(BUILTIN_TOML); + let filter = find_filter_in("make all", &filters).expect("make built-in"); + + let input = r#"make[1]: Entering directory '/home/user/project' +make[2]: Entering directory '/home/user/project/src' +gcc -O2 -Wall -c foo.c -o foo.o + +make[2]: Nothing to be done for 'install'. +make[3]: Entering directory '/home/user/project/src/lib' +ar rcs libfoo.a foo.o bar.o baz.o +make[3]: Leaving directory '/home/user/project/src/lib' +make[2]: Leaving directory '/home/user/project/src' + +make[1]: Leaving directory '/home/user/project' +gcc -O2 -Wall -c bar.c -o bar.o + +gcc -O2 -Wall -c baz.c -o baz.o + +make[1]: Entering directory '/home/user/project/test' +make[2]: Entering directory '/home/user/project/test/unit' +./run_tests --verbose +make[2]: Nothing to be done for 'check'. +make[2]: Leaving directory '/home/user/project/test/unit' +make[1]: Leaving directory '/home/user/project/test' + +ld -o myapp foo.o bar.o baz.o -lfoo + +make[1]: Entering directory '/home/user/project/docs' +doxygen Doxyfile +make[1]: Leaving directory '/home/user/project/docs' +"#; + let out = apply_filter(filter, input); + let input_words = input.split_whitespace().count(); + let out_words = out.split_whitespace().count(); + let savings = 100.0 - (out_words as f64 / input_words as f64 * 100.0); + assert!( + savings >= 60.0, + "make filter: expected >=60% savings, got {:.1}% (in={} out={})", + savings, + input_words, + out_words + ); + } + + #[test] + fn test_spring_boot_match_command_requires_spring_named_jar() { + let filters = make_filters(BUILTIN_TOML); + let spring_boot = filters + .iter() + .find(|f| f.name == "spring-boot") + .expect("spring-boot filter must exist"); + + assert!( + !spring_boot + .match_regex + .is_match("java -jar build/libs/my-other-tool.jar"), + "a non-Spring jar must not activate the spring-boot filter" + ); + + let spring_jar = find_filter_in("java -jar build/libs/my-spring-app.jar", &filters) + .expect("a jar with 'spring' in its filename must still match"); + assert_eq!(spring_jar.name, "spring-boot"); + + let mvn_run = find_filter_in("mvn spring-boot:run", &filters) + .expect("mvn spring-boot:run must still match"); + assert_eq!(mvn_run.name, "spring-boot"); + + let capitalized = find_filter_in("java -jar build/libs/MySpringApp.jar", &filters) + .expect("a jar with 'Spring' capitalized in its filename must still match"); + assert_eq!(capitalized.name, "spring-boot"); + + assert!( + !spring_boot + .match_regex + .is_match("java -jar /opt/spring-cache/other-tool.jar"), + "'spring' appearing only in a directory segment (not the jar filename itself) must not activate the spring-boot filter" + ); + + // Windows paths: '\' must be treated as a path separator too, not swallowed + // into the filename-only match (the exact bug this guard exists to catch). + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\spring-cache\other.jar"), + "'spring' appearing only in a Windows directory segment must not activate the spring-boot filter" + ); + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\dev\spring-workspace\build\other-tool.jar"), + "'spring' appearing only in a nested Windows directory segment must not activate the spring-boot filter" + ); + assert!( + spring_boot + .match_regex + .is_match(r"java -jar C:\dev\my-spring-app.jar"), + "a Windows-path jar with 'spring' in its own filename must still match" + ); + + // argv reaches this regex as one space-joined string, so a path containing spaces + // is indistinguishable from a following argument. Widening the prefix across + // whitespace would let a later 'spring'-named jar argument re-trigger the filter, + // so jars under such a path stay on full passthrough instead. + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\Program Files\app\my-spring-app.jar"), + "a jar under a path containing spaces is deliberately left to full passthrough" + ); + } + + #[test] + fn test_liquibase_match_command_ignores_path_substring() { + let filters = make_filters(BUILTIN_TOML); + let liquibase = filters + .iter() + .find(|f| f.name == "liquibase") + .expect("liquibase filter must exist"); + + assert!( + !liquibase.match_regex.is_match("rm -rf /opt/liquibase"), + "'liquibase' appearing only as a path argument must not activate the liquibase filter" + ); + + let bare = find_filter_in("liquibase status", &filters) + .expect("bare liquibase invocation must still match"); + assert_eq!(bare.name, "liquibase"); + + // Production callers (run_fallback in src/main.rs, strip_absolute_path in + // src/discover/registry.rs) always basename argv[0] before this regex runs, + // so a raw path-qualified string must NOT match on its own — the regex has + // no path-prefix branch to fall back on. + assert!( + !liquibase + .match_regex + .is_match("/usr/local/bin/liquibase update"), + "a raw path-qualified invocation must not match — callers basename argv[0] before matching" + ); + + for wrapped in [ + "timeout 5 /usr/bin/liquibase update", + "nohup /opt/tools/liquibase update", + ] { + assert!( + !liquibase.match_regex.is_match(wrapped), + "a wrapper command must not activate the liquibase filter: {wrapped}" + ); + } + } + + #[test] + fn test_ssh_match_command_excludes_ssh_dash_utilities() { + let filters = make_filters(BUILTIN_TOML); + let ssh = filters + .iter() + .find(|f| f.name == "ssh") + .expect("ssh filter must exist"); + + assert!( + !ssh.match_regex.is_match("ssh-keygen -t ed25519"), + "ssh-keygen must not activate the plain ssh connection filter" + ); + assert!( + !ssh.match_regex.is_match("ssh-add id_ed25519"), + "ssh-add must not activate the plain ssh connection filter" + ); + + let plain = find_filter_in("ssh user@host", &filters) + .expect("plain ssh invocation must still match"); + assert_eq!(plain.name, "ssh"); + + let bare = find_filter_in("ssh", &filters).expect("bare ssh with no args must still match"); + assert_eq!(bare.name, "ssh"); + } + + // --- Edge cases --- + + #[test] + fn test_empty_input() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = [".*"] +"#, + ); + let out = apply_filter(&f, ""); + assert_eq!(out, ""); + } + + #[test] + fn test_unicode_preserved() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["^noise"] +"#, + ); + let out = apply_filter(&f, "日本語テスト\nnoise\n中文内容"); + assert_eq!(out, "日本語テスト\n中文内容"); + } + + // --- match_output tests (PR1) --- + + #[test] + fn test_match_output_basic_short_circuit() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +"#, + ); + let out = apply_filter(&f, "Switched to branch 'main'"); + assert_eq!(out, "ok"); + } + + #[test] + fn test_match_output_second_rule_matches() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "switched" }, + { pattern = "Already on", message = "already" }, +] +"#, + ); + let out = apply_filter(&f, "Already on 'main'"); + assert_eq!(out, "already"); + } + + #[test] + fn test_match_output_no_match_pipeline_continues() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +strip_lines_matching = ["^noise"] +"#, + ); + let out = apply_filter(&f, "noise\nkeep this"); + // No match_output match, pipeline continues and strips noise + assert_eq!(out, "keep this"); + } + + #[test] + fn test_match_output_strip_ansi_before_match() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +"#, + ); + // ANSI stripped before match_output check (stage 1 before stage 3) + let out = apply_filter(&f, "\x1b[32mSwitched to branch\x1b[0m 'main'"); + assert_eq!(out, "ok"); + } + + #[test] + fn test_match_output_no_match_then_on_empty() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched", message = "ok" }, +] +strip_lines_matching = [".*"] +on_empty = "nothing" +"#, + ); + // No match_output match; pipeline strips all lines; on_empty fires + let out = apply_filter(&f, "foo bar baz"); + assert_eq!(out, "nothing"); + } + + #[test] + fn test_match_output_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "[invalid", message = "ok" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- match_output unless tests (PR3) --- + + #[test] + fn test_match_output_unless_blocks_short_circuit_when_errors_present() { + // "total size is" matches, but "error" also matches — unless fires, rule is skipped. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^rsync" +match_output = [ + { pattern = "total size is", message = "ok (synced)", unless = "error|failed" }, +] +"#, + ); + let input = "rsync: [sender] error\ntotal size is 1000 speedup is 3.33\n"; + let out = apply_filter(&f, input); + // Should NOT return "ok (synced)" because "error" matches the unless pattern + assert_ne!( + out.trim(), + "ok (synced)", + "unless should have blocked short-circuit when errors are present" + ); + // The raw lines should pass through (no further strip rules in this filter) + assert!(out.contains("error")); + } + + #[test] + fn test_match_output_unless_allows_short_circuit_when_no_errors() { + // "total size is" matches and "error" does NOT appear — unless does not fire, rule wins. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^rsync" +match_output = [ + { pattern = "total size is", message = "ok (synced)", unless = "error|failed" }, +] +"#, + ); + let input = "file.txt\ntotal size is 98765 speedup is 77.31\n"; + let out = apply_filter(&f, input); + assert_eq!(out.trim(), "ok (synced)"); + } + + #[test] + fn test_match_output_unless_falls_through_to_next_rule() { + // First rule blocked by unless; second rule (no unless) should match. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "success", message = "ok", unless = "error" }, + { pattern = "success", message = "ok with warnings" }, +] +"#, + ); + let input = "success\nerror: something went wrong\n"; + let out = apply_filter(&f, input); + // First rule skipped (unless matched), second rule (no unless) fires + assert_eq!(out.trim(), "ok with warnings"); + } + + #[test] + fn test_match_output_unless_no_field_behaves_like_before() { + // When unless is absent, behaviour is identical to original (no regression). + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Build complete", message = "ok (build complete)" }, +] +"#, + ); + let out = apply_filter(&f, "Build complete!\n"); + assert_eq!(out.trim(), "ok (build complete)"); + } + + #[test] + fn test_match_output_unless_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "success", message = "ok", unless = "[invalid" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- replace tests (PR1) --- + + #[test] + fn test_replace_basic_all_occurrences() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "foo", replacement = "bar" }, +] +"#, + ); + let out = apply_filter(&f, "foo baz foo\nfoo"); + assert_eq!(out, "bar baz bar\nbar"); + } + + #[test] + fn test_replace_chaining_sequential() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "aaa", replacement = "bbb" }, + { pattern = "bbb", replacement = "ccc" }, +] +"#, + ); + // Rule 2 operates on the output of rule 1 + let out = apply_filter(&f, "aaa"); + assert_eq!(out, "ccc"); + } + + #[test] + fn test_replace_backreferences() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "(\\w+):(\\w+)", replacement = "$2:$1" }, +] +"#, + ); + let out = apply_filter(&f, "hello:world"); + assert_eq!(out, "world:hello"); + } + + #[test] + fn test_replace_then_strip_interaction() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "noise", replacement = "DROPPED" }, +] +strip_lines_matching = ["^DROPPED"] +"#, + ); + // replace transforms "noise line" -> "DROPPED line", strip removes it + let out = apply_filter(&f, "noise line\nkeep this"); + assert_eq!(out, "keep this"); + } + + #[test] + fn test_replace_empty_input_noop() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "foo", replacement = "bar" }, +] +"#, + ); + let out = apply_filter(&f, ""); + assert_eq!(out, ""); + } + + #[test] + fn test_replace_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "[invalid", replacement = "bar" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- verify (PR2) --- + + #[test] + fn test_run_filter_tests_passes_on_correct_expected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +strip_lines_matching = ["^make\\[\\d+\\]:"] + +[[tests.make]] +name = "strips entering/leaving lines" +input = """ +make[1]: Entering directory '/home/user' +gcc -O2 foo.c +make[1]: Leaving directory '/home/user' +""" +expected = """ +gcc -O2 foo.c +""" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + assert_eq!(outcomes.len(), 1); + assert!( + outcomes[0].passed, + "test should pass: {:?}", + outcomes[0].actual + ); + } + + #[test] + fn test_run_filter_tests_fails_on_wrong_expected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +strip_lines_matching = ["^make\\[\\d+\\]:"] + +[[tests.make]] +name = "wrong expected" +input = "make[1]: Entering\ngcc foo.c" +expected = "wrong output" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + assert_eq!(outcomes.len(), 1); + assert!(!outcomes[0].passed); + } + + #[test] + fn test_filters_without_tests_detected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + // No tests defined, but filter exists + assert_eq!(outcomes.len(), 0); + assert!(all_names.contains(&"make".to_string())); + assert!(!tested.contains("make")); + } + + // --- Multi-file architecture tests (build.rs) --- + + /// Verify BUILTIN_TOML was generated with the correct schema_version header. + /// build.rs injects it — if the const is somehow stale this fails immediately. + #[test] + fn test_builtin_toml_has_schema_version() { + assert!( + BUILTIN_TOML.contains("schema_version = 1"), + "BUILTIN_TOML must start with 'schema_version = 1' (injected by build.rs)" + ); + } + + /// Verify every expected filter name is present in BUILTIN_TOML. + /// This is the safeguard against accidentally deleting a filter file. + #[test] + fn test_builtin_all_expected_filters_present() { + let filters = make_filters(BUILTIN_TOML); + let names: std::collections::HashSet<&str> = + filters.iter().map(|f| f.name.as_str()).collect(); + + let expected = [ + "ansible-playbook", + "brew-install", + "composer-install", + "df", + "dotnet-build", + "du", + "fail2ban-client", + "gcloud", + "hadolint", + "helm", + "iptables", + "liquibase", + "make", + "markdownlint", + "mix-compile", + "mix-format", + "ping", + "pio-run", + "poetry-install", + "pre-commit", + "ps", + "pulumi-destroy", + "pulumi-preview", + "pulumi-refresh", + "pulumi-stack", + "pulumi-up", + "quarto-render", + "rsync", + "shellcheck", + "shopify-theme", + "sops", + "swift-build", + "systemctl-status", + "terraform-plan", + "tofu-fmt", + "tofu-init", + "tofu-plan", + "tofu-validate", + "trunk-build", + "uv-sync", + "yamllint", + ]; + + for name in &expected { + assert!( + names.contains(name), + "Built-in filter '{}' is missing — was its .toml file deleted from src/filters/?", + name + ); + } + } + + /// Verify the exact count of built-in filters. + /// Fails if a file is added/removed without updating this test. + #[test] + fn test_builtin_filter_count() { + let filters = make_filters(BUILTIN_TOML); + assert_eq!( + filters.len(), + 62, + "Expected exactly 62 built-in filters, got {}. \ + Update this count when adding/removing filters in src/filters/.", + filters.len() + ); + } + + /// Every built-in filter must be anchored on every top-level branch: the + /// match runs against the whole command line, so an unanchored branch + /// activates the filter on a path component or wrapper argument. + #[test] + fn test_builtin_all_filters_match_command_anchored() { + // Read the raw patterns rather than the compiled registry: both loaders + // drop unanchored filters, so a compiled view cannot observe one. + let file: TomlFilterFile = + toml::from_str(BUILTIN_TOML).expect("built-in filters should parse"); + assert!(!file.filters.is_empty()); + for (name, def) in &file.filters { + assert!( + is_fully_anchored(&def.match_command), + "Filter '{}' has match_command '{}' with a top-level branch that does not start with '^'", + name, + def.match_command + ); + } + + assert!(is_fully_anchored(r"^liquibase(?:\s|$)")); + assert!(is_fully_anchored(r"^(liquibase\b|/liquibase\b)")); + assert!(is_fully_anchored(r"^pnpm\b|^npm\b")); + assert!(is_fully_anchored(r"(?i)^liquibase\b")); + assert!(!is_fully_anchored(r"^liquibase\b|/liquibase\b")); + assert!(!is_fully_anchored(r"(?:^|/)liquibase(?:\s|$)")); + } + + /// The invariant above covers `BUILTIN_TOML`; project-local and global + /// filters are user-authored, so both loaders drop unanchored patterns + /// rather than letting them match mid-command. + #[test] + fn test_unanchored_user_filter_is_rejected_by_both_loaders() { + let unanchored = + "schema_version = 1\n[filters.mytool]\nmatch_command = \"(?:^|/)mytool\\\\b\"\n"; + assert!(match_patterns_in(unanchored).is_empty()); + assert!( + TomlFilterRegistry::parse_and_compile(unanchored, "test") + .expect("schema is valid") + .is_empty() + ); + + let anchored = "schema_version = 1\n[filters.mytool]\nmatch_command = \"^mytool\\\\b\"\n"; + assert_eq!(match_patterns_in(anchored), vec!["^mytool\\b".to_string()]); + assert_eq!( + TomlFilterRegistry::parse_and_compile(anchored, "test") + .expect("schema is valid") + .len(), + 1 + ); + } + + /// Verify that every built-in filter has at least one inline test. + /// Prevents shipping filters with zero test coverage. + #[test] + fn test_builtin_all_filters_have_inline_tests() { + let mut all_names: Vec = Vec::new(); + let mut tested: std::collections::HashSet = std::collections::HashSet::new(); + let mut outcomes = Vec::new(); + collect_test_outcomes( + BUILTIN_TOML, + None, + &mut outcomes, + &mut all_names, + &mut tested, + ); + + let untested: Vec<&str> = all_names + .iter() + .filter(|name| !tested.contains(name.as_str())) + .map(|s| s.as_str()) + .collect(); + + assert!( + untested.is_empty(), + "The following built-in filters have no inline tests: {:?}\n\ + Add [[tests.]] entries to the corresponding src/filters/.toml file.", + untested + ); + } + + /// Verify that adding a new filter entry to any TOML content makes it + /// immediately discoverable via find_filter_in — simulating how a new + /// src/filters/my-tool.toml would work after cargo build. + #[test] + fn test_new_filter_discoverable_after_concat() { + // Simulate build.rs: concat BUILTIN_TOML with a brand-new filter block + let new_filter = r#" +[filters.my-new-tool] +description = "Compact my-new-tool output" +match_command = "^my-new-tool\\b" +strip_lines_matching = ["^\\s*$"] +max_lines = 30 +on_empty = "my-new-tool: ok" + +[[tests.my-new-tool]] +name = "strips blank lines" +input = "output line 1\n\noutput line 2" +expected = "output line 1\noutput line 2" +"#; + let combined = format!("{}\n\n{}", BUILTIN_TOML, new_filter); + let filters = make_filters(&combined); + + // All 62 existing filters still present + 1 new = 63 + assert_eq!( + filters.len(), + 63, + "Expected 63 filters after concat (62 built-in + 1 new)" + ); + + // New filter is discoverable + let found = find_filter_in("my-new-tool --verbose", &filters); + assert!( + found.is_some(), + "Newly added filter must be discoverable via find_filter_in" + ); + assert_eq!(found.unwrap().name, "my-new-tool"); + } +} + +pub fn registry() -> &'static TomlFilterRegistry { + ®ISTRY +} + +fn collect_test_outcomes( + content: &str, + filter_name_opt: Option<&str>, + outcomes: &mut Vec, + all_filter_names: &mut Vec, + tested_filter_names: &mut std::collections::HashSet, +) { + let file: TomlFilterFile = match toml::from_str(content) { + Ok(f) => f, + Err(e) => { + eprintln!("[rtk] warning: TOML parse error during verify: {}", e); + return; + } + }; + + // Compile all filters and track their names + let mut compiled_filters: BTreeMap = BTreeMap::new(); + for (name, def) in file.filters { + all_filter_names.push(name.clone()); + match compile_filter(name.clone(), def) { + Ok(f) => { + compiled_filters.insert(name, f); + } + Err(e) => eprintln!("[rtk] warning: filter '{}' compilation error: {}", name, e), + } + } + + // Run tests + for (filter_name, tests) in file.tests { + if let Some(name) = filter_name_opt + && filter_name != name + { + continue; + } + + tested_filter_names.insert(filter_name.clone()); + + let compiled = match compiled_filters.get(&filter_name) { + Some(f) => f, + None => { + eprintln!( + "[rtk] warning: [[tests.{}]] references unknown filter", + filter_name + ); + continue; + } + }; + + for test in tests { + let actual = apply_filter(compiled, &test.input); + // Trim trailing newlines: TOML multiline strings end with a newline + let actual_cmp = actual.trim_end_matches('\n').to_string(); + let expected_cmp = test.expected.trim_end_matches('\n').to_string(); + outcomes.push(TestOutcome { + filter_name: filter_name.clone(), + test_name: test.name, + passed: actual_cmp == expected_cmp, + actual: actual_cmp, + expected: expected_cmp, + }); + } + } +} diff --git a/crates/sparkplane-rtk/src/core/truncate.rs b/crates/sparkplane-rtk/src/core/truncate.rs new file mode 100644 index 0000000..58285f6 --- /dev/null +++ b/crates/sparkplane-rtk/src/core/truncate.rs @@ -0,0 +1,60 @@ +//! Global truncation caps shared by every filter. See `src/core/README.md` +//! ("Truncation Caps") for the cap classes, config policy, and deviation rules. + +/// Errors: most actionable, shown the most. +pub const CAP_ERRORS: usize = 20; +/// Warnings and test failures: lower signal density than errors. +pub const CAP_WARNINGS: usize = 10; +/// Flat lists (PRs, services, packages): one line per item. +pub const CAP_LIST: usize = 20; +/// Inventories (`pip list`, `docker images`): exhaustive lookups. +pub const CAP_INVENTORY: usize = 50; + +/// A cap reduced for a verbose data class. Falls back to `cap` when `by >= cap` +/// so a deviation can never empty the list; `0` stays `0`. `const fn`, underflow-safe. +pub const fn reduced(cap: usize, by: usize) -> usize { + if by < cap { cap - by } else { cap } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reduced_preserves_current_values() { + assert_eq!(reduced(CAP_WARNINGS, 5), 5); + assert_eq!(reduced(CAP_LIST, 5), 15); + } + + #[test] + fn reduced_falls_back_to_cap_when_offset_too_large() { + assert_eq!(reduced(4, 5), 4); + assert_eq!(reduced(5, 5), 5); + } + + #[test] + fn reduced_honors_zero_cap() { + assert_eq!(reduced(0, 5), 0); + } + + // Sweep every plausible (cap, by) a future config could produce and assert + // the invariants that make caps safe: the result never wraps past `cap`, and + // the offset never empties a non-zero cap. `usize::MAX` covers a wraparound bug. + #[test] + fn reduced_is_underflow_safe_across_all_inputs() { + for cap in 0..=64usize { + for by in [0usize, 1, 5, 10, 64, usize::MAX] { + let r = reduced(cap, by); + assert!(r <= cap, "reduced({cap}, {by}) = {r} exceeds cap (wrapped)"); + if cap == 0 { + assert_eq!(r, 0, "zero cap must stay zero"); + } else { + assert!(r >= 1, "reduced({cap}, {by}) = {r} emptied a non-zero cap"); + } + if by < cap { + assert_eq!(r, cap - by, "exact deviation must be preserved"); + } + } + } + } +} diff --git a/crates/sparkplane-rtk/src/core/utils.rs b/crates/sparkplane-rtk/src/core/utils.rs new file mode 100644 index 0000000..f2e89dd --- /dev/null +++ b/crates/sparkplane-rtk/src/core/utils.rs @@ -0,0 +1,26 @@ +use regex::Regex; +use std::sync::LazyLock; +pub fn truncate(s: &str, max_len: usize) -> String { + let char_count = s.chars().count(); + if char_count <= max_len { + s.to_string() + } else if max_len < 3 { + // If max_len is too small, just return "..." + "...".to_string() + } else { + format!("{}...", s.chars().take(max_len - 3).collect::()) + } +} +pub fn strip_ansi(text: &str) -> String { + static ANSI_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\x1b\[[0-9;]*[a-zA-Z]").unwrap()); + + ANSI_RE.replace_all(text, "").to_string() +} +pub fn join_with_overflow(items: &[String], total: usize, max: usize, label: &str) -> String { + let mut out = items.join("\n"); + if total > max { + out.push_str(&format!("\n… +{} more {}", total - max, label)); + } + out +} diff --git a/crates/sparkplane-rtk/src/git.rs b/crates/sparkplane-rtk/src/git.rs new file mode 100644 index 0000000..1ee8b9d --- /dev/null +++ b/crates/sparkplane-rtk/src/git.rs @@ -0,0 +1,35 @@ +pub fn filter_status_with_args(output: &str) -> String { + let mut result = Vec::new(); + + for line in output.lines() { + let trimmed = line.trim(); + + // Skip empty lines + if trimmed.is_empty() { + continue; + } + + // Skip git hints - can appear at start or within line + if trimmed.starts_with("(use \"git") + || trimmed.starts_with("(create/copy files") + || trimmed.contains("(use \"git add") + || trimmed.contains("(use \"git restore") + { + continue; + } + + // Special case: clean working tree + if trimmed.contains("nothing to commit") && trimmed.contains("working tree clean") { + result.push(trimmed.to_string()); + break; + } + + result.push(line.to_string()); + } + + if result.is_empty() { + "ok".to_string() + } else { + result.join("\n") + } +} diff --git a/crates/sparkplane-rtk/src/lib.rs b/crates/sparkplane-rtk/src/lib.rs new file mode 100644 index 0000000..66e42c5 --- /dev/null +++ b/crates/sparkplane-rtk/src/lib.rs @@ -0,0 +1,132 @@ +//! Embedded RTK output filters. Command execution and metrics belong to Sparkplane. +// Vendored modules retain upstream style and public helpers for future filter imports. +#[allow(dead_code, clippy::all)] +mod cargo; +#[allow(dead_code, clippy::all)] +mod git; +#[allow(dead_code, clippy::all)] +mod pytest; +#[allow(dead_code, clippy::all)] +mod core { + pub mod constants; + pub mod guard; + pub mod stream; + pub mod toml_filter; + pub mod truncate; + pub mod utils; + pub mod tracking { + pub fn estimate_tokens(value: &str) -> usize { + value.len().div_ceil(4) + } + } + pub mod tee { + pub fn force_tee_hint(_: &str, _: &str) -> Option { + None + } + pub fn force_tee_tail_hint(_: &str, _: &str, _: usize) -> Option { + None + } + } +} + +pub const REVISION: &str = "6d4b77eadee1c66dc1f68466ad77e96d1b6e4989"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Filter { + GitStatus, + Cargo(&'static str), + Pytest, + Declarative(String), +} + +/// Conservative shell-only profile. Explicit output formats must stay byte exact. +pub fn select(args: &[String]) -> Option { + if args.iter().any(|a| { + a.starts_with('/') + || a.contains('=') + || [ + "--json", + "--porcelain", + "--format", + "--output", + "--raw", + "--null", + "-z", + "--message-format", + "--xml", + "--csv", + "--help", + "--version", + ] + .iter() + .any(|flag| a == flag || a.starts_with(&format!("{flag}="))) + }) { + return None; + } + match (args.first()?.as_str(), args.get(1).map(String::as_str)) { + ("git", Some("status")) => Some(Filter::GitStatus), + ("cargo", Some("test")) => Some(Filter::Cargo("test")), + ("cargo", Some("build")) => Some(Filter::Cargo("build")), + ("cargo", Some("check")) => Some(Filter::Cargo("check")), + ("cargo", Some("clippy")) => Some(Filter::Cargo("clippy")), + ("pytest", _) => Some(Filter::Pytest), + ("jq", _) => None, + _ => { + let command = args.join(" "); + core::toml_filter::find_matching_filter(&command) + .map(|f| Filter::Declarative(f.name.clone())) + } + } +} + +/// No truncation of unsuccessful executions: retain every diagnostic. +pub fn filter(profile: &Filter, output: &[u8], exit_code: i32) -> Vec { + let Ok(raw) = std::str::from_utf8(output) else { + return output.to_vec(); + }; + if exit_code != 0 || output.contains(&0) { + return output.to_vec(); + } + let compact = match profile { + Filter::GitStatus => git::filter_status_with_args(raw), + Filter::Cargo("test") => cargo::filter_cargo_test(raw), + Filter::Cargo(label) => cargo::filter_cargo_build_labeled(raw, label, exit_code), + Filter::Pytest => pytest::filter_pytest_output(raw), + Filter::Declarative(name) => { + let registry = core::toml_filter::registry(); + let Some(filter) = registry.filters.iter().find(|f| &f.name == name) else { + return output.to_vec(); + }; + core::toml_filter::apply_filter(filter, raw) + } + }; + let compact = if compact.ends_with('\n') { + compact + } else { + format!("{compact}\n") + }; + if compact.len() >= output.len() { + output.to_vec() + } else { + compact.into_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn cargo_success_compresses_and_failure_is_exact() { + let raw = b"running 2 tests\ntest first ... ok\ntest second ... ok\ntest result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s\n"; + let compact = filter(&Filter::Cargo("test"), raw, 0); + assert!(compact.len() < raw.len()); + assert!(String::from_utf8(compact).unwrap().contains("2 passed")); + assert_eq!(filter(&Filter::Cargo("test"), raw, 1), raw); + } + #[test] + fn builtins_compile_and_binary_is_exact() { + let registry = core::toml_filter::registry(); + assert!(registry.filters.len() > 50); + assert_eq!(filter(&Filter::GitStatus, b"\xff\x00", 0), b"\xff\x00"); + } +} diff --git a/crates/sparkplane-rtk/src/pytest.rs b/crates/sparkplane-rtk/src/pytest.rs new file mode 100644 index 0000000..b7e0e0d --- /dev/null +++ b/crates/sparkplane-rtk/src/pytest.rs @@ -0,0 +1,290 @@ +const MAX_XFAIL: usize = CAP_WARNINGS; +const MAX_PYTEST_FAILURES: usize = CAP_WARNINGS; +use crate::core::truncate::CAP_WARNINGS; +use crate::core::utils::truncate; +const PYTEST_NO_TESTS: &str = "Pytest: No tests collected"; + +pub(crate) fn filter_pytest_output(output: &str) -> String { + let mut state = ParseState::Header; + let mut test_files: Vec = Vec::new(); + let mut failures: Vec = Vec::new(); + let mut current_failure: Vec = Vec::new(); + let mut xfail_lines: Vec = Vec::new(); + let mut summary_line = String::new(); + + for line in output.lines() { + let trimmed = line.trim(); + + // State transitions + if trimmed.starts_with("===") && trimmed.contains("test session starts") { + state = ParseState::Header; + continue; + } else if trimmed.starts_with("===") && trimmed.contains("FAILURES") { + state = ParseState::Failures; + continue; + } else if trimmed.starts_with("===") && trimmed.contains("short test summary") { + state = ParseState::Summary; + // Save current failure if any + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } + continue; + } else if trimmed.starts_with("===") + && (trimmed.contains("passed") + || trimmed.contains("failed") + || trimmed.contains("skipped")) + { + summary_line = trimmed.to_string(); + continue; + // quiet mode (-q): bare summary without === wrapper, e.g. "5 failed, 1698 passed, 2 skipped in 108.89s" + } else if summary_line.is_empty() + && !trimmed.starts_with("===") + && !trimmed.starts_with("FAILED") + && !trimmed.starts_with("ERROR") + && (trimmed.contains(" passed") + || trimmed.contains(" failed") + || trimmed.contains(" skipped")) + && trimmed.contains(" in ") + { + summary_line = trimmed.to_string(); + continue; + } + + // Process based on state + match state { + ParseState::Header => { + if trimmed.starts_with("collected") { + state = ParseState::TestProgress; + } + } + ParseState::TestProgress => { + // Lines like "tests/test_foo.py .... [ 40%]" + if !trimmed.is_empty() + && !trimmed.starts_with("===") + && (trimmed.contains(".py") || trimmed.contains("%]")) + { + test_files.push(trimmed.to_string()); + } + } + ParseState::Failures => { + // Collect failure details + if trimmed.starts_with("___") { + // New failure section + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } + current_failure.push(trimmed.to_string()); + } else if !trimmed.is_empty() && !trimmed.starts_with("===") { + current_failure.push(trimmed.to_string()); + } + } + ParseState::Summary => { + // FAILED test lines + if trimmed.starts_with("FAILED") || trimmed.starts_with("ERROR") { + failures.push(trimmed.to_string()); + } else if trimmed.starts_with("XFAIL") || trimmed.starts_with("XPASS") { + xfail_lines.push(trimmed.to_string()); + } + } + } + } + + // Save last failure if any + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + } + + // Build compact output + build_pytest_summary(&summary_line, &test_files, &failures, &xfail_lines) +} + +#[derive(Default)] +struct PytestCounts { + passed: usize, + failed: usize, + skipped: usize, + xfailed: usize, + xpassed: usize, + duration: Option, +} + +fn build_pytest_summary( + summary: &str, + _test_files: &[String], + failures: &[String], + xfail_lines: &[String], +) -> String { + let counts = parse_summary_line(summary); + let PytestCounts { + passed, + failed, + skipped, + xfailed, + xpassed, + duration, + } = counts; + + if passed == 0 && failed == 0 && skipped == 0 && xfailed == 0 && xpassed == 0 { + return PYTEST_NO_TESTS.to_string(); + } + + let extras_present = skipped > 0 || xfailed > 0 || xpassed > 0 || !xfail_lines.is_empty(); + let duration_suffix = duration + .as_deref() + .map(|value| format!(" in {value}")) + .unwrap_or_default(); + + if failed == 0 && passed > 0 && !extras_present { + return format!("Pytest: {passed} passed{duration_suffix}"); + } + + let mut result = String::new(); + result.push_str(&format!("Pytest: {} passed, {} failed", passed, failed)); + if skipped > 0 { + result.push_str(&format!(", {} skipped", skipped)); + } + if xfailed > 0 { + result.push_str(&format!(", {} xfailed", xfailed)); + } + if xpassed > 0 { + result.push_str(&format!(", {} xpassed", xpassed)); + } + result.push_str(&duration_suffix); + result.push('\n'); + + // Surface xfail/xpass entries (with their reasons) — XPASS in particular + // signals that something expected-to-fail now passes. + if !xfail_lines.is_empty() { + result.push_str("\nExpected-failure outcomes:\n"); + for line in xfail_lines.iter().take(MAX_XFAIL) { + result.push_str(&format!(" {}\n", truncate(line, 120))); + } + if xfail_lines.len() > MAX_XFAIL { + result.push_str(&format!(" … +{} more\n", xfail_lines.len() - MAX_XFAIL)); + let all_xfail = xfail_lines.join("\n"); + if let Some(hint) = + crate::core::tee::force_tee_tail_hint(&all_xfail, "pytest-xfail", MAX_XFAIL + 1) + { + result.push_str(&format!(" {}\n", hint)); + } + } + } + + if failures.is_empty() { + return result.trim().to_string(); + } + + // Show failures (limit to key information) + result.push_str("\nFailures:\n"); + + for (i, failure) in failures.iter().take(MAX_PYTEST_FAILURES).enumerate() { + // Extract test name and key error info + let lines: Vec<&str> = failure.lines().collect(); + + // First line is usually test name (after ___) + if let Some(first_line) = lines.first() { + if first_line.starts_with("___") { + // Extract test name between ___ + let test_name = first_line.trim_matches('_').trim(); + result.push_str(&format!("{}. [FAIL] {}\n", i + 1, test_name)); + } else if first_line.starts_with("FAILED") { + // Summary format: "FAILED tests/test_foo.py::test_bar - AssertionError" + let parts: Vec<&str> = first_line.split(" - ").collect(); + if let Some(test_path) = parts.first() { + let test_name = test_path.trim_start_matches("FAILED "); + result.push_str(&format!("{}. [FAIL] {}\n", i + 1, test_name)); + } + if parts.len() > 1 { + result.push_str(&format!(" {}\n", truncate(parts[1], 100))); + } + continue; + } + } + + // Show relevant error lines (assertions, errors, file locations) + let mut relevant_lines = 0; + for line in &lines[1..] { + let line_lower = line.to_lowercase(); + let is_relevant = line.trim().starts_with('>') + || line.trim().starts_with('E') + || line_lower.contains("assert") + || line_lower.contains("error") + || line.contains(".py:"); + + if is_relevant && relevant_lines < 3 { + result.push_str(&format!(" {}\n", truncate(line, 100))); + relevant_lines += 1; + } + } + + if i < failures.len() - 1 { + result.push('\n'); + } + } + + if failures.len() > MAX_PYTEST_FAILURES { + result.push_str(&format!( + "\n… +{} more failures\n", + failures.len() - MAX_PYTEST_FAILURES + )); + let all_failures = failures.join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_failures, "pytest-failures") { + result.push_str(&format!(" {}\n", hint)); + } + } + + result.trim().to_string() +} + +fn parse_summary_line(summary: &str) -> PytestCounts { + let mut counts = PytestCounts { + duration: parse_duration(summary), + ..PytestCounts::default() + }; + + // Parse lines like "=== 4 passed, 1 failed, 2 xfailed, 1 xpassed in 0.50s ===" + for part in summary.split(',') { + let words: Vec<&str> = part.split_whitespace().collect(); + for (i, word) in words.iter().enumerate() { + if i == 0 { + continue; + } + let Ok(n) = words[i - 1].parse::() else { + continue; + }; + // Order matters: "xpassed"/"xfailed" contain "passed"/"failed". + if word.contains("xpassed") { + counts.xpassed = n; + } else if word.contains("xfailed") { + counts.xfailed = n; + } else if word.contains("passed") { + counts.passed = n; + } else if word.contains("failed") { + counts.failed = n; + } else if word.contains("skipped") { + counts.skipped = n; + } + } + } + + counts +} + +fn parse_duration(summary: &str) -> Option { + let unwrapped = summary.trim().trim_matches('=').trim(); + let (_, duration_tail) = unwrapped.rsplit_once(" in ")?; + let duration = duration_tail.split_whitespace().next()?; + let seconds = duration.strip_suffix('s')?; + seconds.parse::().ok()?; + Some(duration.to_string()) +} + +#[derive(Clone, Copy)] +enum ParseState { + Header, + TestProgress, + Failures, + Summary, +} diff --git a/crates/sparkplane-rtk/upstream/Cargo.toml b/crates/sparkplane-rtk/upstream/Cargo.toml new file mode 100644 index 0000000..17dd7d8 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/Cargo.toml @@ -0,0 +1,89 @@ +[package] +name = "rtk" +version = "0.49.0" +edition = "2024" +rust-version = "1.91" +authors = ["Patrick Szymkowiak"] +description = "Rust Token Killer - High-performance CLI proxy to minimize LLM token consumption" +license = "Apache-2.0" +homepage = "https://www.rtk-ai.app" +repository = "https://github.com/rtk-ai/rtk" +readme = "README.md" +keywords = ["cli", "llm", "token", "filter", "productivity"] +categories = ["command-line-utilities", "development-tools"] + +[dependencies] +clap = { version = "4", features = ["derive"] } +anyhow = "1.0" +# Console code page decoding for child-process output. Kept off the +# cfg(windows) target so the mapping and the incremental UTF-8 walk stay +# compiled — and unit-tested — on every platform; only the code page *lookup* +# is Windows-specific. codepage covers the ANSI/DBCS pages encoding_rs +# implements, oem_cp the legacy OEM/DOS pages (437, 850, …) it does not. +encoding_rs = "0.8" +codepage = "0.1" +oem_cp = "2" +ignore = "0.4" +walkdir = "2" +regex = "1" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["preserve_order"] } +colored = "3" +dirs = "5" +rusqlite = { version = "0.31", features = ["bundled"] } +toml = "0.8" +toml_edit = "0.22" +chrono = "0.4" +tempfile = "3" +sha2 = "0.10" +ureq = "2" +getrandom = "0.4" +flate2 = "1.0" +quick-xml = "0.37" +which = "8" + +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.59", features = [ + "Win32_System_Console", + "Win32_Globalization", +] } + +[build-dependencies] +toml = "0.8" + +[dev-dependencies] +# Safe, closure-scoped env mutation for tests. std::env::set_var is unsafe +# in edition 2024 and has no safe std replacement; temp-env restores the +# previous value on completion or panic and serializes callers internally. +temp-env = "0.3" + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 +panic = "abort" +strip = true + +# cargo-deb configuration +[package.metadata.deb] +maintainer = "Patrick Szymkowiak" +copyright = "2024 Patrick Szymkowiak" +license-file = ["LICENSE", "0"] +extended-description = "rtk filters and compresses command outputs before they reach your LLM context, saving 60-90% of tokens." +section = "utility" +priority = "optional" +assets = [ + ["target/release/rtk", "usr/bin/", "755"], +] +# cargo-generate-rpm configuration +[package.metadata.generate-rpm] +assets = [ + { source = "target/release/rtk", dest = "/usr/bin/rtk", mode = "755" }, +] + +[lints.rust] +unsafe_code = "deny" +warnings = "deny" diff --git a/crates/sparkplane-rtk/upstream/LICENSE b/crates/sparkplane-rtk/upstream/LICENSE new file mode 100644 index 0000000..0afaf4b --- /dev/null +++ b/crates/sparkplane-rtk/upstream/LICENSE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 rtk-ai and rtk-ai Labs + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/sparkplane-rtk/upstream/src/cmds/git/git_cmd.rs b/crates/sparkplane-rtk/upstream/src/cmds/git/git_cmd.rs new file mode 100644 index 0000000..88b61e1 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/cmds/git/git_cmd.rs @@ -0,0 +1,6833 @@ +//! Filters git output — log, status, diff, and more — keeping just the essential info. + +use crate::core::arg_tokenizer::{ + self, Attachment, Dialect, Token, TokenKind, ValueSpec, is_digit_run, +}; +use crate::core::args_utils; +use crate::core::guard::never_worse; +use crate::core::runner::{self, RunOptions}; +use crate::core::stream::{ + self, CaptureResult, FilterMode, LineHandler, LineStreamFilter, StdinMode, exec_capture, + exec_capture_stdin, +}; +use crate::core::tracking; +use crate::core::truncate::{CAP_LIST, CAP_WARNINGS}; +use crate::core::utils::{exit_code_from_status, join_with_overflow, resolved_command, strip_ansi}; +use anyhow::{Context, Result}; +use std::ffi::OsString; +use std::process::Command; + +#[derive(Debug, Clone)] +pub enum GitCommand { + Diff, + Log, + Status, + Show, + Add, + Commit, + Checkout, + Push, + Pull, + Branch, + Fetch, + Stash { subcommand: Option }, + Worktree, +} + +/// Create a git Command with global options (e.g. -C, -c, --git-dir, --work-tree) +/// prepended before any subcommand arguments. +fn git_cmd(global_args: &[String]) -> Command { + let mut cmd = resolved_command("git"); + for arg in global_args { + cmd.arg(arg); + } + cmd +} + +/// Create a git Command for internal parsing that must be locale-stable. +/// +/// We only use this for non-user-facing parses where RTK depends on git's +/// English status phrases. User-visible passthrough output keeps the user's +/// locale. +fn git_cmd_c_locale(global_args: &[String]) -> Command { + let mut cmd = git_cmd(global_args); + cmd.env("LC_ALL", "C"); + cmd +} + +fn uses_compact_status_path(args: &[String]) -> bool { + let tokens = arg_tokenizer::tokenize(args); + + if tokens.is_empty() { + return true; + } + + let mut saw_branch = false; + let mut saw_flag = false; + for token in &tokens { + match (token.kind, token.text) { + // A `--` with no pathspec after it selects nothing, so `git status -sb --` is + // `git status -sb`, and a lone `--` is plain `git status`. + (TokenKind::DashDash, _) => {} + (TokenKind::Short, "b") | (TokenKind::Long, "branch") => { + saw_branch = true; + saw_flag = true; + } + (TokenKind::Short, "s") | (TokenKind::Long, "short") => saw_flag = true, + _ => return false, + } + } + + saw_branch || !saw_flag +} + +fn build_status_command(args: &[String], global_args: &[String]) -> Command { + let mut cmd = git_cmd(global_args); + cmd.arg("status"); + if uses_compact_status_path(args) { + cmd.args(["--porcelain", "-b"]); + } else { + cmd.args(args); + } + cmd +} + +pub fn run( + cmd: GitCommand, + args: &[String], + max_lines: Option, + verbose: u8, + global_args: &[String], +) -> Result { + // Centralized here, once, rather than ad hoc per handler. `stash` needs the region put + // back together first: clap carves its subcommand positional out of the same trailing args, + // and restore_double_dash requires the whole region (fed only the remainder it slices one + // token short, which turned `git stash -- -p` into an interactive `stash push -p`). + let (cmd, args) = match cmd { + GitCommand::Stash { subcommand } => { + let mut region: Vec = subcommand.into_iter().collect(); + region.extend_from_slice(args); + let region = args_utils::restore_double_dash(®ion); + let (subcommand, rest) = split_stash_region(®ion); + (GitCommand::Stash { subcommand }, rest) + } + other => (other, args_utils::restore_double_dash(args)), + }; + let args = &args; + match cmd { + GitCommand::Diff => run_diff(args, max_lines, verbose, global_args), + GitCommand::Log => run_log(args, max_lines, verbose, global_args), + GitCommand::Status => run_status(args, verbose, global_args), + GitCommand::Show => run_show(args, max_lines, verbose, global_args), + GitCommand::Add => run_add(args, verbose, global_args), + GitCommand::Commit => run_commit(args, verbose, global_args), + GitCommand::Checkout => run_checkout(args, verbose, global_args), + GitCommand::Push => run_push(args, verbose, global_args), + GitCommand::Pull => run_pull(args, verbose, global_args), + GitCommand::Branch => run_branch(args, verbose, global_args), + GitCommand::Fetch => run_fetch(args, verbose, global_args), + GitCommand::Stash { subcommand } => { + run_stash(subcommand.as_deref(), args, verbose, global_args) + } + GitCommand::Worktree => run_worktree(args, verbose, global_args), + } +} + +/// Splits a restored `stash` region back into subcommand and remainder, the way clap did +/// before the `--` came back: a leading token that is neither a flag nor the boundary. +fn split_stash_region(region: &[String]) -> (Option, Vec) { + let tokens = arg_tokenizer::tokenize(region); + match tokens.first() { + Some(token) if token.kind == TokenKind::Positional => { + (Some(region[0].clone()), region[1..].to_vec()) + } + _ => (None, region.to_vec()), + } +} + +/// `-s`/`--no-patch` ask `git diff` for no body at all, so there is nothing to compact and +/// RTK's own `--stat` header would answer a question the user did not ask. On `git show` the +/// same flags ask for the commit summary, which is exactly what the compact form prints, so +/// this is deliberately not part of [`diff_wants_raw_shape`]. +fn suppresses_diff_body(token: &Token<'_>) -> bool { + matches!( + (token.kind, token.text), + (TokenKind::Long, "no-patch" | "quiet") | (TokenKind::Short, "s") + ) +} + +/// True for a token asking git for patch output: `-p`/`-u`/`--patch`, and the context-width +/// flags that imply a patch (`-U3`, `--unified=3`, `-W`/`--function-context`). +/// Whether the body ends up suppressed once git's own arbitration is applied: `-s`, +/// `--no-patch` and `--quiet` lose to a *later* `-p`/`--patch`/`-U` and win over an earlier +/// one (`git show -s -p` prints the diff, `git show -p -s` does not -- git 2.53). Taking the +/// suppressors order-independently swallowed a patch the user had asked for last. +fn body_is_suppressed(tokens: &[Token<'_>]) -> bool { + // `-s`/`--no-patch` and a patch request resolve by last flag wins: `git show -s -p` prints + // the diff, `git show -p -s` does not (git 2.53). + let mut suppressed = false; + for token in tokens { + if hard_suppresses_diff_body(token) { + suppressed = true; + } else if requests_patch_output(token) { + suppressed = false; + } + } + // `--quiet` does not play that game. In `show` it loses to a patch request from either + // side -- `--quiet -p` and `-p --quiet` both print the diff -- and only suppresses when + // nothing else asked for output. Treating it as a third spelling of `-s` dropped a patch + // the user had asked for. + suppressed || (tokens.iter().any(is_quiet_flag) && !tokens.iter().any(requests_patch_output)) +} + +/// `--quiet`, which is `--exit-code`'s companion rather than a shape flag. Only `run_diff` +/// treats it as one, and there it takes the raw route before any of this is consulted. +fn is_quiet_flag(token: &Token<'_>) -> bool { + (token.kind, token.text) == (TokenKind::Long, "quiet") +} + +/// The suppressors that really do replace the body: `-s` and its long spelling. +fn hard_suppresses_diff_body(token: &Token<'_>) -> bool { + matches!( + (token.kind, token.text), + (TokenKind::Long, "no-patch") | (TokenKind::Short, "s") + ) +} + +fn requests_patch_output(token: &Token<'_>) -> bool { + match token.kind { + TokenKind::Long => matches!( + token.text, + "patch" | "patch-with-stat" | "patch-with-raw" | "unified" | "function-context" + ), + TokenKind::Short => matches!(token.text, "p" | "u" | "U" | "W"), + _ => false, + } +} + +/// `args` with the patch-shape flags removed, so a stat-only header cannot be outranked by +/// them wherever git would have read them. Rebuilt per token rather than per arg: every short +/// flag in a `-xyz` cluster shares one `source_index`, so dropping the whole arg would take +/// its siblings with it -- `-pl 100` lost the `-l` and left `100` behind as a bogus revision. +/// `args` with any `--oneline` removed, by the token's own `source_index` so a pathspec of that +/// name past `--` is left alone. +fn args_without_oneline(args: &[String], tokens: &[Token<'_>]) -> Vec { + let dropped: Vec = tokens + .iter() + .filter(|t| t.kind == TokenKind::Long && t.text == "oneline") + .map(|t| t.source_index) + .collect(); + if dropped.is_empty() { + return args.to_vec(); + } + args.iter() + .enumerate() + .filter(|(index, _)| !dropped.contains(index)) + .map(|(_, arg)| arg.clone()) + .collect() +} + +fn args_without_patch_shape(args: &[String], tokens: &[Token<'_>]) -> Vec { + let mut out: Vec = Vec::with_capacity(args.len()); + for (index, arg) in args.iter().enumerate() { + let owned: Vec<&Token<'_>> = tokens + .iter() + .filter(|t| t.source_index == index && t.kind != TokenKind::Positional) + .collect(); + if owned.is_empty() || !owned.iter().any(|t| requests_patch_output(t)) { + out.push(arg.clone()); + continue; + } + // A cluster keeps whatever letters were not shape flags, with their attached value. + let kept: Vec<&&Token<'_>> = owned + .iter() + .filter(|t| t.kind == TokenKind::Short && !requests_patch_output(t)) + .collect(); + if kept.is_empty() { + continue; + } + let mut rebuilt = String::from("-"); + for token in &kept { + rebuilt.push_str(token.text); + } + if let Some(attached) = kept.last().and_then(|t| t.attached) { + rebuilt.push_str(attached); + } + out.push(rebuilt); + } + out +} + +fn run_diff( + args: &[String], + max_lines: Option, + verbose: u8, + global_args: &[String], +) -> Result { + let timer = tracking::TimedExecution::start(); + + let tokens = tokenize_git_diff_args(args); + let wants_stat = tokens + .iter() + .any(|t| diff_wants_raw_shape(t, &tokens) || suppresses_diff_body(t)); + + // Compact diff is the default RTK behavior; --no-compact is RTK's own pseudo-flag. The + // strip below removes the arg this token came from, so detection and removal can't + // disagree -- re-matching the string dropped a pathspec of the same name past `--`. + // Any spelling counts, attached value or not: `--no-compact` is RTK's own, so git would + // only answer `error: invalid option` if a value form leaked through. + let no_compact: Vec = tokens + .iter() + .filter(|t| t.kind == TokenKind::Long && t.text == "no-compact") + .map(|t| t.source_index) + .collect(); + let wants_compact = no_compact.is_empty() && !emits_word_diff(&tokens); + + if wants_stat || !wants_compact { + // User wants stat or explicitly no compacting - pass through directly + let mut cmd = git_cmd(global_args); + cmd.arg("diff"); + for (index, arg) in args.iter().enumerate() { + if no_compact.contains(&index) { + continue; // RTK flag, not a git flag + } + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git diff")?; + + // A non-zero exit does not mean there was nothing to say: `git diff --check` reports + // every whitespace error on stdout and *then* exits 2. Printed verbatim, since the + // report's payload *is* trailing whitespace and `--stat` has a leading column space. + print!("{}", result.stdout); + + if !result.success() { + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr.trim()); + } + timer.track( + &format!("git diff {}", args.join(" ")), + &format!("rtk git diff {} (passthrough)", args.join(" ")), + &result.stdout, + &result.stdout, + ); + return Ok(result.exit_code); + } + + timer.track( + &format!("git diff {}", args.join(" ")), + &format!("rtk git diff {} (passthrough)", args.join(" ")), + &result.stdout, + &result.stdout, + ); + + return Ok(0); + } + + // Default RTK behavior: stat first, then compacted diff. `--no-patch --stat` forces the + // header to be stat-only whatever the user asked for -- `git diff --stat -p` (or -U3, -W, + // ...) emits the patch too, and RTK then printed it again, compacted, for 2.4x the raw + // output. The flags go before the user's own `--`, where git still reads them as options. + // The user's own command runs first, because only it can give git's verdict on what they + // typed. The stat header below runs with the patch-shape flags stripped, so it answers a + // *different* command: `git diff -Uabc nonexistent-ref` is `error: --unified expects a + // numerical value` (129) to git, and the stripped probe reported `ambiguous argument` (128) + // instead. A probe is decoration; it must never be the thing that reports failure. + let mut diff_cmd = git_cmd(global_args); + diff_cmd.arg("diff"); + for arg in args { + diff_cmd.arg(arg); + } + + let diff_result = exec_capture(&mut diff_cmd).context("Failed to run git diff")?; + + if !diff_result.success() { + if !diff_result.stderr.trim().is_empty() { + eprint!("{}", diff_result.stderr); + } + timer.track( + &format!("git diff {}", args.join(" ")), + &format!("rtk git diff {}", args.join(" ")), + &diff_result.stdout, + &diff_result.stdout, + ); + return Ok(diff_result.exit_code); + } + + // `--no-patch --stat` forces the header to be stat-only whatever the user asked for -- + // `git diff --stat -p` (or -U3, -W, ...) emits the patch too, and RTK then printed it + // again, compacted, for 2.4x the raw output. The flags go before the user's own `--`, + // where git still reads them as options. A failure here costs the header, not the command. + let mut cmd = git_cmd(global_args); + cmd.args(["diff", "--no-patch", "--stat"]); + for arg in args_without_patch_shape(args, &tokens) { + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git diff")?; + + if verbose > 0 { + eprintln!("Git diff summary:"); + } + + let printed = if !diff_result.stdout.is_empty() { + let compacted = compact_diff(&strip_ansi(&diff_result.stdout), max_lines.unwrap_or(500)); + format!("{}\n\nChanges:\n{}", result.stdout.trim(), compacted) + } else { + result.stdout.trim().to_string() + }; + + let raw = format!("{}\n{}", result.stdout, diff_result.stdout); + let shown = never_worse(&raw, &printed); + println!("{}", shown); + + timer.track( + &format!("git diff {}", args.join(" ")), + &format!("rtk git diff {}", args.join(" ")), + &raw, + shown, + ); + + Ok(0) +} + +/// `git show` with RTK's own shape flags in front. `drop_patch_shape` removes the user's +/// patch-enabling flags, which the summary and stat steps need (git takes the last one, so a +/// `-p` anywhere would bring the patch back) but the diff step must not -- `-U5` sets the +/// context width of the patch RTK is about to compact. +/// git takes the last output-format flag, so a `-p` left anywhere in the args -- including +/// after a revision, where RTK's flags cannot be placed -- would re-enable the patch in the +/// summary and stat steps and the result stopped being smaller than raw. +/// True when the user asked `git show` for a commit format of their own, which RTK cannot +/// compact around -- its own `--pretty=format:` would be overridden by theirs. +/// +/// `--oneline` is deliberately absent, unlike `run_log`'s equivalent check: it does not replace +/// the summary with something unparseable, so routing the whole command raw over it cost 3.7x +/// the output on a real commit (116 KB against 31 KB), on the one metric this tool exists for. +/// It does outrank RTK's own `--pretty=format:`, which is passed before the user's args, so the +/// stat step drops it from what it forwards (see `args_without_oneline`); otherwise the commit +/// header that step suppresses comes back and the summary prints twice. +fn show_wants_format(tokens: &[Token<'_>]) -> bool { + tokens + .iter() + .any(|t| t.kind == TokenKind::Long && matches!(t.text, "format" | "pretty")) +} + +fn show_cmd( + global_args: &[String], + args: &[String], + tokens: &[Token<'_>], + rtk_flags: &[&str], + drop_patch_shape: bool, +) -> Command { + let mut cmd = git_cmd(global_args); + cmd.arg("show"); + cmd.args(rtk_flags); + let forwarded = if drop_patch_shape { + args_without_patch_shape(args, tokens) + } else { + args.to_vec() + }; + for arg in forwarded { + cmd.arg(arg); + } + cmd +} + +fn run_show( + args: &[String], + max_lines: Option, + verbose: u8, + global_args: &[String], +) -> Result { + let timer = tracking::TimedExecution::start(); + + // Re-insert `--` when clap's trailing_var_arg consumed it (issue #1215), same as + // run_diff/run_checkout. Without this the pathspec separator never reaches + // `show_positionals`, so `git show -- ` would misread the + // colon'd pathspec as a `:` blob and dump it instead of a commit-diff. + let args = &args_utils::restore_double_dash(args); + + // Pick one of three handlers for `git show`. `show_route` decides the blob case + // FIRST (see its docs): the two branches below are mutually exclusive on `route`, + // so their source order does not affect which one runs. + let positionals = show_positionals(args); + // Authoritative blob decision (belt-and-suspenders). `show_route` is the cheap pure + // pre-filter: does ANY positional look like `rev:path`? Only then do we probe with + // `git cat-file -t` — and we probe EVERY colon positional, not just the first. The + // cluster-aware `show_positionals` already drops option-value operands (`-S 'url:1'`, + // the `-G` value in `-wG a:b HEAD:blob`, …), but probing all candidates means that + // even if the walker ever missed some exotic short-flag cluster, a non-object like + // `a:b` is still rejected by `cat-file` and the REAL blob elsewhere on the line is + // rescued instead of being silently dropped or misrouted through lossy decoding. + // * exactly one blob → window it (the byte-safe path below), + // * more than one → git concatenates the objects with no separator, so the + // hint can't reconstruct them → raw passthrough, + // * zero → ordinary commit-diff / --stat classification. + // Any presence of a blob object takes the byte-safe path: its output carries raw + // blob bytes that the commit-diff path's lossy UTF-8 decode would corrupt (a Latin-1 + // `0xF1` becomes the `U+FFFD` replacement char — verified). + let (route, blob_objects) = match show_route(args) { + ShowRoute::Blob => { + let blobs: Vec<&String> = blob_candidates(args) + .into_iter() + .filter(|c| probe_is_blob(global_args, c)) + .collect(); + if blobs.is_empty() { + (commit_or_stat_route(args), blobs) + } else { + (ShowRoute::Blob, blobs) + } + } + other => (other, Vec::new()), + }; + + if route == ShowRoute::StatOrFormat { + let mut cmd = git_cmd(global_args); + cmd.arg("show"); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git show")?; + // Verbatim, and before the exit check: `git show --check` reports on stdout and then + // exits 2, so returning early on failure threw the whole report away. + print!("{}", result.stdout); + if !result.success() { + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr.trim()); + } + return Ok(result.exit_code); + } + + timer.track( + &format!("git show {}", args.join(" ")), + &format!("rtk git show {} (passthrough)", args.join(" ")), + &result.stdout, + &result.stdout, + ); + + return Ok(0); + } + + if route == ShowRoute::Blob { + let mut cmd = git_cmd(global_args); + cmd.arg("show"); + for arg in args { + cmd.arg(arg); + } + // Capture raw bytes: `git show` of an ISO-8859/Latin-1 file (e.g. Oracle + // PL/SQL `.pck`) is not UTF-8, and we must preserve git's exact bytes for the + // passthrough path below — decoding into a `String` first would lose them. + let result = + crate::core::stream::exec_capture_bytes(&mut cmd).context("Failed to run git show")?; + let label = format!("git show {}", args.join(" ")); + let rtk_label = format!("rtk git show {}", args.join(" ")); + if !result.success() { + eprint!( + "{}", + crate::core::utils::decode_process_output(&result.stderr) + ); + return Ok(result.exit_code); + } + // git can warn on stderr (e.g. CRLF / autocrlf notices) while still exiting 0; + // surface it instead of swallowing it just because the command succeeded. + if !result.stderr.is_empty() { + eprint!( + "{}", + crate::core::utils::decode_process_output(&result.stderr) + ); + } + // Fidelity invariant: "byte-identical unless we successfully windowed." + // + // Windowing shows a head and points at the rest with `git show 'rev:path' | + // tail -n +N`. That recovery reconstructs the file EXACTLY only if the head we + // printed is a BYTE-EXACT prefix of what git wrote — which holds only when the + // content is valid UTF-8 as-is. Any transcode (Latin-1 → UTF-8), BOM rewrite, + // UTF-16 (whose `tail`-sliced bytes are UTF-16 that won't concatenate with a + // UTF-8 head), U+FFFD from lossy decoding, or true binary would make the head + // diverge from git's bytes and silently break recovery — and can even inflate + // the output past what git emitted. + // + // So we window ONLY content that is valid UTF-8 recoverable byte-for-byte AND + // whose flags/pathspec don't perturb the dump (see the `can_window` gate below: + // no `--textconv`/`--filters`/`--ext-diff`, no trailing `-- `, a single + // sole blob object); everything else goes through `emit_raw_bytes_passthrough`, + // byte-identical to a plain `git show`. This makes the recovery hint reconstruct + // exactly whenever we DO window, and scopes windowing to the common text-lockfile + // case the filter exists for. + // + // Larger blobs window regardless of whether stdout is a pipe or a TTY: the whole + // point of the filter is to shrink what the agent reads, and the agent reads + // through a pipe. That mirrors the diff/log filters (which also compact in a + // pipe); a consumer that needs the full content follows the `| tail -n +N` + // recovery hint — the same tradeoff `git log | grep` already makes. + // (`is_terminal()` is not a reliable "human is watching" signal anyway: the hook + // always pipes rtk's stdout — verified `isatty: False` — and CI agents hand rtk + // a pseudo-TTY. Had this branch gated on it, it would have been the first gate to + // drop CONTENT; every other `stdout().is_terminal()` gate in the tree governs + // only PRESENTATION — color, line-buffering, curl formatting.) + let text = match std::str::from_utf8(&result.stdout) { + // Valid UTF-8: the string's bytes ARE git's bytes, so a head prefix + tail + // recovery is byte-exact. Below the budget it passes through unchanged; over + // it, `compact_blob_show` windows it (or declines and returns it whole). + Ok(s) => s, + // Latin-1, UTF-16/BOM, or binary: recovery would not be byte-exact, so pass + // the raw bytes through verbatim, tracked as a passthrough. + Err(_) => { + return emit_raw_bytes_passthrough( + &result.stdout, + &label, + &rtk_label, + &timer, + result.exit_code, + ); + } + }; + // A blob dump is unfiltered file content: cap large text blobs to a byte + // budget with a tail-recovery pointer, mirroring how the commit-diff path + // below caps at `max_lines`. `--max-lines` does not apply here — blob output + // is bounded by bytes, not lines. + // + // Window ONLY when git emits exactly this one blob and nothing that makes the + // `git show : | tail` recovery hint diverge from git's bytes: + // * a single positional that is the sole blob object — git concatenates + // multiple objects with no separator, so cutting the first would silently + // drop the rest, and that concatenation is not the single `rev:path` the + // hint reconstructs; + // * no content-transforming flag (`--textconv`/`--filters`/`--ext-diff` + // rewrite the dump, and the hint omits them — see `has_content_transform_flag`); + // * no trailing `-- ` (also omitted from the hint — see + // `has_trailing_pathspec`). + // Everything else passes the raw bytes through, byte-identical to plain git show. + // Below the budget, `compact_blob_show` returns the text unchanged, so a windowed + // single-object print stays byte-identical to git there too. + let can_window = positionals.len() == 1 + && blob_objects.len() == 1 + && !has_content_transform_flag(args) + && !has_trailing_pathspec(args); + if can_window { + let shown = compact_blob_show(text, blob_objects[0], global_args); + print!("{}", shown); + // Track savings against the bytes git actually wrote (`result.stdout`). + timer.track_bytes(&label, &rtk_label, result.stdout.len(), &shown); + return Ok(0); + } + // Not windowable (multiple concatenated objects, a content-transforming flag, or + // a trailing pathspec): pass through byte-identical. + return emit_raw_bytes_passthrough( + &result.stdout, + &label, + &rtk_label, + &timer, + result.exit_code, + ); + } + + // The compacted commit-diff path from here down. `show_route` above already decided the + // blob and summary cases, so what is left is the one shape RTK renders itself. + let tokens = tokenize_git_diff_args(args); + + // Get raw output for tracking + let mut raw_cmd = git_cmd(global_args); + raw_cmd.arg("show"); + for arg in args { + raw_cmd.arg(arg); + } + let raw_result = exec_capture(&mut raw_cmd).context("Failed to run git show")?; + // git's verdict on the command as the user typed it, before the steps below run it again + // with RTK's own flags -- the stat step strips the patch-shape flags, so it succeeds where + // git refused (`git show -pq HEAD` exits 128 raw) and the compaction looked like success. + if !raw_result.success() { + if !raw_result.stderr.trim().is_empty() { + eprint!("{}", raw_result.stderr); + } + timer.track( + &format!("git show {}", args.join(" ")), + &format!("rtk git show {}", args.join(" ")), + &raw_result.stdout, + &raw_result.stdout, + ); + return Ok(raw_result.exit_code); + } + let raw_output = raw_result.stdout; + + // Step 1: one-line commit summary + let mut summary_cmd = show_cmd( + global_args, + args, + &tokens, + &["--no-patch", "--pretty=format:%h %s (%ar) <%an>"], + true, + ); + let summary_result = exec_capture(&mut summary_cmd).context("Failed to run git show")?; + if !summary_result.success() { + eprintln!("{}", summary_result.stderr); + return Ok(summary_result.exit_code); + } + let mut printed = summary_result.stdout.trim().to_string(); + + if body_is_suppressed(&tokens) { + // `git show -s` is the commit summary and nothing else, so the stat and diff steps + // below would print what the user explicitly suppressed. + let shown = never_worse(&raw_output, &printed); + println!("{}", shown); + timer.track( + &format!("git show {}", args.join(" ")), + &format!("rtk git show {}", args.join(" ")), + &raw_output, + shown, + ); + return Ok(0); + } + + // Step 2: --stat summary + // `--oneline` is dropped, not outranked: RTK's `--pretty=format:` suppresses the commit + // header this step must not repeat, and git resolves the two by last flag wins, so a user + // `--oneline` re-enabled it and the summary printed twice. Appending RTK's flag after the + // user's instead would put it past their `--`, where git reads it as a pathspec. + // Re-tokenized, because dropping `--oneline` shifts every later argument down one and + // `tokens` still addresses the unshifted `args`. Stale indices make the patch-shape strip + // below drop the wrong argument: `--oneline -p ` loses the `` and keeps a bare + // `-p`, which outranks RTK's `--no-patch` -- HEAD's stat and patch under another + // commit's header. + let stat_args = args_without_oneline(args, &tokens); + let stat_tokens = tokenize_git_diff_args(&stat_args); + let mut stat_cmd = show_cmd( + global_args, + &stat_args, + &stat_tokens, + &["--no-patch", "--stat", "--pretty=format:"], + true, + ); + let stat_result = exec_capture(&mut stat_cmd).context("Failed to run git show --stat")?; + let stat_text = stat_result.stdout.trim(); + if !stat_text.is_empty() { + printed.push('\n'); + printed.push_str(stat_text); + } + + // Step 3: compacted diff + // No `--patch` here: a patch is `show`'s default, and forcing it would override a user + // `-s`/`--no-patch`, whose whole point is that there is no body to print. + let mut diff_cmd = show_cmd(global_args, args, &tokens, &["--pretty=format:"], false); + let diff_result = exec_capture(&mut diff_cmd).context("Failed to run git show (diff)")?; + let diff_text = diff_result.stdout.trim(); + + if !diff_text.is_empty() { + if verbose > 0 { + printed.push_str("\n\nChanges:"); + } + let compacted = compact_diff(&strip_ansi(diff_text), max_lines.unwrap_or(500)); + printed.push('\n'); + printed.push_str(&compacted); + } + + let shown = never_worse(&raw_output, &printed); + println!("{}", shown); + + timer.track( + &format!("git show {}", args.join(" ")), + &format!("rtk git show {}", args.join(" ")), + &raw_output, + shown, + ); + + Ok(0) +} + +/// Whether these args make git emit a word diff rather than a line diff. +/// +/// `compact_diff` reads a unified or combined diff: a body line's first column +/// (or columns) is a marker and the rest is content. A word diff drops the +/// marker entirely and puts `[-removed-]` / `{+added+}` inline, so its body +/// lines are arbitrary content in the marker position. A line starting with `+` +/// then counts as an addition, one starting with `\` is dropped as a +/// no-newline annotation, and one whose content happens to start `diff --` +/// opens a new file section. There is nothing to compact faithfully, so these +/// modes pass through. +/// +/// `--word-diff=none` is the mode that turns a word diff back off, leaving an +/// ordinary unified diff to compact. Modes are last-one-wins, which is what +/// that mode is for: overriding an alias or an earlier flag on the same line. +/// +/// Takes tokens rather than raw args: a `--word-diff` consumed as another option's value +/// (`--author --word-diff`) or sitting past `--` as a pathspec is not a word diff request, and +/// real git agrees on both. +fn emits_word_diff(tokens: &[Token]) -> bool { + let mut word_diff = false; + for token in tokens { + if token.kind != TokenKind::Long { + continue; + } + match token.text { + "word-diff" => word_diff = token.attached != Some("none"), + // `--color-words[=]` takes a regex rather than a mode, so there is no `none` + // to honour on that spelling. + "color-words" | "word-diff-regex" => word_diff = true, + _ => {} + } + } + word_diff +} + +/// `rev:path` names a blob. The caller filters to free positionals first, so a flag's own +/// value (`--pretty=format:...`) never reaches here. +fn is_blob_show_arg(arg: &str) -> bool { + // Detect `rev:path` style arguments while ignoring flags like `--pretty=format:...`. + // `:/text` is a commit-message search, not a blob, so it is excluded. `:path` and + // `:N:path` (index / merge-stage blobs) start with `:` but ARE blobs, so only the + // `:/` prefix is filtered out there. + // + // Magic pathspecs (`:(exclude)…`, `:(top)…`, `:!…`, and `:^…` — an exact synonym + // of `:!…` for exclude magic) also start with `:` but are NOT blobs: they only + // ever appear as pathspecs, so exclude them too. An option value with a colon + // (`git show -S 'a:b' HEAD`) is handled by `show_positionals`, which skips a + // flag's operand via git's argument grammar, so it never reaches here as a blob. + !arg.starts_with('-') + && !arg.starts_with(":/") + && !arg.starts_with(":(") + && !arg.starts_with(":!") + && !arg.starts_with(":^") + && arg.contains(':') + // A colons-only token (`:`, `::`) is never a blob object: route it to git + // rather than the blob window, which would only surface git's own error. + && arg.chars().any(|c| c != ':') +} + +/// The positional (non-option) arguments of a `git show` — its objects. Options and +/// the value tokens they consume (`-S 'url:1'`, `-L 1,2:file`) are dropped via git's +/// own flag/value grammar ([`flag_token_consumes_next`]), so an option operand that +/// happens to contain a colon is never mistaken for a blob and truncated. This handles +/// short-flag CLUSTERS too (`-wG a:b`, `-pS a:b`), whose value-flag tail git re-parses. +/// Args after a `--` are pathspecs, never objects, so a colon in a filename there +/// (`-- weird:name`) is excluded by scanning only the args before the first `--`; a +/// trailing `-- ` beside a real object arg is thus ignored (git still dumps the +/// blob) rather than emptying the list. +fn show_positionals(args: &[String]) -> Vec<&String> { + let rev_args = match args.iter().position(|a| a == "--") { + Some(sep) => &args[..sep], + None => args, + }; + let mut positionals = Vec::new(); + let mut iter = rev_args.iter(); + while let Some(arg) = iter.next() { + if arg.starts_with('-') { + if flag_token_consumes_next(arg) { + iter.next(); // skip this flag's value token + } + continue; + } + positionals.push(arg); + } + positionals +} + +/// Whether `arg` is a flag that consumes the following argument as its value. +/// +/// Delegates to [`log_takes_value`] so git's flag/value grammar lives in one table rather than +/// two that can drift: the same predicate the tokenizer folds with. `AttachedOnly` flags are +/// excluded because they never take a separate token (`-M50` attaches, `-M 50` does not), and +/// the cluster-position rule for solo-only flags is applied by the caller below. +fn consumes_next_token_as_value(arg: &str) -> bool { + let (kind, name) = match arg.strip_prefix("--") { + Some(rest) => (TokenKind::Long, rest), + None => match arg.strip_prefix('-') { + Some(rest) => (TokenKind::Short, rest), + None => return false, + }, + }; + log_takes_value(kind, name).is_some_and(|spec| spec.attachment != Attachment::AttachedOnly) +} + +/// Whether a flag token consumes the NEXT arg as its value (so `show_positionals` must +/// skip it). Handles long flags (`--grep foo`) via [`consumes_next_token_as_value`] and +/// short-flag CLUSTERS (`-wG foo`, `-pS bar`), which git re-parses char by char. +/// +/// Inside a cluster, the first value-taking short flag (`-S -G -I -L -O -l -n`) takes +/// the REST of the cluster as an INLINE value when more chars follow it (`-Sfoo` == `-S +/// foo`, so it does NOT consume the next arg), or the NEXT arg when it is the cluster's +/// last char (`-wG` == `-w -G`, consuming the next arg). Any earlier char is a boolean +/// flag we skip over. This reuses the single flag/value table rather than re-tokenizing +/// git's whole grammar, so `git show -wG x:y HEAD:big` correctly treats `x:y` as `-G`'s +/// value and `HEAD:big` as the object. +// +// TODO(after #3681): replace this short-cluster walk with the ValueSpec factorization; +// the per-char logic here is exactly what a ValueSpec table subsumes. +fn flag_token_consumes_next(arg: &str) -> bool { + // A short cluster is a single leading `-` followed by non-empty flag chars (not the + // `--long` form and not the bare `-` stdin sentinel). Everything else (`--foo`, `-`) + // uses the exact-match table directly. + match arg.strip_prefix('-') { + Some(cluster) if !cluster.is_empty() && !cluster.starts_with('-') => { + for (i, c) in cluster.char_indices() { + if is_short_value_flag(c) { + // Consumes the next arg only if no inline value follows in-cluster. + return i + c.len_utf8() == cluster.len(); + } + } + false + } + _ => consumes_next_token_as_value(arg), + } +} + +/// Whether a single-letter short flag takes a value (`-S`, `-G`, `-L`, …). Derived from +/// [`consumes_next_token_as_value`] so the flag/value table stays the single source of +/// truth and no parallel list can drift out of sync. +fn is_short_value_flag(c: char) -> bool { + c.is_ascii() && consumes_next_token_as_value(format!("-{c}").as_str()) +} + +/// The `git show` positionals that look like `:` blob objects — the +/// windowing candidates. ALL of them are returned (not just the first) so `run_show` +/// can `cat-file`-probe every one: a value operand a missed exotic cluster might leave +/// behind is rejected by the probe, while the real blob elsewhere on the line is found. +fn blob_candidates(args: &[String]) -> Vec<&String> { + show_positionals(args) + .into_iter() + .filter(|a| is_blob_show_arg(a)) + .collect() +} + +/// Whether a `git show` invocation carries any content-transforming flag +/// (`--textconv`/`--filters`/`--ext-diff`) that rewrites a blob's bytes. The `git show +/// : | tail` recovery hint omits these flags, so its output would not match +/// what git printed; their presence forces byte-identical raw passthrough instead of +/// windowing. The `--no-*` spellings restore the default (no rewrite) and are safe, so +/// only the enabling spellings count. Flags precede `--`, so scan up to it. +fn has_content_transform_flag(args: &[String]) -> bool { + args.iter() + .take_while(|a| *a != "--") + .any(|a| matches!(a.as_str(), "--textconv" | "--filters" | "--ext-diff")) +} + +/// Whether a `-- ` with at least one path after it is present. Empirically +/// git ignores a trailing pathspec for a `rev:path` blob dump (verified: output is +/// byte-identical with and without it), but the recovery hint omits it, so rather than +/// bank on that holding for every git version and pathspec form we force byte-identical +/// raw passthrough whenever one accompanies a blob. +fn has_trailing_pathspec(args: &[String]) -> bool { + matches!(args.iter().position(|a| a == "--"), Some(sep) if args.len() > sep + 1) +} + +/// Which `git show` handler an invocation routes to. +#[derive(Debug, PartialEq, Eq)] +enum ShowRoute { + /// A `:` blob dump → byte-safe decode/window path. + Blob, + /// `--stat`/`--numstat`/`--shortstat`/`--pretty`/`--format` with NO blob target + /// → summary-only passthrough. + StatOrFormat, + /// An ordinary commit → compacted commit-diff. + CommitDiff, +} + +/// Classify a `git show` invocation. Blob detection wins over --stat/--format +/// because git accepts and silently ignores those flags when the argument resolves +/// to a blob (verified against git 2.39 — output byte-identical to a plain blob +/// dump), still emitting the full file. Routing such a call to the --stat passthrough +/// would send it through lossy UTF-8 decoding and reintroduce the blob corruption the +/// byte-safe path fixes, so the blob case is checked first. +/// +/// This is a cheap, pure PRE-FILTER: a `Blob` result here only means SOME positional +/// *looks like* `rev:path`. The authoritative blob decision is a `git cat-file -t` +/// probe run by `run_show` (see `probe_is_blob`) on those candidates — the pre-filter +/// keeps the probe off every other invocation. +fn show_route(args: &[String]) -> ShowRoute { + if !blob_candidates(args).is_empty() { + return ShowRoute::Blob; + } + commit_or_stat_route(args) +} + +/// Classify a `git show` that is NOT a blob dump: `--stat`/`--format`/word-diff +/// summary passthrough vs. a compacted commit-diff. Split out from `show_route` so +/// `run_show` can fall back to it when the `cat-file` probe rejects a `rev:path` +/// candidate (a flag value like `-S 'url:1'`, or a tree/commit/bogus object). +fn commit_or_stat_route(args: &[String]) -> ShowRoute { + // Tokenized, not string-matched: a pathspec named `--stat` past the boundary is not the + // flag, `--prettyish` is not `--pretty`, and the shapes RTK cannot compact are more than + // the three stat spellings -- `--raw`, `--name-only`, `--check` and the rest route here + // too. A word/color-word diff has no unified-diff markers for `compact_diff` to read, so + // it passes through like a summary; it only ever applies to a commit-diff, so it is + // checked after the blob case above. + let tokens = tokenize_git_diff_args(args); + if tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)) + || show_wants_format(&tokens) + || emits_word_diff(&tokens) + { + return ShowRoute::StatOrFormat; + } + ShowRoute::CommitDiff +} + +/// Authoritatively decide whether a `git show` argument is a blob, by asking git +/// instead of mirroring its flag grammar. `git cat-file -t ` prints the object +/// type; only an exact `blob` is a windowing target. A tree/commit/tag, or any +/// non-zero exit (a flag value that isn't an object, e.g. `-S 'url:1'`, or a bogus +/// `rev:path`), is not a blob and routes to the normal commit-diff / passthrough path. +/// +/// Run with the SAME global args as the command (`git_cmd(global_args)`) so `-C`, +/// `-c`, `--git-dir`, `--work-tree` resolve the object in the right repo — the same +/// requirement the recovery hint has. Only called on the path where the arg already +/// looks like `rev:path` (the `show_route` pre-filter), so the ~1 ms subprocess never +/// runs on an ordinary commit show. +/// +// TODO(after #3681): once ValueSpec factorization lands, a flag pre-filter can avoid +// the cat-file probe on the common path. +fn probe_is_blob(global_args: &[String], arg: &str) -> bool { + let mut cmd = git_cmd(global_args); + cmd.args(["cat-file", "-t", arg]); + match exec_capture(&mut cmd) { + Ok(result) if result.success() => result.stdout.trim() == "blob", + _ => false, + } +} + +/// Byte budget for a blob preview before truncation kicks in (~2k tokens). +const MAX_BLOB_BYTES: Budget = Budget(8192); + +/// Byte budget after which a blob preview is truncated. A newtype rather than a bare +/// `usize` so a call site can't silently pass some other length in its place. +#[derive(Clone, Copy)] +struct Budget(usize); + +/// Decide how to window a `git show :` blob. Returns +/// `Some((head, remaining_lines, tail_offset))` when the blob should be truncated, +/// or `None` to pass it through unchanged: a small blob, a binary blob, a tree +/// listing, or a single line too long to cut at a line boundary. +/// +/// Pure and side-effect free so it can be unit-tested against real blob fixtures. +fn blob_truncation(raw: &str, budget: Budget) -> Option<(&str, usize, usize)> { + let Budget(budget) = budget; + // No content-sniffing for object type here: the caller only reaches this function + // for an arg the `git cat-file -t` probe has already confirmed is a blob, so a tree + // listing never arrives (and the old `raw.starts_with("tree ")` heuristic — a false + // positive for Newick/NEXUS blobs that genuinely begin "tree " — is gone with it). + // A defensive binary guard is likewise unnecessary now: `run_show` only calls this + // on valid UTF-8 (`\0`/`U+FFFD` content routes to byte-exact passthrough upstream), + // but keep the cheap check so the pure function stays safe when unit-tested directly. + if raw.contains('\0') || raw.contains('\u{FFFD}') { + return None; + } + if raw.len() <= budget { + return None; + } + // Slicing a `&str` at a byte index that is not a UTF-8 char boundary panics, so + // walk the budget back to the nearest boundary at or below it before cutting. + let mut end = budget; + while end > 0 && !raw.is_char_boundary(end) { + end -= 1; + } + // Cut at the last line boundary inside the window. No newline means a single line + // longer than the budget, which tail-based recovery cannot re-window: pass through. + let cut = raw[..end].rfind('\n')? + 1; + let head = &raw[..cut]; + let head_lines = head.lines().count(); + let total_lines = raw.lines().count(); + let remaining = total_lines.checked_sub(head_lines).filter(|&r| r > 0)?; + // `tail -n +offset` recovers everything from the first un-shown line onward. + Some((head, remaining, head_lines + 1)) +} + +/// POSIX single-quote a blob arg so a hint with a space or shell metachar in the path +/// stays copy-paste safe. +fn shell_single_quote(s: &str) -> String { + format!("'{}'", s.replace('\'', r"'\''")) +} + +/// Window a single blob dump: cap a large text blob and point at the rest with a hint +/// re-derived from the blob arg itself — `git show : | tail -n +N`. +/// +/// The hint deliberately does not lean on a tee file. The tee is capped at +/// `max_file_size` (1 MB by default), so it would refuse to store exactly the giant +/// lockfiles this filter most wants to trim, leaving the biggest blobs un-windowed; +/// re-running `git show` reproduces the tail at any size, with no cap and no I/O. +fn compact_blob_show(raw: &str, blob_arg: &str, global_args: &[String]) -> String { + let Some((head, remaining, offset)) = blob_truncation(raw, MAX_BLOB_BYTES) else { + return raw.to_string(); + }; + // Carry the command's global args (`-C `, `-c k=v`, `--git-dir`, `--work-tree`) + // into the hint so it is runnable from anywhere, not just the repo root: without + // them `rtk git -C /repo show HEAD:big` would emit `git show 'HEAD:big' | tail …`, + // which fails outside /repo. Each is shell-quoted for copy-paste safety. + let mut prefix = String::new(); + for arg in global_args { + prefix.push_str(&shell_single_quote(arg)); + prefix.push(' '); + } + // Through `rtk proxy`, like the `diff` hints: a bare `git show` is what RTK's own hook + // rewrites back into `rtk git show`, which windows the dump a second time and leaves + // `tail` nothing but this hint to print -- one line where the reader asked for the rest + // of the file. `proxy` is the documented way to reach the unfiltered command. + let hint = format!( + "[see remaining: rtk proxy git {}show {} | tail -n +{}]", + prefix, + shell_single_quote(blob_arg), + offset + ); + let out = format!("{}... (+{} lines) {}\n", head, remaining, hint); + never_worse(raw, &out).to_string() +} + +/// Write raw bytes straight to stdout, tracked as a passthrough. Used when the blob +/// must reach the caller byte-for-byte — binary or ambiguously-encoded content that +/// decoding would corrupt, or a passthrough where any rewrite (BOM strip, transcode) +/// would diverge from what plain `git show` emits — so it goes through the locked +/// stdout handle as bytes rather than a `String`. +fn emit_raw_bytes_passthrough( + bytes: &[u8], + label: &str, + rtk_label: &str, + timer: &tracking::TimedExecution, + exit_code: i32, +) -> Result { + use std::io::Write; + let stdout = std::io::stdout(); + let mut handle = stdout.lock(); + handle + .write_all(bytes) + .context("Failed to write blob to stdout")?; + timer.track_passthrough(label, rtk_label); + Ok(exit_code) +} + +/// Path named by a diff section header. +/// +/// `diff --git a/p b/p` carries the path twice; `diff --cc p` and +/// `diff --combined p` carry it once, as the whole remainder of the line. Only +/// the two-path form can be split at its midpoint, so the header kind decides +/// which shape to read: `diff --cc dup dup` names one file called `dup dup`, +/// not the file `dup` twice. +/// +/// Under the default `core.quotepath`, git wraps a path in `"` and escapes any +/// non-ASCII byte, control character, quote or backslash inside it — but not a +/// space. The quoting is undone here, so the header carries the path as it is +/// on disk and a `grep` over the output finds it by name. +fn diff_header_path(line: &str) -> String { + let Some(rest) = line.splitn(3, ' ').nth(2) else { + return "unknown".to_string(); + }; + if !line.starts_with("diff --git ") { + return unquote_path(rest); + } + if let Some(path) = same_path_twice(rest) { + return path; + } + // A rename names two different paths, and the destination is the second. + if let Some(quoted) = rest + .split(" \"b/") + .nth(1) + .and_then(|dst| dst.strip_suffix('"')) + { + return unescape_path(quoted); + } + match rest.split(" b/").nth(1) { + Some(path) => path.to_string(), + None => unquote_path(rest), + } +} + +/// The path a `diff --git` header names twice, split at the midpoint. +/// +/// Anything but a rename names the same path on both sides, so the two halves +/// are the same length and the separating space sits dead centre. Splitting +/// there instead of on the first ` b/` keeps a path that contains that +/// substring — a file under a directory named `x b`. Prefixes are then dropped +/// by matching the halves against each other rather than by name, so +/// `--no-prefix` and any custom `--src-prefix` / `--dst-prefix` read alike. +/// +/// `None` for a rename, whose halves differ past their first component, and for +/// anything else the two halves disagree on; both fall through to the ` b/` +/// split. A `--no-prefix` rename between two directories is the one shape this +/// cannot tell from a prefix pair — space-separated paths with no prefix are +/// ambiguous by construction — and it reads as the shared trailing path. +fn same_path_twice(rest: &str) -> Option { + if rest.len().is_multiple_of(2) { + return None; + } + let mid = rest.len() / 2; + // A space at the midpoint is a char boundary, so both halves are valid. + if rest.as_bytes().get(mid) != Some(&b' ') { + return None; + } + let (left, right) = (unquote_path(&rest[..mid]), unquote_path(&rest[mid + 1..])); + if left == right { + return Some(left); + } + let (_, left_path) = left.split_once('/')?; + let (_, right_path) = right.split_once('/')?; + (left_path == right_path).then(|| right_path.to_string()) +} + +/// Undo git's `core.quotepath` quoting: `"a/\303\251.txt"` becomes `a/é.txt`. +/// +/// A path git did not quote is returned as-is, so either form can be passed. +fn unquote_path(raw: &str) -> String { + match raw.strip_prefix('"').and_then(|r| r.strip_suffix('"')) { + Some(quoted) => unescape_path(quoted), + None => raw.to_string(), + } +} + +/// Decode the C escapes inside a quoted path. +/// +/// The octal escapes spell out the path's bytes one at a time, so a multi-byte +/// character arrives as several of them; they are collected as bytes and +/// decoded once at the end rather than per escape. A path whose bytes are not +/// UTF-8 keeps replacement characters, which is as close as a `String` gets. +fn unescape_path(quoted: &str) -> String { + let bytes = quoted.as_bytes(); + let mut out: Vec = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] != b'\\' || i + 1 == bytes.len() { + out.push(bytes[i]); + i += 1; + continue; + } + let escape = bytes[i + 1]; + if escape.is_ascii_digit() { + let end = (i + 4).min(bytes.len()); + let octal = std::str::from_utf8(&bytes[i + 1..end]) + .ok() + .and_then(|digits| u8::from_str_radix(digits, 8).ok()); + match octal { + Some(byte) => { + out.push(byte); + i = end; + } + // Not an octal escape after all: keep the backslash verbatim. + None => { + out.push(bytes[i]); + i += 1; + } + } + continue; + } + out.push(match escape { + b'a' => 0x07, + b'b' => 0x08, + b't' => b'\t', + b'n' => b'\n', + b'v' => 0x0b, + b'f' => 0x0c, + b'r' => b'\r', + // `\"` and `\\` stand for themselves. + other => other, + }); + i += 2; + } + String::from_utf8_lossy(&out).into_owned() +} + +/// Line budget a hunk header declares, and how wide its body prefix is. +struct HunkHeader { + /// Lines the hunk spans in each parent, in marker-column order. One entry + /// for a unified `@@`, one per parent for a combined `@@@`. + parents: Vec, + /// Lines the hunk spans in the result file. + new: usize, + /// Marker columns: 1 for `@@`, and one per parent for a combined `@@@`. + prefix_width: usize, +} + +impl HunkHeader { + /// Whether every declared line has been accounted for, which is where the + /// hunk body ends. A combined hunk is not done until *every* parent's + /// budget is spent: a line removed from only the second parent spends that + /// parent's budget and neither the first's nor the result's. + fn exhausted(&self) -> bool { + self.new == 0 && self.parents.iter().all(|&remaining| remaining == 0) + } + + /// Charge a body line against the budgets it occupies. + /// + /// Column `i` is the line's marker against parent `i + 1`. `-` there means + /// the line is in that parent and is being removed; a space on a line that + /// is not a removal means the line is in that parent unchanged. Both spend + /// one of that parent's lines. `+` there, or a space on a removal line, + /// means the line is not in that parent at all. + /// + /// Collapsing the columns into one add/delete pair, as an aggregate over + /// the whole prefix does, loses that distinction and leaves a combined + /// hunk's budget unable to converge. + fn consume(&mut self, markers: &[u8]) { + let is_add = markers.contains(&b'+'); + let is_del = markers.contains(&b'-'); + for (i, remaining) in self.parents.iter_mut().enumerate() { + let column = markers.get(i).copied(); + let present = if is_del { + column == Some(b'-') + } else { + // A line shorter than the prefix reads as context, which is + // what a bare blank line in a unified diff body is. + column != Some(b'+') + }; + if present { + *remaining = remaining.saturating_sub(1); + } + } + // In the result file unless the line is a pure deletion. + if is_add || !is_del { + self.new = self.new.saturating_sub(1); + } + } +} + +/// Parse `@@ -a,b +c,d @@` and the combined `@@@ -a,b -c,d +e,f @@@`. +/// +/// The counts bound the hunk body, which is what lets the body end where the +/// hunk ends rather than running on until the next header. Anything after it — +/// an mbox envelope, a `--` signature, trailing prose — is then outside every +/// hunk and cannot be read as diff content. A count is 1 when the header omits +/// it (`@@ -1 +1 @@`). +fn parse_hunk_header(line: &str) -> Option { + let at_run = line.len() - line.trim_start_matches('@').len(); + if at_run < 2 { + return None; + } + let body = line[at_run..].split('@').next()?; + + let mut parents: Vec = Vec::new(); + let mut new = None; + for group in body.split_whitespace() { + let Some(rest) = group.strip_prefix(['-', '+']) else { + continue; + }; + let count = match rest.split_once(',') { + Some((_, c)) => c.parse::().ok()?, + None => 1, + }; + if group.starts_with('-') { + // A combined header lists one range per parent, in the same order + // as the marker columns. + parents.push(count); + } else { + new = Some(count); + } + } + + // `@@` has one marker column, `@@@` two, and so on for more parents. + let prefix_width = at_run - 1; + // A well-formed header lists exactly one range per marker column. When it + // does not, only the columns can be charged, so trust them: an untracked + // parent would otherwise sit at its declared count forever and the hunk + // would never close, while a parent with no column of its own would be + // charged against nothing. A missing range gets `usize::MAX`, which keeps + // the hunk open to the next header rather than dropping its body. + if parents.len() != prefix_width { + parents.resize(prefix_width, usize::MAX); + } + + Some(HunkHeader { + parents, + new: new.unwrap_or(0), + prefix_width, + }) +} + +/// Render the note for change lines dropped past `max_hunk_lines`, split by +/// sign so an anchored `^-` / `^+` audit can tell what it did not see. +fn hunk_truncation_note(deletions: usize, additions: usize) -> Option { + fn count(n: usize, noun: &str) -> String { + if n == 1 { + format!("{} {}", n, noun) + } else { + format!("{} {}s", n, noun) + } + } + match (deletions, additions) { + (0, 0) => None, + (0, a) => Some(format!(" ... ({} truncated)", count(a, "addition"))), + (d, 0) => Some(format!(" ... ({} truncated)", count(d, "deletion"))), + (d, a) => Some(format!( + " ... ({}, {} truncated)", + count(d, "deletion"), + count(a, "addition") + )), + } +} + +/// Emit the buffered leading context, charged against the diff-wide budget. +/// +/// Keeps the lines closest to the change when the budget cannot take all of +/// them. Called wherever a hunk closes as well as at its first change line: +/// context buffered by a hunk that ends without one would otherwise be dropped, +/// leaving a bare hunk header with nothing under it. +fn flush_leading_context( + buffer: &mut Vec, + result: &mut Vec, + total: &mut usize, + cap: usize, +) { + let room = cap.saturating_sub(*total); + let keep = buffer.len().min(room); + let skip = buffer.len() - keep; + for ctx in buffer.drain(..).skip(skip) { + result.push(ctx); + } + *total += keep; +} + +pub(crate) fn compact_diff(diff: &str, max_lines: usize) -> String { + let mut result = Vec::new(); + let mut current_file = String::new(); + let mut added = 0; + let mut removed = 0; + let mut hunk: Option = None; + let mut hunk_shown = 0; + let mut skipped_add = 0usize; + let mut skipped_del = 0usize; + let mut leading_context: Vec = Vec::new(); + let mut leading_context_total = 0usize; + let max_hunk_lines = 100; + // Context before a hunk's first change, up to three lines per hunk and + // `max_lines / 10` across the diff. It does not count against `max_lines`, + // so it cannot displace change lines, and the diff-wide cap is what bounds + // the overrun that exemption would otherwise allow: a diff of many small + // hunks would otherwise spend three exempt lines on every one of them. + let max_leading_context = 3; + let leading_context_cap = max_lines / 10; + let mut was_truncated = false; + + for line in diff.lines() { + // Every diff section header (`--git`, `--cc`, `--combined`) opens a new + // file and closes any open hunk, so the `---` / `+++` headers that + // follow it are never read as hunk content. + if line.starts_with("diff --") { + flush_leading_context( + &mut leading_context, + &mut result, + &mut leading_context_total, + leading_context_cap, + ); + if let Some(note) = hunk_truncation_note(skipped_del, skipped_add) { + result.push(note); + was_truncated = true; + skipped_del = 0; + skipped_add = 0; + } + if !current_file.is_empty() && (added > 0 || removed > 0) { + result.push(format!(" +{} -{}", added, removed)); + } + current_file = diff_header_path(line); + result.push(format!("\n{}", current_file)); + added = 0; + removed = 0; + hunk = None; + hunk_shown = 0; + } else if let Some(header) = parse_hunk_header(line) { + flush_leading_context( + &mut leading_context, + &mut result, + &mut leading_context_total, + leading_context_cap, + ); + if let Some(note) = hunk_truncation_note(skipped_del, skipped_add) { + result.push(note); + was_truncated = true; + skipped_del = 0; + skipped_add = 0; + } + hunk = Some(header); + hunk_shown = 0; + // Preserve the full unified diff hunk header, including trailing + // function / symbol context after the second @@ marker. + result.push(line.to_string()); + } else if let Some(header) = hunk.as_mut() { + if header.exhausted() { + hunk = None; + continue; + } + if line.starts_with('\\') { + // "\ No newline at end of file" annotates the line above and + // occupies no line in either file. + continue; + } + + // Slice the marker columns as bytes. `prefix_width` counts columns, + // and the markers are ASCII by construction, but the body content + // right after them is not: `--word-diff` emits body lines with no + // marker column at all, so a `char`-unaware `&line[..width]` splits + // a leading multi-byte character and panics. + let width = header.prefix_width.min(line.len()); + let markers = &line.as_bytes()[..width]; + let is_add = markers.contains(&b'+'); + let is_del = markers.contains(&b'-'); + header.consume(markers); + + // Hunk bodies emit at column 0 in git's own unified shape, so + // `^+` / `^-` anchor. rtk's own annotations stay indented so those + // same anchors never match them. Inside a hunk every `+`/`-` line + // is content: the `---` / `+++` file headers only ever appear + // before the first hunk header. + if is_add || is_del { + if is_add { + added += 1; + } + if is_del { + removed += 1; + } + if hunk_shown < max_hunk_lines { + // The context immediately preceding the change, so the body + // reads as contiguous with it. The diff-wide budget is + // charged on emit rather than on buffering, so a line the + // ring evicted never costs anything. + flush_leading_context( + &mut leading_context, + &mut result, + &mut leading_context_total, + leading_context_cap, + ); + result.push(line.to_string()); + hunk_shown += 1; + } else if is_del { + skipped_del += 1; + } else { + skipped_add += 1; + } + leading_context.clear(); + } else if hunk_shown > 0 { + if hunk_shown < max_hunk_lines { + result.push(line.to_string()); + hunk_shown += 1; + } + } else if leading_context_total < leading_context_cap { + // Keep the last `max_leading_context` lines rather than the + // first: with `-U10` or `--function-context` the first ones sit + // ten lines above the change and would imply an adjacency the + // file does not have. + if leading_context.len() == max_leading_context { + leading_context.remove(0); + } + leading_context.push(line.to_string()); + } + + if header.exhausted() { + hunk = None; + flush_leading_context( + &mut leading_context, + &mut result, + &mut leading_context_total, + leading_context_cap, + ); + } + } + + if result.len().saturating_sub(leading_context_total) >= max_lines { + result.push("\n... (more changes truncated)".to_string()); + was_truncated = true; + break; + } + } + + // Flush last hunk + flush_leading_context( + &mut leading_context, + &mut result, + &mut leading_context_total, + leading_context_cap, + ); + if let Some(note) = hunk_truncation_note(skipped_del, skipped_add) { + result.push(note); + was_truncated = true; + } + + if !current_file.is_empty() && (added > 0 || removed > 0) { + result.push(format!(" +{} -{}", added, removed)); + } + + if was_truncated { + result.push("[full diff: rtk git diff --no-compact]".to_string()); + } + + result.join("\n") +} + +/// RTK's default `git log` limit, applied whenever the user names none. +const DEFAULT_LOG_LIMIT: usize = 10; +const DEFAULT_LOG_LIMIT_ARG: &str = "-10"; + +/// `git log ` for the raw passthrough, carrying RTK's default limit unless the user named +/// one. [`run_passthrough`] streams straight to the terminal, so the limit has to be in the args +/// or it never applies: a patch request became the whole history, 411k lines against 50 for +/// plain `rtk git log`. +/// +/// The limit goes first, ahead of every user argument. `injection_point` is the wrong tool here: +/// it only guarantees "before the user's `--`", while git requires options to precede *all* +/// positionals ("fatal: -10 option must come before non-option arguments") and a pathspec needs +/// no boundary to be one. Only the limit is injected -- `--no-merges` would gut `--cc`/`-c`, +/// whose entire purpose is the merge diff. +fn raw_log_passthrough_args(args: &[String], capped: bool) -> Vec { + let mut out = vec![OsString::from("log")]; + if capped { + out.push(OsString::from(DEFAULT_LOG_LIMIT_ARG)); + } + out.extend(args.iter().map(OsString::from)); + out +} + +/// True when RTK's default limit applies: the user named no limit of their own and did not +/// bound the walk with a revision range. +fn raw_log_is_capped(tokens: &[Token<'_>]) -> bool { + !has_limit_flag(tokens) && !bounds_the_walk(tokens) +} + +/// Whether the walk holds more commits than RTK's default limit shows -- the question the +/// truncation notice answers. +/// +/// Asks git for the one commit just past the limit and reads only whether anything came back. +/// Nothing about the printed shape can disfigure that answer, which is what counting commits +/// out of the emitted output could never manage: `--oneline` leaves no commit header, +/// `log.decorate` and `--graph` disfigure the one there is, `-z` runs the walk onto a single +/// line, `--line-prefix` puts something in front of it, colour puts an escape there, and a +/// SHA-256 object name is not 40 characters. Each of those silenced the notice on a cap that +/// had taken most of the history away. +/// +/// RTK's flags go in front of every user argument: git refuses an option that follows a +/// positional, so a bare pathspec would otherwise make the probe fail outright. That leaves +/// the user's flags last, where git's last-flag-wins hands the format back to them -- harmless +/// except for `--pretty`/`--format`/`--oneline`, since an empty `--pretty=format:` prints a +/// commit as no bytes at all and would read as "nothing left". Those three are dropped from +/// what is forwarded, along with `--skip`, which RTK's own has to absorb, `--exit-code`, which +/// would make the probe report failure on a walk git was perfectly happy with, `--output`, +/// which is a redirect that would truncate the file the command just wrote, and the +/// patch-shape flags, which are work with no answer in them. +/// +/// `None` when git refuses the command: RTK then says nothing rather than guessing. +fn walk_exceeds_limit( + global_args: &[String], + args: &[String], + tokens: &[Token<'_>], + limit: usize, +) -> Option { + let mut cmd = git_cmd(global_args); + cmd.arg("log"); + // No `--no-patch`: git refuses it beside `--name-only`/`--name-status`, and the strip in + // `log_probe_args` has already taken the patch-shape flags out of what is forwarded. + cmd.args(["--pretty=format:%H"]); + // `--skip` is counted where the walk starts; a diff-based filter is applied after it. So + // for those the skip lands on commits the filter would have dropped and the probe reports + // on a commit the user was never going to see -- `git log -p -S needle` matching twice + // claimed a cap of ten. `--max-count` is applied last, after every filter, so there the + // question becomes how many came back rather than whether one did. + let counting = selects_by_diff(tokens); + if counting { + cmd.arg(format!("--max-count={}", limit.saturating_add(1))); + } else { + cmd.args(["--max-count=1".to_string()]); + cmd.arg(format!( + "--skip={}", + limit.saturating_add(user_skip(tokens)) + )); + } + cmd.args(log_probe_args(args, tokens)); + + let result = exec_capture(&mut cmd).ok()?; + if !result.success() { + return None; + } + if !counting { + return Some(!result.stdout.trim().is_empty()); + } + // One line per commit is RTK's own `--pretty` talking, but the user's `--stat`, + // `--name-only` or `--graph` still add lines of their own, so commits are counted by + // shape. ANSI first: `color.ui=always` puts an escape in front of the rail. + let shown = strip_ansi(&result.stdout) + .lines() + .filter(|line| is_commit_name(line)) + .count(); + Some(shown > limit) +} + +/// True for a `--pretty=format:%H` line, ignoring any `--graph` rail in front of it. No upper +/// bound on the length: a SHA-256 object name is 64 characters, and rejecting it counted every +/// commit in such a repository as none. +fn is_commit_name(line: &str) -> bool { + let name = + line.trim_matches(|c: char| c.is_whitespace() || matches!(c, '*' | '|' | '/' | '\\')); + name.len() >= 7 && name.chars().all(|c| c.is_ascii_hexdigit()) +} + +/// True when the walk is narrowed by what the commits changed rather than by where they are. +/// git applies these after `--skip` and before `--max-count`, which is what makes the two +/// probes above answer different questions. +fn selects_by_diff(tokens: &[Token<'_>]) -> bool { + arg_tokenizer::before_dashdash(tokens).iter().any(|t| { + match t.kind { + TokenKind::Long => matches!( + t.text, + "find-object" | "diff-filter" | "pickaxe-regex" | "pickaxe-all" + ), + // `-S` and `-G`. + TokenKind::Short => matches!(t.text, "S" | "G"), + _ => false, + } + }) +} + +/// The user arguments [`walk_exceeds_limit`] forwards: everything that bounds the walk, and +/// nothing that decides what git does with it. +/// +/// Three classes come out. `--pretty`/`--format`/`--oneline`, because RTK's own `--pretty` is +/// written first and would otherwise lose the last-flag-wins arbitration -- an empty +/// `--pretty=format:` then prints a commit as no bytes at all, which reads as "nothing left" +/// (`--summary` and `--dirstat` print nothing of their own for a plain modification, so the +/// probe would have only that format to go on). `--skip`, because RTK's has to absorb it for +/// the same reason. And `--output=`, which is not a format at all but a redirect: left +/// in, the probe reruns it and truncates the file the command it follows has just written. +/// +/// The patch-shape flags go too, through the same strip the `show` header uses. The probe +/// reads only whether git printed anything, so a patch is work with no answer in it -- and +/// with `diff.external` or a `textconv` driver configured, work that runs the user's own +/// program one more time than they asked for. +fn log_probe_args(args: &[String], tokens: &[Token<'_>]) -> Vec { + let mut dropped: Vec = Vec::new(); + for token in tokens { + if token.kind != TokenKind::Long + || !matches!( + token.text, + "pretty" | "format" | "oneline" | "skip" | "output" | "exit-code" + ) + { + continue; + } + dropped.push(token.source_index); + // `--skip 5` spends a second argument on its value; `--skip=5` does not. + if let Some(index) = token.linked + && let Some(value) = tokens.get(index) + { + dropped.push(value.source_index); + } + } + let kept: Vec = args + .iter() + .enumerate() + .filter(|(index, _)| !dropped.contains(index)) + .map(|(_, arg)| arg.clone()) + .collect(); + // Re-tokenized, because the drop above shifted every index `tokens` holds. + let kept_tokens = tokenize_git_log_args(&kept); + args_without_patch_shape(&kept, &kept_tokens) +} + +/// The user's own `--skip`, which RTK's has to absorb: git takes the last one, and RTK's is +/// written first. Last one wins here too, matching git. Unparseable means git will reject the +/// command anyway, so 0 is as good an answer as any. +fn user_skip(tokens: &[Token<'_>]) -> usize { + tokens + .iter() + .rfind(|t| t.kind == TokenKind::Long && t.text == "skip") + .and_then(|t| t.value(tokens)) + .and_then(|value| value.parse().ok()) + .unwrap_or(0) +} + +/// True when the arguments already bound the walk, so RTK's default limit would only take away +/// what the user explicitly asked for -- `git log -p HEAD~15..HEAD` means those 15 commits. +/// +/// A revision range is the one bound that is exact; `--since` and friends narrow the walk but +/// not to any particular size, so those still get the limit (and now the notice with it). A +/// relative path is excluded because it is a pathspec, not a range. +fn bounds_the_walk(tokens: &[Token<'_>]) -> bool { + arg_tokenizer::before_dashdash(tokens).iter().any(|t| { + t.is_free_positional() + && t.text.contains("..") + && !t.text.starts_with("./") + && !t.text.starts_with("../") + }) +} + +fn run_log( + args: &[String], + _max_lines: Option, + verbose: u8, + global_args: &[String], +) -> Result { + let tokens = tokenize_git_log_args(args); + + if tokens.iter().any(|t| log_wants_raw_shape(t, &tokens)) { + let capped = raw_log_is_capped(&tokens); + let passthrough_args = raw_log_passthrough_args(args, capped); + if !capped { + return run_passthrough(&passthrough_args, global_args, verbose); + } + // Capped, so the output is bounded and worth capturing: it is the only way to say + // whether the limit actually took anything away. Announcing it up-front announced a + // truncation that had not happened, on a two-commit repo, and announced it ahead of + // commands git then rejected outright. + let timer = tracking::TimedExecution::start(); + let mut cmd = git_cmd(global_args); + cmd.args(&passthrough_args); + let result = exec_capture(&mut cmd).context("Failed to run git log")?; + print!("{}", result.stdout); + if !result.stderr.trim().is_empty() { + eprint!("{}", result.stderr); + } + timer.track( + &format!("git log {}", args.join(" ")), + &format!("rtk git log {} (passthrough)", args.join(" ")), + &result.stdout, + &result.stdout, + ); + // Before the exit check, not after: `--exit-code` makes a perfectly successful + // `git log` exit 1, and returning first left that cap unannounced. A command git + // really did refuse silences the notice anyway -- the probe fails on it too. + // + // Asked unconditionally, never skipped on empty output: `--output=` sends the + // whole run to a file and `--summary --pretty=format:` prints nothing for a plain + // modification, and both of those are capped walks with a notice owed. The probe + // walks the history a second time when a filter like `-S` matches little, which is + // the price of an answer that does not depend on what the run happened to print. + if walk_exceeds_limit(global_args, args, &tokens, DEFAULT_LOG_LIMIT) == Some(true) { + eprintln!( + "[rtk] capped at {} commits; pass -n for more", + DEFAULT_LOG_LIMIT + ); + } + if !result.success() { + return Ok(result.exit_code); + } + return Ok(0); + } + + let timer = tracking::TimedExecution::start(); + + let mut cmd = git_cmd(global_args); + cmd.arg("log"); + + // Check if user provided format flags + let has_format_flag = tokens + .iter() + .any(|t| t.kind == TokenKind::Long && matches!(t.text, "format" | "oneline" | "pretty")); + + // Check if user provided limit flag (-N, -n N, --max-count=N, --max-count N) + let has_limit_flag = has_limit_flag(&tokens); + + // Apply RTK defaults only if user didn't specify them + // Use %b (body) to preserve first line of commit body for agent context + // (BREAKING CHANGE, Closes #xxx, design notes) + if !has_format_flag { + cmd.args(["--pretty=format:%h %s (%ar) <%an>%n%b%n---END---"]); + } + + // Determine limit: respect user's explicit -N flag, use sensible defaults otherwise + let (limit, user_set_limit) = if has_limit_flag { + // User explicitly passed -N / -n N / --max-count=N → respect their choice + let n = parse_limit_from_tokens(&tokens).unwrap_or(10); + (n, true) + } else if has_format_flag { + // --oneline / --pretty without -N: user wants compact output, allow more + cmd.arg("-50"); + (50, false) + } else { + // No flags at all: default to 10 + cmd.arg(DEFAULT_LOG_LIMIT_ARG); + (10, false) + }; + + // Only add --no-merges if user didn't explicitly request merge commits. Any + // `--min-parents=N` with N >= 2 asks for merges; pinning it to 2 let `--min-parents=3` + // collect RTK's `--no-merges` as well, and the two constraints select nothing at all. + let wants_merges = tokens.iter().any(|t| { + t.kind == TokenKind::Long + && (t.text == "merges" + || t.text == "no-merges" + || (t.text == "min-parents" + && t.attached + .is_some_and(|v| v.parse::().is_ok_and(|n| n >= 2)))) + }); + // Don't add --no-merges if user explicitly requested merges or an exact count (-n N / --max-count) + if !wants_merges && !has_limit_flag { + cmd.arg("--no-merges"); + } + + // Pass all user arguments + for arg in args { + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git log")?; + + if !result.success() { + eprintln!("{}", result.stderr); + return Ok(result.exit_code); + } + + if verbose > 0 { + eprintln!("Git log output:"); + } + + // Post-process: truncate long messages, cap lines only if RTK set the default + let filtered = filter_log_output(&result.stdout, limit, user_set_limit, has_format_flag); + let filtered = never_worse(&result.stdout, &filtered).to_string(); + println!("{}", filtered); + + timer.track( + &format!("git log {}", args.join(" ")), + &format!("rtk git log {}", args.join(" ")), + &result.stdout, + &filtered, + ); + + Ok(0) +} + +/// The long flags `git log`, `diff` and `show` all take a value for. Shared deliberately: these +/// are revision-walk options all three parse the same way. The *short* grammar is where they +/// differ, so each subcommand states its own below. +/// +/// E.g. `--grep -p` searches messages for the literal string "-p"; it does not request patch +/// output. +fn shared_long_takes_value(name: &str) -> bool { + matches!( + name, + "after" + | "anchored" + | "author" + | "before" + | "color-moved-ws" + | "committer" + | "date" + | "decorate-refs" + | "decorate-refs-exclude" + | "diff-algorithm" + | "diff-filter" + | "diff-merges" + | "dst-prefix" + | "encoding" + | "exclude" + | "find-object" + | "glob" + | "grep" + | "grep-reflog" + | "ignore-matching-lines" + | "inter-hunk-context" + | "line-prefix" + | "max-age" + | "max-count" + | "max-depth" + | "min-age" + | "output" + | "output-indicator-context" + | "output-indicator-new" + | "output-indicator-old" + | "rotate-to" + | "since" + | "since-as-filter" + | "skip" + | "skip-to" + | "src-prefix" + | "stat-count" + | "stat-graph-width" + | "stat-name-width" + | "stat-width" + | "until" + | "word-diff-regex" + | "ws-error-highlight" + ) +} + +/// `git log`'s grammar. `-M`/`-U`/`-C`/`-B` take an optional attached number and never a +/// separate token; `-n` and `-l` take one only when solo (`git log -pn 2` fails against git +/// 2.53, and `-l` is kept solo-only out of caution -- only run_log uses this, where a stray +/// positional is inert). +fn log_takes_value(kind: TokenKind, name: &str) -> Option { + match kind { + TokenKind::Long => shared_long_takes_value(name).then(ValueSpec::value), + TokenKind::Short => match name { + "B" | "C" | "M" | "U" => Some(ValueSpec::attached_only()), + "G" | "I" | "L" | "O" | "S" => Some(ValueSpec::value()), + "l" | "n" => Some(ValueSpec::solo_only()), + _ => None, + }, + _ => None, + } +} + +/// `diff`/`show`'s grammar. Same long list, but `-l` is the rename limit here and *does* +/// cluster: `git diff -wl 100` works where `git log -cl 2` does not. Sharing log's short +/// grammar made RTK read the 100 as a pathspec and splice its own flags in front of it. +fn diff_takes_value(kind: TokenKind, name: &str) -> Option { + match kind { + TokenKind::Long => shared_long_takes_value(name).then(ValueSpec::value), + TokenKind::Short => match name { + "B" | "C" | "M" | "U" => Some(ValueSpec::attached_only()), + "G" | "I" | "L" | "O" | "S" | "l" => Some(ValueSpec::value()), + "n" => Some(ValueSpec::solo_only()), + _ => None, + }, + _ => None, + } +} + +fn tokenize_git_diff_args(args: &[String]) -> Vec> { + arg_tokenizer::tokenize_grammar(args, &diff_takes_value, Dialect::Posix) +} + +fn tokenize_git_log_args(args: &[String]) -> Vec> { + arg_tokenizer::tokenize_grammar(args, &log_takes_value, Dialect::Posix) +} + +#[cfg(test)] +fn real_flag_args(args: &[String]) -> Vec<&str> { + tokenize_git_log_args(args) + .iter() + .filter(|t| matches!(t.kind, TokenKind::Long | TokenKind::Short)) + .map(|t| t.text) + .collect() +} + +/// True for git log flags that change the *shape* of git's raw output (patch text, diffstat, +/// name lists) in a way incompatible with RTK's injected `--pretty=format` markers, requiring +/// the raw passthrough path instead (see [`requests_raw_log_output`]). `diff`/`show` use the +/// narrower [`diff_wants_raw_shape`]/[`show_wants_raw_shape`] instead. +fn log_wants_raw_shape(token: &Token<'_>, tokens: &[Token<'_>]) -> bool { + // Every `--diff-merges` format but `off`/`none` emits a patch (git 2.53), and log's + // one-line-per-commit compaction cannot represent one -- the same reason `-p` is listed + // below. git takes the value attached or as the next token, so both spellings are read. + if token.kind == TokenKind::Long && token.text == "diff-merges" { + return !matches!(token.value(tokens), None | Some("none" | "off")); + } + match token.kind { + TokenKind::Long => matches!( + token.text, + // A binary patch is meant to be fed back to `git apply`; compacting it destroys + // that, so it takes the raw route rather than merely being kept out of the header. + "binary" + // `--cc`/`--remerge-diff` imply `-p` on merge commits (8 diff lines against + // git 2.53 where plain log has none), and the log compaction drops the patch + // with no tee to recover it from. + | "cc" + | "remerge-diff" + | "compact-summary" + | "dirstat" + // Prefixes every output line, including the `diff --git`/`@@` markers the + // compaction keys on, so nothing parses and the body came back empty. + | "line-prefix" + | "name-only" + | "name-status" + | "numstat" + | "patch" + | "patch-with-raw" + | "patch-with-stat" + | "raw" + | "shortstat" + | "stat" + | "summary" + | "unified" + ), + // `-U` makes `git log` emit a patch the one-line-per-commit compaction cannot + // represent, and `-c` is the combined-diff form of `--cc`. `-W`/`--function-context` + // is *not* here: against git 2.53 it leaves `git log` byte-identical to plain log. + TokenKind::Short => matches!(token.text, "U" | "c" | "p" | "u"), + _ => false, + } +} + +/// `diff`'s raw-output grammar: `show`'s, plus `--quiet`. A strict superset, so it composes +/// rather than repeating the list -- `git diff --quiet` prints nothing and exits 1 on a +/// difference (git 2.53), so there is nothing to compact and the exit code has to survive. +fn diff_wants_raw_shape(token: &Token<'_>, tokens: &[Token<'_>]) -> bool { + matches!((token.kind, token.text), (TokenKind::Long, "quiet")) + || show_wants_raw_shape(token, tokens) +} + +/// `show`'s raw-output grammar. `--check` reports whitespace errors and exits 2; `--exit-code` +/// prints the whole patch and exits 1. `--quiet` is deliberately absent: in `show` it is a +/// synonym of `-s` (exit 0, body suppressed, git 2.53), which [`suppresses_diff_body`] renders +/// as the compact summary -- claiming it here made `git show --quiet` raw-pass the header its +/// own synonyms compact. +/// +/// Excludes `--patch`/`-p`/`-u` and `--unified`/`-U`: unlike `log`, `diff`/`show`'s default +/// output already *is* patch text, so those are redundant with the default rather than a shape +/// RTK cannot produce. Delegates the rest to [`log_wants_raw_shape`] so the two can't drift. +fn show_wants_raw_shape(token: &Token<'_>, tokens: &[Token<'_>]) -> bool { + if matches!( + (token.kind, token.text), + (TokenKind::Long, "check" | "exit-code") + ) { + return true; + } + // Only `--diff-merges`' combined formats produce the two marker columns `compact_diff` + // misreads -- the same reason `-c`/`--cc` are raw here. The rest are ordinary + // single-column patches, which is already `show`'s default output. + if token.kind == TokenKind::Long && token.text == "diff-merges" { + return matches!( + token.value(tokens), + Some("c" | "cc" | "combined" | "dense-combined") + ); + } + // `-c` is the exception: it is the combined-diff form of `--cc`, not a patch request, and + // `compact_diff` reads a combined diff's two marker columns as one -- `git show -c` on a + // merge reported `+54 -8` where git's own stat says 156 insertions and 0 deletions. + if (token.kind == TokenKind::Short && token.text != "c") + || (token.kind == TokenKind::Long && matches!(token.text, "patch" | "unified")) + { + return false; + } + log_wants_raw_shape(token, tokens) +} + +/// Test-only convenience wrapper. +#[cfg(test)] +fn requests_raw_log_output(args: &[String]) -> bool { + let tokens = tokenize_git_log_args(args); + tokens.iter().any(|t| log_wants_raw_shape(t, &tokens)) +} + +/// Parse the user-specified limit from git log args. +/// Handles: -20, -n 20, --max-count=20, --max-count 20 +/// `run_log` shares a single tokenization via [`parse_limit_from_tokens`] +/// instead; this convenience wrapper exists for tests. +#[cfg(test)] +fn parse_user_limit(args: &[String]) -> Option { + parse_limit_from_tokens(&tokenize_git_log_args(args)) +} + +/// True if the user explicitly requested a commit-count limit (-N, -n N, --max-count=N, +/// --max-count N). +fn has_limit_flag(tokens: &[Token<'_>]) -> bool { + tokens.iter().any(|t| match t.kind { + TokenKind::Long => t.text == "max-count", + // "n" only counts if it actually captured a value -- e.g. clustered with another short + // flag (log_takes_value's solo_only spec refuses to link a value there), it's + // just an inert letter, not a real limit request. + TokenKind::Short => (t.text == "n" && t.value(tokens).is_some()) || is_digit_run(t.text), + _ => false, + }) +} + +fn parse_limit_from_tokens(tokens: &[Token<'_>]) -> Option { + for token in tokens { + let value = match token.kind { + // --max-count=20 (attached) or --max-count 20 (two-token form). + TokenKind::Long if token.text == "max-count" => token.value(tokens), + // -20 (combined digit form): the token itself is the count. + TokenKind::Short if is_digit_run(token.text) => Some(token.text), + // -n 20 (two-token form) or -n's value if ever attached. + TokenKind::Short if token.text == "n" => token.value(tokens), + _ => None, + }; + if let Some(n) = value.and_then(|v| v.parse::().ok()) { + return Some(n); + } + } + None +} + +/// When `user_set_limit` is true, the user explicitly passed `-N` to git log, +/// so we skip line capping (git already returns exactly N commits) and use a +/// wider truncation threshold (120 chars) to preserve commit context that LLMs +/// need for rebase/squash operations. +pub(crate) fn filter_log_output( + output: &str, + limit: usize, + user_set_limit: bool, + user_format: bool, +) -> String { + let truncate_width = if user_set_limit { 120 } else { 80 }; + + // When user specified their own format (--oneline, --pretty, --format), + // RTK did not inject ---END--- markers. Use simple line-based truncation. + if user_format { + let lines: Vec<&str> = output.lines().collect(); + let max_lines = if user_set_limit { lines.len() } else { limit }; + return lines + .iter() + .take(max_lines) + .map(|l| truncate_line(l, truncate_width)) + .collect::>() + .join("\n"); + } + + // RTK injected format: split output into commit blocks separated by ---END--- + let commits: Vec<&str> = output.split("---END---").collect(); + let max_commits = if user_set_limit { commits.len() } else { limit }; + + let mut result = Vec::new(); + for block in commits.iter().take(max_commits) { + let block = block.trim(); + if block.is_empty() { + continue; + } + let mut lines = block.lines(); + // First line is the header: hash subject (date) + let header = match lines.next() { + Some(h) => truncate_line(h.trim(), truncate_width), + None => continue, + }; + // Remaining lines are the body — keep up to 3 non-empty, non-trailer lines + let all_body_lines: Vec<&str> = lines + .map(|l| l.trim()) + .filter(|l| { + !l.is_empty() + && !l.starts_with("Signed-off-by:") + && !l.starts_with("Co-authored-by:") + }) + .collect(); + let body_omitted = all_body_lines.len().saturating_sub(3); + let body_lines = &all_body_lines[..all_body_lines.len().min(3)]; + + if body_lines.is_empty() { + result.push(header); + } else { + let mut entry = header; + for body in body_lines { + entry.push_str(&format!("\n {}", truncate_line(body, truncate_width))); + } + if body_omitted > 0 { + entry.push_str(&format!("\n [+{} lines omitted]", body_omitted)); + } + result.push(entry); + } + } + + result.join("\n").trim().to_string() +} + +/// Truncate a single line to `width` characters, appending "..." if needed +fn truncate_line(line: &str, width: usize) -> String { + if line.chars().count() > width { + let truncated: String = line.chars().take(width - 3).collect(); + format!("{}...", truncated) + } else { + line.to_string() + } +} + +pub(crate) fn format_status_output(porcelain: &str) -> String { + format_status_inner(porcelain, None) +} + +pub(crate) fn format_status_output_detached(porcelain: &str, detached_ref: &str) -> String { + format_status_inner(porcelain, Some(detached_ref)) +} + +fn format_status_inner(porcelain: &str, detached: Option<&str>) -> String { + let lines: Vec<&str> = porcelain + .lines() + .filter(|line| !line.trim().is_empty()) + .collect(); + + if lines.is_empty() { + return "Clean working tree".to_string(); + } + + let mut output = Vec::new(); + + if let Some(branch_line) = lines.first() { + if branch_line.starts_with("##") { + let branch = branch_line.trim_start_matches("## "); + let display = detached.unwrap_or(branch); + output.push(format!("* {}", display)); + } else { + output.push((*branch_line).to_string()); + } + } + + for line in lines.iter().skip(1) { + output.push((*line).to_string()); + } + + if lines.len() == 1 && lines[0].starts_with("##") { + output.push("clean — nothing to commit".to_string()); + } + + output.join("\n") +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GitStatusState { + Rebase, + MergeConflicts, + MergeReadyToCommit, + CherryPick, + Revert, + Bisect, + Am, + SparseCheckout, +} + +impl GitStatusState { + fn summary(self) -> &'static str { + match self { + Self::Rebase => "rebase in progress", + Self::MergeConflicts => "merge in progress. unresolved conflicts", + Self::MergeReadyToCommit => "merge in progress. no conflicts", + Self::CherryPick => "cherry-pick in progress", + Self::Revert => "revert in progress", + Self::Bisect => "bisect in progress", + Self::Am => "am session in progress", + Self::SparseCheckout => "sparse checkout enabled", + } + } +} + +const REBASE_INDICATORS: &[&str] = &[ + "rebase in progress", + "You are currently rebasing", + "You are currently editing", + "You are currently splitting", + "Last command done", + "Next command to do", + "No commands remaining", +]; + +fn detect_status_state(line: &str) -> Option { + if line.contains("All conflicts fixed but you are still merging") { + Some(GitStatusState::MergeReadyToCommit) + } else if line.contains("You have unmerged paths") { + Some(GitStatusState::MergeConflicts) + } else if line.contains("You are currently cherry-picking") { + Some(GitStatusState::CherryPick) + } else if line.contains("You are currently reverting") { + Some(GitStatusState::Revert) + } else if line.contains("You are currently bisecting") { + Some(GitStatusState::Bisect) + } else if line.contains("You are in the middle of an am session") { + Some(GitStatusState::Am) + } else if line.contains("You are in a sparse checkout") { + Some(GitStatusState::SparseCheckout) + } else if REBASE_INDICATORS.iter().any(|i| line.contains(i)) { + Some(GitStatusState::Rebase) + } else { + None + } +} + +/// `git status --porcelain -b` (compact mode) omits the state header for rebase/merge/ +/// cherry-pick/etc, so an in-progress rebase can look like a clean status. Extracts a compact +/// summary of that state from plain `git status` output instead. `None` if none is in progress. +fn extract_state_header(raw: &str) -> Option { + // Headers of the file-change blocks — everything relevant to state appears + // above these in git's output, so they double as a terminator. + const STOPPERS: &[&str] = &[ + "Changes to be committed:", + "Changes not staged for commit:", + "Untracked files:", + "Unmerged paths:", + "no changes added to commit", + "nothing to commit", + "nothing added to commit", + ]; + + for line in raw.lines() { + let stripped = line.trim(); + + if STOPPERS.iter().any(|s| stripped.starts_with(s)) { + break; + } + + if let Some(state) = detect_status_state(stripped) { + return Some(state.summary().to_string()); + } + } + + None +} + +/// Porcelain `-b` collapses a detached HEAD to the opaque `## HEAD (no branch)`, which can be +/// misread as a branch literally named `HEAD`. Extracts the explicit "HEAD detached at/from +/// " line from plain `git status` output instead. `None` if HEAD is on a branch. +fn extract_detached_head(raw: &str) -> Option { + raw.lines() + .map(str::trim) + .find(|l| l.starts_with("HEAD detached ")) + .map(str::to_string) +} + +/// Minimal filtering for git status with user-provided args +fn filter_status_with_args(output: &str) -> String { + let mut result = Vec::new(); + + for line in output.lines() { + let trimmed = line.trim(); + + // Skip empty lines + if trimmed.is_empty() { + continue; + } + + // Skip git hints - can appear at start or within line + if trimmed.starts_with("(use \"git") + || trimmed.starts_with("(create/copy files") + || trimmed.contains("(use \"git add") + || trimmed.contains("(use \"git restore") + { + continue; + } + + // Special case: clean working tree + if trimmed.contains("nothing to commit") && trimmed.contains("working tree clean") { + result.push(trimmed.to_string()); + break; + } + + result.push(line.to_string()); + } + + if result.is_empty() { + "ok".to_string() + } else { + result.join("\n") + } +} + +fn run_status(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + // Keep a narrow compact path for no-arg status and branch/short-only flags. + // More complex explicit args still use the existing minimal-filter path. + if !uses_compact_status_path(args) { + let mut cmd = build_status_command(args, global_args); + let result = exec_capture(&mut cmd).context("Failed to run git status")?; + + if !result.success() { + if !result.stderr.trim().is_empty() { + eprint!("{}", result.stderr); + } + timer.track( + &format!("git status {}", args.join(" ")), + &format!("rtk git status {}", args.join(" ")), + &result.stdout, + &result.stdout, + ); + return Ok(result.exit_code); + } + + if verbose > 0 || !result.stderr.is_empty() { + eprint!("{}", result.stderr); + } + + // Apply minimal filtering: strip ANSI, remove hints, empty lines + let filtered = filter_status_with_args(&result.stdout); + let filtered = never_worse(&result.stdout, &filtered).to_string(); + print!("{}", filtered); + + timer.track( + &format!("git status {}", args.join(" ")), + &format!("rtk git status {}", args.join(" ")), + &result.stdout, + &filtered, + ); + + return Ok(0); + } + + let mut raw_cmd = git_cmd_c_locale(global_args); + raw_cmd.arg("status"); + raw_cmd.args(args); + let raw_output = exec_capture(&mut raw_cmd) + .map(|r| r.stdout) + .unwrap_or_default(); + + let mut cmd = build_status_command(args, global_args); + let result = exec_capture(&mut cmd).context("Failed to run git status")?; + + if !result.success() { + let message = if result.stderr.contains("not a git repository") { + "Not a git repository".to_string() + } else { + result.stderr.trim().to_string() + }; + if !message.is_empty() { + eprintln!("{}", message); + } + let original_cmd = if args.is_empty() { + "git status".to_string() + } else { + format!("git status {}", args.join(" ")) + }; + let rtk_cmd = if args.is_empty() { + "rtk git status".to_string() + } else { + format!("rtk git status {}", args.join(" ")) + }; + let shown = never_worse(&raw_output, &message); + timer.track(&original_cmd, &rtk_cmd, &raw_output, shown); + return Ok(result.exit_code); + } + + let formatted = match extract_detached_head(&raw_output) { + Some(detached_ref) => format_status_output_detached(&result.stdout, &detached_ref), + None => format_status_output(&result.stdout), + }; + + // Surface in-progress state (rebase/merge/cherry-pick/bisect/am) from the + // plain-status output we already captured for tracking. Porcelain omits it + // and hiding it misleads the user about the true repo state. + let final_output = match extract_state_header(&raw_output) { + Some(state) => format!("{}\n{}", state, formatted), + None => formatted, + }; + + let shown = never_worse(&raw_output, &final_output); + println!("{}", shown); + + let original_cmd = if args.is_empty() { + "git status".to_string() + } else { + format!("git status {}", args.join(" ")) + }; + let rtk_cmd = if args.is_empty() { + "rtk git status".to_string() + } else { + format!("rtk git status {}", args.join(" ")) + }; + + timer.track(&original_cmd, &rtk_cmd, &raw_output, shown); + + Ok(0) +} + +fn run_add(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + let mut cmd = git_cmd(global_args); + cmd.arg("add"); + + // Pass all arguments directly to git (flags like -A, -p, --all, etc.) + if args.is_empty() { + cmd.arg("."); + } else { + for arg in args { + cmd.arg(arg); + } + } + + let result = exec_capture(&mut cmd).context("Failed to run git add")?; + + if verbose > 0 { + eprintln!("git add executed"); + } + + let raw_output = format!("{}\n{}", result.stdout, result.stderr); + + if result.success() { + // Count what was added + let mut stat_cmd = git_cmd(global_args); + stat_cmd.args(["diff", "--cached", "--stat", "--shortstat"]); + let stat_result = exec_capture(&mut stat_cmd).context("Failed to check staged files")?; + + // Mirror git's own behaviour: a no-op `git add` is silent. Emitting a + // generic "ok" here is misleading — an agent can't tell "staged N files" + // from "staged nothing" when both print "ok". + let compact = if stat_result.stdout.trim().is_empty() { + String::new() + } else { + // Parse "1 file changed, 5 insertions(+)" format + let short = stat_result.stdout.lines().last().unwrap_or("").trim(); + if short.is_empty() { + "ok".to_string() + } else { + format!("ok {}", short) + } + }; + + if !compact.is_empty() { + println!("{}", compact); + } else if !result.stderr.trim().is_empty() { + // Nothing staged, but git had something to say about why (`git add --` answers + // "Nothing specified, nothing added" with a hint). Printing neither the count nor + // git's own explanation leaves the agent unable to tell that from a crash. + eprintln!("{}", result.stderr.trim()); + } + + timer.track( + &format!("git add {}", args.join(" ")), + &format!("rtk git add {}", args.join(" ")), + &raw_output, + &compact, + ); + } else { + eprintln!("FAILED: git add"); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + if !result.stdout.trim().is_empty() { + eprintln!("{}", result.stdout); + } + return Ok(result.exit_code); + } + + Ok(0) +} + +fn build_commit_command(args: &[String], global_args: &[String]) -> Command { + let mut cmd = git_cmd(global_args); + cmd.arg("commit"); + for arg in args { + cmd.arg(arg); + } + cmd +} + +/// Parse the first line of `git commit` success output and return a compact token. +/// Handles: `[main abc1234def] message`, `[main (root-commit) abc1234def] msg`, +/// localized variants, and multibyte branch names. +fn parse_commit_output(line: &str) -> String { + // Locate the brackets rather than assume the line starts with '[': git prints hook output + // first, and slicing from byte 1 would panic on a multi-byte leading character. + let (Some(open), Some(bracket_end)) = (line.find('['), line.find(']')) else { + return "ok".to_string(); + }; + if open >= bracket_end { + return "ok".to_string(); + } + + let bracket_content = &line[open + 1..bracket_end]; + let hash = bracket_content.split_whitespace().next_back().unwrap_or(""); + if hash.chars().count() >= 7 { + let short_hash: String = hash.chars().take(7).collect(); + format!("ok {}", short_hash) + } else { + "ok".to_string() + } +} + +fn run_commit(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + let original_cmd = format!("git commit {}", args.join(" ")); + + if verbose > 0 { + eprintln!("{}", original_cmd); + } + + // stdin is inherited so an interactive editor, GPG passphrase prompt or + // credential helper still reaches the terminal. + let CaptureResult { + stdout, + stderr, + exit_code, + } = exec_capture_stdin(&mut build_commit_command(args, global_args)) + .context("Failed to run git commit")?; + let raw_output = format!("{}\n{}", stdout, stderr); + + match classify_commit_outcome(exit_code == 0, &stdout, exit_code) { + CommitOutcome::Ok(compact) => { + println!("{}", compact); + timer.track(&original_cmd, "rtk git commit", &raw_output, &compact); + Ok(0) + } + CommitOutcome::Failed(code) => { + if !stderr.trim().is_empty() { + eprint!("{}", stderr); + } + if !stdout.trim().is_empty() { + eprint!("{}", stdout); + } + timer.track(&original_cmd, "rtk git commit", &raw_output, &raw_output); + Ok(code) + } + } +} + +/// Outcome of a `git commit`: a non-success status propagates the exit code +/// rather than being reported as "ok" (#2494). +enum CommitOutcome { + Ok(String), + Failed(i32), +} + +/// Classify a `git commit` result. +fn classify_commit_outcome(success: bool, stdout: &str, exit_code: i32) -> CommitOutcome { + if success { + // Extract commit hash from output + let compact = stdout + .lines() + .next() + .map(parse_commit_output) + .unwrap_or_else(|| "ok".to_string()); + CommitOutcome::Ok(compact) + } else { + CommitOutcome::Failed(exit_code) + } +} + +fn run_checkout(args: &[String], verbose: u8, global_args: &[String]) -> Result { + if verbose > 0 { + eprintln!("git checkout"); + } + + // The user's locale, per git_cmd_c_locale's own contract: this child's stderr is shown + // verbatim on failure. When the English "Switched to branch ..." scan misses, the + // args-based fallback below still names the branch. + let mut cmd = git_cmd(global_args); + cmd.arg("checkout"); + for arg in args { + cmd.arg(arg); + } + + let args_display = args.join(" "); + let args_for_filter = args.to_vec(); + runner::run_filtered_with_exit( + cmd, + "git checkout", + &args_display, + move |raw, exit_code| format_checkout_output(&args_for_filter, raw, exit_code), + RunOptions::with_tee("git_checkout"), + ) +} + +fn format_checkout_output(args: &[String], raw: &str, exit_code: i32) -> String { + if exit_code == 0 { + format_checkout_success(args, raw) + } else { + filter_checkout_failure(raw) + } +} + +fn format_checkout_success(args: &[String], raw: &str) -> String { + let tokens = arg_tokenizer::tokenize_grammar(args, &checkout_takes_value, Dialect::Posix); + + if let Some(restored) = checkout_restored_count(&tokens) { + return format!( + "ok {} {}", + restored, + pluralize(restored, "file restored", "files restored") + ); + } + for line in raw.lines().map(str::trim) { + if let Some(branch) = quoted_suffix(line, "Switched to a new branch ") { + return format!("ok {} (new)", branch); + } + if let Some(branch) = quoted_suffix(line, "Switched to branch ") { + return format!("ok {}", branch); + } + if let Some(branch) = quoted_suffix(line, "Already on ") { + return format!("ok {}", branch); + } + if let Some(rest) = line.strip_prefix("HEAD is now at ") { + let hash = rest.split_whitespace().next().unwrap_or("HEAD"); + return format!("ok HEAD {}", hash); + } + if line.starts_with("Updated ") && line.contains(" path") { + return format!("ok {}", line.to_ascii_lowercase()); + } + } + + // Both of these are the fallback for when the scan above misses, never a short-circuit + // past it: `-B` creates *or* resets, and only git knows which happened, so claiming the + // name early cost `git checkout -Bfoo` its `(new)` marker. + // + // The scan is English-only and this child deliberately keeps the user's locale (see + // run_checkout), so under another locale `-B` reaches `ok ` here even when it + // created the branch. That is a weaker answer, never a wrong one -- the marker is omitted + // rather than claimed falsely -- but it does mean `(new)` is not guaranteed. + if let Some(branch) = checkout_new_branch_arg(&tokens) { + return format!("ok {} (new)", branch); + } + if let Some(branch) = checkout_reset_branch_arg(&tokens) { + return format!("ok {}", branch); + } + if let Some(branch) = checkout_branch_arg(&tokens) { + return format!("ok {}", branch); + } + + "ok".to_string() +} + +/// The options git consumes a separate token for: `--orphan`/`-b`/`-B` take a branch name, +/// `--conflict` a style, `--pathspec-from-file` a file (all confirmed against git 2.53, which +/// answers "requires a value"). `-t`/`--track`/`--detach` and any other `-`-prefixed token are +/// booleans. Shared by every `checkout_*_arg` helper below via one +/// [`arg_tokenizer::tokenize`] call instead of each hand-rolling its own scan over `args`. +fn checkout_takes_value(kind: TokenKind, name: &str) -> Option { + match kind { + TokenKind::Long => { + matches!(name, "conflict" | "orphan" | "pathspec-from-file").then(ValueSpec::value) + } + TokenKind::Short => matches!(name, "B" | "b").then(ValueSpec::value), + _ => None, + } +} + +fn checkout_restored_count(tokens: &[Token<'_>]) -> Option { + let separator = arg_tokenizer::dashdash_index(tokens)?; + let count = tokens[separator + 1..] + .iter() + .filter(|t| !t.text.is_empty()) + .count(); + (count > 0).then_some(count) +} + +fn checkout_new_branch_arg<'a>(tokens: &[Token<'a>]) -> Option<&'a str> { + tokens.iter().find_map(|t| match t.kind { + TokenKind::Long if t.text == "orphan" => t.value(tokens), + TokenKind::Short if t.text == "b" => t.value(tokens), + _ => None, + }) +} + +fn checkout_reset_branch_arg<'a>(tokens: &[Token<'a>]) -> Option<&'a str> { + tokens + .iter() + .find(|t| t.kind == TokenKind::Short && t.text == "B") + .and_then(|t| t.value(tokens)) +} + +fn checkout_branch_arg<'a>(tokens: &[Token<'a>]) -> Option<&'a str> { + if arg_tokenizer::has_dashdash(tokens) { + return None; + } + tokens + .iter() + // A bare `-` is git's "the branch I was on before", not a branch name to echo back. + .find(|t| t.is_free_positional() && t.text != "-") + .map(|t| t.text) +} + +fn quoted_suffix<'a>(line: &'a str, prefix: &str) -> Option<&'a str> { + line.strip_prefix(prefix) + .and_then(|rest| rest.strip_prefix('\'')) + .and_then(|rest| rest.strip_suffix('\'')) +} + +fn pluralize<'a>(count: usize, singular: &'a str, plural: &'a str) -> &'a str { + if count == 1 { singular } else { plural } +} + +fn filter_checkout_failure(raw: &str) -> String { + let mut important = Vec::new(); + let mut in_file_list = false; + + for line in raw.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + + let is_header = trimmed.starts_with("error:") + || trimmed.starts_with("fatal:") + || trimmed.starts_with("CONFLICT"); + + if is_header { + in_file_list = trimmed.contains("following") + && trimmed.contains("files") + && trimmed.ends_with(':'); + important.push(trimmed.to_string()); + continue; + } + + if in_file_list { + if trimmed.starts_with("Please ") || trimmed.starts_with("Aborting") { + in_file_list = false; + } else if line.starts_with(char::is_whitespace) { + important.push(line.to_string()); + continue; + } + } + + if trimmed.starts_with("Aborting") { + important.push(trimmed.to_string()); + } + } + + if important.is_empty() { + raw.trim().to_string() + } else { + important.join("\n") + } +} + +// Git push progress prefixes (stderr) — dropped from the stream. +const GIT_PUSH_NOISE_PREFIXES: &[&str] = &[ + "Enumerating objects:", + "Counting objects:", + "Compressing objects:", + "Writing objects:", + "Delta compression using", + "Total ", +]; + +#[derive(Default)] +struct GitPushLineHandler { + up_to_date: bool, + pushed_ref: Option, +} + +impl LineHandler for GitPushLineHandler { + fn should_skip(&mut self, line: &str) -> bool { + if line.is_empty() { + return true; + } + let trimmed = line.trim_start(); + GIT_PUSH_NOISE_PREFIXES + .iter() + .any(|p| trimmed.starts_with(p)) + } + + fn observe_line(&mut self, line: &str) { + if line.contains("Everything up-to-date") { + self.up_to_date = true; + } + if self.pushed_ref.is_none() + && let Some(idx) = line.find(" -> ") + { + let after = &line[idx + 4..]; + if let Some(dest) = after.split_whitespace().next() { + self.pushed_ref = Some(dest.to_string()); + } + } + } + + fn format_summary(&self, exit_code: i32, _raw: &str) -> Option { + if exit_code != 0 { + return None; + } + let summary = if self.up_to_date { + "ok (up-to-date)".to_string() + } else if let Some(dest) = &self.pushed_ref { + format!("ok {}", dest) + } else { + "ok".to_string() + }; + Some(format!("{}\n", summary)) + } +} + +fn run_push(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git push"); + } + + let mut cmd = git_cmd(global_args); + cmd.arg("push"); + for arg in args { + cmd.arg(arg); + } + + let cmd_label = format!("git push {}", args.join(" ")); + let filter = LineStreamFilter::new(GitPushLineHandler::default()); + let result = stream::run_streaming( + &mut cmd, + StdinMode::Inherit, + FilterMode::Streaming(Box::new(filter)), + ) + .context("Failed to run git push")?; + + timer.track( + &cmd_label, + &format!("rtk {}", cmd_label), + &result.raw, + &result.filtered, + ); + + Ok(result.exit_code) +} + +fn run_pull(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git pull"); + } + + let mut cmd = git_cmd(global_args); + cmd.arg("pull"); + for arg in args { + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git pull")?; + + let raw_output = format!("{}\n{}", result.stdout, result.stderr); + + if result.success() { + let compact = if result.stdout.contains("Already up to date") + || result.stdout.contains("Already up-to-date") + { + "ok (up-to-date)".to_string() + } else { + // Count files changed + let mut files = 0; + let mut insertions = 0; + let mut deletions = 0; + + for line in result.stdout.lines() { + if line.contains("file") && line.contains("changed") { + // Parse "3 files changed, 10 insertions(+), 2 deletions(-)" + for part in line.split(',') { + let part = part.trim(); + if part.contains("file") { + files = part + .split_whitespace() + .next() + .and_then(|n| n.parse().ok()) + .unwrap_or(0); + } else if part.contains("insertion") { + insertions = part + .split_whitespace() + .next() + .and_then(|n| n.parse().ok()) + .unwrap_or(0); + } else if part.contains("deletion") { + deletions = part + .split_whitespace() + .next() + .and_then(|n| n.parse().ok()) + .unwrap_or(0); + } + } + } + } + + if files > 0 { + format!("ok {} files +{} -{}", files, insertions, deletions) + } else { + "ok".to_string() + } + }; + + println!("{}", compact); + + timer.track( + &format!("git pull {}", args.join(" ")), + &format!("rtk git pull {}", args.join(" ")), + &raw_output, + &compact, + ); + } else { + eprintln!("FAILED: git pull"); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + if !result.stdout.trim().is_empty() { + eprintln!("{}", result.stdout); + } + return Ok(result.exit_code); + } + + Ok(0) +} + +fn branch_takes_value(kind: TokenKind, name: &str) -> Option { + // -c/-C/-m/-M/-d/-D are followed by positional branch names, not a "flag value" in the + // attached/separate-value sense, so they're excluded here. -u IS a genuine value-taking flag + // (its short form takes the same single upstream-ref value as --set-upstream-to) and must be + // included alongside its long form, or `git branch -u origin/main` leaves "origin/main" as + // an unlinked Positional token instead of -u's linked value. + match kind { + TokenKind::Long => matches!( + name, + "contains" + | "format" + | "merged" + | "no-contains" + | "no-merged" + | "points-at" + | "set-upstream-to" + | "sort" + ) + .then(ValueSpec::value), + TokenKind::Short => (name == "u").then(ValueSpec::value), + _ => None, + } +} + +fn run_branch(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git branch"); + } + + let tokens = arg_tokenizer::tokenize_grammar(args, &branch_takes_value, Dialect::Posix); + + // Detect write operations: delete, rename, copy, upstream tracking + let has_action_flag = tokens.iter().any(|t| match t.kind { + TokenKind::Short => matches!(t.text, "C" | "D" | "M" | "c" | "d" | "m" | "u"), + TokenKind::Long => matches!( + t.text, + "edit-description" | "set-upstream-to" | "unset-upstream" + ), + _ => false, + }); + + // Detect flags that produce specific output (not a branch list) + let has_show_flag = tokens + .iter() + .any(|t| t.kind == TokenKind::Long && t.text == "show-current"); + + // Detect list-mode flags + let has_list_flag = tokens.iter().any(|t| match t.kind { + TokenKind::Short => matches!(t.text, "a" | "r"), + TokenKind::Long => matches!( + t.text, + "all" + | "contains" + | "format" + | "list" + | "merged" + | "no-contains" + | "no-merged" + | "points-at" + | "remotes" + | "sort" + ), + _ => false, + }); + + // Detect positional arguments (not flags) — indicates branch creation. A value consumed by + // a preceding flag (e.g. -u/--set-upstream-to's upstream ref) is that flag's value, not an + // independent positional branch name, so linked tokens are excluded. + let has_positional_arg = tokens.iter().any(|t| t.is_free_positional()); + + // --show-current: passthrough with raw stdout (not "ok") + if has_show_flag { + let mut cmd = git_cmd(global_args); + cmd.arg("branch"); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git branch")?; + let combined = result.combined(); + + let trimmed = result.stdout.trim(); + timer.track( + &format!("git branch {}", args.join(" ")), + &format!("rtk git branch {}", args.join(" ")), + &combined, + trimmed, + ); + + if result.success() { + println!("{}", trimmed); + } else { + eprintln!("FAILED: git branch {}", args.join(" ")); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + return Ok(result.exit_code); + } + return Ok(0); + } + + // Write operation: action flags, or positional args without list flags (= branch creation) + if has_action_flag || (has_positional_arg && !has_list_flag) { + let mut cmd = git_cmd(global_args); + cmd.arg("branch"); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git branch")?; + let combined = result.combined(); + + let msg = if result.success() { "ok" } else { &combined }; + + timer.track( + &format!("git branch {}", args.join(" ")), + &format!("rtk git branch {}", args.join(" ")), + &combined, + msg, + ); + + if result.success() { + println!("ok"); + } else { + eprintln!("FAILED: git branch {}", args.join(" ")); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + if !result.stdout.trim().is_empty() { + eprintln!("{}", result.stdout); + } + return Ok(result.exit_code); + } + return Ok(0); + } + + // List mode: show compact branch list + let mut cmd = git_cmd(global_args); + cmd.arg("branch"); + if !has_list_flag { + cmd.arg("-a"); + } + cmd.arg("--no-color"); + for arg in args { + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git branch")?; + + if !result.success() { + if !result.stderr.trim().is_empty() { + eprint!("{}", result.stderr); + } + timer.track( + &format!("git branch {}", args.join(" ")), + &format!("rtk git branch {}", args.join(" ")), + &result.stdout, + &result.stdout, + ); + return Ok(result.exit_code); + } + + let filtered = filter_branch_output(&result.stdout); + let filtered = never_worse(&result.stdout, &filtered).to_string(); + println!("{}", filtered); + + timer.track( + &format!("git branch {}", args.join(" ")), + &format!("rtk git branch {}", args.join(" ")), + &result.stdout, + &filtered, + ); + + Ok(0) +} + +fn filter_branch_output(output: &str) -> String { + let mut current = String::new(); + let mut local: Vec = Vec::new(); + let mut remote: Vec = Vec::new(); + let mut seen_remote: std::collections::HashSet = std::collections::HashSet::new(); + + for line in output.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + + if let Some(branch) = line.strip_prefix("* ") { + current = branch.to_string(); + } else if let Some(rest) = line.strip_prefix("remotes/") { + if let Some(slash_pos) = rest.find('/') { + let branch = &rest[slash_pos + 1..]; + if branch.starts_with("HEAD ") { + continue; + } + if seen_remote.insert(branch.to_string()) { + remote.push(branch.to_string()); + } + } + } else { + local.push(line.to_string()); + } + } + + let mut result = Vec::new(); + result.push(format!("* {}", current)); + + if !local.is_empty() { + for b in &local { + result.push(format!(" {}", b)); + } + } + + if !remote.is_empty() { + let remote_only: Vec<&String> = remote + .iter() + .filter(|r| *r != ¤t && !local.contains(r)) + .collect(); + if !remote_only.is_empty() { + const MAX_REMOTE_BRANCHES: usize = CAP_WARNINGS; + result.push(format!(" remote-only ({}):", remote_only.len())); + for b in remote_only.iter().take(MAX_REMOTE_BRANCHES) { + result.push(format!(" {}", b)); + } + if remote_only.len() > MAX_REMOTE_BRANCHES { + result.push(format!( + " ... +{} more", + remote_only.len() - MAX_REMOTE_BRANCHES + )); + } + } + } + + result.join("\n") +} + +fn run_fetch(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git fetch"); + } + + let mut cmd = git_cmd(global_args); + cmd.arg("fetch"); + for arg in args { + cmd.arg(arg); + } + + let result = exec_capture(&mut cmd).context("Failed to run git fetch")?; + let raw = result.combined(); + + if !result.success() { + eprintln!("FAILED: git fetch"); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + return Ok(result.exit_code); + } + + // Count new refs from stderr (git fetch outputs to stderr) + let new_refs: usize = result + .stderr + .lines() + .filter(|l| l.contains("->") || l.contains("[new")) + .count(); + + let msg = if new_refs > 0 { + format!("ok fetched ({} new refs)", new_refs) + } else { + "ok fetched".to_string() + }; + + println!("{}", msg); + timer.track("git fetch", "rtk git fetch", &raw, &msg); + + Ok(0) +} + +/// Format status message for stash operations. +/// - For create operations (push/save): checks for "No local changes" +/// - For other operations: uses "ok stash " format +fn format_stash_message(subcommand: Option<&str>, result: &CaptureResult) -> String { + match subcommand { + None | Some("push") | Some("save") => { + // A successful stash collapses to "ok stashed" (the WIP ref/sha git + // prints isn't needed to `git stash pop`). But a no-op must NOT look + // like success — pass git's "No local changes to save" through so the + // agent can tell nothing was stashed. + if result.combined().contains("No local changes") { + "No local changes to save".to_string() + } else { + "ok stashed".to_string() + } + } + Some(sub) => format!("ok stash {}", sub), + } +} + +/// True if `-p`/`--patch` was requested. Note: `-u` means `--include-untracked` here, not `-p`. +/// +/// The "nothing takes a value" predicate *is* `stash show`'s grammar for this question, not a +/// placeholder: git parses `-p`/`-u` itself before handing the rest to the revision machinery, +/// so no flag it sees here consumes a following token, and treating one as if it did swallowed +/// the `-p` after it (confirmed against git 2.53 with `git stash show --author -p`). +fn stash_show_wants_patch(args: &[String]) -> bool { + let tokens = arg_tokenizer::tokenize(args); + tokens.iter().any(|t| match t.kind { + TokenKind::Long => t.text == "patch", + TokenKind::Short => t.text == "p", + _ => false, + }) +} + +fn run_stash( + subcommand: Option<&str>, + args: &[String], + verbose: u8, + global_args: &[String], +) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git stash {:?}", subcommand); + } + + match subcommand { + Some("list") => { + let mut cmd = git_cmd(global_args); + cmd.args(["stash", "list"]); + let result = exec_capture(&mut cmd).context("Failed to run git stash list")?; + + if result.stdout.trim().is_empty() { + if !result.success() && !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr.trim()); + } + timer.track("git stash list", "rtk git stash list", &result.stdout, ""); + return Ok(result.exit_code); + } + + let filtered = filter_stash_list(&result.stdout); + let filtered = never_worse(&result.stdout, &filtered).to_string(); + println!("{}", filtered); + timer.track( + "git stash list", + "rtk git stash list", + &result.stdout, + &filtered, + ); + } + Some("show") => { + let asked_for_patch = stash_show_wants_patch(args); + + let mut cmd = git_cmd(global_args); + cmd.args(["stash", "show"]); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git stash show")?; + + if result.stdout.trim().is_empty() { + if !result.success() && !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr.trim()); + } + timer.track("git stash show", "rtk git stash show", &result.stdout, ""); + return Ok(result.exit_code); + } + + // What git actually produced settles it, not what the flags predicted: `git stash + // show --` (or any other extra argument) flips git to diff output, and running the + // stat filter over patch text yields nothing at all. + let patch_mode = asked_for_patch + || result + .stdout + .lines() + .any(|line| line.starts_with("diff --git ") || line.starts_with("diff --cc ")); + + // Log's grammar, unlike `stash_show_wants_patch`'s: `stash show` parses `-p`/`-u` + // itself, but hands the rest to the revision machinery, which does consume a + // following `--word-diff` as `--author`'s value. + let filtered = if patch_mode && !emits_word_diff(&tokenize_git_log_args(args)) { + compact_diff(&result.stdout, 100) + } else if patch_mode { + result.stdout.clone() + } else { + compact_stash_stat(&result.stdout) + }; + let shown = crate::core::runner::emit_guarded(&filtered, None, &result.stdout); + timer.track( + "git stash show", + "rtk git stash show", + &result.stdout, + &shown, + ); + } + Some("apply") | Some("branch") | Some("clear") | Some("create") | Some("drop") + | Some("export") | Some("import") | Some("pop") | Some("store") => { + let sub = subcommand.unwrap(); + let mut cmd = git_cmd(global_args); + cmd.args(["stash", sub]); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git stash")?; + let combined = result.combined(); + + let msg = if result.success() { + let msg = format_stash_message(subcommand, &result); + println!("{}", msg); + msg + } else { + eprintln!("FAILED: git stash {}", sub); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + combined.clone() + }; + + timer.track( + &format!("git stash {}", sub), + &format!("rtk git stash {}", sub), + &combined, + &msg, + ); + + if !result.success() { + return Ok(result.exit_code); + } + } + // Default: "git stash [push] [--] [...]" or "git stash save []" + Some(_) | None => { + let (sub, arg) = match subcommand { + Some("save") => ("save", None), + Some("push") => ("push", None), + Some(s) => ("push", Some(s)), + None => ("push", None), + }; + let mut cmd = git_cmd(global_args); + cmd.args(["stash", sub]); + if let Some(arg) = arg { + cmd.arg(arg); + } + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git stash")?; + let combined = result.combined(); + + let msg = if result.success() { + let msg = format_stash_message(subcommand, &result); + println!("{}", msg); + msg + } else { + eprintln!("FAILED: git stash {}", sub); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + combined.clone() + }; + + timer.track( + &format!("git stash {}", sub), + &format!("rtk git stash {}", sub), + &combined, + &msg, + ); + + if !result.success() { + return Ok(result.exit_code); + } + } + } + + Ok(0) +} + +fn filter_stash_list(output: &str) -> String { + // Format: "stash@{0}: WIP on main: abc1234 commit message" + let mut result = Vec::new(); + for line in output.lines() { + if let Some(colon_pos) = line.find(": ") { + let index = &line[..colon_pos]; + let rest = &line[colon_pos + 2..]; + // Compact: strip "WIP on branch:" prefix if present + let message = if let Some(second_colon) = rest.find(": ") { + rest[second_colon + 2..].trim() + } else { + rest.trim() + }; + result.push(format!("{}: {}", index, message)); + } else { + result.push(line.to_string()); + } + } + result.join("\n") +} + +fn compact_stash_stat(raw: &str) -> String { + let (files, summary) = parse_stash_stat(raw); + if files.is_empty() { + return raw.trim_end().to_string(); + } + let total = files.len(); + let mut out = join_with_overflow(&files[..total.min(CAP_LIST)], total, CAP_LIST, "files"); + if total > CAP_LIST + && let Some(hint) = + crate::core::tee::force_tee_tail_hint(&files.join("\n"), "git-stash-show", CAP_LIST + 1) + { + out.push(' '); + out.push_str(&hint); + } + if !summary.is_empty() { + out.push('\n'); + out.push_str(&compress_stat_summary(&summary)); + } + out +} + +fn compress_stat_summary(summary: &str) -> String { + summary + .replace("insertions(+)", "+") + .replace("insertion(+)", "+") + .replace("deletions(-)", "-") + .replace("deletion(-)", "-") + .replace("files changed", "changed") + .replace("file changed", "changed") + .replace(",", "") +} + +fn parse_stash_stat(stat: &str) -> (Vec, String) { + let stat = strip_ansi(stat); + let mut files = Vec::new(); + let mut summary = String::new(); + + for line in stat.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + match diffstat_row(line) { + Some(row) => files.push(row), + None => summary = line.to_string(), + } + } + + (files, summary) +} + +fn diffstat_row(line: &str) -> Option { + let bar = line.rfind('|')?; + let path = line[..bar].trim(); + let rhs = line[bar + 1..].trim(); + let is_diffstat_row = rhs.starts_with("Bin") || rhs.starts_with(|c: char| c.is_ascii_digit()); + if path.is_empty() || !is_diffstat_row { + return None; + } + if rhs.starts_with("Bin") { + return Some(format!("{} (binary)", path)); + } + let count = rhs.split_whitespace().next().unwrap_or(""); + let sign = match (rhs.contains('+'), rhs.contains('-')) { + (true, true) => " +-", + (true, false) => " +", + (false, true) => " -", + (false, false) => "", + }; + Some(format!("{} {}{}", path, count, sign)) +} + +/// True when a `git worktree` write action was asked to report what it did: `prune --dry-run` +/// names every worktree it would remove and that list is the whole point of the command, while +/// `git worktree add`'s progress lines are exactly what RTK's "ok" replaces. +fn worktree_asked_for_report(tokens: &[Token<'_>]) -> bool { + arg_tokenizer::before_dashdash(tokens) + .iter() + .any(|t| match t.kind { + TokenKind::Long => matches!(t.text, "dry-run" | "verbose"), + TokenKind::Short => matches!(t.text, "n" | "v"), + _ => false, + }) +} + +fn run_worktree(args: &[String], verbose: u8, global_args: &[String]) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git worktree list"); + } + + // The subcommand is the first positional, not any arg that happens to spell one (a + // worktree path named "move", say). + let tokens = arg_tokenizer::tokenize(args); + let subcommand = arg_tokenizer::before_dashdash(&tokens) + .iter() + .find(|t| t.is_free_positional()) + .map(|t| t.text); + + // Only a bare listing is RTK's to compact. A write action gets the terse "ok"; everything + // else -- `list` with flags of its own like `--porcelain`, a subcommand git grows later, + // or a `--` with no subcommand, which real git rejects -- passes through verbatim, since + // RTK cannot know what its output means. + let compact_list = tokens.is_empty() || (subcommand == Some("list") && tokens.len() == 1); + let write_action = matches!( + subcommand, + Some("add" | "remove" | "prune" | "lock" | "unlock" | "move" | "repair") + ); + let asked_for_report = worktree_asked_for_report(&tokens); + + if !compact_list && !write_action { + let mut cmd = git_cmd(global_args); + cmd.arg("worktree"); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git worktree")?; + print!("{}", result.stdout); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr.trim()); + } + timer.track( + &format!("git worktree {}", args.join(" ")), + &format!("rtk git worktree {} (passthrough)", args.join(" ")), + &result.stdout, + &result.stdout, + ); + return Ok(result.exit_code); + } + + if write_action { + let mut cmd = git_cmd(global_args); + cmd.arg("worktree"); + for arg in args { + cmd.arg(arg); + } + let result = exec_capture(&mut cmd).context("Failed to run git worktree")?; + let combined = result.combined(); + + let said = if asked_for_report { + combined.trim() + } else { + "" + }; + // Track what RTK prints, not what git said: on success that is the report or "ok". + let msg = if !result.success() { + combined.as_str() + } else if said.is_empty() { + "ok" + } else { + said + }; + + timer.track( + &format!("git worktree {}", args.join(" ")), + &format!("rtk git worktree {}", args.join(" ")), + &combined, + msg, + ); + + if result.success() { + if said.is_empty() { + println!("ok"); + } else { + println!("{said}"); + } + } else { + eprintln!("FAILED: git worktree {}", args.join(" ")); + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + return Ok(result.exit_code); + } + return Ok(0); + } + + // Default: list mode + let mut cmd = git_cmd(global_args); + cmd.args(["worktree", "list"]); + let result = exec_capture(&mut cmd).context("Failed to run git worktree list")?; + + if !result.success() { + if !result.stderr.trim().is_empty() { + eprintln!("{}", result.stderr); + } + timer.track( + "git worktree list", + "rtk git worktree", + &result.stdout, + &result.stderr, + ); + return Ok(result.exit_code); + } + + let filtered = filter_worktree_list(&result.stdout); + let filtered = never_worse(&result.stdout, &filtered).to_string(); + println!("{}", filtered); + timer.track( + "git worktree list", + "rtk git worktree", + &result.stdout, + &filtered, + ); + + Ok(0) +} + +fn filter_worktree_list(output: &str) -> String { + let home = dirs::home_dir() + .map(|h| h.to_string_lossy().to_string()) + .unwrap_or_default(); + + let mut result = Vec::new(); + for line in output.lines() { + if line.trim().is_empty() { + continue; + } + // Format: "/path/to/worktree abc1234 [branch]" + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() >= 3 { + let mut path = parts[0].to_string(); + if !home.is_empty() && path.starts_with(&home) { + path = format!("~{}", &path[home.len()..]); + } + let hash = parts[1]; + let branch = parts[2..].join(" "); + result.push(format!("{} {} {}", path, hash, branch)); + } else { + result.push(line.to_string()); + } + } + result.join("\n") +} + +/// Runs an unsupported git subcommand by passing it through directly +pub fn run_passthrough(args: &[OsString], global_args: &[String], verbose: u8) -> Result { + let timer = tracking::TimedExecution::start(); + + if verbose > 0 { + eprintln!("git passthrough: {:?}", args); + } + let status = git_cmd(global_args) + .args(args) + .status() + .context("Failed to run git")?; + + let args_str = tracking::args_display(args); + timer.track_passthrough( + &format!("git {}", args_str), + &format!("rtk git {} (passthrough)", args_str), + ); + + if !status.success() { + return Ok(exit_code_from_status(&status, "git")); + } + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_branch_dash_u_links_its_upstream_value_not_a_free_positional() { + // -u's short form must link its value like its long form --set-upstream-to does. + let args = vec!["-u".to_string(), "origin/main".to_string()]; + let tokens = arg_tokenizer::tokenize_grammar(&args, &branch_takes_value, Dialect::Posix); + assert_eq!(tokens[0].kind, TokenKind::Short); + assert_eq!(tokens[0].text, "u"); + assert_eq!(tokens[0].value(&tokens), Some("origin/main")); + assert_eq!(tokens[1].kind, TokenKind::Positional); + assert!( + tokens[1].linked.is_some(), + "origin/main must be linked as -u's value, not a free-standing positional" + ); + } + + #[test] + fn test_checkout_new_branch_arg_accepts_glued_short_flag() { + // `-bmy-branch` (glued) and `-b my-branch` (separate) must both work. + let args = vec!["-bmy-branch".to_string()]; + let tokens = arg_tokenizer::tokenize_grammar(&args, &checkout_takes_value, Dialect::Posix); + assert_eq!(checkout_new_branch_arg(&tokens), Some("my-branch")); + + let args = vec!["-b".to_string(), "my-branch".to_string()]; + let tokens = arg_tokenizer::tokenize_grammar(&args, &checkout_takes_value, Dialect::Posix); + assert_eq!(checkout_new_branch_arg(&tokens), Some("my-branch")); + } + + #[test] + fn test_checkout_reset_branch_arg_accepts_glued_short_flag() { + // Same glued-form guarantee as -b, for -B (force-create/reset). + let args = vec!["-Bmy-branch".to_string()]; + let tokens = arg_tokenizer::tokenize_grammar(&args, &checkout_takes_value, Dialect::Posix); + assert_eq!(checkout_reset_branch_arg(&tokens), Some("my-branch")); + } + + #[test] + fn test_git_cmd_no_global_args() { + let cmd = git_cmd(&[]); + let program = cmd.get_program().to_string_lossy().to_string(); + // On Windows, resolved_command returns full path (e.g. "C:\Program Files\Git\bin\git.exe") + let basename = std::path::Path::new(&program) + .file_stem() + .unwrap() + .to_string_lossy() + .to_string(); + assert_eq!(basename, "git"); + let args: Vec<_> = cmd.get_args().collect(); + assert!(args.is_empty()); + } + + #[test] + fn test_git_cmd_with_directory() { + let global_args = vec!["-C".to_string(), "/tmp".to_string()]; + let cmd = git_cmd(&global_args); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!(args, vec!["-C", "/tmp"]); + } + + #[test] + fn test_git_cmd_with_multiple_global_args() { + let global_args = vec![ + "-C".to_string(), + "/tmp".to_string(), + "-c".to_string(), + "user.name=test".to_string(), + "--git-dir".to_string(), + "/foo/.git".to_string(), + ]; + let cmd = git_cmd(&global_args); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!( + args, + vec![ + "-C", + "/tmp", + "-c", + "user.name=test", + "--git-dir", + "/foo/.git" + ] + ); + } + + #[test] + fn test_git_cmd_with_boolean_flags() { + let global_args = vec!["--no-pager".to_string(), "--bare".to_string()]; + let cmd = git_cmd(&global_args); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!(args, vec!["--no-pager", "--bare"]); + } + + #[test] + fn test_git_cmd_c_locale_sets_stable_env() { + let cmd = git_cmd_c_locale(&[]); + let envs: Vec<_> = cmd + .get_envs() + .map(|(key, value)| { + ( + key.to_string_lossy().to_string(), + value.expect("env value").to_string_lossy().to_string(), + ) + }) + .collect(); + assert!(envs.contains(&("LC_ALL".to_string(), "C".to_string()))); + } + + #[test] + fn test_build_status_command_default_compact() { + let cmd = build_status_command(&[], &[]); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!(args, vec!["status", "--porcelain", "-b"]); + } + + #[test] + fn test_uses_compact_status_path_for_branch_and_short_flags() { + assert!(uses_compact_status_path(&["-b".to_string()])); + assert!(uses_compact_status_path(&["--branch".to_string()])); + assert!(uses_compact_status_path(&["-sb".to_string()])); + assert!(uses_compact_status_path(&[ + "-s".to_string(), + "-b".to_string() + ])); + assert!(uses_compact_status_path(&[ + "--short".to_string(), + "--branch".to_string() + ])); + assert!(!uses_compact_status_path(&["-s".to_string()])); + assert!(!uses_compact_status_path(&["--short".to_string()])); + assert!(!uses_compact_status_path(&["--porcelain".to_string()])); + assert!(!uses_compact_status_path(&["-uno".to_string()])); + } + + #[test] + fn test_build_status_command_with_user_args_passthrough() { + let args = vec!["--short".to_string(), "--branch".to_string()]; + let cmd = build_status_command(&args, &[]); + let cmd_args: Vec<_> = cmd.get_args().collect(); + assert_eq!(cmd_args, vec!["status", "--porcelain", "-b"]); + } + + #[test] + fn test_build_status_command_with_incompatible_user_args_passthrough() { + let args = vec!["--porcelain".to_string(), "-uno".to_string()]; + let cmd = build_status_command(&args, &[]); + let cmd_args: Vec<_> = cmd.get_args().collect(); + assert_eq!(cmd_args, vec!["status", "--porcelain", "-uno"]); + } + + #[test] + fn test_run_status_compact_propagates_non_repo_failure() { + // #2497: a `git status` failure other than "not a git repository" + // (here: a corrupt index) must propagate a non-zero exit, not be + // flattened into "Clean working tree" + exit 0. + let dir = tempfile::tempdir().expect("tempdir"); + let p = dir.path().to_string_lossy().into_owned(); + assert!( + Command::new("git") + .args(["-C", &p, "init", "-q"]) + .status() + .expect("git init") + .success(), + "git init should succeed" + ); + std::fs::write(dir.path().join(".git/index"), "corrupt-index").expect("corrupt index"); + let global = vec!["-C".to_string(), p]; + let code = run_status(&[], 0, &global).expect("run_status"); + assert_ne!( + code, 0, + "corrupt-index git status must not be reported as success" + ); + } + + #[test] + fn test_compact_diff() { + let diff = r#"diff --git a/foo.rs b/foo.rs +--- a/foo.rs ++++ b/foo.rs +@@ -1,3 +1,4 @@ + fn main() { ++ println!("hello"); + } +"#; + let result = compact_diff(diff, 100); + assert!(result.contains("foo.rs")); + assert!(result.contains("+")); + } + + #[test] + fn test_compact_diff_hunk_lines_are_grep_anchorable() { + let diff = "diff --git a/f.txt b/f.txt\n\ + --- a/f.txt\n\ + +++ b/f.txt\n\ + @@ -1,5 +1,4 @@\n\ + \x20keep1\n\ + -DELETED_A\n\ + \x20keep2\n\ + -DELETED_B\n\ + +ADDED\n"; + let result = compact_diff(diff, 100); + + let removed: Vec<&str> = result.lines().filter(|l| l.starts_with('-')).collect(); + let added: Vec<&str> = result.lines().filter(|l| l.starts_with('+')).collect(); + + assert_eq!( + removed, + vec!["-DELETED_A", "-DELETED_B"], + "`^-` must anchor" + ); + assert_eq!(added, vec!["+ADDED"], "`^+` must anchor"); + + // rtk's own tally stays indented so these same greps never count it as + // a diff line. Without this, `^+` would pick up the "+1 -2" summary. + assert!(result.contains(" +1 -2"), "tally must stay indented"); + + // Context lines keep git's leading space, so they are not `^-`/`^+`. + // Both are emitted: the one before the first change as well as the one + // between changes. + assert!( + result.lines().any(|l| l == " keep1"), + "leading context must survive, got:\n{}", + result + ); + assert!(result.lines().any(|l| l == " keep2")); + } + + #[test] + fn test_compact_diff_keeps_content_starting_with_plus_or_minus() { + // `---` / `+++` are file headers only before the first `@@`. Inside a + // hunk, `++i;` and `-- sql comment` are content and must be neither + // dropped from the body nor missing from the tally. + let diff = "diff --git a/f.sql b/f.sql\n\ + --- a/f.sql\n\ + +++ b/f.sql\n\ + @@ -1,2 +1,2 @@\n\ + --- sql comment\n\ + +++i;\n"; + let result = compact_diff(diff, 100); + + assert!( + result.lines().any(|l| l == "--- sql comment"), + "deleted SQL comment must survive, got:\n{}", + result + ); + assert!( + result.lines().any(|l| l == "+++i;"), + "added `++i;` must survive, got:\n{}", + result + ); + assert!( + result.contains(" +1 -1"), + "tally must count both, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_leading_context_has_its_own_budget() { + // Leading context must not consume the 100-line change budget: a hunk + // opening with more context than the budget still shows every change. + let mut diff = String::from("diff --git a/f.rs b/f.rs\n@@ -1,120 +1,120 @@\n"); + for i in 0..20 { + diff.push_str(&format!(" ctx{}\n", i)); + } + for i in 0..100 { + diff.push_str(&format!("-del{}\n", i)); + } + let result = compact_diff(&diff, 1000); + + let ctx = result.lines().filter(|l| l.starts_with(" ctx")).count(); + let dels = result.lines().filter(|l| l.starts_with("-del")).count(); + assert_eq!(ctx, 3, "leading context is capped, got:\n{}", result); + assert_eq!( + dels, 100, + "every change must still be shown, got:\n{}", + result + ); + assert!( + !result.contains("truncated"), + "no change was dropped, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_combined_diff_headers_are_not_hunk_content() { + // `diff --cc` sections do not match `diff --git`, so the reset fires on + // the `diff --` prefix and the `---` / `+++` headers of every section + // stay outside the hunk body. + let diff = "diff --cc a.txt\n\ + index ba2906d,e45c9c2..0000000\n\ + --- a/a.txt\n\ + +++ b/a.txt\n\ + @@@ -1,1 -1,1 +1,5 @@@\n\ + ++<<<<<<< HEAD\n\ + \x20+main\n\ + ++=======\n\ + + side\n\ + ++>>>>>>> side\n\ + diff --cc z.txt\n\ + index ba2906d,e45c9c2..0000000\n\ + --- a/z.txt\n\ + +++ b/z.txt\n\ + @@@ -1,1 -1,1 +1,5 @@@\n\ + ++<<<<<<< HEAD\n\ + \x20+main\n\ + ++=======\n\ + + side\n\ + ++>>>>>>> side\n"; + let result = compact_diff(diff, 500); + + assert!( + !result.lines().any(|l| l.starts_with("+++ b/")), + "file headers must not reach the hunk body, got:\n{}", + result + ); + assert!(result.contains("z.txt"), "got:\n{}", result); + // A combined diff carries one marker column per parent, so ` +main` is + // an addition against the second parent. The tally counts all five + // added lines per file; an anchored `^+` sees only the four whose + // marker sits in column 1. That gap is documented in FEATURES.md. + assert_eq!( + result.matches(" +5 -0").count(), + 2, + "column-2 markers must be counted, got:\n{}", + result + ); + let anchored = result.lines().filter(|l| l.starts_with('+')).count(); + assert_eq!(anchored, 8, "four per file anchor, got:\n{}", result); + } + + #[test] + fn test_compact_diff_mbox_signature_is_not_a_deletion() { + // `gh pr diff --patch` yields an mbox: a bare `---` before the diffstat + // and a `-- ` signature after each patch, both at column 0. The hunk + // ends where its declared line counts run out, so neither is read as + // hunk content. + let diff = "From abc Mon Sep 17 00:00:00 2001\n\ + Subject: [PATCH 1/2] one\n\ + \n\ + ---\n\ + \x20f.txt | 2 +-\n\ + \n\ + diff --git a/f.txt b/f.txt\n\ + --- a/f.txt\n\ + +++ b/f.txt\n\ + @@ -1,2 +1,2 @@\n\ + -old1\n\ + +new1\n\ + \x20tail1\n\ + -- \n\ + 2.40.0\n\ + \n\ + From def Mon Sep 17 00:00:00 2001\n\ + Subject: [PATCH 2/2] two\n\ + \n\ + ---\n\ + diff --git a/g.txt b/g.txt\n\ + --- a/g.txt\n\ + +++ b/g.txt\n\ + @@ -1,2 +1,2 @@\n\ + -old2\n\ + +new2\n\ + \x20tail2\n\ + -- \n\ + 2.40.0\n"; + let result = compact_diff(diff, 500); + + let removed: Vec<&str> = result.lines().filter(|l| l.starts_with('-')).collect(); + assert_eq!( + removed, + vec!["-old1", "-old2"], + "only real deletions anchor, got:\n{}", + result + ); + assert!( + !result.contains("Subject:"), + "mbox envelope must stay out of the body, got:\n{}", + result + ); + assert!( + result.contains(" +1 -1"), + "tally counts real changes only, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_leading_context_is_adjacent_to_the_change() { + // With `-U10` the first context lines sit ten lines above the change. + // Emitting those would tell the reader that ctx3 precedes the deletion + // when ctx10 does. + let mut diff = String::from("diff --git a/f.rs b/f.rs\n@@ -1,11 +1,11 @@\n"); + for i in 1..=10 { + diff.push_str(&format!(" ctx{}\n", i)); + } + diff.push_str("-old\n+new\n"); + let result = compact_diff(&diff, 500); + + let ctx: Vec<&str> = result.lines().filter(|l| l.starts_with(" ctx")).collect(); + assert_eq!( + ctx, + vec![" ctx8", " ctx9", " ctx10"], + "the last context lines, not the first, got:\n{}", + result + ); + } + + #[test] + fn test_diff_header_path_keeps_spaces() { + assert_eq!( + diff_header_path("diff --git a/my file.txt b/my file.txt"), + "my file.txt" + ); + assert_eq!(diff_header_path("diff --cc my file.txt"), "my file.txt"); + assert_eq!( + diff_header_path("diff --combined my file.txt"), + "my file.txt" + ); + } + + #[test] + fn test_diff_header_path_handles_gits_quoted_paths() { + // Under the default `core.quotepath`, git escapes a non-ASCII path and + // wraps it in quotes, which removes the ` b/` separator the plain form + // is split on. Without the quoted form handled, the fallback returned + // the whole remainder — both paths — as the section header. + assert_eq!( + diff_header_path(r#"diff --git "a/été.txt" "b/été.txt""#), + r"été.txt" + ); + assert_eq!(diff_header_path(r#"diff --cc "été.txt""#), r"été.txt"); + // A rename quotes each side on its own. + assert_eq!( + diff_header_path(r#"diff --git a/plain.txt "b/ét.txt""#), + r"ét.txt" + ); + assert_eq!( + diff_header_path(r#"diff --git "a/ét.txt" b/plain.txt"#), + "plain.txt" + ); + } + + #[test] + fn test_diff_header_path_unescapes_gits_default_quoting() { + // What git actually emits under the default `core.quotepath`: one octal + // escape per byte, so the header has to be decoded rather than merely + // unwrapped, or `rtk git diff | grep été` finds nothing. + assert_eq!( + diff_header_path(r#"diff --git "a/\303\251t\303\251.txt" "b/\303\251t\303\251.txt""#), + "été.txt" + ); + assert_eq!( + diff_header_path(r#"diff --cc "\303\251t\303\251.txt""#), + "été.txt" + ); + assert_eq!( + diff_header_path(r#"diff --git a/plain.txt "b/\303\251t.txt""#), + "ét.txt" + ); + // The single-character escapes, and a backslash standing for itself. + assert_eq!( + diff_header_path(r#"diff --cc "tab\there.txt""#), + "tab\there.txt" + ); + assert_eq!( + diff_header_path(r#"diff --cc "quote\"here.txt""#), + "quote\"here.txt" + ); + assert_eq!( + diff_header_path(r#"diff --cc "back\\slash.txt""#), + r"back\slash.txt" + ); + } + + fn word_diff_from(args: &[&str]) -> bool { + let args: Vec = args.iter().map(|a| a.to_string()).collect(); + emits_word_diff(&tokenize_git_log_args(&args)) + } + + #[test] + fn test_emits_word_diff_detects_every_form() { + for flag in [ + "--word-diff", + "--word-diff=plain", + "--word-diff=porcelain", + "--word-diff-regex=.", + "--color-words", + "--color-words=.", + ] { + assert!(word_diff_from(&[flag]), "{} must pass through", flag); + } + assert!(!word_diff_from(&["--stat"])); + assert!(!word_diff_from(&["-U10"])); + assert!(!word_diff_from(&[])); + } + + #[test] + fn test_emits_word_diff_honours_the_none_mode() { + // `--word-diff=none` leaves an ordinary unified diff, which compacts + // like any other. Treating it as a word diff passed the whole raw diff + // through, so a defensive `--word-diff=none` lost every saving. + assert!(!word_diff_from(&["--word-diff=none"])); + // Modes are last-one-wins, which is what `none` exists to do. + assert!(!word_diff_from(&["--word-diff", "--word-diff=none"])); + assert!(word_diff_from(&["--word-diff=none", "--word-diff"])); + // `--color-words` takes a regex, so `none` there is a pattern. + assert!(word_diff_from(&["--color-words=none"])); + } + + #[test] + fn test_emits_word_diff_ignores_a_consumed_or_pathspec_word_diff() { + // Real git 2.53.0: `git diff --author --word-diff` emits an ordinary line diff (the + // flag is `--author`'s value), and `git diff -- --word-diff` treats it as a pathspec. + assert!(!word_diff_from(&["--author", "--word-diff"])); + // `-M` takes an attached value only, so it consumes nothing and git does word-diff: + // reading this as `-M`'s value would hand compact_diff a word diff to mangle. + assert!(word_diff_from(&["-M", "--word-diff"])); + assert!(!word_diff_from(&["--", "--word-diff"])); + // `--word-diff-regex` still requests one when its own value is flag-shaped. + assert!(word_diff_from(&["--word-diff-regex", "--stat"])); + } + + #[test] + fn test_parse_hunk_header_reconciles_ranges_with_marker_columns() { + // A well-formed header lists one range per marker column. When it does + // not, only the columns can be charged. A missing range must not leave + // an untracked parent holding the hunk open forever, and an extra one + // must not sit at its declared count with no column to spend it. + let h = parse_hunk_header("@@@ -1 +1 @@@").expect("two columns, one range"); + assert_eq!(h.prefix_width, 2); + assert_eq!(h.parents, vec![1, usize::MAX]); + + let h = parse_hunk_header("@@@ -1 -1 -1 +0,0 @@@").expect("two columns, three ranges"); + assert_eq!(h.prefix_width, 2); + assert_eq!(h.parents, vec![1, 1]); + } + + #[test] + fn test_compact_diff_extra_range_does_not_strand_a_hunk() { + // With the third range untracked, `--x` left it at 1 forever, so the + // hunk never closed and the mbox signature became its content. + let out = compact_diff( + "diff --cc f\n@@@ -1 -1 -1 +0,0 @@@\n--x\n-- \n2.40.0\n", + 100, + ); + assert!(out.contains("--x"), "got:\n{}", out); + assert!(!out.contains("2.40.0"), "got:\n{}", out); + assert!(!out.contains("-- "), "got:\n{}", out); + // One line, removed from both parents, is one deletion. + assert!(out.contains("+0 -1"), "got:\n{}", out); + } + + #[test] + fn test_compact_diff_missing_range_keeps_the_body() { + // One range for two columns: the untracked parent gets `usize::MAX`, so + // the hunk stays open to the next header rather than closing early and + // dropping ` -lost`. + let out = compact_diff("diff --cc f\n@@@ -1 +1 @@@\n +kept\n -lost\n", 100); + assert!(out.contains(" +kept"), "got:\n{}", out); + assert!(out.contains(" -lost"), "got:\n{}", out); + assert!(out.contains("+1 -1"), "got:\n{}", out); + } + + #[test] + fn test_diff_header_path_splits_the_pair_at_its_midpoint() { + // A file under a directory named `x b` puts the ` b/` separator inside + // the path, so the first match is the wrong one. + assert_eq!(diff_header_path("diff --git a/x b/y b/x b/y"), "x b/y"); + // `--no-prefix` and custom prefixes leave no ` b/` at all. + assert_eq!(diff_header_path("diff --git x x"), "x"); + assert_eq!( + diff_header_path("diff --git src/main.rs src/main.rs"), + "src/main.rs" + ); + // Prefixes are matched against each other, not by name, so a custom + // `--dst-prefix` reads like any other pair. + assert_eq!(diff_header_path("diff --git a/f.txt w/f.txt"), "f.txt"); + assert_eq!(diff_header_path("diff --git i/f.txt w/f.txt"), "f.txt"); + // A rename's halves disagree past their first component, so the ` b/` + // split still names the destination. + assert_eq!( + diff_header_path("diff --git a/old.txt b/new.txt"), + "new.txt" + ); + } + + #[test] + fn test_diff_header_path_does_not_split_single_path_headers() { + // `diff --cc` names one path. Splitting its remainder at the midpoint + // would read a file called `dup dup` as the file `dup` named twice. + assert_eq!(diff_header_path("diff --cc dup dup"), "dup dup"); + assert_eq!(diff_header_path("diff --combined dup dup"), "dup dup"); + assert_eq!(diff_header_path("diff --cc a/x b/x"), "a/x b/x"); + } + + #[test] + fn test_parse_hunk_header_counts() { + let h = parse_hunk_header("@@ -10,3 +10,4 @@ fn ctx() {").expect("unified header"); + assert_eq!( + (h.parents.as_slice(), h.new, h.prefix_width), + (&[3][..], 4, 1) + ); + + // Omitted counts mean one line. + let h = parse_hunk_header("@@ -1 +1 @@").expect("single-line header"); + assert_eq!((h.parents.as_slice(), h.new), (&[1][..], 1)); + + // A combined header lists one range per parent, in marker-column order. + // Every one of them bounds the hunk body. + let h = parse_hunk_header("@@@ -1,1 -1,4 +1,5 @@@").expect("combined header"); + assert_eq!( + (h.parents.as_slice(), h.new, h.prefix_width), + (&[1, 4][..], 5, 2) + ); + + assert!(parse_hunk_header("@ -1,1 +1,1 @").is_none()); + assert!(parse_hunk_header("-- ").is_none()); + assert!(parse_hunk_header("---").is_none()); + } + + #[test] + fn test_compact_diff_non_ascii_body_line_without_a_marker_does_not_panic() { + // `--word-diff` / `--color-words` emit body lines with no marker column, + // so content lands where the markers are sliced. Slicing by byte index + // split a leading multi-byte character and aborted the process. + let out = compact_diff( + "diff --git a/f.txt b/f.txt\n@@ -1,3 +1,3 @@\n-old\n+new\nécole ancienne ligne\n", + 100, + ); + assert!(out.contains("-old"), "got:\n{}", out); + assert!(out.contains("+new"), "got:\n{}", out); + assert!(out.contains("école ancienne ligne"), "got:\n{}", out); + } + + #[test] + fn test_compact_diff_combined_hunk_ends_at_its_declared_length() { + // Every parent's declared range bounds the body. Charging only the + // first parent left `old` unable to converge on real conflict output, + // so the hunk never closed by count and the mbox / signature / prose + // guard did not apply to combined sections at all. + let conflict = "diff --cc f.txt\n@@@ -1,1 -1,1 +1,5 @@@\n++<<<<<<<\n +main\n++=======\n+ side\n++>>>>>>>\n-- \ntrailing signature\n"; + let out = compact_diff(conflict, 100); + assert!(!out.contains("trailing signature"), "got:\n{}", out); + assert!(!out.contains("-- "), "got:\n{}", out); + assert!(out.contains("+5 -0"), "got:\n{}", out); + } + + #[test] + fn test_compact_diff_combined_hunk_keeps_second_parent_removals() { + // `-1,2 -1,4 +1,2`: two removals spend only the second parent's budget. + // Closing on the first parent and the result alone dropped them with no + // tally and no truncation note — a silent loss. + let out = compact_diff( + "diff --cc f.txt\n@@@ -1,2 -1,4 +1,2 @@@\n a\n b\n -x\n -y\n", + 100, + ); + assert!(out.contains(" -x"), "got:\n{}", out); + assert!(out.contains(" -y"), "got:\n{}", out); + assert!(out.contains("+0 -2"), "got:\n{}", out); + } + + #[test] + fn test_compact_diff_flushes_context_from_a_hunk_with_no_change_line() { + // The buffer drained only on the first change line, so a hunk that ends + // without one rendered as a bare header with nothing under it. + let out = compact_diff( + "diff --git a/g.txt b/g.txt\n@@ -1,3 +1,3 @@\n ctx1\n ctx2\n ctx3\n", + 100, + ); + assert!(out.contains(" ctx1"), "got:\n{}", out); + assert!(out.contains(" ctx3"), "got:\n{}", out); + } + + #[test] + fn test_compact_diff_leading_context_does_not_displace_change_lines() { + // Leading context is exempt from `max_lines`, so the same number of + // change lines survives whether or not the hunks open with context. + let build = |with_context: bool| { + let mut diff = String::new(); + for f in 0..30 { + diff.push_str(&format!("diff --git a/f{}.rs b/f{}.rs\n", f, f)); + diff.push_str("@@ -1,20 +1,20 @@\n"); + if with_context { + for c in 0..3 { + diff.push_str(&format!(" ctx{}_{}\n", f, c)); + } + } + for i in 0..12 { + diff.push_str(&format!("-del{}_{}\n", f, i)); + } + } + diff + }; + let count_changes = |out: &str| out.lines().filter(|l| l.starts_with("-del")).count(); + + let without = compact_diff(&build(false), 500); + let with = compact_diff(&build(true), 500); + assert_eq!( + count_changes(&with), + count_changes(&without), + "leading context displaced change lines:\n{}", + with + ); + } + + #[test] + fn test_compact_diff_leading_context_is_capped_across_the_diff() { + // The diff-wide cap is what bounds the exemption: without it, a diff of + // many small hunks would spend three exempt lines on each one. + let mut diff = String::new(); + for f in 0..200 { + diff.push_str(&format!("diff --git a/f{}.rs b/f{}.rs\n", f, f)); + diff.push_str("@@ -1,4 +1,4 @@\n"); + for c in 0..3 { + diff.push_str(&format!(" ctx{}_{}\n", f, c)); + } + diff.push_str(&format!("-del{}\n", f)); + } + let result = compact_diff(&diff, 500); + + let ctx = result.lines().filter(|l| l.starts_with(" ctx")).count(); + assert!( + ctx <= 50, + "leading context must stay within max_lines / 10, got {} in:\n{}", + ctx, + result + ); + } + + #[test] + fn test_hunk_truncation_note_counts_one_as_singular() { + assert_eq!( + hunk_truncation_note(1, 0).as_deref(), + Some(" ... (1 deletion truncated)") + ); + assert_eq!( + hunk_truncation_note(0, 1).as_deref(), + Some(" ... (1 addition truncated)") + ); + assert_eq!( + hunk_truncation_note(1, 2).as_deref(), + Some(" ... (1 deletion, 2 additions truncated)") + ); + assert_eq!(hunk_truncation_note(0, 0), None); + } + + #[test] + fn test_compact_diff_truncation_note_splits_by_sign() { + // An anchored `^-` audit needs to know how many deletions it did not + // see, which a merged "N lines truncated" cannot tell it. + let mut diff = String::from("diff --git a/f.rs b/f.rs\n@@ -1,160 +1,160 @@\n"); + for i in 0..80 { + diff.push_str(&format!("-del{}\n", i)); + diff.push_str(&format!("+add{}\n", i)); + } + let result = compact_diff(&diff, 1000); + + assert!( + result.contains(" ... (30 deletions, 30 additions truncated)"), + "expected per-sign truncation note, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_preserves_full_hunk_header_context() { + let diff = r#"diff --git a/foo.rs b/foo.rs +--- a/foo.rs ++++ b/foo.rs +@@ -10,3 +10,4 @@ fn important_context() { + fn main() { ++ println!("hello"); + } +"#; + let result = compact_diff(diff, 100); + assert!( + result.contains("@@ -10,3 +10,4 @@ fn important_context() {"), + "Expected full hunk header with trailing context, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_increased_hunk_limit() { + // Build a hunk with 25 changed lines — should NOT be truncated with limit 30 + let mut diff = + "diff --git a/big.rs b/big.rs\n--- a/big.rs\n+++ b/big.rs\n@@ -1,25 +1,25 @@\n" + .to_string(); + for i in 1..=25 { + diff.push_str(&format!("+line{}\n", i)); + } + let result = compact_diff(&diff, 500); + assert!( + !result.contains("... (truncated)"), + "25 lines should not be truncated with max_hunk_lines=30" + ); + assert!(result.contains("+line25")); + } + + #[test] + fn test_compact_diff_increased_total_limit() { + // Build a diff with 150 output result lines across multiple files — should NOT be cut at 100 + let mut diff = String::new(); + for f in 1..=5 { + diff.push_str(&format!("diff --git a/file{f}.rs b/file{f}.rs\n--- a/file{f}.rs\n+++ b/file{f}.rs\n@@ -1,20 +1,20 @@\n")); + for i in 1..=20 { + diff.push_str(&format!("+line{f}_{i}\n")); + } + } + let result = compact_diff(&diff, 500); + assert!( + !result.contains("more changes truncated"), + "5 files × 20 lines should not exceed max_lines=500" + ); + } + + #[test] + fn test_is_blob_show_arg() { + assert!(is_blob_show_arg("develop:modules/pairs_backtest.py")); + assert!(is_blob_show_arg("HEAD:src/main.rs")); + // Index / merge-stage blobs start with `:` but are still blobs. + assert!(is_blob_show_arg(":Cargo.toml")); + assert!(is_blob_show_arg(":2:conflict.rs")); + assert!(!is_blob_show_arg("--pretty=format:%h")); + assert!(!is_blob_show_arg("--format=short")); + assert!(!is_blob_show_arg("HEAD")); + // `:/text` is a commit-message search, not a blob. + assert!(!is_blob_show_arg(":/fix the bug")); + // Magic pathspecs are pathspecs, never blobs. + assert!(!is_blob_show_arg(":(exclude)b.txt")); + assert!(!is_blob_show_arg(":(top,glob)*.rs")); + assert!(!is_blob_show_arg(":!b.txt")); + // `:^` is an exact synonym of `:!` (exclude magic), also a pathspec. + assert!(!is_blob_show_arg(":^b.txt")); + // A colons-only token is not a blob object. + assert!(!is_blob_show_arg(":")); + assert!(!is_blob_show_arg("::")); + } + + /// Helper: build an args slice from string literals. + fn show_args(v: &[&str]) -> Vec { + v.iter().map(|x| x.to_string()).collect() + } + + #[test] + fn test_show_route_blob_wins_over_stat_and_format() { + // Blocking bug #1: git accepts and silently ignores --stat/--numstat/--pretty/ + // --format for a blob target and still dumps the file, so these must NOT shadow + // the blob path (which would send the blob through lossy UTF-8 decoding). + assert_eq!( + show_route(&show_args(&["--stat", "HEAD:legacy.pck"])), + ShowRoute::Blob + ); + assert_eq!( + show_route(&show_args(&["--numstat", "HEAD:legacy.pck"])), + ShowRoute::Blob + ); + assert_eq!( + show_route(&show_args(&["--format=medium", "HEAD:legacy.pck"])), + ShowRoute::Blob + ); + assert_eq!( + show_route(&show_args(&["--pretty=oneline", "HEAD:legacy.pck"])), + ShowRoute::Blob + ); + } + + #[test] + fn test_show_route_trailing_pathspec_stays_a_blob() { + // Blocking bug #2: a trailing `-- ` beside a blob arg is ignored by git + // (still a blob dump), so it must not disable blob handling. + let args = show_args(&["HEAD:Cargo.toml", "--", "Cargo.toml"]); + assert_eq!(show_route(&args), ShowRoute::Blob); + assert_eq!(blob_candidates(&args), vec![&"HEAD:Cargo.toml".to_string()]); + } + + #[test] + fn test_blob_candidates_skip_flag_operand_with_colon() { + // Blocking bug #2: `-S 'url:1'`'s operand contains a colon but is the value of + // a pickaxe flag, not an object — it must not be read as a blob and truncated. + // The commit is the real (colon-free) object, so this is a commit-diff. + let args = show_args(&["-S", "url:1", "HEAD"]); + assert!(blob_candidates(&args).is_empty()); + assert_eq!(show_route(&args), ShowRoute::CommitDiff); + // `-L :` operand likewise carries a colon. + let args = show_args(&["-L", "1,2:file.rs", "HEAD"]); + assert!(blob_candidates(&args).is_empty()); + // A real blob still routes as a blob even with a preceding value flag. + let args = show_args(&["-S", "needle", "HEAD:src/main.rs"]); + assert_eq!( + blob_candidates(&args), + vec![&"HEAD:src/main.rs".to_string()] + ); + // A value operand that IS a valid-looking blob (`-S 'HEAD:real'`) is the pickaxe + // value, not an object: the walker excludes it, so only the trailing commit + // remains and nothing is offered as a windowing candidate. + let args = show_args(&["-S", "HEAD:real", "HEAD"]); + assert!(blob_candidates(&args).is_empty()); + } + + #[test] + fn test_blob_candidates_short_flag_clusters() { + // BLOCKER: a short-flag cluster whose value-taking tail consumes the NEXT token + // (`-wG x:y` == `-w -G x:y`) must skip `x:y` and expose the real blob object, not + // mistake `x:y` for the blob. Covers the clusters git re-parses. + for cluster in ["-wG", "-pS", "-pI", "-pwG", "-wpG"] { + let args = show_args(&[cluster, "x:y", "HEAD:big.txt"]); + assert_eq!( + blob_candidates(&args), + vec![&"HEAD:big.txt".to_string()], + "cluster {cluster}: x:y is the value flag's operand, HEAD:big.txt the object", + ); + } + // An INLINE cluster value (`-Sfoo` == `-S foo`) does NOT consume the next token, + // so the following object is still exposed. + let args = show_args(&["-Sneedle", "HEAD:big.txt"]); + assert_eq!(blob_candidates(&args), vec![&"HEAD:big.txt".to_string()]); + // `-Gx:y` (value flag NOT last, inline value `x:y`) consumes no next token. + let args = show_args(&["-Gx:y", "HEAD:big.txt"]); + assert_eq!(blob_candidates(&args), vec![&"HEAD:big.txt".to_string()]); + // A boolean-only cluster (`-wp`) consumes nothing: the object stays a candidate. + let args = show_args(&["-wp", "HEAD:big.txt"]); + assert_eq!(blob_candidates(&args), vec![&"HEAD:big.txt".to_string()]); + } + + #[test] + fn test_show_route_colon_pathspec_after_dashdash_is_not_a_blob() { + // A filename containing a colon AFTER `--` is a pathspec, not a blob target. + // This `--` is representative of the real CLI path: clap strips the separator, + // but `run_show` restores it via `restore_double_dash` before calling + // `show_route`/`show_positionals` (verified live: `git show HEAD -- a:b.txt` + // renders a commit-diff, not a blob dump). + let args = show_args(&["HEAD", "--", "weird:name.txt"]); + assert_eq!(show_route(&args), ShowRoute::CommitDiff); + assert!(blob_candidates(&args).is_empty()); + } + + #[test] + fn test_show_route_plain_cases() { + assert_eq!( + show_route(&show_args(&["--stat", "HEAD"])), + ShowRoute::StatOrFormat + ); + assert_eq!(show_route(&show_args(&["HEAD"])), ShowRoute::CommitDiff); + assert_eq!( + show_route(&show_args(&["HEAD:src/main.rs"])), + ShowRoute::Blob + ); + } + + #[test] + fn test_blob_windowing_token_savings() { + // A filter must verify its savings claim with a real fixture. The enforced floor + // is 20% (CONTRIBUTING.md), and `blob_large.txt` (a real `git show + // HEAD:src/main.rs | head -350`, ~10.7 KB) has always cleared it: windowing to + // the 8 KiB head plus a few-token recovery hint is the savings floor. (An earlier + // revision introduced a synthetic ~107 KB fixture to chase a stale 60% figure + // that only `.claude/rules/cli-testing.md` still cites; that churn is reverted.) + fn count_tokens(text: &str) -> usize { + text.split_whitespace().count() + } + let raw = include_str!("../../../tests/fixtures/git/blob_large.txt"); + assert!(raw.len() > MAX_BLOB_BYTES.0); + let (head, _remaining, _offset) = + blob_truncation(raw, MAX_BLOB_BYTES).expect("large blob should window"); + let savings = 100.0 - (count_tokens(head) as f64 / count_tokens(raw) as f64 * 100.0); + assert!( + savings >= 20.0, + "expected ≥20% token savings, got {:.1}%", + savings + ); + } + + #[test] + fn test_blob_truncation_small_passthrough() { + // Real committed file, ~1.7KB < budget: passed through unchanged. + let small = include_str!("../../../Cargo.toml"); + assert!(blob_truncation(small, MAX_BLOB_BYTES).is_none()); + } + + #[test] + fn test_blob_truncation_large_windowed() { + // Real blob fixture (`git show HEAD:src/main.rs | head -350`), ~10.7KB. + let large = include_str!("../../../tests/fixtures/git/blob_large.txt"); + let (head, remaining, offset) = + blob_truncation(large, MAX_BLOB_BYTES).expect("large blob should window"); + assert!(head.len() <= MAX_BLOB_BYTES.0); + assert!(head.ends_with('\n'), "head must end at a line boundary"); + let head_lines = head.lines().count(); + // N formula: remaining == total - head_lines, offset == head_lines + 1. + assert_eq!(offset, head_lines + 1); + assert_eq!(remaining, large.lines().count() - head_lines); + assert!(remaining > 0); + } + + #[test] + fn test_blob_truncation_tree_passthrough() { + // Real tree listing (`git show HEAD:src`): `tree :\n\n...`. In the live + // path the `git cat-file -t` probe classifies a tree as non-blob before this + // function is ever reached, so the old `starts_with("tree ")` content-sniff was + // removed; this small (87 B) listing still declines here via the size check. + let tree = include_str!("../../../tests/fixtures/git/tree_listing.txt"); + assert!(tree.starts_with("tree ")); + assert!(blob_truncation(tree, MAX_BLOB_BYTES).is_none()); + } + + #[test] + fn test_blob_truncation_binary_passthrough() { + let mut binary = "x".repeat(9000); + binary.push('\0'); + assert!(blob_truncation(&binary, MAX_BLOB_BYTES).is_none()); + // Replacement char from lossy UTF-8 decoding of a binary blob. + let lossy = format!("{}\u{FFFD}", "y".repeat(9000)); + assert!(blob_truncation(&lossy, MAX_BLOB_BYTES).is_none()); + } + + #[test] + fn test_blob_truncation_giant_single_line_passthrough() { + // A single line longer than the budget has no line boundary to cut at. + let giant = "a".repeat(20_000); + assert!(blob_truncation(&giant, MAX_BLOB_BYTES).is_none()); + } + + #[test] + fn test_blob_truncation_utf8_boundary_no_panic() { + // Multibyte content so the byte budget can land mid-codepoint: must not panic. + let mut s = String::new(); + while s.len() < 9000 { + s.push_str("áéíóú-ñ-日本語"); + s.push('\n'); + } + // Exercise budgets straddling a multibyte char around the window edge. + let _ = blob_truncation(&s, MAX_BLOB_BYTES); + let _ = blob_truncation(&s, Budget(8191)); + let _ = blob_truncation(&s, Budget(8193)); + // Should still window (multi-line, over budget) without crashing. + assert!(blob_truncation(&s, MAX_BLOB_BYTES).is_some()); + } + + #[test] + fn test_blob_truncation_no_trailing_newline() { + let mut s = String::new(); + for i in 0..500 { + s.push_str(&format!("line number {i} with a bit of content here\n")); + } + s.pop(); // drop the final newline + let (head, remaining, offset) = blob_truncation(&s, MAX_BLOB_BYTES).expect("should window"); + assert_eq!(offset, head.lines().count() + 1); + assert_eq!(remaining, s.lines().count() - head.lines().count()); + } + + #[test] + fn test_compact_blob_show_hint_is_tee_independent() { + // The recovery hint re-derives the tail from the blob arg itself — no tee file, + // so it works at any size (N2) and quotes a path with a space for safe paste. + let mut s = String::new(); + for i in 0..2000 { + s.push_str(&format!( + "line {i} with enough content to exceed the byte budget\n" + )); + } + let offset = blob_truncation(&s, MAX_BLOB_BYTES) + .expect("should window") + .2; + let out = compact_blob_show(&s, "HEAD:my dir/big.lock", &[]); + assert!(out.len() < s.len(), "windowing must shrink the output"); + let expected = format!( + "[see remaining: rtk proxy git show 'HEAD:my dir/big.lock' | tail -n +{offset}]" + ); + assert!(out.contains(&expected), "hint missing/unquoted: {out:?}"); + assert!(!out.contains("tail -n +0")); + + // A path containing a single quote must be escaped `'\''` so the hint stays + // copy-paste safe (the dangerous branch of `shell_single_quote`). + let out_q = compact_blob_show(&s, "HEAD:it's/a.lock", &[]); + let expected_q = format!( + "[see remaining: rtk proxy git show 'HEAD:it'\\''s/a.lock' | tail -n +{offset}]" + ); + assert!( + out_q.contains(&expected_q), + "single-quote path unescaped: {out_q:?}" + ); + } + + #[test] + fn test_compact_blob_show_hint_survives_rtks_own_hook() { + // The hook rewrites a bare `git ...` into `rtk git ...`, which windows the dump a + // second time -- `tail` then has only the hint left to print. The hint must name a + // command the hook leaves alone. + let mut s = String::new(); + for i in 0..2000 { + s.push_str(&format!( + "line {i} with enough content to exceed the byte budget\n" + )); + } + let out = compact_blob_show(&s, "HEAD:big.lock", &[]); + let hint = out + .lines() + .next_back() + .and_then(|l| l.split_once("[see remaining: ")) + .map(|(_, rest)| rest.trim_end_matches(']')) + .expect("hint"); + assert!( + hint.starts_with("rtk proxy git "), + "hint must go through proxy: {hint:?}" + ); + assert_eq!( + crate::discover::registry::rewrite_command(hint, &[], &[]), + None, + "the hook must leave the hint alone: {hint:?}" + ); + } + + #[test] + fn test_compact_blob_show_small_passes_through() { + // Below the byte budget: returned unchanged, no hint. + let small = "a few\nshort\nlines\n"; + assert_eq!(compact_blob_show(small, "HEAD:x.txt", &[]), small); + } + + #[test] + fn test_compact_blob_show_hint_carries_global_args() { + // `rtk git -C /repo -c core.x=y show HEAD:big` must produce a hint that is + // runnable from OUTSIDE the repo: the global args go between `git` and `show`, + // each shell-quoted, so the recovery command targets the same repo. + let mut s = String::new(); + for i in 0..2000 { + s.push_str(&format!( + "line {i} with enough content to exceed the byte budget\n" + )); + } + let offset = blob_truncation(&s, MAX_BLOB_BYTES) + .expect("should window") + .2; + let globals = vec![ + "-C".to_string(), + "/tmp/my repo".to_string(), + "-c".to_string(), + "core.autocrlf=false".to_string(), + ]; + let out = compact_blob_show(&s, "HEAD:big.lock", &globals); + // Every token is shell-quoted (same policy as the blob arg) — quoting a flag + // like `-C` is a harmless no-op and keeps the hint copy-paste safe. + let expected = format!( + "[see remaining: rtk proxy git '-C' '/tmp/my repo' '-c' 'core.autocrlf=false' show 'HEAD:big.lock' | tail -n +{offset}]" + ); + assert!(out.contains(&expected), "global-args hint wrong: {out:?}"); + } + + #[test] + // `from_utf8` on a `include_bytes!` literal is exactly the point here (asserting the + // fixture is NOT valid UTF-8, matching `run_show`'s decision predicate), so silence + // clippy's "literal always errors" lint rather than obscure the intent. + #[allow(invalid_from_utf8)] + fn test_blob_latin1_fixture_passes_through() { + // Real ISO-8859-1 slice of an Oracle PL/SQL `.pck` (14 KB, contains "MÉTODO", + // NOT valid UTF-8). Earlier revisions transcoded it to UTF-8 and windowed it, + // but the head shown was then no longer a byte-exact prefix of git's bytes, so + // `git show 'rev:path' | tail -n +N` could not reconstruct the original. Under + // the fidelity invariant, `run_show` windows ONLY content that is valid UTF-8 + // byte-for-byte; anything that would need transcoding passes through unchanged. + // + // The decision predicate is `std::str::from_utf8(git_bytes).is_ok()`, so assert + // the fixture is NOT valid UTF-8 — which is exactly why it takes the byte-exact + // passthrough path rather than being windowed. + let bytes = include_bytes!("../../../tests/fixtures/git/latin1_blob.pck"); + assert!(bytes.len() > MAX_BLOB_BYTES.0); + assert!( + std::str::from_utf8(bytes).is_err(), + "a Latin-1 .pck is not valid UTF-8, so it must pass through byte-identically \ + (exact recovery is impossible once transcoded)" + ); + } + + #[test] + fn test_filter_branch_output() { + let output = "* main\n feature/auth\n fix/bug-123\n remotes/origin/HEAD -> origin/main\n remotes/origin/main\n remotes/origin/feature/auth\n remotes/origin/release/v2\n"; + let result = filter_branch_output(output); + assert!(result.contains("* main")); + assert!(result.contains("feature/auth")); + assert!(result.contains("fix/bug-123")); + // remote-only should show release/v2 but not main or feature/auth (already local) + assert!(result.contains("remote-only")); + assert!(result.contains("release/v2")); + } + + #[test] + fn test_filter_branch_no_remotes() { + let output = "* main\n develop\n"; + let result = filter_branch_output(output); + assert!(result.contains("* main")); + assert!(result.contains("develop")); + assert!(!result.contains("remote-only")); + } + + #[test] + fn test_filter_branch_multi_remote() { + let output = "* main\n develop\n remotes/origin/HEAD -> origin/main\n remotes/origin/main\n remotes/origin/feature-x\n remotes/upstream/main\n remotes/upstream/release-v3\n remotes/fork/main\n remotes/fork/experiment\n"; + let result = filter_branch_output(output); + assert!(result.contains("* main")); + assert!(result.contains("develop")); + assert!( + result.contains("feature-x"), + "origin branch shown: {}", + result + ); + assert!( + result.contains("release-v3"), + "upstream branch shown: {}", + result + ); + assert!( + result.contains("experiment"), + "fork branch shown: {}", + result + ); + assert!( + !result.contains("remotes/"), + "remote prefix stripped: {}", + result + ); + let main_count = result.matches("main").count(); + assert!( + main_count <= 2, + "main deduplicated across remotes (found {} occurrences): {}", + main_count, + result + ); + } + + #[test] + fn test_filter_stash_list() { + let output = + "stash@{0}: WIP on main: abc1234 fix login\nstash@{1}: On feature: def5678 wip\n"; + let result = filter_stash_list(output); + assert!(result.contains("stash@{0}: abc1234 fix login")); + assert!(result.contains("stash@{1}: def5678 wip")); + } + + #[test] + fn test_parse_stash_stat_strips_decorations() { + let raw = " del.md | 2 --\n keep.md | 5 ++++-\n logo.bin | Bin 0 -> 1024 bytes\n \ + new.rs | 40 ++++++++\n 4 files changed, 44 insertions(+), 3 deletions(-)\n"; + let (files, summary) = parse_stash_stat(raw); + assert_eq!( + files, + vec![ + "del.md 2 -", + "keep.md 5 +-", + "logo.bin (binary)", + "new.rs 40 +" + ] + ); + assert_eq!(summary, "4 files changed, 44 insertions(+), 3 deletions(-)"); + } + + #[test] + fn test_parse_stash_stat_collapsed_bar() { + let (files, _) = parse_stash_stat(" .claude/CLAUDE.md | 234 +-\n"); + assert_eq!(files, vec![".claude/CLAUDE.md 234 +-"]); + } + + #[test] + fn test_compact_stash_stat_passthrough_numstat() { + let raw = "0\t1\tdel.md\n3\t2\tkeep.md\n1\t0\tn1.rs\n"; + assert_eq!( + compact_stash_stat(raw), + "0\t1\tdel.md\n3\t2\tkeep.md\n1\t0\tn1.rs" + ); + } + + #[test] + fn test_compact_stash_stat_passthrough_name_only() { + let raw = "del.md\nkeep.md\nn1.rs\n"; + assert_eq!(compact_stash_stat(raw), "del.md\nkeep.md\nn1.rs"); + } + + #[test] + fn test_compress_stat_summary_variants() { + assert_eq!( + compress_stat_summary("4 files changed, 60 insertions(+), 313 deletions(-)"), + "4 changed 60 + 313 -" + ); + assert_eq!( + compress_stat_summary("1 file changed, 1 insertion(+)"), + "1 changed 1 +" + ); + assert_eq!( + compress_stat_summary("1 file changed, 1 deletion(-)"), + "1 changed 1 -" + ); + assert_eq!( + compress_stat_summary("2 files changed, 4 insertions(+), 1 deletion(-)"), + "2 changed 4 + 1 -" + ); + } + + #[test] + fn test_compact_stash_stat_compresses_summary() { + let raw = " a.txt | 2 ++\n 1 file changed, 2 insertions(+)\n"; + assert_eq!(compact_stash_stat(raw), "a.txt 2 +\n1 changed 2 +"); + } + + #[test] + fn test_parse_stash_stat_last_pipe_is_separator() { + let (files, _) = parse_stash_stat(" weird|name.txt | 3 +++\n"); + assert_eq!(files, vec!["weird|name.txt 3 +"]); + } + + #[test] + fn test_parse_stash_stat_strips_ansi() { + let (files, _) = parse_stash_stat(" a.txt | 2 \x1b[32m++\x1b[m\n"); + assert_eq!(files, vec!["a.txt 2 +"]); + } + + #[test] + fn test_parse_stash_stat_empty() { + let (files, summary) = parse_stash_stat(""); + assert!(files.is_empty()); + assert!(summary.is_empty()); + } + + #[test] + fn test_parse_stash_stat_unicode_and_malformed_never_panic() { + let _ = parse_stash_stat("not a diffstat at all"); + let _ = parse_stash_stat("| | |"); + let (files, _) = parse_stash_stat(" 日本語.md | 5 +++--\n"); + assert_eq!(files, vec!["日本語.md 5 +-"]); + } + + #[test] + fn test_parse_stash_stat_savings() { + use crate::core::tracking::estimate_tokens; + let raw = " CONTRIBUTING.md | 305 \ + ----------------------------------------------------------\n \ + README.md | 28 ++++--\n logo.bin | Bin 0 -> 2048 bytes\n \ + newfeature.rs | 40 ++++++++\n \ + 4 files changed, 60 insertions(+), 313 deletions(-)\n"; + let (files, summary) = parse_stash_stat(raw); + let compact = format!("{}\n{}", files.join("\n"), summary); + let savings = + 100.0 - (estimate_tokens(&compact) as f64 / estimate_tokens(raw) as f64 * 100.0); + assert!( + savings >= 40.0, + "expected >=40% savings, got {:.1}%", + savings + ); + } + + #[test] + fn test_run_stash_list_propagates_failure() { + let dir = tempfile::tempdir().expect("tempdir"); + let global = vec!["-C".to_string(), dir.path().to_string_lossy().into_owned()]; + let code = run_stash(Some("list"), &[], 0, &global).expect("run_stash list"); + assert_ne!(code, 0, "git stash list failure must propagate"); + } + + #[test] + fn test_run_stash_show_propagates_failure() { + let dir = tempfile::tempdir().expect("tempdir"); + let global = vec!["-C".to_string(), dir.path().to_string_lossy().into_owned()]; + let code = run_stash(Some("show"), &[], 0, &global).expect("run_stash show"); + assert_ne!(code, 0, "git stash show failure must propagate"); + } + + #[test] + fn test_filter_worktree_list() { + let output = + "/home/user/project abc1234 [main]\n/home/user/worktrees/feat def5678 [feature]\n"; + let result = filter_worktree_list(output); + assert!(result.contains("abc1234")); + assert!(result.contains("[main]")); + assert!(result.contains("[feature]")); + } + + #[test] + fn test_run_worktree_list_propagates_failure() { + // #2497: `git worktree list` outside a repo exits non-zero; rtk must not + // report success (empty output + exit 0). + let dir = tempfile::tempdir().expect("tempdir"); + let global = vec!["-C".to_string(), dir.path().to_string_lossy().into_owned()]; + let code = run_worktree(&[], 0, &global).expect("run_worktree"); + assert_ne!(code, 0, "git worktree list failure must propagate"); + } + + #[test] + fn test_worktree_asked_for_report() { + let report = |args: &[&str]| { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + worktree_asked_for_report(&arg_tokenizer::tokenize(&owned)) + }; + // `add` writes two progress lines that "ok" exists to replace, and has no --dry-run or + // --verbose of its own. + assert!(!report(&["add", "/tmp/w", "-b", "topic"])); + assert!(!report(&["prune"])); + assert!(!report(&["remove", "/tmp/w"])); + for spelling in [ + &["prune", "-n"][..], + &["prune", "--dry-run"][..], + &["prune", "-v"][..], + ] { + assert!(report(spelling), "{spelling:?} asks for a report"); + } + // A worktree path spelled like the flag is a path, not a request. + assert!(!report(&["remove", "--", "-n"])); + } + + #[test] + fn test_format_status_output_clean() { + let porcelain = "## main...origin/main\n"; + let result = format_status_output(porcelain); + assert_eq!(result, "* main...origin/main\nclean — nothing to commit"); + } + + #[test] + fn test_extract_state_header_clean_returns_none() { + let raw = "On branch main\nYour branch is up to date with 'origin/main'.\n\nnothing to commit, working tree clean\n"; + assert_eq!(extract_state_header(raw), None); + } + + #[test] + fn test_extract_state_header_no_state_with_changes_returns_none() { + let raw = "On branch main\nChanges not staged for commit:\n (use \"git add ...\" to update what will be committed)\n\tmodified: src/main.rs\n\nno changes added to commit\n"; + assert_eq!(extract_state_header(raw), None); + } + + #[test] + fn test_extract_state_header_editing_while_rebasing() { + let raw = "On branch feature\n\ninteractive rebase in progress; onto abc1234\nLast command done (1 command done):\n edit abc123 some message\nNo commands remaining.\nYou are currently editing a commit while rebasing branch 'feature' on 'abc1234'.\n (use \"git commit --amend\" to amend the current commit)\n (use \"git rebase --continue\" once you are satisfied with your changes)\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "rebase in progress"); + } + + #[test] + fn test_extract_state_header_merge_unresolved() { + let raw = "On branch main\nYou have unmerged paths.\n (fix conflicts and run \"git commit\")\n (use \"git merge --abort\" to abort the merge)\n\nUnmerged paths:\n\tboth modified: src/main.rs\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "merge in progress. unresolved conflicts"); + } + + #[test] + fn test_extract_state_header_cherry_pick() { + let raw = "On branch main\n\nYou are currently cherry-picking commit abc1234.\n (fix conflicts and run \"git cherry-pick --continue\")\n (use \"git cherry-pick --abort\" to cancel the cherry-pick operation)\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "cherry-pick in progress"); + } + + #[test] + fn test_extract_state_header_bisect() { + let raw = "On branch main\n\nYou are currently bisecting, started from branch 'main'.\n (use \"git bisect reset\" to get back to the original branch)\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "bisect in progress"); + } + + #[test] + fn test_extract_state_header_revert() { + let raw = "On branch main\n\nYou are currently reverting commit abc1234.\n (fix conflicts and run \"git revert --continue\")\n (use \"git revert --abort\" to cancel the revert operation)\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "revert in progress"); + } + + #[test] + fn test_extract_state_header_merge_in_middle() { + let raw = "On branch main\n\nAll conflicts fixed but you are still merging.\n (use \"git commit\" to conclude merge)\n\nChanges to be committed:\n\tmodified: src/main.rs\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "merge in progress. no conflicts"); + } + + #[test] + fn test_extract_state_header_am_session() { + let raw = "On branch main\n\nYou are in the middle of an am session.\n (use \"git am --continue\" to continue)\n (use \"git am --abort\" to restore the original branch)\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "am session in progress"); + } + + #[test] + fn test_extract_state_header_sparse_checkout() { + let raw = "On branch main\n\nYou are in a sparse checkout with 17% of tracked files present.\n\nnothing to commit, working tree clean\n"; + let out = extract_state_header(raw).expect("state expected"); + assert_eq!(out, "sparse checkout enabled"); + } + + #[test] + fn test_format_status_output_preserves_nested_untracked_paths() { + let porcelain = "## main\n?? tmp/c.txt\n?? tmp/nested/d.txt\n"; + let result = format_status_output(porcelain); + assert!(result.contains("* main")); + assert!(result.contains("?? tmp/c.txt")); + assert!(result.contains("?? tmp/nested/d.txt")); + assert!( + result.lines().all(|line| line != "?? tmp/"), + "Nested untracked files must not collapse back to a directory marker:\n{}", + result + ); + } + + #[test] + fn test_format_status_output_mixed_changes() { + let porcelain = r#"## main +M staged.rs + M modified.rs +A added.rs +?? untracked.txt +"#; + let result = format_status_output(porcelain); + assert!(result.contains("* main")); + assert!(result.contains("M staged.rs")); + assert!(result.contains(" M modified.rs")); + assert!(result.contains("A added.rs")); + assert!(result.contains("?? untracked.txt")); + assert!(!result.contains("Staged")); + assert!(!result.contains("Modified")); + assert!(!result.contains("Untracked")); + } + + #[test] + fn test_format_status_output_preserves_rename_and_conflict_lines() { + let porcelain = "## main\nR old.rs -> new.rs\nUU conflict.rs\nMM mixed.rs\n"; + let result = format_status_output(porcelain); + assert!(result.contains("* main")); + assert!(result.contains("R old.rs -> new.rs")); + assert!(result.contains("UU conflict.rs")); + assert!(result.contains("MM mixed.rs")); + assert!(!result.contains("conflicts:")); + } + + #[test] + fn test_run_passthrough_accepts_args() { + // Test that run_passthrough compiles and has correct signature + let _args: Vec = vec![OsString::from("tag"), OsString::from("--list")]; + // Compile-time verification that the function exists with correct signature + } + + #[test] + fn test_filter_log_output() { + let output = "abc1234 This is a commit message (2 days ago) \n\n---END---\ndef5678 Another commit (1 week ago) \n\n---END---\n"; + let result = filter_log_output(output, 10, false, false); + assert!(result.contains("abc1234")); + assert!(result.contains("def5678")); + assert_eq!(result.lines().count(), 2); + } + + #[test] + fn test_filter_log_output_with_body() { + // Commit with body: first non-trailer body line should appear indented + let output = "abc1234 feat: add feature (2 days ago) \nBREAKING CHANGE: removed old API\nSigned-off-by: Author \n---END---\ndef5678 fix: typo (1 day ago) \n\n---END---\n"; + let result = filter_log_output(output, 10, false, false); + assert!(result.contains("abc1234")); + assert!(result.contains("BREAKING CHANGE: removed old API")); + assert!(!result.contains("Signed-off-by:")); + // def5678 has no body — just header + assert!(result.contains("def5678")); + // 3 lines: header1, body1 indented, header2 + assert_eq!(result.lines().count(), 3); + } + + #[test] + fn test_filter_log_output_skips_trailers() { + // Body with only trailers should not produce a body line + let output = "abc1234 chore: bump (1 day ago) \nSigned-off-by: Bot \nCo-authored-by: Human \n---END---\n"; + let result = filter_log_output(output, 10, false, false); + assert!(result.contains("abc1234")); + assert!(!result.contains("Signed-off-by:")); + assert!(!result.contains("Co-authored-by:")); + assert_eq!(result.lines().count(), 1); + } + + #[test] + fn test_filter_log_output_truncate_long() { + let long_line = "abc1234 ".to_string() + &"x".repeat(100) + " (2 days ago) "; + let result = filter_log_output(&long_line, 10, false, false); + assert!(result.chars().count() < long_line.chars().count()); + assert!(result.contains("...")); + assert!(result.chars().count() <= 80); + } + + #[test] + fn test_filter_log_output_cap_lines() { + let output = (0..20) + .map(|i| format!("hash{} message {} (1 day ago) \n\n---END---", i, i)) + .collect::>() + .join("\n"); + let result = filter_log_output(&output, 5, false, false); + assert_eq!(result.lines().count(), 5); + } + + #[test] + fn test_filter_log_output_user_limit_no_cap() { + // When user explicitly passes -N, all N lines should be returned (no re-truncation) + let output = (0..20) + .map(|i| format!("hash{} message {} (1 day ago) \n\n---END---", i, i)) + .collect::>() + .join("\n"); + let result = filter_log_output(&output, 20, true, false); + assert_eq!( + result.lines().count(), + 20, + "User's -20 should return all 20 lines" + ); + } + + #[test] + fn test_filter_log_output_user_limit_wider_truncation() { + // When user explicitly passes -N, lines up to 120 chars should NOT be truncated + let line_90_chars = format!("abc1234 {} (2 days ago) ", "x".repeat(60)); + assert!(line_90_chars.chars().count() > 80); + assert!(line_90_chars.chars().count() < 120); + + let result_default = filter_log_output(&line_90_chars, 10, false, false); + let result_user = filter_log_output(&line_90_chars, 10, true, false); + + // Default truncates at 80 chars + assert!( + result_default.contains("..."), + "Default should truncate at 80 chars" + ); + // User-set limit uses wider threshold (120 chars) + assert!( + !result_user.contains("..."), + "User limit should not truncate 90-char line" + ); + } + + #[test] + fn test_parse_user_limit_combined() { + let args: Vec = vec!["-20".into()]; + assert_eq!(parse_user_limit(&args), Some(20)); + } + + #[test] + fn test_parse_user_limit_n_space() { + let args: Vec = vec!["-n".into(), "15".into()]; + assert_eq!(parse_user_limit(&args), Some(15)); + } + + #[test] + fn test_parse_user_limit_max_count_eq() { + let args: Vec = vec!["--max-count=30".into()]; + assert_eq!(parse_user_limit(&args), Some(30)); + } + + #[test] + fn test_parse_user_limit_max_count_space() { + let args: Vec = vec!["--max-count".into(), "25".into()]; + assert_eq!(parse_user_limit(&args), Some(25)); + } + + #[test] + fn test_parse_user_limit_none() { + let args: Vec = vec!["--oneline".into()]; + assert_eq!(parse_user_limit(&args), None); + } + + #[test] + fn test_parse_user_limit_malformed_combined_digit_run() { + // "-5x" isn't a valid git log limit (real git rejects it outright), but the digit-run + // rule only looks at the leading run and parses 5 anyway -- harmless since run_log bails + // out on git's own failure before ever using this value. + let args: Vec = vec!["-5x".into()]; + assert_eq!(parse_user_limit(&args), Some(5)); + } + + #[test] + fn test_patch_log_flags_request_raw_output() { + for flag in [ + "-p", + "-u", + "--patch", + "--patch-with-raw", + "--patch-with-stat", + ] { + let args = vec![flag.to_string()]; + assert!(requests_raw_log_output(&args), "{flag} should pass through"); + } + } + + #[test] + fn test_patch_flag_after_pathspec_separator_is_ignored() { + // `git log -- -p` means "show history for a path literally named -p", + // not "show patches" — the flag lookalike appears after `--`. + let args = vec!["--".to_string(), "-p".to_string()]; + assert!( + !requests_raw_log_output(&args), + "-p after -- is a pathspec, not a patch flag, and should stay on the filtered path" + ); + } + + #[test] + fn test_non_patch_log_flags_remain_filtered() { + for flag in ["--no-patch", "--oneline", "--format=%H"] { + let args = vec![flag.to_string()]; + assert!( + !requests_raw_log_output(&args), + "{flag} should remain on the filtered log path" + ); + } + } + + #[test] + fn test_diff_shape_flags_request_raw_output() { + // These change the shape of git's raw output (diffstat, name lists) + // the same way -p does — RTK's injected --pretty=format markers + // can't coexist with them, so they must stay on the raw path too. + for flag in [ + "--dirstat", + "--dirstat=files", + "--name-only", + "--name-status", + "--numstat", + "--raw", + "--shortstat", + "--stat", + "--stat=80", + "--summary", + ] { + let args = vec![flag.to_string()]; + assert!( + requests_raw_log_output(&args), + "{flag} changes output shape and should request raw output" + ); + } + } + + #[test] + fn test_diff_show_raw_shape_excludes_patch_flags() { + for flag in ["--patch", "-p", "-u"] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_log_args(&args); + assert!( + !tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)) + && !tokens.iter().any(|t| diff_wants_raw_shape(t, &tokens)), + "{flag} must stay on diff/show's compact path, not the raw passthrough path" + ); + } + } + + #[test] + fn test_diff_show_raw_shape_still_includes_other_shape_flags() { + for flag in [ + "--dirstat", + "--name-only", + "--name-status", + "--numstat", + "--patch-with-raw", + "--patch-with-stat", + "--raw", + "--shortstat", + "--stat", + "--summary", + ] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_log_args(&args); + assert!( + tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)) + && tokens.iter().any(|t| diff_wants_raw_shape(t, &tokens)), + "{flag} changes output shape and should still request the raw passthrough path" + ); + } + } + + #[test] + fn test_diff_shape_flag_as_value_of_grep_is_not_misdetected() { + // `git log --grep --stat` searches for the literal string + // "--stat"; git consumes it as --grep's value, not the --stat flag. + let args = vec!["--grep".to_string(), "--stat".to_string()]; + assert!( + !requests_raw_log_output(&args), + "--stat as the value of --grep should stay on the filtered path" + ); + } + + #[test] + fn test_patch_flag_as_value_of_grep_is_not_misdetected() { + // `git log --grep -p` searches commit messages for the literal + // string "-p"; git does not treat it as the patch flag. + for opt in [ + "--author", + "--committer", + "--diff-algorithm", + "--diff-filter", + "--grep", + "-G", + "-S", + ] { + let args = vec![opt.to_string(), "-p".to_string()]; + assert!( + !requests_raw_log_output(&args), + "-p as the value of {opt} should stay on the filtered path" + ); + } + } + + #[test] + fn test_patch_flag_still_detected_after_value_taking_option() { + // The value-taking option consumes only its own value token; + // a genuine -p later in the args still triggers the raw path. + let args = vec!["--grep".to_string(), "fix".to_string(), "-p".to_string()]; + assert!( + requests_raw_log_output(&args), + "a real -p after --grep's value should still request raw output" + ); + } + + #[test] + fn test_optional_value_options_do_not_consume_next_token() { + // These options only take an attached value (-U3, --unified=3, --expand-tabs=4, + // --max-parents=2); a bare separate token after them is not their value. + for opt in [ + "-U", + "--unified", + "--expand-tabs", + "--max-parents", + "--min-parents", + ] { + let args = vec![opt.to_string(), "-p".to_string()]; + assert!( + requests_raw_log_output(&args), + "a real -p after {opt} should still request raw output" + ); + } + } + + #[test] + fn test_glued_diff_shape_short_flag_is_not_misread_as_a_limit() { + // -M/-C/-B/-U take only an attached optional numeric value ("-M50"); the "50" must not + // decompose into a stray digit-run token misread as a "-5" limit flag. + for glued in ["-M50", "-C5", "-B10", "-U3"] { + let args = vec![glued.to_string()]; + assert_eq!( + parse_user_limit(&args), + None, + "{glued} must not be misread as a commit-count limit" + ); + } + } + + #[test] + fn test_clustered_short_flag_does_not_consume_separate_value() { + // `-n 2` consumes the separate "2", but clustered with another short flag (`-cn 2`), + // -n's value is only the (empty) remainder of the same arg, never the next token. + let args = vec!["-cn".to_string(), "2".to_string()]; + assert_eq!( + parse_user_limit(&args), + None, + "-n clustered with -c must not consume \"2\" as its value" + ); + + // The bare, standalone form must still work as documented. + let args = vec!["-n".to_string(), "2".to_string()]; + assert_eq!(parse_user_limit(&args), Some(2)); + } + + #[test] + fn test_diff_grammar_differs_from_logs_where_git_does() { + // `git diff -wl 100` clusters (rename limit); `git log -cl 2` does not. Sharing log's + // predicate made RTK read the 100 as a pathspec and splice its own flags before it. + let args = vec!["-wl".to_string(), "100".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert_eq!(tokens[1].text, "l"); + assert_eq!(tokens[1].value(&tokens), Some("100")); + + // Options git's own completion helper lists as value-taking that RTK had missed: their + // values were read as free positionals, which is what the project-path and pathspec + // logic keys on. + for opt in [ + "ignore-matching-lines", + "stat-graph-width", + "max-age", + "min-age", + ] { + let args = vec![format!("--{opt}"), "x".to_string(), "HEAD".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert_eq!(tokens[0].value(&tokens), Some("x"), "--{opt}"); + assert!( + !tokens[1].is_free_positional(), + "--{opt}'s value must not read as a positional" + ); + } + } + + #[test] + fn test_patch_shape_removal_keeps_the_rest_of_a_cluster() { + // Every short flag in `-pl` shares one source_index, so dropping the whole arg took + // `-l` with it and left its `100` behind as a bogus revision. + let rebuild = |args: &[&str]| -> Vec { + let args: Vec = args.iter().map(|a| a.to_string()).collect(); + let tokens = tokenize_git_diff_args(&args); + args_without_patch_shape(&args, &tokens) + }; + + assert_eq!(rebuild(&["-pl", "100"]), vec!["-l", "100"]); + assert_eq!(rebuild(&["-pw"]), vec!["-w"]); + assert_eq!(rebuild(&["-wU2"]), vec!["-w"]); + assert_eq!(rebuild(&["-p"]), Vec::::new()); + // Nothing to strip: flags and positionals pass through untouched. + assert_eq!( + rebuild(&["-w", "HEAD~1", "f.txt"]), + vec!["-w", "HEAD~1", "f.txt"] + ); + assert_eq!(rebuild(&["--author", "-p"]), vec!["--author", "-p"]); + } + + #[test] + fn test_stat_step_retokenizes_after_dropping_oneline() { + // The stat step drops `--oneline` and then strips patch-shape flags. Both steps read + // token indices, and the first shifts them, so the second must re-tokenize or it + // deletes the argument that inherited the patch flag's index -- here the revision. + let args: Vec = ["--oneline", "-p", "deadbee"] + .iter() + .map(|a| a.to_string()) + .collect(); + let tokens = tokenize_git_diff_args(&args); + let stat_args = args_without_oneline(&args, &tokens); + assert_eq!(stat_args, vec!["-p", "deadbee"]); + + let stat_tokens = tokenize_git_diff_args(&stat_args); + assert_eq!( + args_without_patch_shape(&stat_args, &stat_tokens), + vec!["deadbee"], + "the revision must survive; only the patch flag is dropped" + ); + assert_eq!( + args_without_patch_shape(&stat_args, &tokens), + vec!["-p"], + "stale indices invert the result: revision dropped, patch flag kept" + ); + } + + #[test] + fn test_diff_header_drops_the_users_patch_flags() { + // git takes the last shape flag and accepts options after a revision, so a `-p` + // written there outranks RTK's header flags wherever they are placed -- the header + // then carried the whole patch and RTK printed it again, compacted. + let args = vec!["HEAD~1".to_string(), "-p".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert_eq!(args_without_patch_shape(&args, &tokens), vec!["HEAD~1"]); + + for flag in ["-p", "-u", "--patch", "-U5", "-W", "--function-context"] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!( + args_without_patch_shape(&args, &tokens).is_empty(), + "{flag}" + ); + } + // A flag that only tunes the diff must survive into the header. + let args = vec!["-w".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert_eq!(args_without_patch_shape(&args, &tokens), vec!["-w"]); + } + + #[test] + fn test_suppression_flags_are_raw_for_diff_but_compact_for_show() { + // `-s` asks diff for no body (nothing to compact) but asks show for the commit + // summary, which is exactly what the compact form prints. + for flag in ["-s", "--no-patch"] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!(tokens.iter().any(suppresses_diff_body), "{flag}"); + assert!( + !tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)), + "{flag} must stay on show's compact path" + ); + } + // --check replaces the body with a whitespace report in both. + let args = vec!["--check".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!(tokens.iter().any(|t| show_wants_raw_shape(t, &tokens))); + assert!(tokens.iter().any(|t| diff_wants_raw_shape(t, &tokens))); + } + + #[test] + fn test_quiet_is_raw_for_diff_but_compact_for_show() { + // Verified against git 2.53: `git diff --quiet` prints nothing and exits 1 on a + // difference, while `git show --quiet` exits 0 and prints the header its synonyms + // `-s`/`--no-patch` compact. Claiming it for show raw-passed that header. + let args = vec!["--quiet".to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!( + tokens.iter().any(|t| diff_wants_raw_shape(t, &tokens)), + "diff needs the exit code" + ); + assert!( + !tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)), + "show's --quiet is -s, which suppresses_diff_body renders as the summary" + ); + assert!(tokens.iter().any(suppresses_diff_body)); + } + + #[test] + fn test_raw_log_passthrough_keeps_rtks_default_limit() { + let built = |args: &[&str]| { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + let tokens = tokenize_git_log_args(&owned); + assert!( + tokens.iter().any(|t| log_wants_raw_shape(t, &tokens)), + "{args:?} must route raw" + ); + raw_log_passthrough_args(&owned, raw_log_is_capped(&tokens)) + .iter() + .map(|a| a.to_string_lossy().into_owned()) + .collect::>() + }; + + // The limit precedes every user argument. git rejects an option that follows a + // positional -- "fatal: -10 option must come before non-option arguments" -- and a + // pathspec needs no `--` to be a positional, so anchoring on the boundary is not enough. + assert_eq!(built(&["-p"]), ["log", "-10", "-p"]); + assert_eq!( + built(&["-p", "src/main.rs"]), + ["log", "-10", "-p", "src/main.rs"], + "a bare pathspec still has to come after the limit" + ); + assert_eq!( + built(&["--stat", "--", "src/main.rs"]), + ["log", "-10", "--stat", "--", "src/main.rs"] + ); + + // A limit the user set is left alone, in every spelling has_limit_flag knows. + for limit in [&["-5"][..], &["-n", "5"][..], &["--max-count=5"][..]] { + let args: Vec<&str> = std::iter::once("-p").chain(limit.iter().copied()).collect(); + let got = built(&args); + assert!( + !got.contains(&DEFAULT_LOG_LIMIT_ARG.to_string()), + "{args:?} already names a limit, got {got:?}" + ); + } + } + + #[test] + fn test_show_keeps_compacting_under_oneline() { + let gate = |args: &[&str]| { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + show_wants_format(&tokenize_git_diff_args(&owned)) + }; + // The user's own format displaces RTK's summary entirely: nothing to compact around. + assert!(gate(&["--pretty=format:%H"])); + assert!(gate(&["--format=%H"])); + // `--oneline` only outranks RTK's own one-line summary. Compaction stays on, or one + // display flag turns the whole command into a raw passthrough. + assert!(!gate(&["--oneline"])); + assert!(!gate(&[])); + } + + #[test] + fn test_body_suppression_follows_gits_last_flag_wins() { + // git 2.53: `git show -s -p` prints the diff, `git show -p -s` does not. Taking the + // suppressors order-independently swallowed a patch the user asked for last. + // `--quiet` is deliberately not in the last-wins group: git show ignores it for shape + // whenever a patch is requested, from either side, and honours it only when nothing + // else asked for output. Every row measured against git 2.53. + let cases: [(&[&str], bool); 13] = [ + (&["-s"], true), + (&["--no-patch"], true), + (&["--quiet"], true), + (&["-s", "-p"], false), + (&["-p", "-s"], true), + (&["--no-patch", "--patch"], false), + (&["--patch", "--no-patch"], true), + (&["--quiet", "-p"], false), + (&["-p", "--quiet"], false), + (&["--quiet", "-U0"], false), + (&["-U0", "--quiet"], false), + (&["--quiet", "-s", "-p"], false), + (&["-p", "-s", "--quiet"], true), + ]; + for (args, expected) in cases { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + let tokens = tokenize_git_diff_args(&owned); + assert_eq!(body_is_suppressed(&tokens), expected, "{args:?}"); + } + } + + #[test] + fn test_raw_log_limit_leaves_an_explicitly_bounded_walk_alone() { + let built = |args: &[&str]| { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + let tokens = tokenize_git_log_args(&owned); + raw_log_passthrough_args(&owned, raw_log_is_capped(&tokens)) + .iter() + .any(|a| a == DEFAULT_LOG_LIMIT_ARG) + }; + // A revision range is an exact bound the user chose; capping it takes away what they + // asked for -- `git log -p HEAD~15..HEAD` came back with 10 of the 15. + assert!(!built(&["-p", "HEAD~15..HEAD"])); + assert!(!built(&["-p", "HEAD~3...HEAD"])); + assert!(!built(&["-p", "origin/main..HEAD"])); + // Unbounded still gets the limit, or a patch request prints the whole history. + assert!(built(&["-p"])); + assert!(built(&["-p", "HEAD"])); + // A relative pathspec is not a range. + assert!(built(&["-p", "../sibling"])); + assert!(built(&["-p", "./a..b"])); + // And an explicit limit still wins over both. + assert!(!built(&["-p", "-5"])); + } + + #[test] + fn test_line_prefix_takes_the_raw_route() { + // It prefixes every output line, `diff --git` and `@@` markers included, so the + // compaction parses nothing and printed a diffstat over an empty `Changes:` section. + for args in [ + &["--line-prefix=XX"][..], + &["--line-prefix", "XX"][..], + // git takes the next token as the value, so a following flag is the prefix text. + &["--line-prefix", "--stat"][..], + ] { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + let log = tokenize_git_log_args(&owned); + assert!( + log.iter().any(|t| log_wants_raw_shape(t, &log)), + "{args:?} on log" + ); + let diff = tokenize_git_diff_args(&owned); + assert!( + diff.iter().any(|t| diff_wants_raw_shape(t, &diff)), + "{args:?} on diff" + ); + } + } + + #[test] + fn test_diff_merges_routes_by_the_format_it_names() { + // Verified against git 2.53 on a conflict-resolved merge: every format but off/none + // emits a patch, and only the combined ones use the two `@@@` marker columns. + let route = |args: &[&str]| { + let owned: Vec = args.iter().map(|a| a.to_string()).collect(); + let log = tokenize_git_log_args(&owned); + let show = tokenize_git_diff_args(&owned); + ( + log.iter().any(|t| log_wants_raw_shape(t, &log)), + show.iter().any(|t| show_wants_raw_shape(t, &show)), + ) + }; + + for fmt in ["c", "cc", "combined", "dense-combined"] { + // Combined: raw for both -- compact_diff reads the two marker columns as one. + assert_eq!( + route(&[&format!("--diff-merges={fmt}")]), + (true, true), + "={fmt}" + ); + // git takes the value as the next token too, and that spelling must route alike. + assert_eq!( + route(&["--diff-merges", fmt]), + (true, true), + "separate {fmt}" + ); + } + + for fmt in ["1", "first-parent", "m", "on", "r", "remerge", "separate"] { + // A single-column patch: log still cannot represent one, show's default already is + // one, so the two subcommands disagree here on purpose. + assert_eq!( + route(&[&format!("--diff-merges={fmt}")]), + (true, false), + "={fmt}" + ); + assert_eq!( + route(&["--diff-merges", fmt]), + (true, false), + "separate {fmt}" + ); + } + + for fmt in ["none", "off"] { + // No patch at all, so nothing to escape the compact path for. + assert_eq!( + route(&[&format!("--diff-merges={fmt}")]), + (false, false), + "={fmt}" + ); + assert_eq!( + route(&["--diff-merges", fmt]), + (false, false), + "separate {fmt}" + ); + } + + // No value is a git error either way; RTK must not read it as a patch request. + assert_eq!(route(&["--diff-merges"]), (false, false)); + } + + #[test] + fn test_combined_diff_short_flag_is_raw_for_show_like_its_long_form() { + // `-c` is `--cc`'s combined-diff form, not a redundant patch request: compact_diff + // reads a combined diff's two marker columns as one, so `git show -c ` came + // back as `+54 -8` against git's own 156 insertions / 0 deletions. + for flag in ["-c", "--cc"] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!( + tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)), + "{flag} must take show's raw route" + ); + assert!( + tokens.iter().any(|t| diff_wants_raw_shape(t, &tokens)), + "{flag} for diff too" + ); + } + // The other short flags stay on the compact path: they only restate the default. + for flag in ["-p", "-u", "-U3"] { + let args = vec![flag.to_string()]; + let tokens = tokenize_git_diff_args(&args); + assert!( + !tokens.iter().any(|t| show_wants_raw_shape(t, &tokens)), + "{flag}" + ); + } + } + + #[test] + fn test_log_output_flags_are_not_a_patch_request() { + // Neither changes `git log`'s output at all (byte-identical to plain `git log` + // against git 2.53), so routing them raw skipped RTK's own -10 and printed the + // entire history. + for flag in ["--quiet", "--exit-code"] { + let args = vec![flag.to_string()]; + assert!( + !requests_raw_log_output(&args), + "{flag} leaves git log's output unchanged, so it has no shape to escape to" + ); + } + } + + #[test] + fn test_stat_header_flags_land_before_the_users_pathspec_boundary() { + // `--no-patch --stat` after the user's `--` would be pathspecs, not options. + let args = vec!["--".to_string(), "src/".to_string()]; + let tokens = tokenize_git_log_args(&args); + assert_eq!(arg_tokenizer::injection_point(&tokens, args.len()), 0); + + let args = vec!["-p".to_string()]; + let tokens = tokenize_git_log_args(&args); + assert_eq!(arg_tokenizer::injection_point(&tokens, args.len()), 1); + } + + #[test] + fn test_blob_show_detection_ignores_flag_values_and_pathspecs() { + // Only a free positional before `--` can name a blob: `--author 'a:b'` is that flag's + // value and `-- a:b` is a pathspec, and neither should force raw passthrough. + let blob = |args: &[&str]| -> bool { + let args: Vec = args.iter().map(|a| a.to_string()).collect(); + // diff's grammar, the one run_show actually uses: log's disagrees on clustered + // `-l`, so the test would assert a classification that never ships. + let tokens = tokenize_git_diff_args(&args); + arg_tokenizer::before_dashdash(&tokens) + .iter() + .any(|t| t.is_free_positional() && is_blob_show_arg(t.text)) + }; + + assert!(blob(&["HEAD:src/main.rs"])); + assert!(!blob(&["--author", "a:b", "HEAD"])); + assert!(!blob(&["--", "a:b"])); + assert!(!blob(&["--pretty=format:%h"])); + } + + #[test] + fn test_status_compact_path_survives_a_bare_double_dash() { + // `git status --` selects no pathspec, so it is `git status` -- and must keep the + // compact porcelain path rather than falling through to git's prose output. The + // separator has to be ignored wherever it sits, not only when it is the only token. + assert!(uses_compact_status_path(&[])); + assert!(uses_compact_status_path(&["--".to_string()])); + assert!(uses_compact_status_path(&[ + "-sb".to_string(), + "--".to_string() + ])); + assert!(uses_compact_status_path(&[ + "-s".to_string(), + "-b".to_string(), + "--".to_string() + ])); + assert!(uses_compact_status_path(&["-sb".to_string()])); + assert!(uses_compact_status_path(&["-b".to_string()])); + // A real pathspec still passes through. + assert!(!uses_compact_status_path(&[ + "--".to_string(), + "f.txt".to_string() + ])); + } + + #[test] + fn test_split_stash_region_keeps_the_boundary_out_of_the_subcommand() { + // `git stash -- -p` is a pathspec, not interactive patch mode: the restored region + // starts at the boundary, so nothing is taken as the subcommand. + let owned = |args: &[&str]| -> Vec { args.iter().map(|a| a.to_string()).collect() }; + + let (subcommand, rest) = split_stash_region(&owned(&["--", "-p"])); + assert_eq!(subcommand, None); + assert_eq!(rest, owned(&["--", "-p"])); + + let (subcommand, rest) = split_stash_region(&owned(&["show", "-p"])); + assert_eq!(subcommand.as_deref(), Some("show")); + assert_eq!(rest, owned(&["-p"])); + + let (subcommand, rest) = split_stash_region(&owned(&["push", "--", "file.txt"])); + assert_eq!(subcommand.as_deref(), Some("push")); + assert_eq!(rest, owned(&["--", "file.txt"])); + + let (subcommand, rest) = split_stash_region(&owned(&["-p"])); + assert_eq!(subcommand, None); + assert_eq!(rest, owned(&["-p"])); + } + + #[test] + fn test_stash_show_wants_patch_ignores_dash_p_pathspec_after_double_dash() { + // A pathspec literally named "-p" after `--` must not be mistaken for the flag. + let args = vec!["--".to_string(), "-p".to_string()]; + assert!(!stash_show_wants_patch(&args)); + + let args = vec!["-p".to_string()]; + assert!(stash_show_wants_patch(&args)); + let args = vec!["--patch".to_string()]; + assert!(stash_show_wants_patch(&args)); + } + + #[test] + fn test_stash_show_wants_patch_does_not_swallow_dash_p_as_log_only_option_value() { + // git log's grammar (where --author takes a separate value) must not apply here. + let args = vec!["--author".to_string(), "-p".to_string()]; + assert!(stash_show_wants_patch(&args)); + + let args = vec!["--grep".to_string(), "-p".to_string()]; + assert!(stash_show_wants_patch(&args)); + } + + #[test] + fn test_stash_show_wants_patch_does_not_treat_dash_u_as_patch() { + // -u means --include-untracked for stash show, not -p (unlike git log, where -u is a + // -p synonym) -- conflating them routed -u's stat-only output through compact_diff, + // which only renders patch content, producing silently empty output. + let args = vec!["-u".to_string()]; + assert!(!stash_show_wants_patch(&args)); + + let args = vec!["-u".to_string(), "-p".to_string()]; + assert!(stash_show_wants_patch(&args)); + } + + #[test] + fn test_has_limit_flag_ignores_a_clustered_n_with_no_captured_value() { + // A clustered "n" with no captured value (e.g. "-cn") must not count as a limit. + let args = vec!["-cn".to_string(), "2".to_string()]; + let tokens = tokenize_git_log_args(&args); + assert!(!has_limit_flag(&tokens)); + + // The bare, standalone form still counts. + let args = vec!["-n".to_string(), "2".to_string()]; + let tokens = tokenize_git_log_args(&args); + assert!(has_limit_flag(&tokens)); + } + + #[test] + fn test_real_flag_args_drops_value_taking_option_values() { + // `--grep`'s value is not itself a flag and must not appear in the + // filtered set, even when it looks like -N, --pretty, or --merges. + let args = vec!["--grep".to_string(), "-5".to_string()]; + assert_eq!(real_flag_args(&args), vec!["grep"]); + } + + #[test] + fn test_real_flag_args_keeps_limit_flag_drops_its_value() { + let args = vec!["-n".to_string(), "15".to_string()]; + assert_eq!(real_flag_args(&args), vec!["n"]); + + let args = vec!["--max-count".to_string(), "25".to_string()]; + assert_eq!(real_flag_args(&args), vec!["max-count"]); + } + + #[test] + fn test_real_flag_args_keeps_genuine_flags() { + let args = vec![ + "--grep".to_string(), + "fix".to_string(), + "--oneline".to_string(), + ]; + assert_eq!(real_flag_args(&args), vec!["grep", "oneline"]); + } + + #[test] + fn test_grep_value_looking_like_limit_flag_is_not_misdetected() { + // `git log --grep -5` searches commit messages for the literal + // string "-5"; it is not a request to limit output to 5 commits. + let args = vec!["--grep".to_string(), "-5".to_string()]; + assert!( + !real_flag_args(&args).iter().any(|arg| is_digit_run(arg)), + "-5 as the value of --grep should not be seen as a limit flag" + ); + assert_eq!( + parse_user_limit(&args), + None, + "-5 as the value of --grep should not be parsed as a limit" + ); + } + + #[test] + fn test_grep_value_looking_like_format_flag_is_not_misdetected() { + // `git log --grep --pretty` searches for the literal string + // "--pretty"; git consumes it as --grep's value, not a format flag. + let args = vec!["--grep".to_string(), "--pretty".to_string()]; + assert!( + !real_flag_args(&args).contains(&"pretty"), + "--pretty as the value of --grep should not be seen as a format flag" + ); + } + + #[test] + fn test_grep_value_looking_like_merges_flag_is_not_misdetected() { + // `git log --grep --merges` searches for the literal string + // "--merges"; git consumes it as --grep's value, not --merges. + let args = vec!["--grep".to_string(), "--merges".to_string()]; + assert!( + !real_flag_args(&args).contains(&"merges"), + "--merges as the value of --grep should not be seen as the --merges flag" + ); + } + + #[test] + fn test_parse_user_limit_skips_foreign_option_values() { + // A real limit later in the args is still found after a + // value-taking option's value is skipped. + let args = vec!["--grep".to_string(), "-5".to_string(), "-20".to_string()]; + assert_eq!(parse_user_limit(&args), Some(20)); + } + + #[test] + fn test_log_arg_tokens_stop_at_pathspec_separator() { + // `git log -- -5` means "history for the path literally named -5", + // not a limit flag — tokens after `--` must be ignored entirely. + let args = vec!["--".to_string(), "-5".to_string()]; + assert!( + real_flag_args(&args).is_empty(), + "-5 after -- is a pathspec, not a flag" + ); + assert_eq!( + parse_user_limit(&args), + None, + "-5 after -- should not be parsed as a limit" + ); + } + + #[test] + fn test_filter_log_output_token_savings() { + fn count_tokens(text: &str) -> usize { + text.split_whitespace().count() + } + // Simulate verbose git log output (default format with full metadata) + let input = (0..20) + .map(|i| { + format!( + "commit abc123{:02x}\nAuthor: User Name \nDate: Mon Mar 10 10:00:00 2026 +0000\n\n fix: commit message number {}\n\n Extended body with details about the change.\n", + i, i + ) + }) + .collect::>() + .join("\n"); + let output = filter_log_output(&input, 10, false, false); + let savings = 100.0 - (count_tokens(&output) as f64 / count_tokens(&input) as f64 * 100.0); + assert!( + savings >= 60.0, + "Expected ≥60% token savings, got {:.1}%", + savings + ); + } + + #[test] + fn test_filter_status_with_args() { + let output = r#"On branch main +Your branch is up to date with 'origin/main'. + +Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + modified: src/main.rs + +no changes added to commit (use "git add" and/or "git commit -a") +"#; + let result = filter_status_with_args(output); + eprintln!("Result:\n{}", result); + assert!(result.contains("On branch main")); + assert!(result.contains("modified: src/main.rs")); + assert!( + !result.contains("(use \"git"), + "Result should not contain git hints" + ); + } + + #[test] + fn test_filter_status_with_args_clean() { + let output = "nothing to commit, working tree clean\n"; + let result = filter_status_with_args(output); + assert!(result.contains("nothing to commit")); + } + + #[test] + fn test_filter_log_output_multibyte() { + // Thai characters: each is 3 bytes. A line with >80 bytes but few chars + let thai_msg = format!("abc1234 {} (2 days ago) ", "ก".repeat(30)); + let result = filter_log_output(&thai_msg, 10, false, false); + // Should not panic + assert!(result.contains("abc1234")); + // The line has 30 Thai chars + other text, so > 80 chars total + // truncate_line now counts chars, not bytes + // 30 Thai + ~33 other = 63 chars < 80 threshold, so no truncation + assert!(result.contains("abc1234")); + } + + #[test] + fn test_filter_log_output_emoji() { + let emoji_msg = "abc1234 🎉🎊🎈🎁🎂🎄🎃🎆🎇✨🎉🎊🎈🎁🎂🎄🎃🎆🎇✨ (1 day ago) "; + let result = filter_log_output(emoji_msg, 10, false, false); + // Should not panic + // 20 emoji + ~30 other chars = ~50 chars < 80, no truncation needed + assert!(result.contains("abc1234")); + } + + #[test] + fn test_format_status_output_thai_filename() { + let porcelain = "## main\n M สวัสดี.txt\n?? ทดสอบ.rs\n"; + let result = format_status_output(porcelain); + // Should not panic + assert!(result.contains("* main")); + assert!(result.contains("สวัสดี.txt")); + assert!(result.contains("ทดสอบ.rs")); + } + + #[test] + fn test_format_status_output_emoji_filename() { + let porcelain = "## main\nA 🎉-party.txt\n M 日本語ファイル.rs\n"; + let result = format_status_output(porcelain); + assert!(result.contains("* main")); + } + + // --- commit output parsing --- + + #[test] + fn test_parse_commit_output_normal() { + let line = "[main abc1234def] add feature"; + assert_eq!(parse_commit_output(line), "ok abc1234"); + } + + #[test] + fn test_parse_commit_output_root_commit() { + let line = "[main (root-commit) abc1234def] initial commit"; + assert_eq!(parse_commit_output(line), "ok abc1234"); + } + + /// Regression test: multibyte branch name must not panic (was byte-slicing before fix) + #[test] + fn test_parse_commit_output_multibyte_branch() { + let line = "[分支名 abc1234def] 提交消息"; + assert_eq!(parse_commit_output(line), "ok abc1234"); + } + + /// Regression test: Thai branch name (3 bytes per char) + #[test] + fn test_parse_commit_output_thai_branch() { + let line = "[สาขา abc1234def] commit message"; + assert_eq!(parse_commit_output(line), "ok abc1234"); + } + + /// Regression: git prints hook output before its own summary. A first line + /// that opens with a multi-byte character and contains ']' used to panic on + /// `line[1..]` ("byte index 1 is not a char boundary"). + #[test] + fn test_parse_commit_output_multibyte_prefix_does_not_panic() { + assert_eq!(parse_commit_output("✅ lint passed]"), "ok"); + assert_eq!(parse_commit_output("→ hook] done"), "ok"); + } + + /// The same shape as above, but with a real summary after the hook text — + /// the hash must still be found via the bracket pair. + #[test] + fn test_parse_commit_output_after_multibyte_hook_prefix() { + assert_eq!( + parse_commit_output("✅ [main abc1234def] add feature"), + "ok abc1234" + ); + } + + /// A U+FFFD from lossily decoded output is itself multi-byte. + #[test] + fn test_parse_commit_output_replacement_char_prefix() { + assert_eq!(parse_commit_output("\u{FFFD}oops]"), "ok"); + } + + /// A closing bracket before any opening one must not slice backwards. + #[test] + fn test_parse_commit_output_close_before_open() { + assert_eq!(parse_commit_output("] stray [main abc1234def]"), "ok"); + } + + #[test] + fn test_parse_commit_output_no_bracket() { + let line = "some other output"; + assert_eq!(parse_commit_output(line), "ok"); + } + + #[test] + fn test_parse_commit_output_short_hash() { + // Hash shorter than 7 chars — treat as "ok" (no hash shown) + let line = "[main abc12] message"; + assert_eq!(parse_commit_output(line), "ok"); + } + + #[test] + fn test_parse_commit_output_empty() { + assert_eq!(parse_commit_output(""), "ok"); + } + + // --- commit outcome classification (issue #2494) --- + + #[test] + fn test_classify_commit_success_extracts_hash() { + match classify_commit_outcome(true, "[main abc1234def] add feature", 0) { + CommitOutcome::Ok(s) => assert_eq!(s, "ok abc1234"), + CommitOutcome::Failed(_) => panic!("successful commit must be Ok"), + } + } + + #[test] + fn test_classify_commit_success_empty_stdout() { + match classify_commit_outcome(true, "", 0) { + CommitOutcome::Ok(s) => assert_eq!(s, "ok"), + CommitOutcome::Failed(_) => panic!("successful commit must be Ok"), + } + } + + #[test] + fn test_classify_commit_nothing_to_commit_is_failure() { + match classify_commit_outcome( + false, + "On branch main\nnothing to commit, working tree clean", + 1, + ) { + CommitOutcome::Failed(code) => assert_eq!(code, 1), + CommitOutcome::Ok(s) => panic!("nothing-to-commit must not be ok: {}", s), + } + } + + #[test] + fn test_classify_commit_hook_abort_propagates_exit_code() { + match classify_commit_outcome(false, "pre-commit hook failed", 2) { + CommitOutcome::Failed(code) => assert_eq!(code, 2), + CommitOutcome::Ok(_) => panic!("hook abort must be a failure"), + } + } + + /// Regression test: --oneline and other user format flags must preserve all commits. + /// Before fix, filter_log_output split on ---END--- which doesn't exist when + /// the user specifies their own format, resulting in only 2 commits surviving. + #[test] + fn test_filter_log_output_user_format_oneline() { + let oneline_output = "abc1234 feat: add feature\n\ + def5678 fix: typo\n\ + ghi9012 chore: bump deps\n\ + jkl3456 docs: update readme\n\ + mno7890 test: add tests\n"; + + let result = filter_log_output(oneline_output, 10, false, true); + // All 5 lines must survive — no ---END--- splitting + assert_eq!(result.lines().count(), 5); + assert!(result.contains("abc1234")); + assert!(result.contains("mno7890")); + } + + #[test] + fn test_filter_log_output_user_format_with_limit() { + let oneline_output = "abc1234 feat: add feature\n\ + def5678 fix: typo\n\ + ghi9012 chore: bump deps\n\ + jkl3456 docs: update readme\n\ + mno7890 test: add tests\n"; + + // user_set_limit=true means respect all lines (no cap) + let result = filter_log_output(oneline_output, 3, true, true); + assert_eq!(result.lines().count(), 5); + + // user_set_limit=false means cap at limit + let result = filter_log_output(oneline_output, 3, false, true); + assert_eq!(result.lines().count(), 3); + } + + /// Regression test: `git branch ` must create, not list. + /// Before fix, positional args fell into list mode which added `-a`, + /// turning creation into a pattern-filtered listing (silent no-op). + #[test] + #[ignore] // Integration test: requires git repo + fn test_branch_creation_not_swallowed() { + let branch = "test-rtk-create-branch-regression"; + // Create branch via run_branch + run_branch(&[branch.to_string()], 0, &[]).expect("run_branch should succeed"); + // Verify it exists + let output = Command::new("git") + .args(["branch", "--list", branch]) + .output() + .expect("git branch --list should work"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(branch), + "Branch '{}' was not created. run_branch silently swallowed the creation.", + branch + ); + // Cleanup + let _ = Command::new("git").args(["branch", "-d", branch]).output(); + } + + /// Regression test: `git branch ` must create from commit. + #[test] + #[ignore] // Integration test: requires git repo + fn test_branch_creation_from_commit() { + let branch = "test-rtk-create-from-commit"; + run_branch(&[branch.to_string(), "HEAD".to_string()], 0, &[]) + .expect("run_branch with start-point should succeed"); + let output = Command::new("git") + .args(["branch", "--list", branch]) + .output() + .expect("git branch --list should work"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(branch), + "Branch '{}' was not created from commit.", + branch + ); + let _ = Command::new("git").args(["branch", "-d", branch]).output(); + } + + #[test] + fn test_commit_single_message() { + let args = vec!["-m".to_string(), "fix: typo".to_string()]; + let cmd = build_commit_command(&args, &[]); + let cmd_args: Vec<_> = cmd + .get_args() + .map(|a| a.to_string_lossy().to_string()) + .collect(); + assert_eq!(cmd_args, vec!["commit", "-m", "fix: typo"]); + } + + #[test] + fn test_commit_multiple_messages() { + let args = vec![ + "-m".to_string(), + "feat: add multi-paragraph support".to_string(), + "-m".to_string(), + "This allows git commit -m \"title\" -m \"body\".".to_string(), + ]; + let cmd = build_commit_command(&args, &[]); + let cmd_args: Vec<_> = cmd + .get_args() + .map(|a| a.to_string_lossy().to_string()) + .collect(); + assert_eq!( + cmd_args, + vec![ + "commit", + "-m", + "feat: add multi-paragraph support", + "-m", + "This allows git commit -m \"title\" -m \"body\"." + ] + ); + } + + // #327: git commit -am "msg" must pass -am through to git + #[test] + fn test_commit_am_flag() { + let args = vec!["-am".to_string(), "quick fix".to_string()]; + let cmd = build_commit_command(&args, &[]); + let cmd_args: Vec<_> = cmd + .get_args() + .map(|a| a.to_string_lossy().to_string()) + .collect(); + assert_eq!(cmd_args, vec!["commit", "-am", "quick fix"]); + } + + #[test] + fn test_commit_amend() { + let args = vec![ + "--amend".to_string(), + "-m".to_string(), + "new msg".to_string(), + ]; + let cmd = build_commit_command(&args, &[]); + let cmd_args: Vec<_> = cmd + .get_args() + .map(|a| a.to_string_lossy().to_string()) + .collect(); + assert_eq!(cmd_args, vec!["commit", "--amend", "-m", "new msg"]); + } + + #[test] + #[ignore] // Requires `cargo build` first — run with `cargo test --ignored` + fn test_git_status_not_a_repo_exits_nonzero() { + // Run rtk git status in a directory that is not a git repo + let tmp = std::env::temp_dir().join("rtk_test_not_a_repo"); + let _ = std::fs::create_dir_all(&tmp); + + // Build the path to the test binary + let bin_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join("debug") + .join("rtk"); + assert!( + bin_path.exists(), + "Debug binary not found at {:?} — run `cargo build` first", + bin_path + ); + let output = std::process::Command::new(&bin_path) + .args(["git", "status"]) + .current_dir(&tmp) + .output() + .expect("Failed to run rtk"); + + // Should exit with non-zero (128 from git) + assert!( + !output.status.success(), + "Expected non-zero exit code for git status outside a repo, got {:?}", + output.status.code() + ); + + // Message should be on stderr, not stdout + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stderr.to_lowercase().contains("not a git repository"), + "Expected 'not a git repository' on stderr, got stderr={:?}, stdout={:?}", + stderr, + stdout + ); + + let _ = std::fs::remove_dir_all(&tmp); + } + + // --- truncation accuracy --- + + #[test] + fn test_format_status_output_shows_every_file_when_many_are_dirty() { + let mut porcelain = String::from("## main...origin/main\n"); + for i in 0..25 { + porcelain.push_str(&format!("M staged_file_{}.rs\n", i)); + } + let result = format_status_output(&porcelain); + assert!( + result.contains("staged_file_24.rs"), + "Expected the last staged file to remain visible, got:\n{}", + result + ); + assert!( + result.lines().count() == 26, + "Expected branch + all 25 staged files, got:\n{}", + result + ); + assert!( + !result.contains("... +"), + "Status output must not hide dirty paths behind overflow markers:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_recovery_hint_present() { + // A hunk with 110 lines exceeds max_hunk_lines (100), triggers truncation + // The recovery hint must appear so LLMs can re-fetch the full diff + let mut diff = String::new(); + diff.push_str("diff --git a/large.rs b/large.rs\n"); + diff.push_str("--- a/large.rs\n"); + diff.push_str("+++ b/large.rs\n"); + diff.push_str("@@ -1,150 +1,150 @@\n"); + for i in 0..110 { + diff.push_str(&format!("+added line {}\n", i)); + } + let result = compact_diff(&diff, 500); + assert!( + result.contains("[full diff: rtk git diff --no-compact]"), + "Expected recovery hint when hunk is truncated, got:\n{}", + result + ); + } + + #[test] + fn test_compact_diff_hunk_truncation_count_accurate() { + // 150 change lines in one hunk: 100 shown, 50 silently dropped + // Must report the exact count, not just "(truncated)" + let mut diff = String::from( + "diff --git a/large.rs b/large.rs\n--- a/large.rs\n+++ b/large.rs\n@@ -1,150 +1,150 @@\n", + ); + for i in 0..150 { + diff.push_str(&format!("+line {}\n", i)); + } + let result = compact_diff(&diff, 500); + assert!( + result.contains("50 additions truncated"), + "Expected '50 additions truncated' (150 - 100 = 50), got:\n{}", + result + ); + } + + #[test] + fn test_extract_detached_head_returns_line() { + let raw = "HEAD detached at abc1234\nnothing to commit, working tree clean\n"; + assert_eq!( + extract_detached_head(raw), + Some("HEAD detached at abc1234".to_string()) + ); + } + + #[test] + fn test_extract_detached_head_on_branch_is_none() { + let raw = "On branch main\nnothing to commit, working tree clean\n"; + assert!(extract_detached_head(raw).is_none()); + } + + #[test] + fn test_format_status_output_detached_head() { + let porcelain = "## HEAD (no branch)\n M src/main.rs\n"; + let result = format_status_output_detached(porcelain, "HEAD detached at abc1234"); + assert!( + result.contains("HEAD detached at abc1234"), + "should use explicit detached ref, got: {result}" + ); + assert!( + !result.contains("HEAD (no branch)"), + "should not show opaque porcelain string, got: {result}" + ); + } + + #[test] + fn test_filter_log_output_body_omission_indicator() { + // Commit with 6 meaningful body lines: only 3 shown, must signal "+3 lines omitted" + let body_lines = (1..=6) + .map(|i| format!("body line {}", i)) + .collect::>() + .join("\n"); + let output = format!( + "abc1234 feat: big change (1 day ago) \n{}\n---END---\n", + body_lines + ); + let result = filter_log_output(&output, 10, false, false); + assert!( + result.contains("+3 lines omitted"), + "Expected '+3 lines omitted' when 6 body lines truncated to 3, got:\n{}", + result + ); + } + + fn run_push_filter(input: &str, exit_code: i32) -> String { + use crate::core::stream::StreamFilter; + let mut f = LineStreamFilter::new(GitPushLineHandler::default()); + let mut out = String::new(); + for line in input.lines() { + if let Some(s) = f.feed_line(line) { + out.push_str(&s); + } + } + out.push_str(&f.flush()); + if let Some(s) = f.on_exit(exit_code, input) { + out.push_str(&s); + } + out + } + + #[test] + fn test_push_filter_drops_progress_phases() { + let input = "\ +Enumerating objects: 5, done. +Counting objects: 100% (5/5), done. +Delta compression using up to 8 threads +Compressing objects: 100% (3/3), done. +Writing objects: 100% (3/3), 312 bytes | 312.00 KiB/s, done. +Total 3 (delta 2), reused 0 (delta 0) +To https://github.com/foo/bar.git + abc1234..def5678 master -> master +"; + let result = run_push_filter(input, 0); + for prefix in GIT_PUSH_NOISE_PREFIXES { + assert!( + !result.contains(prefix), + "noise prefix '{}' leaked through, got: {}", + prefix, + result + ); + } + assert!(result.contains("To https://github.com/foo/bar.git")); + assert!(result.contains("master -> master")); + assert!(result.ends_with("ok master\n"), "got: {}", result); + } + + #[test] + fn test_push_filter_up_to_date_summary() { + let input = "Everything up-to-date\n"; + let result = run_push_filter(input, 0); + assert!(result.contains("Everything up-to-date")); + assert!(result.ends_with("ok (up-to-date)\n"), "got: {}", result); + } + + #[test] + fn test_push_filter_passes_remote_messages_through() { + let input = "\ +remote: Resolving deltas: 100% (2/2), completed with 2 local objects. +remote: GitHub found 1 vulnerability on foo/bar's default branch (1 moderate). +To https://github.com/foo/bar.git + abc1234..def5678 feature -> feature +"; + let result = run_push_filter(input, 0); + assert!(result.contains("remote: Resolving deltas")); + assert!(result.contains("remote: GitHub found 1 vulnerability")); + assert!(result.ends_with("ok feature\n"), "got: {}", result); + } + + #[test] + fn test_push_filter_no_summary_on_failure() { + let input = "\ +To https://github.com/foo/bar.git + ! [rejected] master -> master (non-fast-forward) +error: failed to push some refs to 'https://github.com/foo/bar.git' +"; + let result = run_push_filter(input, 1); + assert!(result.contains("[rejected]")); + assert!(result.contains("error: failed to push")); + assert!( + !result.contains("ok "), + "summary leaked on failure, got: {}", + result + ); + } + + #[test] + fn test_push_filter_first_ref_wins_for_summary() { + let input = "\ +To https://github.com/foo/bar.git + abc1234..def5678 feat/a -> feat/a + 1111111..2222222 feat/b -> feat/b +"; + let result = run_push_filter(input, 0); + assert!(result.ends_with("ok feat/a\n"), "got: {}", result); + } + + #[test] + fn test_push_filter_token_savings_on_verbose_output() { + let input = "\ +Enumerating objects: 142, done. +Counting objects: 100% (142/142), done. +Delta compression using up to 8 threads +Compressing objects: 100% (88/88), done. +Writing objects: 100% (104/104), 28.50 KiB | 14.25 MiB/s, done. +Total 104 (delta 64), reused 0 (delta 0), pack-reused 0 +remote: Resolving deltas: 100% (64/64), completed with 24 local objects. +To https://github.com/foo/bar.git + abc1234..def5678 master -> master +"; + let result = run_push_filter(input, 0); + let count_tokens = |s: &str| s.split_whitespace().count(); + let input_tokens = count_tokens(input); + let output_tokens = count_tokens(&result); + let savings = 100.0 - (output_tokens as f64 / input_tokens as f64 * 100.0); + assert!( + savings >= 60.0, + "expected >=60% savings, got {:.1}% (in={}, out={})", + savings, + input_tokens, + output_tokens + ); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/cmds/python/pytest_cmd.rs b/crates/sparkplane-rtk/upstream/src/cmds/python/pytest_cmd.rs new file mode 100644 index 0000000..f32ac7d --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/cmds/python/pytest_cmd.rs @@ -0,0 +1,604 @@ +//! Filters pytest output to show only failures and the summary line. + +use crate::core::config; +use crate::core::runner; +use crate::core::truncate::CAP_WARNINGS; +use crate::core::utils::{resolved_command, strip_ansi, tool_exists, truncate}; +use anyhow::Result; + +const MAX_XFAIL: usize = CAP_WARNINGS; +const MAX_PYTEST_FAILURES: usize = CAP_WARNINGS; + +#[derive(Debug, PartialEq)] +enum ParseState { + Header, + TestProgress, + Failures, + Summary, +} + +pub fn run(args: &[String], verbose: u8) -> Result { + let mut cmd = if tool_exists("pytest") { + resolved_command("pytest") + } else { + let mut c = resolved_command("python"); + c.arg("-m").arg("pytest"); + c + }; + + let has_tb_flag = args.iter().any(|a| a.starts_with("--tb")); + let has_quiet_flag = args.iter().any(|a| a == "-q" || a == "--quiet"); + // Only treat a short `-r…` as pytest's report flag (not `--randomly-seed` etc.) + let has_report_flag = args + .iter() + .any(|a| a.starts_with("-r") && !a.starts_with("--")); + + if !has_tb_flag { + cmd.arg("--tb=short"); + } + if !has_quiet_flag { + cmd.arg("-q"); + } + // Surface xfailed/xpassed (and their reasons) in the short summary section + // so the compact output can report expected failures and — crucially — + // unexpected passes (XPASS), which signal a behavior change. + if !has_report_flag { + cmd.arg("-rxX"); + } + + for arg in args { + cmd.arg(arg); + } + + if verbose > 0 { + eprintln!("Running: pytest --tb=short -q {}", args.join(" ")); + } + + runner::run_filtered_with_exit( + cmd, + "pytest", + &args.join(" "), + |raw, exit_code| { + let clean = strip_ansi(raw); + let filtered = filter_pytest_output(&clean); + // Any other failure parsed as empty means the run broke before reporting. + if exit_code != 0 && exit_code != PYTEST_EXIT_NO_TESTS && filtered == PYTEST_NO_TESTS { + return truncate(clean.trim(), config::limits().passthrough_max_chars); + } + filtered + }, + runner::RunOptions::stdout_only().tee("pytest"), + ) +} + +const PYTEST_NO_TESTS: &str = "Pytest: No tests collected"; +const PYTEST_EXIT_NO_TESTS: i32 = 5; + +pub(crate) fn filter_pytest_output(output: &str) -> String { + let mut state = ParseState::Header; + let mut test_files: Vec = Vec::new(); + let mut failures: Vec = Vec::new(); + let mut current_failure: Vec = Vec::new(); + let mut xfail_lines: Vec = Vec::new(); + let mut summary_line = String::new(); + + for line in output.lines() { + let trimmed = line.trim(); + + // State transitions + if trimmed.starts_with("===") && trimmed.contains("test session starts") { + state = ParseState::Header; + continue; + } else if trimmed.starts_with("===") && trimmed.contains("FAILURES") { + state = ParseState::Failures; + continue; + } else if trimmed.starts_with("===") && trimmed.contains("short test summary") { + state = ParseState::Summary; + // Save current failure if any + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } + continue; + } else if trimmed.starts_with("===") + && (trimmed.contains("passed") + || trimmed.contains("failed") + || trimmed.contains("skipped")) + { + summary_line = trimmed.to_string(); + continue; + // quiet mode (-q): bare summary without === wrapper, e.g. "5 failed, 1698 passed, 2 skipped in 108.89s" + } else if summary_line.is_empty() + && !trimmed.starts_with("===") + && !trimmed.starts_with("FAILED") + && !trimmed.starts_with("ERROR") + && (trimmed.contains(" passed") + || trimmed.contains(" failed") + || trimmed.contains(" skipped")) + && trimmed.contains(" in ") + { + summary_line = trimmed.to_string(); + continue; + } + + // Process based on state + match state { + ParseState::Header => { + if trimmed.starts_with("collected") { + state = ParseState::TestProgress; + } + } + ParseState::TestProgress => { + // Lines like "tests/test_foo.py .... [ 40%]" + if !trimmed.is_empty() + && !trimmed.starts_with("===") + && (trimmed.contains(".py") || trimmed.contains("%]")) + { + test_files.push(trimmed.to_string()); + } + } + ParseState::Failures => { + // Collect failure details + if trimmed.starts_with("___") { + // New failure section + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } + current_failure.push(trimmed.to_string()); + } else if !trimmed.is_empty() && !trimmed.starts_with("===") { + current_failure.push(trimmed.to_string()); + } + } + ParseState::Summary => { + // FAILED test lines + if trimmed.starts_with("FAILED") || trimmed.starts_with("ERROR") { + failures.push(trimmed.to_string()); + } else if trimmed.starts_with("XFAIL") || trimmed.starts_with("XPASS") { + xfail_lines.push(trimmed.to_string()); + } + } + } + } + + // Save last failure if any + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + } + + // Build compact output + build_pytest_summary(&summary_line, &test_files, &failures, &xfail_lines) +} + +#[derive(Default)] +struct PytestCounts { + passed: usize, + failed: usize, + skipped: usize, + xfailed: usize, + xpassed: usize, + duration: Option, +} + +fn build_pytest_summary( + summary: &str, + _test_files: &[String], + failures: &[String], + xfail_lines: &[String], +) -> String { + let counts = parse_summary_line(summary); + let PytestCounts { + passed, + failed, + skipped, + xfailed, + xpassed, + duration, + } = counts; + + if passed == 0 && failed == 0 && skipped == 0 && xfailed == 0 && xpassed == 0 { + return PYTEST_NO_TESTS.to_string(); + } + + let extras_present = skipped > 0 || xfailed > 0 || xpassed > 0 || !xfail_lines.is_empty(); + let duration_suffix = duration + .as_deref() + .map(|value| format!(" in {value}")) + .unwrap_or_default(); + + if failed == 0 && passed > 0 && !extras_present { + return format!("Pytest: {passed} passed{duration_suffix}"); + } + + let mut result = String::new(); + result.push_str(&format!("Pytest: {} passed, {} failed", passed, failed)); + if skipped > 0 { + result.push_str(&format!(", {} skipped", skipped)); + } + if xfailed > 0 { + result.push_str(&format!(", {} xfailed", xfailed)); + } + if xpassed > 0 { + result.push_str(&format!(", {} xpassed", xpassed)); + } + result.push_str(&duration_suffix); + result.push('\n'); + + // Surface xfail/xpass entries (with their reasons) — XPASS in particular + // signals that something expected-to-fail now passes. + if !xfail_lines.is_empty() { + result.push_str("\nExpected-failure outcomes:\n"); + for line in xfail_lines.iter().take(MAX_XFAIL) { + result.push_str(&format!(" {}\n", truncate(line, 120))); + } + if xfail_lines.len() > MAX_XFAIL { + result.push_str(&format!(" … +{} more\n", xfail_lines.len() - MAX_XFAIL)); + let all_xfail = xfail_lines.join("\n"); + if let Some(hint) = + crate::core::tee::force_tee_tail_hint(&all_xfail, "pytest-xfail", MAX_XFAIL + 1) + { + result.push_str(&format!(" {}\n", hint)); + } + } + } + + if failures.is_empty() { + return result.trim().to_string(); + } + + // Show failures (limit to key information) + result.push_str("\nFailures:\n"); + + for (i, failure) in failures.iter().take(MAX_PYTEST_FAILURES).enumerate() { + // Extract test name and key error info + let lines: Vec<&str> = failure.lines().collect(); + + // First line is usually test name (after ___) + if let Some(first_line) = lines.first() { + if first_line.starts_with("___") { + // Extract test name between ___ + let test_name = first_line.trim_matches('_').trim(); + result.push_str(&format!("{}. [FAIL] {}\n", i + 1, test_name)); + } else if first_line.starts_with("FAILED") { + // Summary format: "FAILED tests/test_foo.py::test_bar - AssertionError" + let parts: Vec<&str> = first_line.split(" - ").collect(); + if let Some(test_path) = parts.first() { + let test_name = test_path.trim_start_matches("FAILED "); + result.push_str(&format!("{}. [FAIL] {}\n", i + 1, test_name)); + } + if parts.len() > 1 { + result.push_str(&format!(" {}\n", truncate(parts[1], 100))); + } + continue; + } + } + + // Show relevant error lines (assertions, errors, file locations) + let mut relevant_lines = 0; + for line in &lines[1..] { + let line_lower = line.to_lowercase(); + let is_relevant = line.trim().starts_with('>') + || line.trim().starts_with('E') + || line_lower.contains("assert") + || line_lower.contains("error") + || line.contains(".py:"); + + if is_relevant && relevant_lines < 3 { + result.push_str(&format!(" {}\n", truncate(line, 100))); + relevant_lines += 1; + } + } + + if i < failures.len() - 1 { + result.push('\n'); + } + } + + if failures.len() > MAX_PYTEST_FAILURES { + result.push_str(&format!( + "\n… +{} more failures\n", + failures.len() - MAX_PYTEST_FAILURES + )); + let all_failures = failures.join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_failures, "pytest-failures") { + result.push_str(&format!(" {}\n", hint)); + } + } + + result.trim().to_string() +} + +fn parse_summary_line(summary: &str) -> PytestCounts { + let mut counts = PytestCounts { + duration: parse_duration(summary), + ..PytestCounts::default() + }; + + // Parse lines like "=== 4 passed, 1 failed, 2 xfailed, 1 xpassed in 0.50s ===" + for part in summary.split(',') { + let words: Vec<&str> = part.split_whitespace().collect(); + for (i, word) in words.iter().enumerate() { + if i == 0 { + continue; + } + let Ok(n) = words[i - 1].parse::() else { + continue; + }; + // Order matters: "xpassed"/"xfailed" contain "passed"/"failed". + if word.contains("xpassed") { + counts.xpassed = n; + } else if word.contains("xfailed") { + counts.xfailed = n; + } else if word.contains("passed") { + counts.passed = n; + } else if word.contains("failed") { + counts.failed = n; + } else if word.contains("skipped") { + counts.skipped = n; + } + } + } + + counts +} + +fn parse_duration(summary: &str) -> Option { + let unwrapped = summary.trim().trim_matches('=').trim(); + let (_, duration_tail) = unwrapped.rsplit_once(" in ")?; + let duration = duration_tail.split_whitespace().next()?; + let seconds = duration.strip_suffix('s')?; + seconds.parse::().ok()?; + Some(duration.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_filter_pytest_all_pass() { + let output = r#"=== test session starts === +platform darwin -- Python 3.11.0 +collected 5 items + +tests/test_foo.py ..... [100%] + +=== 5 passed in 0.50s ==="#; + + let result = filter_pytest_output(output); + assert!(result.contains("Pytest")); + assert!(result.contains("5 passed")); + assert_eq!(result, "Pytest: 5 passed in 0.50s"); + } + + #[test] + fn test_filter_pytest_with_failures() { + let output = r#"=== test session starts === +collected 5 items + +tests/test_foo.py ..F.. [100%] + +=== FAILURES === +___ test_something ___ + + def test_something(): +> assert False +E assert False + +tests/test_foo.py:10: AssertionError + +=== short test summary info === +FAILED tests/test_foo.py::test_something - assert False +=== 4 passed, 1 failed in 0.50s ==="#; + + let result = filter_pytest_output(output); + assert!(result.contains("4 passed, 1 failed")); + assert!(result.starts_with("Pytest: 4 passed, 1 failed in 0.50s")); + assert!(result.contains("test_something")); + assert!(result.contains("assert False")); + } + + #[test] + fn test_filter_pytest_multiple_failures() { + let output = r#"=== test session starts === +collected 3 items + +tests/test_foo.py FFF [100%] + +=== FAILURES === +___ test_one ___ +E AssertionError: expected 5 + +___ test_two ___ +E ValueError: invalid value + +=== short test summary info === +FAILED tests/test_foo.py::test_one - AssertionError: expected 5 +FAILED tests/test_foo.py::test_two - ValueError: invalid value +FAILED tests/test_foo.py::test_three - KeyError +=== 3 failed in 0.20s ==="#; + + let result = filter_pytest_output(output); + assert!(result.contains("3 failed")); + assert!(result.contains("test_one")); + assert!(result.contains("test_two")); + assert!(result.contains("expected 5")); + } + + #[test] + fn test_filter_pytest_no_tests() { + let output = r#"=== test session starts === +collected 0 items + +=== no tests ran in 0.00s ==="#; + + let result = filter_pytest_output(output); + assert!(result.contains("No tests collected")); + } + + #[test] + fn test_parse_summary_line() { + let c = parse_summary_line("=== 5 passed in 0.50s ==="); + assert_eq!((c.passed, c.failed, c.skipped), (5, 0, 0)); + assert_eq!(c.duration.as_deref(), Some("0.50s")); + + let c = parse_summary_line("=== 4 passed, 1 failed in 0.50s ==="); + assert_eq!((c.passed, c.failed, c.skipped), (4, 1, 0)); + assert_eq!(c.duration.as_deref(), Some("0.50s")); + + let c = parse_summary_line("=== 3 passed, 1 failed, 2 skipped in 1.0s ==="); + assert_eq!((c.passed, c.failed, c.skipped), (3, 1, 2)); + assert_eq!(c.duration.as_deref(), Some("1.0s")); + + let c = parse_summary_line("=== 2 passed, 1 failed, 2 xfailed, 1 xpassed in 1.0s ==="); + assert_eq!((c.passed, c.failed, c.xfailed, c.xpassed), (2, 1, 2, 1)); + assert_eq!(c.duration.as_deref(), Some("1.0s")); + } + + #[test] + fn test_filter_pytest_quiet_mode_preserves_duration() { + let output = "2 passed in 3.04s"; + + assert_eq!(filter_pytest_output(output), "Pytest: 2 passed in 3.04s"); + } + + #[test] + fn test_filter_pytest_long_run_preserves_seconds_duration() { + let output = "2 passed in 65.12s (0:01:05)"; + + assert_eq!(filter_pytest_output(output), "Pytest: 2 passed in 65.12s"); + } + + #[test] + fn test_parse_duration_rejects_tails_that_are_not_seconds() { + assert_eq!( + parse_duration("=== 2 passed in 0.50s ===").as_deref(), + Some("0.50s") + ); + assert_eq!( + parse_duration("2 passed in 65.12s (0:01:05)").as_deref(), + Some("65.12s") + ); + assert_eq!(parse_duration("=== 2 passed ==="), None); + assert_eq!(parse_duration("2 passed in seconds"), None); + assert_eq!(parse_duration("2 passed in 0.50"), None); + } + + #[test] + fn test_filter_pytest_duration_on_every_summary_shape() { + // Wrapped long-run form with a warnings part before the duration. + let output = "=== 5 passed, 2 warnings in 65.12s (0:01:05) ==="; + assert_eq!(filter_pytest_output(output), "Pytest: 5 passed in 65.12s"); + + // Only-skipped run takes the multi-line emit path. + let output = "=== 3 skipped in 0.10s ==="; + assert_eq!( + filter_pytest_output(output), + "Pytest: 0 passed, 0 failed, 3 skipped in 0.10s" + ); + + // Quiet-mode failures line keeps its duration on the multi-line path too. + let output = "5 failed, 1698 passed, 2 skipped in 108.89s"; + assert_eq!( + filter_pytest_output(output), + "Pytest: 1698 passed, 5 failed, 2 skipped in 108.89s" + ); + } + + #[test] + fn test_filter_pytest_xfail_caps_and_tee_hint() { + let mut lines = String::from("=== test session starts ===\ncollected 30 items\n\n"); + lines.push_str("test_x.py "); + for _ in 0..15 { + lines.push('x'); + } + lines.push_str("\n\n=== short test summary info ===\n"); + for i in 0..15 { + lines.push_str(&format!( + "XFAIL test_x.py::test_case_{i} - known issue #{i}\n" + )); + } + lines.push_str("=== 0 passed, 15 xfailed in 0.05s ===\n"); + + let result = filter_pytest_output(&lines); + let xfail_in_section = result + .split("Expected-failure outcomes:") + .nth(1) + .unwrap_or(""); + let listed = xfail_in_section + .lines() + .filter(|l| l.trim().starts_with("XFAIL")) + .count(); + assert!(listed <= 10, "MAX_XFAIL cap not enforced: listed {listed}"); + assert!(result.contains("… +5 more"), "missing '+N more': {result}"); + } + + #[test] + fn test_filter_pytest_xfail_xpass() { + let output = r#"=== test session starts === +collected 5 items + +test_math.py ..xxX [100%] + +=== short test summary info === +XFAIL test_math.py::test_division_by_zero - known bug in division +XFAIL test_math.py::test_float_precision - float precision issue — bug #42 +XPASS test_math.py::test_unexpected_pass - this should fail but currently passes +=== 2 passed, 2 xfailed, 1 xpassed in 0.05s ==="#; + + let result = filter_pytest_output(output); + assert!(result.contains("xfailed"), "got: {result}"); + assert!(result.contains("xpassed"), "got: {result}"); + assert!(result.contains("XPASS"), "got: {result}"); + assert!(result.contains("float precision"), "got: {result}"); + assert!(result.contains("test_division_by_zero"), "got: {result}"); + } + + #[test] + fn test_filter_pytest_quiet_mode_failures() { + // In -q mode, the final summary line has NO === wrapper + // This was causing "No tests collected" to be reported incorrectly + let output = r#"=== test session starts === +platform linux -- Python 3.12.11, pytest-8.1.0 +collected 1705 items + +.......F....... + +=== FAILURES === +___ test_something ___ + +E AssertionError: expected True + +=== short test summary info === +FAILED tests/test_foo.py::test_something - AssertionError +5 failed, 1698 passed, 2 skipped in 108.89s"#; + + let result = filter_pytest_output(output); + assert!( + !result.contains("No tests collected"), + "Should not report 'No tests collected' when tests ran. Got: {}", + result + ); + assert!( + result.contains("1698") || result.contains("5 failed"), + "Should show actual test counts. Got: {}", + result + ); + } + + #[test] + fn test_filter_pytest_only_skipped() { + // If only skipped tests, should NOT say "No tests collected" + let output = r#"=== test session starts === +collected 3 items + +=== 3 skipped in 0.10s ==="#; + + let result = filter_pytest_output(output); + assert!( + !result.contains("No tests collected"), + "Should not say 'No tests collected' when tests were skipped. Got: {}", + result + ); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/cmds/rust/cargo_cmd.rs b/crates/sparkplane-rtk/upstream/src/cmds/rust/cargo_cmd.rs new file mode 100644 index 0000000..3960418 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/cmds/rust/cargo_cmd.rs @@ -0,0 +1,2899 @@ +//! Filters cargo output — build errors, test results, clippy warnings. + +use crate::core::args_utils; +use crate::core::runner; +use crate::core::stream::{BlockHandler, BlockStreamFilter, StreamFilter}; +use crate::core::truncate::{CAP_ERRORS, CAP_LIST, CAP_WARNINGS}; +use crate::core::utils::{join_with_overflow, resolved_command, truncate}; +use anyhow::Result; +use serde::Deserialize; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::ffi::OsString; +use std::sync::LazyLock; + +#[derive(Debug, Clone)] +pub enum CargoCommand { + Build, + Test, + Clippy, + Check, + Install, + Nextest, +} + +pub fn run(cmd: CargoCommand, args: &[String], verbose: u8) -> Result { + match cmd { + CargoCommand::Build => run_build(args, verbose), + CargoCommand::Test => run_test(args, verbose), + CargoCommand::Clippy => run_clippy(args, verbose), + CargoCommand::Check => run_check(args, verbose), + CargoCommand::Install => run_install(args, verbose), + CargoCommand::Nextest => run_nextest(args, verbose), + } +} + +// --- Stream handlers --- + +struct CargoBuildHandler { + compiled: usize, + warnings: usize, + error_count: usize, + finished_line: Option, + label: &'static str, +} + +impl CargoBuildHandler { + fn with_label(label: &'static str) -> Self { + Self { + compiled: 0, + warnings: 0, + error_count: 0, + finished_line: None, + label, + } + } +} + +impl BlockHandler for CargoBuildHandler { + fn should_skip(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if trimmed.starts_with("Compiling") || trimmed.starts_with("Checking") { + self.compiled += 1; + return true; + } + if trimmed.starts_with("Downloading") || trimmed.starts_with("Downloaded") { + return true; + } + if trimmed.starts_with("Finished") { + self.finished_line = Some(trimmed.to_string()); + return true; + } + if line.starts_with("warning:") && line.contains("generated") && line.contains("warning") { + return true; + } + if (line.starts_with("error:") || line.starts_with("error[")) + && (line.contains("aborting due to") || line.contains("could not compile")) + { + return true; + } + false + } + + fn is_block_start(&mut self, line: &str) -> bool { + if line.starts_with("error[") || line.starts_with("error:") { + self.error_count += 1; + return true; + } + if line.starts_with("warning:") || line.starts_with("warning[") { + self.warnings += 1; + return true; + } + false + } + + fn is_block_continuation(&mut self, line: &str, block: &[String]) -> bool { + !(line.trim().is_empty() && block.len() > 3) + } + + fn format_summary(&self, exit_code: i32, raw: &str) -> Option { + if self.error_count == 0 && self.warnings == 0 && exit_code == 0 { + let summary = + cargo_build_success_line(self.compiled, self.finished_line.as_deref(), self.label); + return Some(crate::core::guard::never_worse(raw, &summary).to_string()); + } + // The streamed path only runs for non-json build/check; error blocks are + // emitted live, so the summary carries no rendered diagnostics. + let empty = JsonDiagnostics { + errors: Vec::new(), + warnings: Vec::new(), + }; + Some(cargo_build_failure_summary( + self.compiled, + self.error_count, + self.warnings, + &empty, + self.label, + exit_code, + )) + } +} + +struct CargoTestHandler { + in_failure_section: bool, + in_failure_names: bool, + summary_lines: Vec, + has_compile_errors: bool, +} + +impl CargoTestHandler { + fn new() -> Self { + Self { + in_failure_section: false, + in_failure_names: false, + summary_lines: Vec::new(), + has_compile_errors: false, + } + } + + /// Compacted `cargo test` summary, before the never-worse guard. + fn compute_test_summary(&self, raw: &str) -> Option { + if self.summary_lines.is_empty() { + let json = extract_json_diagnostics(raw); + if self.has_compile_errors || !json.errors.is_empty() { + // Content-based (exit 0): a real compile error yields "cargo test: N + // errors"; a bare "could not compile" leaves the raw tail fallback. + let build_filtered = filter_cargo_build_labeled(raw, "test", 0); + if build_filtered.contains("cargo test:") { + return Some(format!("{}\n", build_filtered)); + } + // Fallback: last 5 meaningful lines + let meaningful: Vec<&str> = raw + .lines() + .filter(|l| !l.trim().is_empty() && !l.trim_start().starts_with("Compiling")) + .collect(); + let last5: Vec<&str> = meaningful.iter().rev().take(5).rev().copied().collect(); + return Some(format!("{}\n", last5.join("\n"))); + } + } + + // No failures emitted — aggregate pass results + let mut aggregated: Option = None; + let mut all_parsed = true; + + for line in &self.summary_lines { + if let Some(parsed) = AggregatedTestResult::parse_line(line) { + if let Some(ref mut agg) = aggregated { + agg.merge(&parsed); + } else { + aggregated = Some(parsed); + } + } else { + all_parsed = false; + break; + } + } + + if all_parsed + && let Some(agg) = aggregated + && agg.suites > 0 + { + return Some(format!("{}\n", agg.format_compact())); + } + + // Fallback: show raw summary lines + if !self.summary_lines.is_empty() { + let mut s = String::new(); + for line in &self.summary_lines { + s.push_str(line); + s.push('\n'); + } + return Some(s); + } + + None + } +} + +impl BlockHandler for CargoTestHandler { + fn should_skip(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if trimmed.starts_with("Compiling") + || trimmed.starts_with("Downloading") + || trimmed.starts_with("Downloaded") + || trimmed.starts_with("Finished") + { + return true; + } + if line.starts_with("running ") { + return true; + } + if line.starts_with("test ") && line.ends_with("... ok") { + return true; + } + // Track compile errors for fallback + if trimmed.starts_with("error[") || trimmed.starts_with("error:") { + self.has_compile_errors = true; + } + // "failures:" toggles section state + if line == "failures:" { + if self.in_failure_section { + // Second "failures:" = list of failure names — skip them + self.in_failure_names = true; + } + self.in_failure_section = true; + return true; + } + // Skip the failure name listing section + if self.in_failure_names { + if line.starts_with("test result:") { + self.in_failure_names = false; + self.in_failure_section = false; + self.summary_lines.push(line.to_string()); + return true; + } + return true; + } + if line.starts_with("test result:") { + self.summary_lines.push(line.to_string()); + self.in_failure_section = false; + return true; + } + false + } + + fn is_block_start(&mut self, line: &str) -> bool { + self.in_failure_section && line.starts_with("---- ") + } + + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + self.in_failure_section && !line.starts_with("---- ") + } + + fn format_summary(&self, _exit_code: i32, raw: &str) -> Option { + // Same never-worse guard as CargoBuildHandler (#3430 review): if the + // compacted summary ends up larger than the raw output (e.g. a tiny + // `cargo test` run), keep the raw output instead of "compacting" it + // into something bigger. + let summary = self.compute_test_summary(raw)?; + Some(crate::core::guard::never_worse(raw, &summary).to_string()) + } +} + +/// Generic cargo command runner with filtering. +/// Builds the Command with restored `--` separator, then delegates to shared runner. +fn run_cargo_filtered( + subcommand: &str, + args: &[String], + verbose: u8, + filter_fn: F, +) -> Result +where + F: Fn(&str) -> String, +{ + let mut cmd = resolved_command("cargo"); + cmd.arg(subcommand); + + let restored_args = args_utils::restore_double_dash(args); + for arg in &restored_args { + cmd.arg(arg); + } + + if verbose > 0 { + eprintln!("Running: cargo {} {}", subcommand, restored_args.join(" ")); + } + + runner::run_filtered( + cmd, + &format!("cargo {}", subcommand), + &restored_args.join(" "), + filter_fn, + runner::RunOptions::with_tee(&format!("cargo_{}", subcommand)), + ) +} + +/// Same as `run_cargo_filtered` but the filter also receives the child exit code, +/// so it can tell a genuine failure from a clean run when no diagnostics parse. +fn run_cargo_filtered_with_exit( + subcommand: &str, + args: &[String], + verbose: u8, + filter_fn: F, +) -> Result +where + F: Fn(&str, i32) -> String, +{ + let mut cmd = resolved_command("cargo"); + cmd.arg(subcommand); + + let restored_args = args_utils::restore_double_dash(args); + for arg in &restored_args { + cmd.arg(arg); + } + + if verbose > 0 { + eprintln!("Running: cargo {} {}", subcommand, restored_args.join(" ")); + } + + runner::run_filtered_with_exit( + cmd, + &format!("cargo {}", subcommand), + &restored_args.join(" "), + filter_fn, + runner::RunOptions::with_tee(&format!("cargo_{}", subcommand)), + ) +} + +fn run_cargo_streamed( + subcommand: &str, + args: &[String], + verbose: u8, + filter: Box, +) -> Result { + let mut cmd = resolved_command("cargo"); + cmd.arg(subcommand); + + let restored_args = args_utils::restore_double_dash(args); + for arg in &restored_args { + cmd.arg(arg); + } + + if verbose > 0 { + eprintln!("Running: cargo {} {}", subcommand, restored_args.join(" ")); + } + + runner::run_streamed( + cmd, + &format!("cargo {}", subcommand), + &restored_args.join(" "), + filter, + runner::RunOptions::with_tee(&format!("cargo_{}", subcommand)), + ) +} + +fn has_json_message_format(args: &[String]) -> bool { + let mut json = false; + let mut iter = args.iter(); + while let Some(arg) = iter.next() { + if let Some(val) = arg.strip_prefix("--message-format=") { + json = val.contains("json"); + } else if arg == "--message-format" + && let Some(val) = iter.next() + { + json = val.contains("json"); + } + } + json +} + +fn run_build(args: &[String], verbose: u8) -> Result { + if has_json_message_format(args) { + return run_cargo_filtered_with_exit("build", args, verbose, |o, exit| { + filter_cargo_build_labeled(o, "build", exit) + }); + } + run_cargo_streamed( + "build", + args, + verbose, + Box::new(BlockStreamFilter::new(CargoBuildHandler::with_label( + "build", + ))), + ) +} + +fn run_test(args: &[String], verbose: u8) -> Result { + // No json branch here on purpose: --message-format=json only reformats the + // build phase, the test harness output stays human-readable. CargoTestHandler + // reads both — it aggregates the `test result:` lines and, on a compile error, + // parses the json diagnostics in format_summary. + run_cargo_streamed( + "test", + args, + verbose, + Box::new(BlockStreamFilter::new(CargoTestHandler::new())), + ) +} + +fn run_clippy(args: &[String], verbose: u8) -> Result { + if has_json_message_format(args) { + return run_cargo_filtered_with_exit("clippy", args, verbose, filter_cargo_clippy_json); + } + run_cargo_filtered("clippy", args, verbose, filter_cargo_clippy) +} + +fn run_check(args: &[String], verbose: u8) -> Result { + if has_json_message_format(args) { + return run_cargo_filtered_with_exit("check", args, verbose, |o, exit| { + filter_cargo_build_labeled(o, "check", exit) + }); + } + run_cargo_streamed( + "check", + args, + verbose, + Box::new(BlockStreamFilter::new(CargoBuildHandler::with_label( + "check", + ))), + ) +} + +fn run_install(args: &[String], verbose: u8) -> Result { + run_cargo_filtered("install", args, verbose, filter_cargo_install) +} + +fn run_nextest(args: &[String], verbose: u8) -> Result { + run_cargo_filtered("nextest", args, verbose, filter_cargo_nextest) +} + +/// Format crate name + version into a display string +fn format_crate_info(name: &str, version: &str, fallback: &str) -> String { + if name.is_empty() { + fallback.to_string() + } else if version.is_empty() { + name.to_string() + } else { + format!("{} {}", name, version) + } +} + +/// Filter cargo install output - strip dep compilation, keep installed/replaced/errors +fn filter_cargo_install(output: &str) -> String { + let mut errors: Vec = Vec::new(); + let mut error_count = 0; + let mut compiled = 0; + let mut in_error = false; + let mut current_error = Vec::new(); + let mut installed_crate = String::new(); + let mut installed_version = String::new(); + let mut replaced_lines: Vec = Vec::new(); + let mut already_installed = false; + let mut ignored_line = String::new(); + + for line in output.lines() { + let trimmed = line.trim_start(); + + // Strip noise: dep compilation, downloading, locking, etc. + if trimmed.starts_with("Compiling") { + compiled += 1; + continue; + } + if trimmed.starts_with("Downloading") + || trimmed.starts_with("Downloaded") + || trimmed.starts_with("Locking") + || trimmed.starts_with("Updating") + || trimmed.starts_with("Adding") + || trimmed.starts_with("Finished") + || trimmed.starts_with("Blocking waiting for file lock") + { + continue; + } + + // Keep: Installing line (extract crate name + version) + if trimmed.starts_with("Installing") { + let rest = trimmed.strip_prefix("Installing").unwrap_or("").trim(); + if !rest.is_empty() && !rest.starts_with('/') { + if let Some((name, version)) = rest.split_once(' ') { + installed_crate = name.to_string(); + installed_version = version.to_string(); + } else { + installed_crate = rest.to_string(); + } + } + continue; + } + + // Keep: Installed line (extract crate + version if not already set) + if trimmed.starts_with("Installed") { + let rest = trimmed.strip_prefix("Installed").unwrap_or("").trim(); + if !rest.is_empty() && installed_crate.is_empty() { + let mut parts = rest.split_whitespace(); + if let (Some(name), Some(version)) = (parts.next(), parts.next()) { + installed_crate = name.to_string(); + installed_version = version.to_string(); + } + } + continue; + } + + // Keep: Replacing/Replaced lines + if trimmed.starts_with("Replacing") || trimmed.starts_with("Replaced") { + replaced_lines.push(trimmed.to_string()); + continue; + } + + // Keep: "Ignored package" (already up to date) + if trimmed.starts_with("Ignored package") { + already_installed = true; + ignored_line = trimmed.to_string(); + continue; + } + + // Keep: actionable warnings (e.g., "be sure to add `/path` to your PATH") + // Skip summary lines like "warning: `crate` generated N warnings" + if line.starts_with("warning:") { + if !(line.contains("generated") && line.contains("warning")) { + replaced_lines.push(line.to_string()); + } + continue; + } + + // Detect error blocks + if line.starts_with("error[") || line.starts_with("error:") { + if line.contains("aborting due to") || line.contains("could not compile") { + continue; + } + if in_error && !current_error.is_empty() { + errors.push(current_error.join("\n")); + current_error.clear(); + } + error_count += 1; + in_error = true; + current_error.push(line.to_string()); + } else if in_error { + if line.trim().is_empty() && current_error.len() > 3 { + errors.push(current_error.join("\n")); + current_error.clear(); + in_error = false; + } else { + current_error.push(line.to_string()); + } + } + } + + if !current_error.is_empty() { + errors.push(current_error.join("\n")); + } + + // Already installed / up to date + if already_installed { + let info = ignored_line.split('`').nth(1).unwrap_or(&ignored_line); + return format!("cargo install: {} already installed", info); + } + + // Errors + if error_count > 0 { + let crate_info = format_crate_info(&installed_crate, &installed_version, ""); + let deps_info = if compiled > 0 { + format!(", {} deps compiled", compiled) + } else { + String::new() + }; + + let mut result = String::new(); + if crate_info.is_empty() { + result.push_str(&format!( + "cargo install: {} error{}{}\n", + error_count, + if error_count > 1 { "s" } else { "" }, + deps_info + )); + } else { + result.push_str(&format!( + "cargo install: {} error{} ({}{})\n", + error_count, + if error_count > 1 { "s" } else { "" }, + crate_info, + deps_info + )); + } + + const MAX_INSTALL_ERRORS: usize = CAP_ERRORS; + for (i, err) in errors.iter().enumerate().take(MAX_INSTALL_ERRORS) { + result.push_str(err); + result.push('\n'); + if i < errors.len() - 1 { + result.push('\n'); + } + } + + if errors.len() > MAX_INSTALL_ERRORS { + result.push_str(&format!( + "\n… +{} more issues\n", + errors.len() - MAX_INSTALL_ERRORS + )); + let all_errors = errors.join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_errors, "cargo-build-errors") + { + result.push_str(&format!(" {}\n", hint)); + } + } + + return result.trim().to_string(); + } + + // Success + let crate_info = format_crate_info(&installed_crate, &installed_version, "package"); + + let mut result = format!("cargo install ({}, {} deps compiled)", crate_info, compiled); + + for line in &replaced_lines { + result.push_str(&format!("\n {}", line)); + } + + result +} + +/// Push a completed failure block (header + body) into the failures list, then clear the buffers. +fn flush_failure_block(header: &mut String, body: &mut Vec, failures: &mut Vec) { + if header.is_empty() { + return; + } + let mut block = header.clone(); + if !body.is_empty() { + block.push('\n'); + block.push_str(&body.join("\n")); + } + failures.push(block); + header.clear(); + body.clear(); +} + +/// Filter cargo nextest output - show failures + compact summary +fn filter_cargo_nextest(output: &str) -> String { + let summary_re = regex::Regex::new( + r"Summary \[\s*([\d.]+)s\]\s+(\d+) tests? run:\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) skipped)?" + ).expect("invalid nextest summary regex"); + + let starting_re = regex::Regex::new(r"Starting \d+ tests? across (\d+) binar(?:y|ies)") + .expect("invalid nextest starting regex"); + + let mut failures: Vec = Vec::new(); + let mut in_failure_block = false; + let mut past_summary = false; + let mut current_failure_header = String::new(); + let mut current_failure_body = Vec::new(); + let mut summary_line = String::new(); + let mut binaries: u32 = 0; + let mut has_cancel_line = false; + + for line in output.lines() { + let trimmed = line.trim(); + + // Strip compilation noise + if trimmed.starts_with("Compiling") + || trimmed.starts_with("Downloading") + || trimmed.starts_with("Downloaded") + || trimmed.starts_with("Finished") + || trimmed.starts_with("Locking") + || trimmed.starts_with("Updating") + { + continue; + } + + // Strip separator lines (────) + if trimmed.starts_with("────") { + continue; + } + + // Skip post-summary recap lines (FAIL duplicates + "error: test run failed") + if past_summary { + continue; + } + + // Parse binary count from Starting line + if trimmed.starts_with("Starting") { + if let Some(caps) = starting_re.captures(trimmed) + && let Some(m) = caps.get(1) + { + binaries = m.as_str().parse().unwrap_or(0); + } + continue; + } + + // Strip PASS lines + if trimmed.starts_with("PASS") { + if in_failure_block { + flush_failure_block( + &mut current_failure_header, + &mut current_failure_body, + &mut failures, + ); + in_failure_block = false; + } + continue; + } + + // Detect FAIL lines + if trimmed.starts_with("FAIL") { + // Close previous failure block if any + if in_failure_block { + flush_failure_block( + &mut current_failure_header, + &mut current_failure_body, + &mut failures, + ); + } + current_failure_header = trimmed.to_string(); + in_failure_block = true; + continue; + } + + // Cancellation notice + if trimmed.starts_with("Cancelling") || trimmed.starts_with("Canceling") { + has_cancel_line = true; + continue; + } + + // Nextest run ID line + if trimmed.starts_with("Nextest run ID") { + continue; + } + + // Parse summary + if trimmed.starts_with("Summary") { + summary_line = trimmed.to_string(); + if in_failure_block { + flush_failure_block( + &mut current_failure_header, + &mut current_failure_body, + &mut failures, + ); + in_failure_block = false; + } + past_summary = true; + continue; + } + + // Collect failure body lines (stdout/stderr sections) + if in_failure_block { + current_failure_body.push(line.to_string()); + } + } + + // Close last failure block + if in_failure_block { + flush_failure_block( + &mut current_failure_header, + &mut current_failure_body, + &mut failures, + ); + } + + // Parse summary with regex + if let Some(caps) = summary_re.captures(&summary_line) { + let duration = caps.get(1).map_or("?", |m| m.as_str()); + let passed: u32 = caps + .get(3) + .and_then(|m| m.as_str().parse().ok()) + .unwrap_or(0); + let failed: u32 = caps + .get(4) + .and_then(|m| m.as_str().parse().ok()) + .unwrap_or(0); + let skipped: u32 = caps + .get(5) + .and_then(|m| m.as_str().parse().ok()) + .unwrap_or(0); + + let binary_text = match binaries.cmp(&1) { + Ordering::Greater => format!("{} binaries", binaries), + Ordering::Equal => "1 binary".to_string(), + Ordering::Less => String::new(), + }; + + if failed == 0 { + // All pass - compact single line + let mut parts = vec![format!("{} passed", passed)]; + if skipped > 0 { + parts.push(format!("{} skipped", skipped)); + } + let meta = if binary_text.is_empty() { + format!("{}s", duration) + } else { + format!("{}, {}s", binary_text, duration) + }; + return format!("cargo nextest: {} ({})", parts.join(", "), meta); + } + + // With failures - show failure details then summary + let mut result = String::new(); + + for failure in &failures { + result.push_str(failure); + result.push('\n'); + } + + if has_cancel_line { + result.push_str("Cancelling due to test failure\n"); + } + + let mut summary_parts = vec![format!("{} passed", passed)]; + if failed > 0 { + summary_parts.push(format!("{} failed", failed)); + } + if skipped > 0 { + summary_parts.push(format!("{} skipped", skipped)); + } + let meta = if binary_text.is_empty() { + format!("{}s", duration) + } else { + format!("{}, {}s", binary_text, duration) + }; + result.push_str(&format!( + "cargo nextest: {} ({})", + summary_parts.join(", "), + meta + )); + + return result.trim().to_string(); + } + + // Fallback: if summary regex didn't match, show what we have + if !failures.is_empty() { + let mut result = String::new(); + for failure in &failures { + result.push_str(failure); + result.push('\n'); + } + if !summary_line.is_empty() { + result.push_str(&summary_line); + } + return result.trim().to_string(); + } + + if !summary_line.is_empty() { + return summary_line; + } + + // Empty or unrecognized + String::new() +} + +struct JsonDiagnostics { + errors: Vec, + warnings: Vec, +} + +#[derive(Deserialize)] +struct CargoJsonLine { + reason: String, + message: Option, +} + +#[derive(Deserialize)] +struct CargoDiagnostic { + level: String, + #[serde(default)] + message: String, + rendered: Option, +} + +fn extract_json_diagnostics(raw: &str) -> JsonDiagnostics { + let mut errors = Vec::new(); + let mut warnings = Vec::new(); + for line in raw.lines() { + let line = line.trim_start(); + if !line.starts_with('{') { + continue; + } + let Ok(entry) = serde_json::from_str::(line) else { + continue; + }; + if entry.reason != "compiler-message" { + continue; + } + let Some(msg) = entry.message else { + continue; + }; + let bucket = match msg.level.as_str() { + "error" | "error: internal compiler error" => &mut errors, + "warning" => &mut warnings, + _ => continue, + }; + if msg.message.starts_with("aborting due to") + || msg.message.starts_with("could not compile") + || (msg.message.contains("warning") && msg.message.contains("generated")) + { + continue; + } + if let Some(rendered) = msg.rendered { + bucket.push( + crate::core::utils::strip_ansi(&rendered) + .trim_end() + .to_string(), + ); + } else if !msg.message.is_empty() { + bucket.push(msg.message); + } + } + JsonDiagnostics { errors, warnings } +} + +fn merge_diag_counts( + error_count: usize, + warnings: usize, + json: &JsonDiagnostics, +) -> (usize, usize) { + ( + error_count.max(json.errors.len()), + warnings.max(json.warnings.len()), + ) +} + +fn cargo_build_success_line(compiled: usize, finished: Option<&str>, label: &str) -> String { + match finished { + Some(f) => format!("cargo {} ({} crates compiled)\n{}\n", label, compiled, f), + None => format!("cargo {} ({} crates compiled)\n", label, compiled), + } +} + +fn cargo_build_failure_summary( + compiled: usize, + errors: usize, + warnings: usize, + json: &JsonDiagnostics, + label: &str, + exit_code: i32, +) -> String { + let mut out = if errors == 0 && warnings == 0 { + format!("cargo {}: failed (exit {})\n", label, exit_code) + } else { + format!( + "cargo {}: {} errors, {} warnings ({} crates)\n", + label, errors, warnings, compiled + ) + }; + if !json.errors.is_empty() { + let shown: Vec = json.errors.iter().take(CAP_ERRORS).cloned().collect(); + out.push_str(&join_with_overflow( + &shown, + json.errors.len(), + CAP_ERRORS, + "errors", + )); + out.push('\n'); + } + if !json.warnings.is_empty() { + let shown: Vec = json.warnings.iter().take(CAP_WARNINGS).cloned().collect(); + out.push_str(&join_with_overflow( + &shown, + json.warnings.len(), + CAP_WARNINGS, + "warnings", + )); + out.push('\n'); + } + if json.errors.len() > CAP_ERRORS || json.warnings.len() > CAP_WARNINGS { + let full = json + .errors + .iter() + .chain(json.warnings.iter()) + .map(String::as_str) + .collect::>() + .join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&full, "cargo-json-issues") { + out.push_str(&format!(" {}\n", hint)); + } + } + out +} + +#[cfg(test)] +fn filter_cargo_build(output: &str) -> String { + filter_cargo_build_labeled(output, "build", 0) +} + +fn filter_cargo_build_labeled(output: &str, label: &'static str, exit_code: i32) -> String { + let mut handler = CargoBuildHandler::with_label(label); + let mut blocks: Vec> = Vec::new(); + let mut current_block: Vec = Vec::new(); + let mut in_block = false; + + for line in output.lines() { + if handler.should_skip(line) { + continue; + } + if handler.is_block_start(line) { + if in_block && !current_block.is_empty() { + blocks.push(std::mem::take(&mut current_block)); + } + in_block = true; + current_block.push(line.to_string()); + } else if in_block { + if handler.is_block_continuation(line, ¤t_block) { + current_block.push(line.to_string()); + } else { + blocks.push(std::mem::take(&mut current_block)); + in_block = false; + } + } + } + if !current_block.is_empty() { + blocks.push(current_block); + } + + let json = extract_json_diagnostics(output); + let (errors, warnings) = merge_diag_counts(handler.error_count, handler.warnings, &json); + + if errors == 0 && warnings == 0 && exit_code == 0 { + let summary = + cargo_build_success_line(handler.compiled, handler.finished_line.as_deref(), label); + return crate::core::guard::never_worse(output, &summary).to_string(); + } + + let mut result = + cargo_build_failure_summary(handler.compiled, errors, warnings, &json, label, exit_code); + const MAX_CHECK_BLOCKS: usize = CAP_ERRORS; + for (i, blk) in blocks.iter().enumerate().take(MAX_CHECK_BLOCKS) { + result.push_str(&blk.join("\n")); + result.push('\n'); + if i < blocks.len() - 1 { + result.push('\n'); + } + } + if blocks.len() > MAX_CHECK_BLOCKS { + result.push_str(&format!( + "\n… +{} more issues\n", + blocks.len() - MAX_CHECK_BLOCKS + )); + let all_blocks: String = blocks + .iter() + .map(|b| b.join("\n")) + .collect::>() + .join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_blocks, "cargo-check-issues") { + result.push_str(&format!(" {}\n", hint)); + } + } + result.trim().to_string() +} + +/// Aggregated test results for compact display +#[derive(Debug, Default, Clone)] +struct AggregatedTestResult { + passed: usize, + failed: usize, + ignored: usize, + measured: usize, + filtered_out: usize, + suites: usize, + duration_secs: f64, + has_duration: bool, +} + +impl AggregatedTestResult { + /// Parse a test result summary line + /// Format: "test result: ok. 15 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s" + fn parse_line(line: &str) -> Option { + static RE: LazyLock = LazyLock::new(|| { + regex::Regex::new( + r"test result: (\w+)\.\s+(\d+) passed;\s+(\d+) failed;\s+(\d+) ignored;\s+(\d+) measured;\s+(\d+) filtered out(?:;\s+finished in ([\d.]+)s)?" + ).unwrap() + }); + + let caps = RE.captures(line)?; + let status = caps.get(1)?.as_str(); + + // Only aggregate if status is "ok" (all tests passed) + if status != "ok" { + return None; + } + + let passed = caps.get(2)?.as_str().parse().ok()?; + let failed = caps.get(3)?.as_str().parse().ok()?; + let ignored = caps.get(4)?.as_str().parse().ok()?; + let measured = caps.get(5)?.as_str().parse().ok()?; + let filtered_out = caps.get(6)?.as_str().parse().ok()?; + + let (duration_secs, has_duration) = if let Some(duration_match) = caps.get(7) { + (duration_match.as_str().parse().unwrap_or(0.0), true) + } else { + (0.0, false) + }; + + Some(Self { + passed, + failed, + ignored, + measured, + filtered_out, + suites: 1, + duration_secs, + has_duration, + }) + } + + /// Merge another test result into this one + fn merge(&mut self, other: &Self) { + self.passed += other.passed; + self.failed += other.failed; + self.ignored += other.ignored; + self.measured += other.measured; + self.filtered_out += other.filtered_out; + self.suites += other.suites; + self.duration_secs += other.duration_secs; + self.has_duration = self.has_duration && other.has_duration; + } + + /// Format as compact single line + fn format_compact(&self) -> String { + let mut parts = vec![format!("{} passed", self.passed)]; + + if self.ignored > 0 { + parts.push(format!("{} ignored", self.ignored)); + } + if self.filtered_out > 0 { + parts.push(format!("{} filtered out", self.filtered_out)); + } + + let counts = parts.join(", "); + + let suite_text = if self.suites == 1 { + "1 suite".to_string() + } else { + format!("{} suites", self.suites) + }; + + if self.has_duration { + format!( + "cargo test: {} ({}, {:.2}s)", + counts, suite_text, self.duration_secs + ) + } else { + format!("cargo test: {} ({})", counts, suite_text) + } + } +} + +pub(crate) fn filter_cargo_test(output: &str) -> String { + let mut failures: Vec = Vec::new(); + let mut summary_lines: Vec = Vec::new(); + let mut in_failure_section = false; + let mut current_failure = Vec::new(); + + for line in output.lines() { + // Skip compilation lines + if line.trim_start().starts_with("Compiling") + || line.trim_start().starts_with("Downloading") + || line.trim_start().starts_with("Downloaded") + || line.trim_start().starts_with("Finished") + { + continue; + } + + // Skip "running N tests" and individual "test ... ok" lines + if line.starts_with("running ") || (line.starts_with("test ") && line.ends_with("... ok")) { + continue; + } + + // Detect failures section + if line == "failures:" { + in_failure_section = true; + continue; + } + + if in_failure_section { + if line.starts_with("test result:") { + in_failure_section = false; + summary_lines.push(line.to_string()); + } else if line.starts_with(" ") || line.starts_with("---- ") { + current_failure.push(line.to_string()); + } else if line.trim().is_empty() && !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + current_failure.clear(); + } else if !line.trim().is_empty() { + current_failure.push(line.to_string()); + } + } + + // Capture test result summary + if !in_failure_section && line.starts_with("test result:") { + summary_lines.push(line.to_string()); + } + } + + if !current_failure.is_empty() { + failures.push(current_failure.join("\n")); + } + + let mut result = String::new(); + + if failures.is_empty() && !summary_lines.is_empty() { + // All passed - try to aggregate + let mut aggregated: Option = None; + let mut all_parsed = true; + + for line in &summary_lines { + if let Some(parsed) = AggregatedTestResult::parse_line(line) { + if let Some(ref mut agg) = aggregated { + agg.merge(&parsed); + } else { + aggregated = Some(parsed); + } + } else { + all_parsed = false; + break; + } + } + + // If all lines parsed successfully and we have at least one suite, return compact format + if all_parsed + && let Some(agg) = aggregated + && agg.suites > 0 + { + return agg.format_compact(); + } + + // Fallback: use original behavior if regex failed + for line in &summary_lines { + result.push_str(&format!("{}\n", line)); + } + return result.trim().to_string(); + } + + if !failures.is_empty() { + result.push_str(&format!("FAILURES ({}):\n", failures.len())); + const MAX_FAILURES: usize = CAP_WARNINGS; + for (i, failure) in failures.iter().enumerate().take(MAX_FAILURES) { + result.push_str(&format!("{}. {}\n", i + 1, truncate(failure, 200))); + } + if failures.len() > MAX_FAILURES { + result.push_str(&format!( + "\n… +{} more failures\n", + failures.len() - MAX_FAILURES + )); + let all_failures = failures.join("\n\n"); + if let Some(hint) = + crate::core::tee::force_tee_hint(&all_failures, "cargo-test-failures") + { + result.push_str(&format!(" {}\n", hint)); + } + } + result.push('\n'); + } + + for line in &summary_lines { + result.push_str(&format!("{}\n", line)); + } + + if result.trim().is_empty() { + let json = extract_json_diagnostics(output); + let has_compile_errors = !json.errors.is_empty() + || output.lines().any(|line| { + let trimmed = line.trim_start(); + trimmed.starts_with("error[") || trimmed.starts_with("error:") + }); + + if has_compile_errors { + let build_filtered = filter_cargo_build_labeled(output, "test", 0); + if build_filtered.contains("cargo test:") { + return build_filtered; + } + } + + // Fallback: show last meaningful lines + let meaningful: Vec<&str> = output + .lines() + .filter(|l| !l.trim().is_empty() && !l.trim_start().starts_with("Compiling")) + .collect(); + for line in meaningful.iter().rev().take(5).rev() { + result.push_str(&format!("{}\n", line)); + } + } + + result.trim().to_string() +} + +/// Filter cargo clippy output - show full error blocks, group warnings by lint rule +fn filter_cargo_clippy(output: &str) -> String { + let mut by_rule: HashMap> = HashMap::new(); + let mut error_count = 0; + let mut warning_count = 0; + // Each entry is a full multi-line error block (headline + location + code context) + let mut error_blocks: Vec> = Vec::new(); + + let mut current_rule = String::new(); + let mut in_error = false; + let mut current_block: Vec = Vec::new(); + + for line in output.lines() { + // Skip compilation progress lines + if line.trim_start().starts_with("Compiling") + || line.trim_start().starts_with("Checking") + || line.trim_start().starts_with("Downloading") + || line.trim_start().starts_with("Downloaded") + || line.trim_start().starts_with("Finished") + { + if in_error && !current_block.is_empty() { + error_blocks.push(current_block.clone()); + current_block.clear(); + in_error = false; + } + continue; + } + + // Skip noise: summary counts and abort lines + if (line.contains("generated") && line.contains("warning")) + || line.contains("aborting due to") + || line.contains("could not compile") + { + continue; + } + + let is_error_line = line.starts_with("error:") || line.starts_with("error["); + let is_warning_line = line.starts_with("warning:") || line.starts_with("warning["); + + if is_error_line || is_warning_line { + // Flush any in-progress error block before starting a new diagnostic + if in_error && !current_block.is_empty() { + error_blocks.push(current_block.clone()); + current_block.clear(); + } + in_error = false; + + if is_error_line { + error_count += 1; + in_error = true; + current_block.push(line.to_string()); + } else { + warning_count += 1; + } + + // Extract rule/error-code from brackets for warning grouping + current_rule = if let Some(bracket_start) = line.rfind('[') { + if let Some(bracket_end) = line.rfind(']') { + line[bracket_start + 1..bracket_end].to_string() + } else { + line.to_string() + } + } else { + let prefix = if is_error_line { + "error: " + } else { + "warning: " + }; + line.strip_prefix(prefix).unwrap_or(line).to_string() + }; + } else if line.trim_start().starts_with("--> ") { + let location = line.trim_start().trim_start_matches("--> ").to_string(); + if !current_rule.is_empty() { + by_rule + .entry(current_rule.clone()) + .or_default() + .push(location); + } + if in_error { + current_block.push(line.to_string()); + } + } else if in_error { + if line.trim().is_empty() { + // Blank line terminates the error block + if !current_block.is_empty() { + error_blocks.push(current_block.clone()); + current_block.clear(); + } + in_error = false; + } else if current_block.len() < 15 { + // Collect code-context lines (|, ^, = note:, help:, etc.) + current_block.push(line.to_string()); + } + } + } + + // Flush final error block + if in_error && !current_block.is_empty() { + error_blocks.push(current_block); + } + + if error_count == 0 && warning_count == 0 { + return "cargo clippy: No issues found".to_string(); + } + + let mut result = String::new(); + result.push_str(&format!( + "cargo clippy: {} errors, {} warnings\n", + error_count, warning_count + )); + + // Show full error blocks so developers can see what needs fixing + if !error_blocks.is_empty() { + const MAX_CLIPPY_ERRORS: usize = CAP_WARNINGS; + result.push_str("\nErrors:\n"); + for block in error_blocks.iter().take(MAX_CLIPPY_ERRORS) { + for block_line in block { + result.push_str(&format!(" {}\n", truncate(block_line, 160))); + } + result.push('\n'); + } + if error_blocks.len() > MAX_CLIPPY_ERRORS { + result.push_str(&format!( + " … +{} more errors\n", + error_blocks.len() - MAX_CLIPPY_ERRORS + )); + let all_blocks: String = error_blocks + .iter() + .map(|b| b.join("\n")) + .collect::>() + .join("\n\n"); + if let Some(hint) = crate::core::tee::force_tee_hint(&all_blocks, "cargo-clippy-errors") + { + result.push_str(&format!(" {}\n", hint)); + } + } + } + + // Sort warning rules by frequency + let mut rule_counts: Vec<_> = by_rule.iter().collect(); + rule_counts.sort_by_key(|b| std::cmp::Reverse(b.1.len())); + + const MAX_RULES: usize = CAP_LIST; + for (rule, locations) in rule_counts.iter().take(MAX_RULES) { + result.push_str(&format!(" {} ({}x)\n", rule, locations.len())); + for loc in locations.iter().take(3) { + result.push_str(&format!(" {}\n", loc)); + } + if locations.len() > 3 { + result.push_str(&format!(" … +{} more\n", locations.len() - 3)); + } + } + + if by_rule.len() > MAX_RULES { + result.push_str(&format!("\n… +{} more rules\n", by_rule.len() - MAX_RULES)); + let all_rules = rule_counts + .iter() + .map(|(rule, locs)| format!("{} ({}x)", rule, locs.len())) + .collect::>() + .join("\n"); + if let Some(hint) = + crate::core::tee::force_tee_tail_hint(&all_rules, "cargo-clippy-rules", MAX_RULES + 1) + { + result.push_str(&format!(" {}\n", hint)); + } + } + + result.trim().to_string() +} + +fn filter_cargo_clippy_json(output: &str, exit_code: i32) -> String { + let json = extract_json_diagnostics(output); + if json.errors.is_empty() && json.warnings.is_empty() && exit_code == 0 { + return "cargo clippy: No issues found".to_string(); + } + filter_cargo_build_labeled(output, "clippy", exit_code) +} + +pub fn run_passthrough(args: &[OsString], verbose: u8) -> Result { + crate::core::runner::run_passthrough("cargo", args, verbose) +} + +#[cfg(test)] +mod tests { + const TINY_BUILD_SUCCESS_OUTPUT: &str = + " Finished dev [unoptimized + debuginfo] target(s) in 0.01s\n"; + + use super::*; + use crate::core::args_utils::restore_double_dash_with_raw; + + #[test] + fn test_restore_double_dash_with_separator() { + // rtk cargo test -- --nocapture → clap gives ["--nocapture"] + let args: Vec = vec!["--nocapture".into()]; + let raw = vec![ + "rtk".into(), + "cargo".into(), + "test".into(), + "--".into(), + "--nocapture".into(), + ]; + let result = restore_double_dash_with_raw(&args, &raw); + assert_eq!(result, vec!["--", "--nocapture"]); + } + + #[test] + fn test_restore_double_dash_with_test_name() { + // rtk cargo test my_test -- --nocapture → clap gives ["my_test", "--nocapture"] + let args: Vec = vec!["my_test".into(), "--nocapture".into()]; + let raw = vec![ + "rtk".into(), + "cargo".into(), + "test".into(), + "my_test".into(), + "--".into(), + "--nocapture".into(), + ]; + let result = restore_double_dash_with_raw(&args, &raw); + assert_eq!(result, vec!["my_test", "--", "--nocapture"]); + } + + #[test] + fn test_restore_double_dash_without_separator() { + // rtk cargo test my_test → no --, args unchanged + let args: Vec = vec!["my_test".into()]; + let raw = vec![ + "rtk".into(), + "cargo".into(), + "test".into(), + "my_test".into(), + ]; + let result = restore_double_dash_with_raw(&args, &raw); + assert_eq!(result, vec!["my_test"]); + } + + #[test] + fn test_restore_double_dash_empty_args() { + let args: Vec = vec![]; + let raw = vec!["rtk".into(), "cargo".into(), "test".into()]; + let result = restore_double_dash_with_raw(&args, &raw); + assert!(result.is_empty()); + } + + #[test] + fn test_restore_double_dash_clippy() { + // rtk cargo clippy -- -D warnings → clap gives ["-D", "warnings"] + let args: Vec = vec!["-D".into(), "warnings".into()]; + let raw = vec![ + "rtk".into(), + "cargo".into(), + "clippy".into(), + "--".into(), + "-D".into(), + "warnings".into(), + ]; + let result = restore_double_dash_with_raw(&args, &raw); + assert_eq!(result, vec!["--", "-D", "warnings"]); + } + + #[test] + fn test_restore_double_dash_clippy_with_package_flags() { + // rtk cargo clippy -p my-service -p my-crate -- -D warnings + // Clap with trailing_var_arg preserves "--" when args precede it + // → clap gives ["-p", "my-service", "-p", "my-crate", "--", "-D", "warnings"] + let args: Vec = vec![ + "-p".into(), + "my-service".into(), + "-p".into(), + "my-crate".into(), + "--".into(), + "-D".into(), + "warnings".into(), + ]; + let raw = vec![ + "rtk".into(), + "cargo".into(), + "clippy".into(), + "-p".into(), + "my-service".into(), + "-p".into(), + "my-crate".into(), + "--".into(), + "-D".into(), + "warnings".into(), + ]; + let result = restore_double_dash_with_raw(&args, &raw); + // Should NOT double the "--" + assert_eq!( + result, + vec!["-p", "my-service", "-p", "my-crate", "--", "-D", "warnings"] + ); + // Verify only one "--" exists + assert_eq!(result.iter().filter(|a| *a == "--").count(), 1); + } + + #[test] + fn test_filter_cargo_build_success() { + let output = r#" Compiling libc v0.2.153 + Compiling cfg-if v1.0.0 + Compiling rtk v0.5.0 + Finished dev [unoptimized + debuginfo] target(s) in 15.23s +"#; + let result = filter_cargo_build(output); + assert!(result.contains("cargo build")); + assert!(result.contains("3 crates compiled")); + } + + #[test] + fn test_filter_cargo_build_success_uses_raw_when_summary_is_larger() { + let output = TINY_BUILD_SUCCESS_OUTPUT; + let result = filter_cargo_build(output); + assert_eq!(result, output); + } + + #[test] + fn test_cargo_test_summary_uses_raw_when_summary_is_larger() { + // A one-line compile failure is already minimal: prefixing it with the + // "cargo test: N errors, ..." header emits more tokens than the raw + // output, so the never-worse guard must keep the raw output. + const CARGO_COMPILE_FAILURE_EXIT_CODE: i32 = 101; + let raw = "error[E0433]: failed to resolve: use of undeclared type `Foo`\n"; + + let mut handler = CargoTestHandler::new(); + for line in raw.lines() { + handler.should_skip(line); + } + + let unguarded = handler + .compute_test_summary(raw) + .expect("compile failure yields a summary"); + assert!( + unguarded.len() > raw.len(), + "expected the compacted summary to be larger than the raw output, got {:?}", + unguarded + ); + assert_eq!( + handler + .format_summary(CARGO_COMPILE_FAILURE_EXIT_CODE, raw) + .as_deref(), + Some(raw) + ); + } + + #[test] + fn test_filter_cargo_build_errors() { + let output = r#" Compiling rtk v0.5.0 +error[E0308]: mismatched types + --> src/main.rs:10:5 + | +10| "hello" + | ^^^^^^^ expected `i32`, found `&str` + +error: aborting due to 1 previous error +"#; + let result = filter_cargo_build(output); + assert!(result.contains("1 errors")); + assert!(result.contains("E0308")); + assert!(result.contains("mismatched types")); + } + + #[test] + fn test_filter_cargo_test_all_pass() { + let output = r#" Compiling rtk v0.5.0 + Finished test [unoptimized + debuginfo] target(s) in 2.53s + Running target/debug/deps/rtk-abc123 + +running 15 tests +test utils::tests::test_truncate_short_string ... ok +test utils::tests::test_truncate_long_string ... ok +test utils::tests::test_strip_ansi_simple ... ok + +test result: ok. 15 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s +"#; + let result = filter_cargo_test(output); + assert!( + result.contains("cargo test: 15 passed (1 suite, 0.01s)"), + "Expected compact format, got: {}", + result + ); + assert!(!result.contains("Compiling")); + assert!(!result.contains("test utils")); + } + + #[test] + fn test_filter_cargo_test_failures() { + let output = r#"running 5 tests +test foo::test_a ... ok +test foo::test_b ... FAILED +test foo::test_c ... ok + +failures: + +---- foo::test_b stdout ---- +thread 'foo::test_b' panicked at 'assert_eq!(1, 2)' + +failures: + foo::test_b + +test result: FAILED. 4 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +"#; + let result = filter_cargo_test(output); + assert!(result.contains("FAILURES")); + assert!(result.contains("test_b")); + assert!(result.contains("test result:")); + } + + #[test] + fn test_filter_cargo_test_multi_suite_all_pass() { + let output = r#" Compiling rtk v0.5.0 + Finished test [unoptimized + debuginfo] target(s) in 2.53s + Running unittests src/lib.rs (target/debug/deps/rtk-abc123) + +running 50 tests +test result: ok. 50 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.45s + + Running unittests src/main.rs (target/debug/deps/rtk-def456) + +running 30 tests +test result: ok. 30 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.30s + + Running tests/integration.rs (target/debug/deps/integration-ghi789) + +running 25 tests +test result: ok. 25 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.25s + + Doc-tests rtk + +running 32 tests +test result: ok. 32 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.45s +"#; + let result = filter_cargo_test(output); + assert!( + result.contains("cargo test: 137 passed (4 suites, 1.45s)"), + "Expected aggregated format, got: {}", + result + ); + assert!(!result.contains("running")); + } + + #[test] + fn test_filter_cargo_test_multi_suite_with_failures() { + let output = r#" Running unittests src/lib.rs + +running 20 tests +test result: ok. 20 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.10s + + Running unittests src/main.rs + +running 15 tests +test foo::test_bad ... FAILED + +failures: + +---- foo::test_bad stdout ---- +thread panicked at 'assertion failed' + +test result: FAILED. 14 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s + + Running tests/integration.rs + +running 10 tests +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s +"#; + let result = filter_cargo_test(output); + // Should NOT aggregate when there are failures + assert!(result.contains("FAILURES"), "got: {}", result); + assert!(result.contains("test_bad"), "got: {}", result); + assert!(result.contains("test result:"), "got: {}", result); + // Should show individual summaries + assert!(result.contains("20 passed"), "got: {}", result); + assert!(result.contains("14 passed"), "got: {}", result); + assert!(result.contains("10 passed"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_test_all_suites_zero_tests() { + let output = r#" Running unittests src/empty1.rs + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running unittests src/empty2.rs + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/empty3.rs + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s +"#; + let result = filter_cargo_test(output); + assert!( + result.contains("cargo test: 0 passed (3 suites, 0.00s)"), + "Expected compact format for zero tests, got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_test_with_ignored_and_filtered() { + let output = r#" Running unittests src/lib.rs + +running 50 tests +test result: ok. 45 passed; 0 failed; 3 ignored; 0 measured; 2 filtered out; finished in 0.50s + + Running tests/integration.rs + +running 20 tests +test result: ok. 18 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.20s +"#; + let result = filter_cargo_test(output); + assert!( + result.contains("cargo test: 63 passed, 5 ignored, 2 filtered out (2 suites, 0.70s)"), + "Expected compact format with ignored and filtered, got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_test_single_suite_compact() { + let output = r#" Running unittests src/main.rs + +running 15 tests +test result: ok. 15 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s +"#; + let result = filter_cargo_test(output); + assert!( + result.contains("cargo test: 15 passed (1 suite, 0.01s)"), + "Expected singular 'suite', got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_test_regex_fallback() { + let output = r#" Running unittests src/main.rs + +running 15 tests +test result: MALFORMED LINE WITHOUT PROPER FORMAT +"#; + let result = filter_cargo_test(output); + // Should fallback to original behavior (show line without checkmark) + assert!( + result.contains("test result: MALFORMED"), + "Expected fallback format, got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_test_compile_error_preserves_error_header() { + let output = r#" Compiling rtk v0.31.0 (/workspace/projects/rtk) +error[E0425]: cannot find value `missing_symbol` in this scope + --> tests/repro_compile_fail.rs:3:13 + | +3 | let _ = missing_symbol; + | ^^^^^^^^^^^^^^ not found in this scope + +For more information about this error, try `rustc --explain E0425`. +error: could not compile `rtk` (test "repro_compile_fail") due to 1 previous error +"#; + let result = filter_cargo_test(output); + assert!(result.contains("cargo test: 1 errors, 0 warnings (1 crates)")); + assert!(result.contains("error[E0425]"), "got: {}", result); + assert!( + result.contains("--> tests/repro_compile_fail.rs:3:13"), + "got: {}", + result + ); + assert!(!result.starts_with('|'), "got: {}", result); + } + + #[test] + fn test_filter_cargo_clippy_clean() { + let output = r#" Checking rtk v0.5.0 + Finished dev [unoptimized + debuginfo] target(s) in 1.53s +"#; + let result = filter_cargo_clippy(output); + assert!(result.contains("cargo clippy: No issues found")); + } + + #[test] + fn test_filter_cargo_clippy_warnings() { + let output = r#" Checking rtk v0.5.0 +warning: unused variable: `x` [unused_variables] + --> src/main.rs:10:9 + | +10| let x = 5; + | ^ help: if this is intentional, prefix it with an underscore: `_x` + +warning: this function has too many arguments [clippy::too_many_arguments] + --> src/git.rs:16:1 + | +16| pub fn run(a: i32, b: i32, c: i32, d: i32, e: i32, f: i32, g: i32, h: i32) {} + | + +warning: `rtk` (bin) generated 2 warnings + Finished dev [unoptimized + debuginfo] target(s) in 1.53s +"#; + let result = filter_cargo_clippy(output); + assert!(result.contains("0 errors, 2 warnings")); + assert!(result.contains("unused_variables")); + assert!(result.contains("clippy::too_many_arguments")); + } + + #[test] + fn test_filter_cargo_clippy_includes_error_details() { + let output = r#" Checking rtk v0.5.0 +error: struct literals are not allowed here +warning: unused variable: `x` [unused_variables] + Finished dev [unoptimized + debuginfo] target(s) in 1.53s +"#; + let result = filter_cargo_clippy(output); + assert!(result.contains("cargo clippy: 1 errors, 1 warnings")); + assert!(result.contains("Errors:")); + assert!(result.contains("struct literals are not allowed here")); + } + + #[test] + fn test_filter_cargo_clippy_shows_full_error_block() { + // Full multi-line error block must be shown so the developer can debug + let output = r#" Checking rtk v0.5.0 +error[E0308]: mismatched types + --> src/main.rs:10:5 + | +9 | fn foo() -> i32 { + | --- expected `i32` because of return type +10| "hello" + | ^^^^^^^ expected `i32`, found `&str` + +error: aborting due to 1 previous error +"#; + let result = filter_cargo_clippy(output); + assert!( + result.contains("cargo clippy: 1 errors, 0 warnings"), + "got: {}", + result + ); + assert!( + result.contains("error[E0308]: mismatched types"), + "got: {}", + result + ); + assert!(result.contains("src/main.rs:10:5"), "got: {}", result); + assert!( + result.contains("expected `i32`, found `&str`"), + "got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_clippy_multiple_errors_show_all_blocks() { + let output = r#"error[E0308]: mismatched types + --> src/foo.rs:5:3 + +error[E0425]: cannot find value `x` + --> src/bar.rs:12:9 + +error: aborting due to 2 previous errors +"#; + let result = filter_cargo_clippy(output); + assert!(result.contains("2 errors"), "got: {}", result); + assert!(result.contains("src/foo.rs:5:3"), "got: {}", result); + assert!(result.contains("src/bar.rs:12:9"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_success() { + let output = r#" Installing rtk v0.11.0 + Downloading crates ... + Downloaded anyhow v1.0.80 + Downloaded clap v4.5.0 + Compiling libc v0.2.153 + Compiling cfg-if v1.0.0 + Compiling anyhow v1.0.80 + Compiling clap v4.5.0 + Compiling rtk v0.11.0 + Finished `release` profile [optimized] target(s) in 45.23s + Replacing /Users/user/.cargo/bin/rtk + Replaced package `rtk v0.9.4` with `rtk v0.11.0` (/Users/user/.cargo/bin/rtk) +"#; + let result = filter_cargo_install(output); + assert!(result.contains("cargo install"), "got: {}", result); + assert!(result.contains("rtk v0.11.0"), "got: {}", result); + assert!(result.contains("5 deps compiled"), "got: {}", result); + assert!(result.contains("Replaced"), "got: {}", result); + assert!(!result.contains("Compiling"), "got: {}", result); + assert!(!result.contains("Downloading"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_replace() { + let output = r#" Installing rtk v0.11.0 + Compiling rtk v0.11.0 + Finished `release` profile [optimized] target(s) in 10.0s + Replacing /Users/user/.cargo/bin/rtk + Replaced package `rtk v0.9.4` with `rtk v0.11.0` (/Users/user/.cargo/bin/rtk) +"#; + let result = filter_cargo_install(output); + assert!(result.contains("cargo install"), "got: {}", result); + assert!(result.contains("Replacing"), "got: {}", result); + assert!(result.contains("Replaced"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_error() { + let output = r#" Installing rtk v0.11.0 + Compiling rtk v0.11.0 +error[E0308]: mismatched types + --> src/main.rs:10:5 + | +10| "hello" + | ^^^^^^^ expected `i32`, found `&str` + +error: aborting due to 1 previous error +"#; + let result = filter_cargo_install(output); + assert!(result.contains("cargo install: 1 error"), "got: {}", result); + assert!(result.contains("E0308"), "got: {}", result); + assert!(result.contains("mismatched types"), "got: {}", result); + assert!(!result.contains("aborting"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_already_installed() { + let output = r#" Ignored package `rtk v0.11.0`, is already installed +"#; + let result = filter_cargo_install(output); + assert!(result.contains("already installed"), "got: {}", result); + assert!(result.contains("rtk v0.11.0"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_up_to_date() { + let output = r#" Ignored package `cargo-deb v2.1.0 (/Users/user/cargo-deb)`, is already installed +"#; + let result = filter_cargo_install(output); + assert!(result.contains("already installed"), "got: {}", result); + assert!(result.contains("cargo-deb v2.1.0"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_empty_output() { + let result = filter_cargo_install(""); + assert!(result.contains("cargo install"), "got: {}", result); + assert!(result.contains("0 deps compiled"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_path_warning() { + let output = r#" Installing rtk v0.11.0 + Compiling rtk v0.11.0 + Finished `release` profile [optimized] target(s) in 10.0s + Replacing /Users/user/.cargo/bin/rtk + Replaced package `rtk v0.9.4` with `rtk v0.11.0` (/Users/user/.cargo/bin/rtk) +warning: be sure to add `/Users/user/.cargo/bin` to your PATH +"#; + let result = filter_cargo_install(output); + assert!(result.contains("cargo install"), "got: {}", result); + assert!( + result.contains("be sure to add"), + "PATH warning should be kept: {}", + result + ); + assert!(result.contains("Replaced"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_multiple_errors() { + let output = r#" Installing rtk v0.11.0 + Compiling rtk v0.11.0 +error[E0308]: mismatched types + --> src/main.rs:10:5 + | +10| "hello" + | ^^^^^^^ expected `i32`, found `&str` + +error[E0425]: cannot find value `foo` + --> src/lib.rs:20:9 + | +20| foo + | ^^^ not found in this scope + +error: aborting due to 2 previous errors +"#; + let result = filter_cargo_install(output); + assert!( + result.contains("2 errors"), + "should show 2 errors: {}", + result + ); + assert!(result.contains("E0308"), "got: {}", result); + assert!(result.contains("E0425"), "got: {}", result); + assert!(!result.contains("aborting"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_locking_and_blocking() { + let output = r#" Locking 45 packages to latest compatible versions + Blocking waiting for file lock on package cache + Downloading crates ... + Downloaded serde v1.0.200 + Compiling serde v1.0.200 + Compiling rtk v0.11.0 + Finished `release` profile [optimized] target(s) in 30.0s + Installing rtk v0.11.0 +"#; + let result = filter_cargo_install(output); + assert!(result.contains("cargo install"), "got: {}", result); + assert!(!result.contains("Locking"), "got: {}", result); + assert!(!result.contains("Blocking"), "got: {}", result); + assert!(!result.contains("Downloading"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_install_from_path() { + let output = r#" Installing /Users/user/projects/rtk + Compiling rtk v0.11.0 + Finished `release` profile [optimized] target(s) in 10.0s +"#; + let result = filter_cargo_install(output); + // Path-based install: crate info not extracted from path + assert!(result.contains("cargo install"), "got: {}", result); + assert!(result.contains("1 deps compiled"), "got: {}", result); + } + + #[test] + fn test_format_crate_info() { + assert_eq!(format_crate_info("rtk", "v0.11.0", ""), "rtk v0.11.0"); + assert_eq!(format_crate_info("rtk", "", ""), "rtk"); + assert_eq!(format_crate_info("", "", "package"), "package"); + assert_eq!(format_crate_info("", "v0.1.0", "fallback"), "fallback"); + } + + #[test] + fn test_filter_cargo_nextest_all_pass() { + let output = r#" Compiling rtk v0.15.2 + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.04s +──────────────────────────── + Starting 301 tests across 1 binary + PASS [ 0.009s] (1/301) rtk::bin/rtk cargo_cmd::tests::test_one + PASS [ 0.008s] (2/301) rtk::bin/rtk cargo_cmd::tests::test_two + PASS [ 0.007s] (301/301) rtk::bin/rtk cargo_cmd::tests::test_last +──────────────────────────── + Summary [ 0.192s] 301 tests run: 301 passed, 0 skipped +"#; + let result = filter_cargo_nextest(output); + assert_eq!( + result, "cargo nextest: 301 passed (1 binary, 0.192s)", + "got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_with_failures() { + let output = r#" Starting 4 tests across 1 binary (1 test skipped) + PASS [ 0.006s] (1/4) test-proj tests::passing_test + FAIL [ 0.006s] (2/4) test-proj tests::failing_test + + stderr ─── + + thread 'tests::failing_test' panicked at src/lib.rs:15:9: + assertion `left == right` failed + left: 1 + right: 2 + + Cancelling due to test failure: 2 tests still running + PASS [ 0.007s] (3/4) test-proj tests::another_passing + FAIL [ 0.006s] (4/4) test-proj tests::another_failing + + stderr ─── + + thread 'tests::another_failing' panicked at src/lib.rs:20:9: + something went wrong + +──────────────────────────── + Summary [ 0.007s] 4 tests run: 2 passed, 2 failed, 1 skipped + FAIL [ 0.006s] (2/4) test-proj tests::failing_test + FAIL [ 0.006s] (4/4) test-proj tests::another_failing +error: test run failed +"#; + let result = filter_cargo_nextest(output); + assert!( + result.contains("tests::failing_test"), + "should contain first failure: {}", + result + ); + assert!( + result.contains("tests::another_failing"), + "should contain second failure: {}", + result + ); + assert!( + result.contains("panicked"), + "should contain stderr detail: {}", + result + ); + assert!( + result.contains("2 passed, 2 failed, 1 skipped"), + "should contain summary: {}", + result + ); + assert!( + !result.contains("PASS"), + "should not contain PASS lines: {}", + result + ); + // Post-summary FAIL recaps must not create duplicate FAIL header entries + // (test names may appear in both header and stderr body naturally) + assert_eq!( + result.matches("FAIL [").count(), + 2, + "should have exactly 2 FAIL headers (no post-summary duplicates): {}", + result + ); + assert!( + !result.contains("error: test run failed"), + "should not contain post-summary error line: {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_with_skipped() { + let output = r#" Starting 50 tests across 2 binaries (3 tests skipped) + PASS [ 0.010s] (1/50) rtk::bin/rtk test_one + PASS [ 0.010s] (50/50) rtk::bin/rtk test_last +──────────────────────────── + Summary [ 0.500s] 50 tests run: 50 passed, 3 skipped +"#; + let result = filter_cargo_nextest(output); + assert_eq!( + result, "cargo nextest: 50 passed, 3 skipped (2 binaries, 0.500s)", + "got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_single_failure_detail() { + let output = r#" Starting 2 tests across 1 binary + PASS [ 0.005s] (1/2) proj tests::good + FAIL [ 0.005s] (2/2) proj tests::bad + + stderr ─── + + thread 'tests::bad' panicked at src/lib.rs:5:9: + assertion failed: false + +──────────────────────────── + Summary [ 0.010s] 2 tests run: 1 passed, 1 failed + FAIL [ 0.005s] (2/2) proj tests::bad +error: test run failed +"#; + let result = filter_cargo_nextest(output); + assert!( + result.contains("assertion failed: false"), + "should show panic message: {}", + result + ); + assert!( + result.contains("1 passed, 1 failed"), + "should show summary: {}", + result + ); + // Post-summary recap must not duplicate FAIL headers + assert_eq!( + result.matches("FAIL [").count(), + 1, + "should have exactly 1 FAIL header (no post-summary duplicate): {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_multiple_binaries() { + let output = r#" Starting 100 tests across 5 binaries + PASS [ 0.010s] (100/100) test_last +──────────────────────────── + Summary [ 1.234s] 100 tests run: 100 passed, 0 skipped +"#; + let result = filter_cargo_nextest(output); + assert_eq!( + result, "cargo nextest: 100 passed (5 binaries, 1.234s)", + "got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_compilation_stripped() { + let output = r#" Compiling serde v1.0.200 + Compiling rtk v0.15.2 + Downloading crates ... + Finished `test` profile [unoptimized + debuginfo] target(s) in 5.00s +──────────────────────────── + Starting 10 tests across 1 binary + PASS [ 0.010s] (10/10) test_last +──────────────────────────── + Summary [ 0.050s] 10 tests run: 10 passed, 0 skipped +"#; + let result = filter_cargo_nextest(output); + assert!( + !result.contains("Compiling"), + "should strip Compiling: {}", + result + ); + assert!( + !result.contains("Downloading"), + "should strip Downloading: {}", + result + ); + assert!( + !result.contains("Finished"), + "should strip Finished: {}", + result + ); + assert!( + result.contains("cargo nextest: 10 passed"), + "got: {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_empty() { + let result = filter_cargo_nextest(""); + assert!(result.is_empty(), "got: {}", result); + } + + #[test] + fn test_filter_cargo_nextest_cancellation_notice() { + let output = r#" Starting 3 tests across 1 binary + FAIL [ 0.005s] (1/3) proj tests::bad + + stderr ─── + + thread panicked at 'oops' + + Cancelling due to test failure: 2 tests still running +──────────────────────────── + Summary [ 0.010s] 3 tests run: 2 passed, 1 failed + FAIL [ 0.005s] (1/3) proj tests::bad +error: test run failed +"#; + let result = filter_cargo_nextest(output); + assert!( + result.contains("Cancelling due to test failure"), + "should include cancel notice: {}", + result + ); + assert!( + result.contains("1 failed"), + "should show failure count: {}", + result + ); + // Post-summary recap must not duplicate FAIL headers + assert_eq!( + result.matches("FAIL [").count(), + 1, + "should have exactly 1 FAIL header (no post-summary duplicate): {}", + result + ); + } + + #[test] + fn test_filter_cargo_nextest_summary_regex_fallback() { + let output = r#" Starting 5 tests across 1 binary + PASS [ 0.005s] (5/5) test_last +──────────────────────────── + Summary MALFORMED LINE +"#; + let result = filter_cargo_nextest(output); + assert!( + result.contains("Summary MALFORMED"), + "should fall back to raw summary: {}", + result + ); + } + + // --- Streaming handler tests --- + + use crate::core::stream::tests::run_block_filter; + + #[test] + fn test_cargo_build_stream_success() { + let input = " Compiling libc v0.2.153\n Compiling cfg-if v1.0.0\n Compiling rtk v0.5.0\n Finished dev [unoptimized + debuginfo] target(s) in 15.23s\n"; + let mut f = BlockStreamFilter::new(CargoBuildHandler::with_label("build")); + let result = run_block_filter(&mut f, input, 0); + assert!(result.contains("3 crates compiled"), "got: {}", result); + assert!(result.contains("Finished"), "got: {}", result); + assert!(!result.contains("Compiling"), "got: {}", result); + } + + #[test] + fn test_cargo_build_stream_success_uses_raw_when_summary_is_larger() { + let input = TINY_BUILD_SUCCESS_OUTPUT; + let mut f = BlockStreamFilter::new(CargoBuildHandler::with_label("build")); + let result = run_block_filter(&mut f, input, 0); + assert_eq!(result, input); + } + + #[test] + fn test_cargo_build_stream_json_success() { + let input = concat!( + " Compiling demo v0.1.0 (/tmp/demo)\n", + r#"{"reason":"compiler-artifact","package_id":"demo 0.1.0","target":{"name":"demo"},"executable":"/tmp/demo/target/debug/demo"}"#, + "\n", + " Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.39s\n", + r#"{"reason":"build-finished","success":true}"#, + "\n", + ); + let mut f = BlockStreamFilter::new(CargoBuildHandler::with_label("build")); + let result = run_block_filter(&mut f, input, 0); + assert!(result.contains("1 crates compiled"), "got: {}", result); + assert!(result.contains("Finished"), "got: {}", result); + assert!(!result.contains("error"), "got: {}", result); + assert!(!result.contains("Compiling"), "got: {}", result); + } + + #[test] + fn test_cargo_build_stream_errors() { + let input = r#" Compiling rtk v0.5.0 +error[E0308]: mismatched types + --> src/main.rs:10:5 + | +10| "hello" + | ^^^^^^^ expected `i32`, found `&str` + +error: aborting due to 1 previous error +"#; + let mut f = BlockStreamFilter::new(CargoBuildHandler::with_label("build")); + let result = run_block_filter(&mut f, input, 1); + assert!(result.contains("E0308"), "got: {}", result); + assert!(result.contains("mismatched types"), "got: {}", result); + assert!(result.contains("1 errors"), "got: {}", result); + assert!(!result.contains("aborting"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_build_json_warning_rendered() { + let input = concat!( + " Compiling v_cargo v0.1.0 (/tmp/v_cargo)\n", + r#"{"reason":"compiler-message","package_id":"v_cargo 0.1.0","message":{"level":"warning","message":"unused variable: `x`","rendered":"warning: unused variable: `x`\n --> src/main.rs:2:9"}}"#, + "\n", + r#"{"reason":"build-finished","success":true}"#, + "\n", + ); + let result = filter_cargo_build_labeled(input, "build", 0); + assert!(result.contains("unused variable"), "got: {}", result); + assert!(result.contains("1 warnings"), "got: {}", result); + assert!(!result.contains("crates compiled"), "got: {}", result); + } + + #[test] + fn test_cargo_check_stream_success_label() { + let input = " Checking demo v0.1.0 (/tmp/demo)\n Finished dev [unoptimized + debuginfo] target(s) in 0.42s\n"; + let mut f = BlockStreamFilter::new(CargoBuildHandler::with_label("check")); + let result = run_block_filter(&mut f, input, 0); + assert!(result.contains("cargo check"), "got: {}", result); + assert!(!result.contains("cargo build"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_build_labeled_clippy_json_not_clean() { + let input = concat!( + " Checking demo v0.1.0 (/tmp/demo)\n", + r#"{"reason":"compiler-message","message":{"code":{"code":"E0308"},"level":"error","message":"mismatched types","rendered":"error[E0308]: mismatched types\n --> src/main.rs:2:18"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let result = filter_cargo_build_labeled(input, "clippy", 1); + assert!(result.contains("cargo clippy:"), "got: {}", result); + assert!(result.contains("1 errors"), "got: {}", result); + assert!( + !result.contains("No issues found"), + "json clippy errors must not be swallowed: {}", + result + ); + } + + #[test] + fn test_filter_cargo_clippy_json_clean() { + let input = concat!( + " Checking demo v0.1.0 (/tmp/demo)\n", + r#"{"reason":"compiler-artifact","package_id":"demo 0.1.0","target":{"name":"demo"}}"#, + "\n", + r#"{"reason":"build-finished","success":true}"#, + "\n", + ); + assert_eq!( + filter_cargo_clippy_json(input, 0), + "cargo clippy: No issues found" + ); + } + + #[test] + fn test_filter_cargo_clippy_json_warnings() { + let input = concat!( + " Checking demo v0.1.0 (/tmp/demo)\n", + r#"{"reason":"compiler-message","message":{"level":"warning","message":"unused variable: `x`","rendered":"warning: unused variable: `x`\n --> src/main.rs:2:9"}}"#, + "\n", + r#"{"reason":"build-finished","success":true}"#, + "\n", + ); + let result = filter_cargo_clippy_json(input, 0); + assert!(result.contains("unused variable"), "got: {}", result); + assert!( + result.contains("cargo clippy: 0 errors, 1 warnings"), + "got: {}", + result + ); + } + + #[test] + fn test_batch_filter_nonzero_exit_without_diagnostics_is_not_compiled() { + // build-script failure: exit 101, no compiler-message diagnostics parsed. + let input = concat!( + " Compiling demo v0.1.0 (/tmp/demo)\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let result = filter_cargo_build_labeled(input, "build", 101); + assert!( + !result.contains("crates compiled"), + "a failed build must not be reported as compiled: {}", + result + ); + assert!( + result.contains("cargo build: failed (exit 101)"), + "got: {}", + result + ); + } + + #[test] + fn test_failure_summary_line_is_first() { + let json = JsonDiagnostics { + errors: vec!["error[E0308]: mismatched types".to_string()], + warnings: Vec::new(), + }; + let out = cargo_build_failure_summary(1, 1, 0, &json, "build", 101); + assert!( + out.starts_with("cargo build: 1 errors"), + "summary line must come first: {}", + out + ); + } + + #[test] + fn test_filter_cargo_build_labeled_check() { + let input = concat!( + r#"{"reason":"compiler-message","message":{"level":"error","message":"mismatched types","rendered":"error[E0308]: mismatched types\n --> src/main.rs:2:18"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let result = filter_cargo_build_labeled(input, "check", 1); + assert!(result.contains("cargo check:"), "got: {}", result); + assert!(!result.contains("cargo build:"), "got: {}", result); + } + + #[test] + fn test_extract_json_diagnostics_strips_ansi() { + let output = concat!( + r#"{"reason":"compiler-message","message":{"level":"error","message":"mismatched types","rendered":"\u001b[1m\u001b[31merror[E0308]\u001b[0m: mismatched types"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let json = extract_json_diagnostics(output); + assert_eq!(json.errors.len(), 1); + assert!( + !json.errors[0].contains('\u{1b}'), + "ansi escapes must be stripped: {:?}", + json.errors[0] + ); + assert!( + json.errors[0].contains("error[E0308]"), + "got: {:?}", + json.errors[0] + ); + } + + #[test] + fn test_extract_json_diagnostics_skips_generated_summary() { + let output = concat!( + r#"{"reason":"compiler-message","message":{"level":"warning","message":"unused variable: `x`","rendered":"warning: unused variable: `x`"}}"#, + "\n", + r#"{"reason":"compiler-message","message":{"level":"warning","message":"`demo` (bin \"demo\") generated 1 warning","rendered":"warning: `demo` (bin \"demo\") generated 1 warning"}}"#, + "\n", + r#"{"reason":"build-finished","success":true}"#, + "\n", + ); + let json = extract_json_diagnostics(output); + assert_eq!( + json.warnings.len(), + 1, + "generated summary must not inflate the count" + ); + assert!( + !json.warnings.iter().any(|w| w.contains("generated")), + "the generated summary line must not appear in the output" + ); + } + + #[test] + fn test_extract_json_diagnostics_counts_ice_as_error() { + let output = concat!( + r#"{"reason":"compiler-message","message":{"level":"error: internal compiler error","message":"unexpected panic","rendered":"error: internal compiler error: unexpected panic"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let json = extract_json_diagnostics(output); + assert_eq!(json.errors.len(), 1, "an ICE must count as an error"); + assert!( + json.errors[0].contains("internal compiler error"), + "got: {:?}", + json.errors[0] + ); + } + + #[test] + fn test_json_format_inline_eq() { + assert!(has_json_message_format(&["--message-format=json".into()])); + } + + #[test] + fn test_json_format_space_separated() { + assert!(has_json_message_format(&[ + "--message-format".into(), + "json".into(), + ])); + } + + #[test] + fn test_json_format_rendered_ansi() { + assert!(has_json_message_format(&[ + "--message-format=json-diagnostic-rendered-ansi".into() + ])); + } + + #[test] + fn test_json_format_render_diagnostics() { + assert!(has_json_message_format(&[ + "--message-format=json-render-diagnostics".into() + ])); + } + + #[test] + fn test_json_format_space_rendered() { + assert!(has_json_message_format(&[ + "--message-format".into(), + "json-diagnostic-rendered-ansi".into(), + ])); + } + + #[test] + fn test_non_json_format_not_detected() { + assert!(!has_json_message_format(&["--message-format=human".into()])); + } + + #[test] + fn test_no_format_flag() { + assert!(!has_json_message_format(&[ + "--release".into(), + "-p".into(), + "my-crate".into(), + ])); + } + + #[test] + fn test_json_format_among_other_args() { + assert!(has_json_message_format(&[ + "--release".into(), + "-p".into(), + "my-crate".into(), + "--message-format=json".into(), + ])); + } + + #[test] + fn test_json_format_trailing_message_format_no_value() { + assert!(!has_json_message_format(&["--message-format".into()])); + } + + #[test] + fn test_json_format_last_occurrence_wins() { + assert!(!has_json_message_format(&[ + "--message-format=json".into(), + "--message-format=human".into(), + ])); + assert!(has_json_message_format(&[ + "--message-format=human".into(), + "--message-format=json".into(), + ])); + } + + #[test] + fn test_filter_cargo_build_json_failure_not_compiled() { + let output = concat!( + r#"{"reason":"compiler-message","message":{"rendered":"error[E0277]: the trait bound is not satisfied","level":"error","message":"trait bound"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let result = filter_cargo_build(output); + assert!(result.contains("E0277"), "got: {}", result); + assert!(result.contains("1 errors"), "got: {}", result); + assert!(!result.contains("crates compiled"), "got: {}", result); + } + + #[test] + fn test_filter_cargo_build_json_errors_capped() { + let total = CAP_ERRORS + 5; + let mut output = String::new(); + for i in 0..total { + output.push_str(&format!( + r#"{{"reason":"compiler-message","message":{{"code":{{"code":"E0308"}},"level":"error","message":"mismatched types","rendered":"error[E0308]: mismatched types ({})"}}}}"#, + i + )); + output.push('\n'); + } + output.push_str(r#"{"reason":"build-finished","success":false}"#); + output.push('\n'); + + let result = filter_cargo_build(&output); + let rendered = result.matches("error[E0308]").count(); + assert_eq!( + rendered, CAP_ERRORS, + "json errors must be capped: {}", + result + ); + assert!( + result.contains(&format!("… +{} more errors", total - CAP_ERRORS)), + "expected overflow hint: {}", + result + ); + assert!( + result.contains(&format!("{} errors", total)), + "summary must report the real total: {}", + result + ); + } + + #[test] + fn test_filter_cargo_build_json_savings() { + // Real --message-format=json lines carry a verbose envelope (spans, + // children, code, message) around the human `rendered` text. The filter + // keeps only `rendered` and caps the list, so savings come from both + // dropping the envelope and capping a large failing build. + let template = r#"{"reason":"compiler-message","package_id":"demo 0.1.0 (path+file:///tmp/demo)","manifest_path":"/tmp/demo/Cargo.toml","target":{"kind":["bin"],"crate_types":["bin"],"name":"demo","src_path":"/tmp/demo/src/main.rs","edition":"2021","doc":true,"doctest":false,"test":true},"message":{"rendered":"error[E0308]: mismatched types\n --> src/main.rs:IDX:18\n |\nIDX | let _xIDX: i32 = \"errIDX\";\n | --- ^^^^^^^ expected `i32`, found `&str`\n | |\n | expected due to this\n\n","$message_type":"diagnostic","children":[{"children":[],"code":null,"level":"note","message":"expected due to the type annotation here","rendered":null,"spans":[]}],"code":{"code":"E0308","explanation":null},"level":"error","message":"mismatched types","spans":[{"byte_end":40,"byte_start":33,"column_end":25,"column_start":18,"expansion":null,"file_name":"src/main.rs","is_primary":true,"label":"expected `i32`, found `&str`","line_end":IDX,"line_start":IDX,"suggested_replacement":null,"suggestion_applicability":null,"text":[{"highlight_end":25,"highlight_start":18,"text":" let _xIDX: i32 = \"errIDX\";"}]}]}}"#; + + let total = CAP_ERRORS + 30; + let mut input = String::new(); + for i in 0..total { + input.push_str(&template.replace("IDX", &i.to_string())); + input.push('\n'); + } + input.push_str(r#"{"reason":"build-finished","success":false}"#); + input.push('\n'); + + let result = filter_cargo_build(&input); + + // The cap is what bounds the output, so assert it holds here too. + let rendered = result.matches("error[E0308]").count(); + assert_eq!( + rendered, CAP_ERRORS, + "json errors must be capped: {}", + result + ); + assert!( + result.contains(&format!("… +{} more errors", total - CAP_ERRORS)), + "expected overflow hint: {}", + result + ); + + let raw = input.split_whitespace().count(); + let out = result.split_whitespace().count(); + let savings = 100.0 - (out as f64 / raw as f64) * 100.0; + assert!( + savings >= 60.0, + "token savings dropped below 60%: {savings:.1}%" + ); + } + + #[test] + fn test_cargo_test_stream_all_pass() { + let input = r#" Compiling rtk v0.5.0 + Finished test [unoptimized + debuginfo] target(s) in 2.53s + Running target/debug/deps/rtk-abc123 + +running 15 tests +test utils::tests::test_truncate_short_string ... ok +test utils::tests::test_truncate_long_string ... ok +test utils::tests::test_strip_ansi_simple ... ok + +test result: ok. 15 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s +"#; + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 0); + assert!( + result.contains("cargo test: 15 passed (1 suite, 0.01s)"), + "got: {}", + result + ); + assert!(!result.contains("Compiling"), "got: {}", result); + } + + #[test] + fn test_cargo_test_stream_failures() { + let input = r#"running 5 tests +test foo::test_a ... ok +test foo::test_b ... FAILED +test foo::test_c ... ok + +failures: + +---- foo::test_b stdout ---- +thread 'foo::test_b' panicked at 'assert_eq!(1, 2)' + +failures: + foo::test_b + +test result: FAILED. 4 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +"#; + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 1); + assert!(result.contains("test_b"), "got: {}", result); + assert!(result.contains("panicked"), "got: {}", result); + } + + #[test] + fn test_cargo_test_stream_multi_suite() { + let input = r#" Running unittests src/lib.rs (target/debug/deps/rtk-abc123) + +running 50 tests +test result: ok. 50 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.45s + + Running unittests src/main.rs (target/debug/deps/rtk-def456) + +running 30 tests +test result: ok. 30 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.30s +"#; + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 0); + assert!( + result.contains("cargo test: 80 passed (2 suites, 0.75s)"), + "got: {}", + result + ); + } + + #[test] + fn test_cargo_test_stream_compile_error() { + let input = r#" Compiling rtk v0.31.0 (/workspace/projects/rtk) +error[E0425]: cannot find value `missing_symbol` in this scope + --> tests/repro_compile_fail.rs:3:13 + | +3 | let _ = missing_symbol; + | ^^^^^^^^^^^^^^ not found in this scope + +For more information about this error, try `rustc --explain E0425`. +error: could not compile `rtk` (test "repro_compile_fail") due to 1 previous error +"#; + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 1); + assert!(result.contains("cargo test:"), "got: {}", result); + assert!(result.contains("1 errors"), "got: {}", result); + } + + #[test] + fn test_cargo_test_stream_json_compile_error() { + let input = concat!( + " Compiling v_cargo v0.1.0 (/tmp/v_cargo)\n", + r#"{"reason":"compiler-message","package_id":"v_cargo 0.1.0","message":{"code":{"code":"E0425"},"level":"error","message":"cannot find value `missing_symbol` in this scope","rendered":"error[E0425]: cannot find value `missing_symbol` in this scope\n --> tests/repro.rs:3:13"}}"#, + "\n", + r#"{"reason":"build-finished","success":false}"#, + "\n", + ); + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 101); + assert!( + !result.trim().is_empty(), + "json compile error must not be silent" + ); + assert!(result.contains("cargo test:"), "got: {}", result); + assert!(result.contains("1 errors"), "got: {}", result); + assert!( + result.contains("E0425"), + "diagnostic must be surfaced: {}", + result + ); + } + + #[test] + fn test_cargo_test_stream_no_diagnostic_falls_back_to_raw() { + // "could not compile" sets has_compile_errors but is skipped by the build + // re-scan, so the reused filter yields a success-shaped line. We must not + // trust it — fall back to the raw tail instead of a bogus "compiled". + let input = concat!( + " Compiling foo v0.1.0 (/tmp/foo)\n", + "error: could not compile `foo` (test \"bar\") due to previous error\n", + ); + let mut f = BlockStreamFilter::new(CargoTestHandler::new()); + let result = run_block_filter(&mut f, input, 101); + assert!( + !result.contains("crates compiled"), + "must not report a failed test run as compiled: {}", + result + ); + assert!(result.contains("could not compile"), "got: {}", result); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/core/constants.rs b/crates/sparkplane-rtk/upstream/src/core/constants.rs new file mode 100644 index 0000000..e7cc757 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/constants.rs @@ -0,0 +1,38 @@ +pub const RTK_DATA_DIR: &str = "rtk"; +pub const HISTORY_DB: &str = "history.db"; +pub const RECALL_DB: &str = "recall.db"; +pub const CONFIG_TOML: &str = "config.toml"; +pub const FILTERS_TOML: &str = "filters.toml"; +pub const TRUSTED_FILTERS_JSON: &str = "trusted_filters.json"; +pub const DEFAULT_HISTORY_DAYS: i64 = 90; + +/// RTK-only subcommands that should never fall back to raw execution. +/// When adding a new RTK-only subcommand to `Commands`, add its clap name here. +pub const RTK_META_COMMANDS: &[&str] = &[ + "gain", + "discover", + "learn", + "init", + "config", + "proxy", + "recall", + "run", + "hook", + "hook-audit", + "pipe", + "cc-economics", + "verify", + "trust", + "untrust", + "session", + "rewrite", + "telemetry", + "smart", + "deps", + "json", + // `err` and `summary` name no real binary, and since they grew `--shell` + // they have a flag to get wrong. Falling through would try to exec a + // program called `err` instead of reporting the flag error (#4125 review). + "err", + "summary", +]; diff --git a/crates/sparkplane-rtk/upstream/src/core/guard.rs b/crates/sparkplane-rtk/upstream/src/core/guard.rs new file mode 100644 index 0000000..3824d44 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/guard.rs @@ -0,0 +1,71 @@ +//! Never-worse output guard: RTK never emits more tokens than the raw command. +//! +//! Two callers are allowed past it. `rtk diff` prints a one-line message for a +//! difference `str::lines()` cannot render — CRLF against LF, or a missing +//! final newline — where the raw fallback is two blobs that look identical and +//! answer the question worse at any size. The exception is bounded by the case +//! rather than by a token count: it fires only when the bytes differ and the +//! line vectors do not, so the message never competes with a change list. A +//! fixed allowance above raw was tried and could not be met by construction: +//! the shortest form of the message is ~20 tokens and a one-line pair is ~2, so +//! the ceiling sat under the message's own floor and dropped it on 90% of +//! one-line pairs. +//! +//! The other is a clean run of a command whose output format rtk injected +//! (`runner::RunOptions::clean_outputs`): ruff's `[]` answers a question the +//! user never asked, so the filter's summary is shown although it is longer. + +use crate::core::tracking::estimate_tokens; + +/// Returns `filtered`, or `raw` when `filtered` would emit more tokens. +pub fn never_worse<'a>(raw: &'a str, filtered: &'a str) -> &'a str { + if estimate_tokens(filtered) > estimate_tokens(raw) { + raw + } else { + filtered + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_filtered_when_smaller() { + let raw = "a".repeat(400); + assert_eq!(never_worse(&raw, "ok"), "ok"); + } + + #[test] + fn falls_back_to_raw_when_filtered_bigger() { + let raw = "{}"; + let filtered = "{\n \"pretty\": true\n}"; + assert_eq!(never_worse(raw, filtered), raw); + } + + #[test] + fn tie_keeps_filtered() { + assert_eq!(never_worse("abcd", "wxyz"), "wxyz"); + } + + #[test] + fn token_boundary_follows_estimate_tokens() { + assert_eq!(never_worse("abcd", "abcde"), "abcd"); + assert_eq!(never_worse("abcdefgh", "ijklmnop"), "ijklmnop"); + } + + #[test] + fn empty_raw_returns_raw() { + assert_eq!(never_worse("", "0 matches"), ""); + } + + #[test] + fn empty_filtered_returns_filtered() { + assert_eq!(never_worse("data", ""), ""); + } + + #[test] + fn both_empty_returns_filtered() { + assert_eq!(never_worse("", ""), ""); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/core/stream.rs b/crates/sparkplane-rtk/upstream/src/core/stream.rs new file mode 100644 index 0000000..986aa22 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/stream.rs @@ -0,0 +1,1469 @@ +use anyhow::{Context, Result}; +use std::borrow::Cow; +use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; +use std::process::{Command, Stdio}; +use std::sync::mpsc; + +#[cfg(test)] +use regex::Regex; + +/// Read `reader` line by line, decoding each line through the console code +/// page and falling back to lossy UTF-8 (invalid bytes become U+FFFD) instead +/// of erroring. +/// +/// `BufRead::lines()` returns `Err` for a non-UTF-8 line, and callers +/// commonly chain `.map_while(Result::ok)` to skip bad lines — but +/// `map_while` stops at the *first* `None`, so one invalid-UTF-8 line (e.g. +/// OEM/ANSI bytes from a non-English-locale Windows tool) silently discards +/// every line after it too, not just the bad one. This reads raw bytes and +/// never fails on the source encoding, so a garbled line still surfaces +/// instead of vanishing along with everything downstream of it. +/// +/// The OEM/ANSI lines this guards against are exactly what +/// [`decode_process_output`](super::utils::decode_process_output) exists to +/// read, so the streamed path decodes them the same way the captured path +/// does rather than going straight to U+FFFD. +fn read_lines_lossy(reader: impl Read) -> impl Iterator { + BufReader::new(reader).split(b'\n').filter_map(|res| { + let mut buf = match res { + Ok(buf) => buf, + Err(e) => { + eprintln!("rtk: stream read error: {}", e); + return None; + } + }; + if buf.last() == Some(&b'\r') { + buf.pop(); + } + Some(super::utils::decode_process_output(&buf)) + }) +} + +pub trait StreamFilter { + fn feed_line(&mut self, line: &str) -> Option; + fn flush(&mut self) -> String; + fn on_exit(&mut self, _exit_code: i32, _raw: &str) -> Option { + None + } +} + +pub trait BlockHandler { + /// Rewrite a raw line before it is matched or emitted — the place to + /// strip ANSI escapes for tools that colour by default. Identity unless + /// a handler opts in, so the rest of the pipeline sees raw bytes. + fn normalize_line<'a>(&self, line: &'a str) -> Cow<'a, str> { + Cow::Borrowed(line) + } + fn should_skip(&mut self, line: &str) -> bool; + fn is_block_start(&mut self, line: &str) -> bool; + fn is_block_continuation(&mut self, line: &str, block: &[String]) -> bool; + fn format_summary(&self, exit_code: i32, raw: &str) -> Option; +} + +pub struct BlockStreamFilter { + handler: H, + in_block: bool, + current_block: Vec, + blocks_emitted: usize, +} + +impl BlockStreamFilter { + pub fn new(handler: H) -> Self { + Self { + handler, + in_block: false, + current_block: Vec::new(), + blocks_emitted: 0, + } + } + + fn emit_block(&mut self) -> Option { + if self.current_block.is_empty() { + return None; + } + let block = self.current_block.join("\n"); + self.current_block.clear(); + self.blocks_emitted += 1; + Some(format!("{}\n", block)) + } +} + +impl StreamFilter for BlockStreamFilter { + fn feed_line(&mut self, line: &str) -> Option { + let line = self.handler.normalize_line(line); + let line = line.as_ref(); + if self.handler.should_skip(line) { + return None; + } + + if self.handler.is_block_start(line) { + let prev = self.emit_block(); + self.current_block.push(line.to_string()); + self.in_block = true; + prev + } else if self.in_block { + if self + .handler + .is_block_continuation(line, &self.current_block) + { + self.current_block.push(line.to_string()); + None + } else { + self.in_block = false; + self.emit_block() + } + } else { + None + } + } + + fn flush(&mut self) -> String { + self.emit_block().unwrap_or_default() + } + + fn on_exit(&mut self, exit_code: i32, raw: &str) -> Option { + self.handler.format_summary(exit_code, raw) + } +} + +/// Counterpart to [`BlockHandler`] for line-oriented streams. +/// +/// Default behaviour is KEEP — every line is emitted unchanged. Implementors +/// opt in to dropping noise via [`LineHandler::should_skip`] and may capture +/// state for the final summary via [`LineHandler::observe_line`]. +pub trait LineHandler { + fn should_skip(&mut self, _line: &str) -> bool { + false + } + + fn observe_line(&mut self, _line: &str) {} + + fn format_summary(&self, exit_code: i32, raw: &str) -> Option; +} + +pub struct LineStreamFilter { + handler: H, +} + +impl LineStreamFilter { + pub fn new(handler: H) -> Self { + Self { handler } + } +} + +impl StreamFilter for LineStreamFilter { + fn feed_line(&mut self, line: &str) -> Option { + if self.handler.should_skip(line) { + return None; + } + self.handler.observe_line(line); + Some(format!("{}\n", line)) + } + + fn flush(&mut self) -> String { + String::new() + } + + fn on_exit(&mut self, exit_code: i32, raw: &str) -> Option { + self.handler.format_summary(exit_code, raw) + } +} + +#[cfg(test)] // available for command modules; currently used in tests only +pub struct RegexBlockFilter { + start_re: Regex, + skip_prefixes: Vec, + tool_name: String, + block_count: usize, +} + +#[cfg(test)] +impl RegexBlockFilter { + pub fn new(tool_name: &str, start_pattern: &str) -> Self { + Self { + start_re: Regex::new(start_pattern).unwrap_or_else(|e| { + panic!("RegexBlockFilter: bad pattern '{}': {}", start_pattern, e) + }), + skip_prefixes: Vec::new(), + tool_name: tool_name.to_string(), + block_count: 0, + } + } + + pub fn skip_prefix(mut self, prefix: &str) -> Self { + self.skip_prefixes.push(prefix.to_string()); + self + } + + pub fn skip_prefixes(mut self, prefixes: &[&str]) -> Self { + self.skip_prefixes + .extend(prefixes.iter().map(|s| s.to_string())); + self + } +} + +#[cfg(test)] +impl BlockHandler for RegexBlockFilter { + fn should_skip(&mut self, line: &str) -> bool { + self.skip_prefixes.iter().any(|p| line.starts_with(p)) + } + + fn is_block_start(&mut self, line: &str) -> bool { + if self.start_re.is_match(line) { + self.block_count += 1; + true + } else { + false + } + } + + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + line.starts_with(' ') || line.starts_with('\t') + } + + fn format_summary(&self, _exit_code: i32, _raw: &str) -> Option { + if self.block_count == 0 { + Some(format!("{}: no errors found\n", self.tool_name)) + } else { + Some(format!( + "{}: {} blocks in output\n", + self.tool_name, self.block_count + )) + } + } +} + +pub trait StdinFilter: Send { + fn feed_line(&mut self, line: &str) -> Option; + fn flush(&mut self) -> String; +} + +pub enum FilterMode<'a> { + Streaming(Box), + #[allow(dead_code)] + Buffered(Box String + 'a>), + CaptureOnly, + Passthrough, +} + +pub enum StdinMode { + Inherit, + #[allow(dead_code)] // future API: stdin filtering for interactive commands + Filter(Box), + Null, +} + +pub struct StreamResult { + pub exit_code: i32, + pub raw: String, + pub raw_stdout: String, + pub raw_stderr: String, + pub filtered: String, +} + +impl StreamResult { + #[cfg(test)] + pub fn success(&self) -> bool { + self.exit_code == 0 + } +} + +// #2375 +#[cfg(unix)] +mod signal_relay { + use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; + use std::thread; + use std::time::Duration; + + const POLL: Duration = Duration::from_millis(25); + const KILL_GRACE: Duration = Duration::from_millis(750); + const EXIT_GRACE: Duration = Duration::from_millis(750); + + static CHILD_PID: AtomicU32 = AtomicU32::new(0); + static RELAYED: AtomicI32 = AtomicI32::new(0); + static FINISHED: AtomicBool = AtomicBool::new(false); + + #[allow(unsafe_code)] + unsafe extern "C" fn relay(sig: libc::c_int) { + let pid = CHILD_PID.load(Ordering::SeqCst); + if pid == 0 || RELAYED.swap(sig, Ordering::SeqCst) != 0 { + // nosemgrep: unsafe-block + unsafe { + libc::signal(sig, libc::SIG_DFL); + libc::raise(sig); + } + return; + } + // nosemgrep: unsafe-block + unsafe { + libc::kill(pid as libc::pid_t, sig); + } + } + + fn escalate(pid: u32) { + thread::sleep(KILL_GRACE); + if FINISHED.load(Ordering::SeqCst) { + return; + } + #[allow(unsafe_code)] + // nosemgrep: unsafe-block + unsafe { + libc::kill(pid as libc::pid_t, libc::SIGKILL); + } + thread::sleep(EXIT_GRACE); + if FINISHED.load(Ordering::SeqCst) { + return; + } + let sig = RELAYED.load(Ordering::SeqCst); + #[allow(unsafe_code)] + // nosemgrep: unsafe-block + unsafe { + libc::signal(sig, libc::SIG_DFL); + libc::raise(sig); + } + } + + pub fn relayed() -> Option { + match RELAYED.load(Ordering::SeqCst) { + 0 => None, + sig => Some(sig), + } + } + + pub struct Relay; + + impl Relay { + pub fn install(pid: u32) -> Self { + CHILD_PID.store(pid, Ordering::SeqCst); + RELAYED.store(0, Ordering::SeqCst); + FINISHED.store(false, Ordering::SeqCst); + #[allow(unsafe_code)] + // nosemgrep: unsafe-block + unsafe { + for sig in [libc::SIGINT, libc::SIGTERM] { + let previous = libc::signal(sig, relay as *const () as libc::sighandler_t); + if previous == libc::SIG_IGN { + libc::signal(sig, libc::SIG_IGN); + } + } + } + thread::spawn(move || { + while !FINISHED.load(Ordering::SeqCst) { + if RELAYED.load(Ordering::SeqCst) != 0 { + escalate(pid); + return; + } + thread::sleep(POLL); + } + }); + Relay + } + } + + impl Drop for Relay { + fn drop(&mut self) { + FINISHED.store(true, Ordering::SeqCst); + CHILD_PID.store(0, Ordering::SeqCst); + #[allow(unsafe_code)] + // nosemgrep: unsafe-block + unsafe { + for sig in [libc::SIGINT, libc::SIGTERM] { + if libc::signal(sig, libc::SIG_DFL) == libc::SIG_IGN { + libc::signal(sig, libc::SIG_IGN); + } + } + } + } + } +} + +#[cfg(not(unix))] +mod signal_relay { + pub struct Relay; + + impl Relay { + pub fn install(_pid: u32) -> Self { + Relay + } + } +} + +// #2375 +#[cfg(unix)] +pub fn die_by_relayed_signal() { + let Some(sig) = signal_relay::relayed() else { + return; + }; + #[allow(unsafe_code)] + // nosemgrep: unsafe-block + unsafe { + libc::signal(sig, libc::SIG_DFL); + libc::raise(sig); + } +} + +#[cfg(not(unix))] +pub fn die_by_relayed_signal() {} + +pub fn status_to_exit_code(status: std::process::ExitStatus) -> i32 { + if let Some(code) = status.code() { + return code; + } + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if let Some(sig) = status.signal() { + return 128 + sig; + } + } + 1 +} + +// ISSUE #897: ChildGuard RAII prevents zombie processes that caused kernel panic +pub const RAW_CAP: usize = 10_485_760; // 10 MiB + +pub fn run_streaming( + cmd: &mut Command, + stdin_mode: StdinMode, + stdout_mode: FilterMode<'_>, +) -> Result { + if matches!(stdout_mode, FilterMode::Passthrough) { + match &stdin_mode { + StdinMode::Inherit => { + cmd.stdin(Stdio::inherit()); + } + _ => { + cmd.stdin(Stdio::null()); + } + }; + cmd.stdout(Stdio::inherit()); + cmd.stderr(Stdio::inherit()); + let status = cmd.status().context("Failed to spawn process")?; + return Ok(StreamResult { + exit_code: status_to_exit_code(status), + raw: String::new(), + raw_stdout: String::new(), + raw_stderr: String::new(), + filtered: String::new(), + }); + } + + match &stdin_mode { + StdinMode::Inherit => { + cmd.stdin(Stdio::inherit()); + } + StdinMode::Filter(_) | StdinMode::Null => { + cmd.stdin(Stdio::piped()); + } + } + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + struct ChildGuard(std::process::Child); + impl Drop for ChildGuard { + fn drop(&mut self) { + self.0.wait().ok(); + } + } + + let is_streaming = matches!(stdout_mode, FilterMode::Streaming(_)); + + let mut child = ChildGuard(cmd.spawn().context("Failed to spawn process")?); + let _signal_relay = signal_relay::Relay::install(child.0.id()); + + let stdin_thread: Option> = match stdin_mode { + StdinMode::Filter(mut filter) => { + let child_stdin = child.0.stdin.take().context("No child stdin handle")?; + Some(std::thread::spawn(move || { + let mut writer = BufWriter::new(child_stdin); + let stdin_handle = io::stdin(); + for line in read_lines_lossy(stdin_handle.lock()) { + if let Some(out) = filter.feed_line(&line) + && writeln!(writer, "{}", out).is_err() + { + break; + } + } + let tail = filter.flush(); + if !tail.is_empty() { + write!(writer, "{}", tail).ok(); + } + })) + } + StdinMode::Null => { + child.0.stdin.take(); + None + } + StdinMode::Inherit => None, + }; + + let stdout = child.0.stdout.take().context("No child stdout handle")?; + let stderr = child.0.stderr.take().context("No child stderr handle")?; + let mut raw_stdout = String::new(); + let mut raw_stderr = String::new(); + let mut filtered = String::new(); + let mut capped_out = false; + let mut capped_err = false; + let mut saved_filter: Option> = None; + let mut filter_fd_is_stderr = false; + + if is_streaming { + enum StreamLine { + Stdout(String), + Stderr(String), + } + + let (tx, rx) = mpsc::channel(); + let tx_out = tx.clone(); + let stdout_thread = std::thread::spawn(move || { + for line in read_lines_lossy(stdout) { + if tx_out.send(StreamLine::Stdout(line)).is_err() { + break; + } + } + }); + let tx_err = tx; + let stderr_thread = std::thread::spawn(move || { + for line in read_lines_lossy(stderr) { + if tx_err.send(StreamLine::Stderr(line)).is_err() { + break; + } + } + }); + + if let FilterMode::Streaming(mut filter) = stdout_mode { + let stdout_handle = io::stdout(); + let mut out = stdout_handle.lock(); + let stderr_handle = io::stderr(); + let mut err_out = stderr_handle.lock(); + + for msg in rx { + let (line, is_stderr) = match msg { + StreamLine::Stderr(l) => (l, true), + StreamLine::Stdout(l) => (l, false), + }; + if is_stderr { + if !capped_err { + if raw_stderr.len() + line.len() < RAW_CAP { + raw_stderr.push_str(&line); + raw_stderr.push('\n'); + } else { + capped_err = true; + eprintln!("[rtk] warning: stderr exceeds 10 MiB — capture truncated"); + } + } + } else if !capped_out { + if raw_stdout.len() + line.len() < RAW_CAP { + raw_stdout.push_str(&line); + raw_stdout.push('\n'); + } else { + capped_out = true; + eprintln!("[rtk] warning: stdout exceeds 10 MiB — filter input truncated"); + } + } + filter_fd_is_stderr = is_stderr; + if let Some(output) = filter.feed_line(&line) { + filtered.push_str(&output); + let dest: &mut dyn Write = if is_stderr { &mut err_out } else { &mut out }; + match write!(dest, "{}", output) { + Err(e) if e.kind() == io::ErrorKind::BrokenPipe => break, + Err(e) => return Err(e.into()), + Ok(_) => {} + } + } + } + let tail = filter.flush(); + filtered.push_str(&tail); + let flush_dest: &mut dyn Write = if filter_fd_is_stderr { + &mut err_out + } else { + &mut out + }; + match write!(flush_dest, "{}", tail) { + Err(e) if e.kind() == io::ErrorKind::BrokenPipe => {} + Err(e) => return Err(e.into()), + Ok(_) => {} + } + saved_filter = Some(filter); + } + + stdout_thread.join().ok(); + stderr_thread.join().ok(); + } else { + let stderr_thread = std::thread::spawn(move || -> String { + let mut raw_err = String::new(); + let mut capped = false; + for line in read_lines_lossy(stderr) { + if raw_err.len() + line.len() < RAW_CAP { + raw_err.push_str(&line); + raw_err.push('\n'); + } else if !capped { + capped = true; + } + } + raw_err + }); + + { + let stdout_handle = io::stdout(); + let mut out = stdout_handle.lock(); + + match stdout_mode { + FilterMode::Passthrough => unreachable!("handled by early-return above"), + FilterMode::Streaming(_) => unreachable!("handled by is_streaming branch"), + FilterMode::Buffered(filter_fn) => { + for line in read_lines_lossy(stdout) { + if raw_stdout.len() + line.len() < RAW_CAP { + raw_stdout.push_str(&line); + raw_stdout.push('\n'); + } else if !capped_out { + capped_out = true; + eprintln!( + "[rtk] warning: output exceeds 10 MiB — filter input truncated" + ); + } + } + filtered = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + filter_fn(&raw_stdout) + })) + .unwrap_or_else(|_| { + eprintln!("[rtk] warning: filter panicked — passing through raw output"); + raw_stdout.clone() + }); + match write!(out, "{}", filtered) { + Err(e) if e.kind() == io::ErrorKind::BrokenPipe => {} + Err(e) => return Err(e.into()), + Ok(_) => {} + } + } + FilterMode::CaptureOnly => { + for line in read_lines_lossy(stdout) { + if raw_stdout.len() + line.len() < RAW_CAP { + raw_stdout.push_str(&line); + raw_stdout.push('\n'); + } else if !capped_out { + capped_out = true; + eprintln!( + "[rtk] warning: output exceeds 10 MiB — filter input truncated" + ); + } + } + filtered = raw_stdout.clone(); + } + } + } + + raw_stderr = stderr_thread.join().unwrap_or_else(|e| { + eprintln!("[rtk] warning: stderr reader thread panicked: {:?}", e); + String::new() + }); + } + if let Some(t) = stdin_thread { + t.join().ok(); + } + + let status = child.0.wait().context("Failed to wait for child")?; + let exit_code = status_to_exit_code(status); + let raw = format!("{}{}", raw_stdout, raw_stderr); + + if let Some(mut f) = saved_filter + && let Some(post) = f.on_exit(exit_code, &raw) + { + filtered.push_str(&post); + let mut dest: Box = if filter_fd_is_stderr { + Box::new(io::stderr().lock()) + } else { + Box::new(io::stdout().lock()) + }; + match write!(dest, "{}", post) { + Err(e) if e.kind() == io::ErrorKind::BrokenPipe => {} + Err(e) => return Err(e.into()), + Ok(_) => {} + } + } + + Ok(StreamResult { + exit_code, + raw, + raw_stdout, + raw_stderr, + filtered, + }) +} + +pub struct CaptureResult { + pub stdout: String, + pub stderr: String, + pub exit_code: i32, +} + +impl CaptureResult { + pub fn success(&self) -> bool { + self.exit_code == 0 + } + + pub fn combined(&self) -> String { + format!("{}{}", self.stdout, self.stderr) + } +} + +pub fn exec_capture(cmd: &mut Command) -> Result { + cmd.stdin(Stdio::null()); + capture(cmd) +} + +/// Like [`exec_capture`] but inherits stdin so a wrapped engine can read a piped stdin. +pub fn exec_capture_stdin(cmd: &mut Command) -> Result { + cmd.stdin(Stdio::inherit()); + capture(cmd) +} + +/// Run `cmd` to completion, decode what it wrote, and report the exit code. +/// +/// A process killed by a signal has no exit code of its own, and returning +/// only the synthesized `128 + signal` hides why it died. `exit_code_from_output` +/// announces that on stderr, so callers moving here from a hand-rolled +/// `.output()` keep the diagnostic instead of losing it. The program name is +/// used as the label so no call site has to pass one. +fn capture(cmd: &mut Command) -> Result { + let raw = capture_raw(cmd)?; + Ok(CaptureResult { + stdout: super::utils::decode_process_output(&raw.stdout), + stderr: super::utils::decode_process_output(&raw.stderr), + exit_code: raw.exit_code, + }) +} + +/// Run `cmd` to completion and return its raw bytes plus a signal-aware exit code. +/// The single spot that turns an `ExitStatus` into a code via +/// [`exit_code_from_output`](super::utils::exit_code_from_output) — so the +/// `process terminated by signal N` diagnostic is emitted uniformly whether the +/// caller decodes the bytes ([`capture`]) or keeps them raw ([`exec_capture_bytes`]), +/// instead of the raw path silently dropping it. +fn capture_raw(cmd: &mut Command) -> Result { + let program = cmd.get_program().to_string_lossy().into_owned(); + let output = cmd.output().context("Failed to execute command")?; + let exit_code = super::utils::exit_code_from_output(&output, &program); + Ok(CaptureBytes { + stdout: output.stdout, + stderr: output.stderr, + exit_code, + }) +} + +/// Raw-byte capture result, for callers that must control decoding themselves — +/// e.g. non-UTF-8 output that [`exec_capture`]'s `from_utf8_lossy` would corrupt. +pub struct CaptureBytes { + pub stdout: Vec, + pub stderr: Vec, + pub exit_code: i32, +} + +impl CaptureBytes { + pub fn success(&self) -> bool { + self.exit_code == 0 + } +} + +/// Like [`exec_capture`] but returns raw bytes so the caller decides how to decode. +pub fn exec_capture_bytes(cmd: &mut Command) -> Result { + cmd.stdin(Stdio::null()); + capture_raw(cmd) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::process::Command; + + #[test] + fn test_read_lines_lossy_preserves_lines_after_invalid_utf8() { + // Line 2 contains a lone 0xE3 byte (the cp850 case from the bug report). + // `BufRead::lines().map_while(Result::ok)` would stop at that Err and + // silently lose line 3 as well — not just the bad byte. + let mut data = Vec::new(); + data.extend_from_slice(b"ERRO A: ascii\n"); + data.extend_from_slice(&[b'E', b'R', b'R', b'O', b' ', b'B', b':', b' ', 0xE3, b'\n']); + data.extend_from_slice(b"ERRO C: ascii again\n"); + + let lines: Vec = read_lines_lossy(data.as_slice()).collect(); + assert_eq!(lines.len(), 3, "got: {:?}", lines); + assert_eq!(lines[0], "ERRO A: ascii"); + assert!(lines[1].starts_with("ERRO B: "), "got: {:?}", lines[1]); + assert!(lines[1].contains('\u{FFFD}'), "got: {:?}", lines[1]); + assert_eq!(lines[2], "ERRO C: ascii again"); + } + + #[test] + fn test_read_lines_lossy_strips_crlf() { + let lines: Vec = read_lines_lossy(&b"a\r\nb\n"[..]).collect(); + assert_eq!(lines, vec!["a".to_string(), "b".to_string()]); + } + + #[test] + fn test_read_lines_lossy_no_trailing_newline() { + let lines: Vec = read_lines_lossy(&b"only line"[..]).collect(); + assert_eq!(lines, vec!["only line".to_string()]); + } + + #[test] + fn test_read_lines_lossy_empty_input() { + let lines: Vec = read_lines_lossy(&b""[..]).collect(); + assert!(lines.is_empty()); + } + + /// A `Read` that yields some good lines, then a genuine I/O error -- + /// distinct from clean EOF (`Ok(0)`). Before this fix, `Ok(0) | Err(_)` + /// treated both the same way, silently truncating output on a real read + /// failure instead of surfacing it. + struct FailingReader { + data: std::io::Cursor>, + failed: bool, + } + + impl Read for FailingReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.data.position() as usize >= self.data.get_ref().len() { + if !self.failed { + self.failed = true; + return Err(io::Error::other("simulated read failure")); + } + return Ok(0); + } + std::io::Read::read(&mut self.data, buf) + } + } + + #[test] + fn test_read_lines_lossy_stops_on_io_error_without_panicking() { + let reader = FailingReader { + data: std::io::Cursor::new(b"line one\nline two\n".to_vec()), + failed: false, + }; + // The two good lines are still yielded; the simulated failure after + // them must not panic or hang -- it just ends the iterator, same as + // clean EOF would, but via the Err(_) arm instead of Ok(0). + let lines: Vec = read_lines_lossy(reader).collect(); + assert_eq!(lines, vec!["line one".to_string(), "line two".to_string()]); + } + + struct LineFilter Option> { + f: F, + } + + impl Option> LineFilter { + pub fn new(f: F) -> Self { + Self { f } + } + } + + impl Option> StreamFilter for LineFilter { + fn feed_line(&mut self, line: &str) -> Option { + (self.f)(line) + } + + fn flush(&mut self) -> String { + String::new() + } + } + + #[test] + fn test_exit_code_zero() { + let status = Command::new("true").status().unwrap(); + assert_eq!(status_to_exit_code(status), 0); + } + + #[test] + fn test_exit_code_nonzero() { + let status = Command::new("false").status().unwrap(); + assert_eq!(status_to_exit_code(status), 1); + } + + #[cfg(unix)] + #[test] + fn test_exit_code_signal_kill() { + let mut child = Command::new("sleep").arg("60").spawn().unwrap(); + child.kill().unwrap(); + let status = child.wait().unwrap(); + assert_eq!(status_to_exit_code(status), 137); + } + + #[test] + fn test_exec_capture_decodes_and_reports_exit_code() { + let captured = exec_capture(&mut Command::new("false")).expect("spawn"); + assert_eq!(captured.exit_code, 1); + assert!(!captured.success()); + } + + /// A signal-killed child keeps the `128 + signal` code that + /// `exit_code_from_output` reports, so callers that moved off a hand-rolled + /// `.output()` neither lose the code nor the stderr diagnostic with it. + #[cfg(unix)] + #[test] + fn test_exec_capture_reports_signal_exit_code() { + // `kill -TERM $$` makes the shell terminate itself by signal 15. + let mut cmd = Command::new("sh"); + cmd.arg("-c").arg("kill -TERM $$"); + let captured = exec_capture(&mut cmd).expect("spawn"); + assert_eq!(captured.exit_code, 128 + 15); + } + + #[test] + fn test_line_filter_passes_lines() { + let mut f = LineFilter::new(|l| Some(format!("{}\n", l.to_uppercase()))); + assert_eq!(f.feed_line("hello"), Some("HELLO\n".to_string())); + } + + #[test] + fn test_line_filter_drops_lines() { + let mut f = LineFilter::new(|l| { + if l.starts_with('#') { + None + } else { + Some(l.to_string()) + } + }); + assert_eq!(f.feed_line("# comment"), None); + assert_eq!(f.feed_line("code"), Some("code".to_string())); + } + + #[test] + fn test_line_filter_flush_empty() { + let mut f = LineFilter::new(|l| Some(l.to_string())); + assert_eq!(f.flush(), String::new()); + } + + #[test] + fn test_stream_result_success() { + let r = StreamResult { + exit_code: 0, + raw: String::new(), + raw_stdout: String::new(), + raw_stderr: String::new(), + filtered: String::new(), + }; + assert!(r.success()); + } + + #[test] + fn test_stream_result_failure() { + let r = StreamResult { + exit_code: 1, + raw: String::new(), + raw_stdout: String::new(), + raw_stderr: String::new(), + filtered: String::new(), + }; + assert!(!r.success()); + } + + #[test] + fn test_stream_result_signal_not_success() { + let r = StreamResult { + exit_code: 137, + raw: String::new(), + raw_stdout: String::new(), + raw_stderr: String::new(), + filtered: String::new(), + }; + assert!(!r.success()); + } + + #[test] + fn test_run_streaming_passthrough_echo() { + let mut cmd = Command::new("echo"); + cmd.arg("hello"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::Passthrough).unwrap(); + assert_eq!(result.exit_code, 0); + // Passthrough inherits TTY — raw/filtered are empty + assert!(result.raw.is_empty()); + } + + #[test] + fn test_run_streaming_exit_code_preserved() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + cmd.args(["-c", "exit 42"]); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::Passthrough).unwrap(); + assert_eq!(result.exit_code, 42); + } + + #[test] + fn test_run_streaming_exit_code_zero() { + let mut cmd = Command::new("true"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::Passthrough).unwrap(); + assert_eq!(result.exit_code, 0); + assert!(result.success()); + } + + #[test] + fn test_run_streaming_exit_code_one() { + let mut cmd = Command::new("false"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::Passthrough).unwrap(); + assert_eq!(result.exit_code, 1); + assert!(!result.success()); + } + + #[cfg(not(windows))] + #[test] + fn test_run_streaming_streaming_filter_drops_lines() { + let mut cmd = Command::new("printf"); + cmd.arg("a\nb\nc\n"); + let filter = LineFilter::new(|l| { + if l == "b" { + None + } else { + Some(format!("{}\n", l)) + } + }); + let result = run_streaming( + &mut cmd, + StdinMode::Null, + FilterMode::Streaming(Box::new(filter)), + ) + .unwrap(); + assert!(result.filtered.contains('a')); + assert!(!result.filtered.contains('b')); + assert!(result.filtered.contains('c')); + assert_eq!(result.exit_code, 0); + } + + #[cfg(not(windows))] + #[test] + fn test_run_streaming_buffered_filter() { + let mut cmd = Command::new("printf"); + cmd.arg("line1\nline2\nline3\n"); + let result = run_streaming( + &mut cmd, + StdinMode::Null, + FilterMode::Buffered(Box::new(|s: &str| s.to_uppercase())), + ) + .unwrap(); + assert!(result.filtered.contains("LINE1")); + assert!(result.filtered.contains("LINE2")); + assert_eq!(result.exit_code, 0); + } + + #[test] + fn test_run_streaming_raw_cap_at_10mb() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + // ~11 MiB of 80-char lines (fast: fewer lines than `yes | head -6M`) + cmd.args([ + "-c", + "dd if=/dev/zero bs=1024 count=11264 2>/dev/null | tr '\\0' 'a' | fold -w 80", + ]); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::CaptureOnly).unwrap(); + assert!( + result.raw.len() <= 10_485_760 + 100, + "raw should be capped at ~10 MiB, got {} bytes", + result.raw.len() + ); + assert!( + result.raw.len() > 1_000_000, + "Should have captured significant data" + ); + } + + #[test] + fn test_run_streaming_stderr_cap_at_10mb() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + // ~11 MiB on stderr, nothing on stdout + cmd.args([ + "-c", + "dd if=/dev/zero bs=1024 count=11264 2>/dev/null | tr '\\0' 'a' | fold -w 80 1>&2", + ]); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::CaptureOnly).unwrap(); + // raw = raw_stdout + raw_stderr; stdout is empty so raw ≈ stderr size + assert!( + result.raw.len() <= RAW_CAP + 200, + "stderr in raw should be capped at ~10 MiB, got {} bytes", + result.raw.len() + ); + } + + #[test] + fn test_child_guard_prevents_zombie() { + let mut cmd = Command::new("true"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::CaptureOnly); + assert!(result.is_ok()); + assert_eq!(result.unwrap().exit_code, 0); + } + + #[test] + fn test_run_streaming_null_stdin_cat() { + let mut cmd = Command::new("cat"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::Passthrough).unwrap(); + assert_eq!(result.exit_code, 0); + } + + #[test] + fn test_run_streaming_raw_contains_stdout() { + let mut cmd = Command::new("echo"); + cmd.arg("test_output_xyz"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::CaptureOnly).unwrap(); + assert!(result.raw.contains("test_output_xyz")); + } + + #[test] + fn test_run_streaming_capture_only_filtered_equals_raw() { + let mut cmd = Command::new("echo"); + cmd.arg("check_equality"); + let result = run_streaming(&mut cmd, StdinMode::Null, FilterMode::CaptureOnly).unwrap(); + assert_eq!(result.filtered.trim(), result.raw_stdout.trim()); + } + + #[test] + fn test_exec_capture_success() { + let mut cmd = Command::new("echo"); + cmd.arg("hello_capture"); + let result = exec_capture(&mut cmd).unwrap(); + assert!(result.success()); + assert_eq!(result.exit_code, 0); + assert!(result.stdout.contains("hello_capture")); + } + + #[test] + fn test_exec_capture_failure() { + let mut cmd = Command::new("false"); + let result = exec_capture(&mut cmd).unwrap(); + assert!(!result.success()); + assert_eq!(result.exit_code, 1); + } + + #[test] + fn test_exec_capture_stderr() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo err_msg >&2"]); + let result = exec_capture(&mut cmd).unwrap(); + assert!(result.stderr.contains("err_msg")); + } + + #[test] + fn test_exec_capture_combined() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo out_msg; echo err_msg >&2"]); + let result = exec_capture(&mut cmd).unwrap(); + let combined = result.combined(); + assert!(combined.contains("out_msg")); + assert!(combined.contains("err_msg")); + } + + #[test] + fn test_capture_result_combined_empty() { + let r = CaptureResult { + stdout: String::new(), + stderr: String::new(), + exit_code: 0, + }; + assert_eq!(r.combined(), ""); + } + + pub fn run_block_filter(filter: &mut dyn StreamFilter, input: &str, exit_code: i32) -> String { + let mut output = String::new(); + for line in input.lines() { + if let Some(s) = filter.feed_line(line) { + output.push_str(&s); + } + } + output.push_str(&filter.flush()); + if let Some(post) = filter.on_exit(exit_code, input) { + output.push_str(&post); + } + output + } + + struct UpperHandler; + + impl BlockHandler for UpperHandler { + fn normalize_line<'a>(&self, line: &'a str) -> Cow<'a, str> { + Cow::Owned(line.to_uppercase()) + } + fn should_skip(&mut self, _line: &str) -> bool { + false + } + fn is_block_start(&mut self, line: &str) -> bool { + line.starts_with("ERR") + } + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + line.starts_with(" ") + } + fn format_summary(&self, _exit_code: i32, _raw: &str) -> Option { + None + } + } + + #[test] + fn block_handler_normalize_line_feeds_matching_and_emission() { + // Both the match and the emitted block see the normalized line. + let mut f = BlockStreamFilter::new(UpperHandler); + let out = run_block_filter(&mut f, "err: one\n detail\nnoise\n", 0); + assert_eq!(out, "ERR: ONE\n DETAIL\n"); + } + + struct TestHandler; + + impl BlockHandler for TestHandler { + fn should_skip(&mut self, line: &str) -> bool { + line.starts_with("SKIP") + } + fn is_block_start(&mut self, line: &str) -> bool { + line.starts_with("ERROR") + } + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + line.starts_with(" ") + } + fn format_summary(&self, _exit_code: i32, _raw: &str) -> Option { + Some("DONE\n".to_string()) + } + } + + #[test] + fn test_block_filter_emits_blocks() { + let mut f = BlockStreamFilter::new(TestHandler); + let input = "SKIP noise\nERROR first\n detail1\nnon-block\nERROR second\n detail2\n"; + let result = run_block_filter(&mut f, input, 0); + assert!(result.contains("ERROR first\n detail1"), "got: {}", result); + assert!( + result.contains("ERROR second\n detail2"), + "got: {}", + result + ); + assert!(!result.contains("SKIP"), "got: {}", result); + assert!(result.ends_with("DONE\n"), "got: {}", result); + } + + #[test] + fn test_block_filter_no_blocks() { + let mut f = BlockStreamFilter::new(TestHandler); + let result = run_block_filter(&mut f, "nothing here\njust text\n", 0); + assert_eq!(result, "DONE\n"); + } + + #[test] + fn test_regex_block_filter_emits_blocks() { + let handler = RegexBlockFilter::new("test", r"^error\["); + let mut f = BlockStreamFilter::new(handler); + let input = "ok line\nerror[E0308]: mismatched types\n expected `u32`\nok again\nerror[E0599]: no method\n help: try\n"; + let result = run_block_filter(&mut f, input, 1); + assert!( + result.contains("error[E0308]: mismatched types\n expected `u32`"), + "got: {}", + result + ); + assert!( + result.contains("error[E0599]: no method\n help: try"), + "got: {}", + result + ); + assert!( + result.contains("test: 2 blocks in output"), + "got: {}", + result + ); + } + + #[test] + fn test_regex_block_filter_skip_prefix() { + let handler = RegexBlockFilter::new("test", r"^error").skip_prefix("warning:"); + let mut f = BlockStreamFilter::new(handler); + let input = "warning: unused var\nerror: bad type\n detail\nwarning: dead code\n"; + let result = run_block_filter(&mut f, input, 1); + assert!(result.contains("error: bad type"), "got: {}", result); + assert!(!result.contains("warning:"), "got: {}", result); + } + + #[test] + fn test_regex_block_filter_no_blocks() { + let handler = RegexBlockFilter::new("mytest", r"^FAIL"); + let mut f = BlockStreamFilter::new(handler); + let result = run_block_filter(&mut f, "all passed\nok\n", 0); + assert_eq!(result, "mytest: no errors found\n"); + } + + #[test] + fn test_regex_block_filter_indent_continuation() { + let handler = RegexBlockFilter::new("test", r"^ERR"); + let mut f = BlockStreamFilter::new(handler); + let input = "ERR space indent\n two spaces\n\ttab indent\nnon-indent\n"; + let result = run_block_filter(&mut f, input, 1); + assert!( + result.contains("ERR space indent\n two spaces\n\ttab indent"), + "got: {}", + result + ); + assert!(!result.contains("non-indent"), "got: {}", result); + } + + #[test] + fn test_regex_block_filter_multiple_skip_prefixes() { + let handler = + RegexBlockFilter::new("test", r"^error").skip_prefixes(&["note:", "warning:", "help:"]); + let mut f = BlockStreamFilter::new(handler); + let input = "note: see docs\nwarning: unused\nhelp: try this\nerror: fatal\n details\n"; + let result = run_block_filter(&mut f, input, 1); + assert!(!result.contains("note:"), "got: {}", result); + assert!(!result.contains("warning:"), "got: {}", result); + assert!(!result.contains("help:"), "got: {}", result); + assert!( + result.contains("error: fatal\n details"), + "got: {}", + result + ); + } + + #[cfg(not(windows))] + #[test] + fn test_streaming_filters_both_fds_and_routes_to_correct_fd() { + // nosemgrep: interpreter-execution + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo 'error[E0308]: type mismatch'; echo ' Compiling foo v1.0' >&2; echo ' Downloading bar v2.0' >&2; echo ' Finished dev' >&2; echo 'real error on stderr' >&2"]); + + struct CargoLikeHandler; + impl BlockHandler for CargoLikeHandler { + fn should_skip(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + trimmed.starts_with("Compiling") + || trimmed.starts_with("Downloading") + || trimmed.starts_with("Finished") + } + fn is_block_start(&mut self, line: &str) -> bool { + line.starts_with("error") + } + fn is_block_continuation(&mut self, line: &str, _block: &[String]) -> bool { + line.starts_with(' ') + } + fn format_summary(&self, _: i32, _: &str) -> Option { + None + } + } + + let filter = BlockStreamFilter::new(CargoLikeHandler); + let result = run_streaming( + &mut cmd, + StdinMode::Null, + FilterMode::Streaming(Box::new(filter)), + ) + .unwrap(); + + assert!( + result.filtered.contains("error[E0308]"), + "filtered should contain stdout errors, got: {}", + result.filtered + ); + assert!( + !result.filtered.contains("Compiling"), + "cargo noise should be filtered out, got: {}", + result.filtered + ); + assert!( + !result.filtered.contains("Downloading"), + "cargo noise should be filtered out, got: {}", + result.filtered + ); + assert!( + result.raw_stderr.contains("Compiling"), + "raw_stderr should capture all stderr lines" + ); + assert!( + result.raw_stderr.contains("real error on stderr"), + "raw_stderr should capture all stderr lines" + ); + } + + struct CountingLineHandler { + observed: Vec, + skip_prefixes: Vec, + summary_tag: &'static str, + } + + impl LineHandler for CountingLineHandler { + fn should_skip(&mut self, line: &str) -> bool { + self.skip_prefixes.iter().any(|p| line.starts_with(p)) + } + + fn observe_line(&mut self, line: &str) { + self.observed.push(line.to_string()); + } + + fn format_summary(&self, exit_code: i32, _raw: &str) -> Option { + Some(format!( + "{}: {} kept, exit={}\n", + self.summary_tag, + self.observed.len(), + exit_code + )) + } + } + + fn run_line_filter(filter: &mut dyn StreamFilter, input: &str, exit_code: i32) -> String { + let mut out = String::new(); + for line in input.lines() { + if let Some(s) = filter.feed_line(line) { + out.push_str(&s); + } + } + out.push_str(&filter.flush()); + if let Some(post) = filter.on_exit(exit_code, input) { + out.push_str(&post); + } + out + } + + #[test] + fn test_line_filter_defaults_keep_all() { + struct DefaultHandler; + impl LineHandler for DefaultHandler { + fn format_summary(&self, _: i32, _: &str) -> Option { + None + } + } + let mut f = LineStreamFilter::new(DefaultHandler); + let result = run_line_filter(&mut f, "a\nb\nc\n", 0); + assert_eq!(result, "a\nb\nc\n"); + } + + #[test] + fn test_line_filter_skip_drops_matching_lines() { + let handler = CountingLineHandler { + observed: Vec::new(), + skip_prefixes: vec!["NOISE:".to_string()], + summary_tag: "demo", + }; + let mut f = LineStreamFilter::new(handler); + let input = "NOISE: progress 10%\nkeep me\nNOISE: progress 90%\nalso keep\n"; + let result = run_line_filter(&mut f, input, 0); + assert!(!result.contains("NOISE:"), "got: {}", result); + assert!(result.contains("keep me\n")); + assert!(result.contains("also keep\n")); + assert!(result.contains("demo: 2 kept, exit=0\n")); + } + + #[test] + fn test_line_filter_summary_propagates_exit_code() { + let handler = CountingLineHandler { + observed: Vec::new(), + skip_prefixes: Vec::new(), + summary_tag: "demo", + }; + let mut f = LineStreamFilter::new(handler); + let result = run_line_filter(&mut f, "one\n", 42); + assert!(result.contains("exit=42"), "got: {}", result); + } + + #[test] + fn test_line_filter_observe_only_called_for_kept_lines() { + let handler = CountingLineHandler { + observed: Vec::new(), + skip_prefixes: vec!["DROP".to_string()], + summary_tag: "demo", + }; + let mut f = LineStreamFilter::new(handler); + let result = run_line_filter(&mut f, "DROP a\nDROP b\nkeep\n", 0); + // Only "keep" was observed, so summary says "1 kept" + assert!(result.contains("demo: 1 kept"), "got: {}", result); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/core/toml_filter.rs b/crates/sparkplane-rtk/upstream/src/core/toml_filter.rs new file mode 100644 index 0000000..6850b59 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/toml_filter.rs @@ -0,0 +1,2229 @@ +//! Applies TOML-defined filter rules to command output. +/// +/// Provides a declarative pipeline of 8 stages that can be configured +/// via TOML files. Lookup priority (first match wins): +/// 1. `.rtk/filters.toml` — project-local, committable with the repo +/// 2. `~/.config/rtk/filters.toml` — user-global, applies to all projects +/// 3. Built-in TOML — `src/filters/*.toml`, concatenated by build.rs and embedded at compile time +/// 4. Passthrough — no match, handled by caller +/// +/// `rtk init` generates a commented template for both levels (project or global). +/// +/// Environment variables: +/// - `RTK_NO_TOML=1` — bypass TOML engine entirely +/// - `RTK_TOML_DEBUG=1` — print which filter matched and line counts to stderr +/// +/// Pipeline stages (applied in order): +/// 1. strip_ansi — remove ANSI escape codes +/// 2. replace — regex substitutions, line-by-line, chainable +/// 3. match_output — short-circuit: if blob matches a pattern, return message immediately +/// 4. strip/keep_lines — filter lines by regex +/// 5. truncate_lines_at — truncate each line to N chars +/// 6. head/tail_lines — keep first/last N lines +/// 7. max_lines — absolute line cap +/// 8. on_empty — message if result is empty +use super::constants::RTK_META_COMMANDS; +use regex::{Regex, RegexSet}; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::sync::LazyLock; + +// Built-in filters: concatenated from src/filters/*.toml by build.rs at compile time. +const BUILTIN_TOML: &str = include_str!(concat!(env!("OUT_DIR"), "/builtin_filters.toml")); + +// --------------------------------------------------------------------------- +// Deserialization types (TOML schema) +// --------------------------------------------------------------------------- + +/// A match-output rule: if `pattern` matches anywhere in the full output blob, +/// the filter short-circuits and returns `message` immediately. +/// First matching rule wins; remaining rules are not evaluated. +/// Optional `unless`: if this regex also matches the blob, the rule is skipped +/// (prevents short-circuiting when errors or warnings are present). +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct MatchOutputRule { + pattern: String, + message: String, + #[serde(default)] + unless: Option, +} + +/// A regex substitution applied line-by-line. Rules are chained sequentially: +/// rule N+1 operates on the output of rule N. +/// Backreferences (`$1`, `$2`, ...) are supported via the `regex` crate. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ReplaceRule { + pattern: String, + replacement: String, +} + +/// An inline test case attached to a filter in the TOML. +/// Lives in `[[tests.]]` sections, separate from `[filters.*]`. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TomlFilterTestDef { + pub name: String, + pub input: String, + pub expected: String, +} + +#[derive(Deserialize)] +struct TomlFilterFile { + schema_version: u32, + #[serde(default)] + filters: BTreeMap, + /// Inline tests keyed by filter name. Kept separate from `filters` so that + /// `TomlFilterDef` can keep `deny_unknown_fields` without touching test data. + #[serde(default)] + tests: BTreeMap>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TomlFilterDef { + description: Option, + match_command: String, + #[serde(default)] + strip_ansi: bool, + /// Regex substitutions, applied line-by-line before match_output (stage 2). + #[serde(default)] + replace: Vec, + /// Short-circuit rules: if the full output blob matches, return the message (stage 3). + #[serde(default)] + match_output: Vec, + #[serde(default)] + strip_lines_matching: Vec, + #[serde(default)] + keep_lines_matching: Vec, + truncate_lines_at: Option, + head_lines: Option, + tail_lines: Option, + max_lines: Option, + on_empty: Option, + /// When true, stderr is captured and merged with stdout before filtering. + /// Use for tools like liquibase that emit banners/logs to stderr. + #[serde(default)] + filter_stderr: bool, +} + +// --------------------------------------------------------------------------- +// Compiled types (post-validation, ready to use) +// --------------------------------------------------------------------------- + +#[derive(Debug)] +struct CompiledMatchOutputRule { + pattern: Regex, + message: String, + /// If set and matches the blob, this rule is skipped (prevents swallowing errors). + unless: Option, +} + +#[derive(Debug)] +struct CompiledReplaceRule { + pattern: Regex, + replacement: String, +} + +#[derive(Debug)] +enum LineFilter { + None, + Strip(RegexSet), + Keep(RegexSet), +} + +/// A filter that has been parsed and compiled — all regexes are ready. +#[derive(Debug)] +pub struct CompiledFilter { + pub name: String, + #[allow(dead_code)] + pub description: Option, + match_regex: Regex, + strip_ansi: bool, + replace: Vec, + match_output: Vec, + line_filter: LineFilter, + truncate_lines_at: Option, + head_lines: Option, + tail_lines: Option, + pub max_lines: Option, + on_empty: Option, + /// When true, the runner should capture stderr and merge it with stdout. + pub filter_stderr: bool, +} + +// --------------------------------------------------------------------------- +// Results for `rtk verify` +// --------------------------------------------------------------------------- + +/// Outcome of running a single inline test. +pub struct TestOutcome { + pub filter_name: String, + pub test_name: String, + pub passed: bool, + pub actual: String, + pub expected: String, +} + +/// Aggregated results from `run_filter_tests`. +pub struct VerifyResults { + /// Individual test outcomes (all filters, or just the requested one). + pub outcomes: Vec, + /// Filter names that have no inline tests (used by `--require-all`). + pub filters_without_tests: Vec, +} + +// --------------------------------------------------------------------------- +// Registry +// --------------------------------------------------------------------------- + +pub struct TomlFilterRegistry { + pub filters: Vec, +} + +impl TomlFilterRegistry { + /// Load registry from disk + built-in. Emits warnings to stderr on parse + /// errors but never panics — bad files are silently ignored. + fn load() -> Self { + let mut filters = Vec::new(); + + for path in crate::hooks::trust::gated_filter_paths() { + Self::extend_with_trusted(&mut filters, &path); + } + + match Self::parse_and_compile(BUILTIN_TOML, "builtin") { + Ok(f) => filters.extend(f), + Err(e) => eprintln!("[rtk] warning: builtin filters: {}", e), + } + + TomlFilterRegistry { filters } + } + + fn extend_with_trusted(filters: &mut Vec, path: &std::path::Path) { + if !path.exists() { + return; + } + let label = path.display().to_string(); + let (status, content) = crate::hooks::trust::check_trust_with_content(path) + .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); + match status { + crate::hooks::trust::TrustStatus::Trusted + | crate::hooks::trust::TrustStatus::EnvOverride => { + if let Some(content) = content { + match Self::parse_and_compile(&content, &label) { + Ok(f) => filters.extend(f), + Err(e) => eprintln!("[rtk] warning: {}: {}", label, e), + } + } + } + crate::hooks::trust::TrustStatus::Untrusted + | crate::hooks::trust::TrustStatus::ContentChanged { .. } => {} + } + } + + fn parse_and_compile(content: &str, source: &str) -> Result, String> { + let file: TomlFilterFile = toml::from_str(content) + .map_err(|e| format!("TOML parse error in {}: {}", source, e))?; + + if file.schema_version != 1 { + return Err(format!( + "unsupported schema_version {} in {} (expected 1)", + file.schema_version, source + )); + } + + let mut compiled = Vec::new(); + for (name, def) in file.filters { + if !is_fully_anchored(&def.match_command) { + eprintln!( + "[rtk] warning: filter '{}' in {}: match_command '{}' has a top-level branch \ + that does not start with '^'; it would match a path component mid-command. \ + Filter ignored.", + name, source, def.match_command + ); + continue; + } + match compile_filter(name.clone(), def) { + Ok(f) => compiled.push(f), + Err(e) => eprintln!("[rtk] warning: filter '{}' in {}: {}", name, source, e), + } + } + Ok(compiled) + } +} + +/// Commands already handled by dedicated Rust modules (routed by Clap before TOML). +/// A TOML filter whose match_command matches one of these will never activate — +/// Clap routes the command before `run_fallback()` is reached. +const RUST_HANDLED_COMMANDS: &[&str] = &[ + "ls", + "tree", + "read", + "smart", + "git", + "gh", + "aws", + "psql", + "pnpm", + "err", + "test", + "json", + "deps", + "env", + "find", + "diff", + "log", + "docker", + "kubectl", + "summary", + "grep", + "init", + "wget", + "wc", + "gain", + "config", + "ctest", + "vitest", + "prisma", + "tsc", + "next", + "lint", + "prettier", + "format", + "playwright", + "cargo", + "npm", + "npx", + "curl", + "discover", + "ruff", + "pytest", + "mypy", + "pip", + "sqlfluff", + "go", + "golangci-lint", + "rewrite", + "proxy", + "verify", + "learn", +]; + +pub fn is_rtk_reserved_command(name: &str) -> bool { + RUST_HANDLED_COMMANDS.contains(&name) || RTK_META_COMMANDS.contains(&name) +} + +fn compile_filter(name: String, def: TomlFilterDef) -> Result { + // Mutual exclusion: strip and keep cannot both be set + if !def.strip_lines_matching.is_empty() && !def.keep_lines_matching.is_empty() { + return Err("strip_lines_matching and keep_lines_matching are mutually exclusive".into()); + } + + let match_regex = Regex::new(&def.match_command) + .map_err(|e| format!("invalid match_command regex: {}", e))?; + + // Shadow warning: if match_command matches a Rust-handled command, this filter + // will never activate (Clap routes before run_fallback). Warn the author. + for cmd in RUST_HANDLED_COMMANDS { + if match_regex.is_match(cmd) { + eprintln!( + "[rtk] warning: filter '{}' match_command matches '{}' which is already \ + handled by a Rust module — this filter will never activate for that command", + name, cmd + ); + break; + } + } + + let replace = def + .replace + .into_iter() + .map(|r| { + let pat = r.pattern.clone(); + Regex::new(&r.pattern) + .map(|pattern| CompiledReplaceRule { + pattern, + replacement: r.replacement, + }) + .map_err(|e| format!("invalid replace pattern '{}': {}", pat, e)) + }) + .collect::, _>>()?; + + let match_output = def + .match_output + .into_iter() + .map(|r| -> Result { + let pat = r.pattern.clone(); + let pattern = Regex::new(&r.pattern) + .map_err(|e| format!("invalid match_output pattern '{}': {}", pat, e))?; + let unless = r + .unless + .as_deref() + .map(|u| { + Regex::new(u) + .map_err(|e| format!("invalid match_output unless pattern '{}': {}", u, e)) + }) + .transpose()?; + Ok(CompiledMatchOutputRule { + pattern, + message: r.message, + unless, + }) + }) + .collect::, _>>()?; + + let line_filter = if !def.strip_lines_matching.is_empty() { + let set = RegexSet::new(&def.strip_lines_matching) + .map_err(|e| format!("invalid strip_lines_matching regex: {}", e))?; + LineFilter::Strip(set) + } else if !def.keep_lines_matching.is_empty() { + let set = RegexSet::new(&def.keep_lines_matching) + .map_err(|e| format!("invalid keep_lines_matching regex: {}", e))?; + LineFilter::Keep(set) + } else { + LineFilter::None + }; + + Ok(CompiledFilter { + name, + description: def.description, + match_regex, + strip_ansi: def.strip_ansi, + replace, + match_output, + line_filter, + truncate_lines_at: def.truncate_lines_at, + head_lines: def.head_lines, + tail_lines: def.tail_lines, + max_lines: def.max_lines, + on_empty: def.on_empty, + filter_stderr: def.filter_stderr, + }) +} + +// --------------------------------------------------------------------------- +// Singleton (lazy-loaded, one-time cost) +// --------------------------------------------------------------------------- + +static REGISTRY: LazyLock = LazyLock::new(TomlFilterRegistry::load); + +pub fn toml_disabled() -> bool { + std::env::var("RTK_NO_TOML").ok().as_deref() == Some("1") +} + +pub fn active_filter_summaries(content: &str) -> Vec<(String, String)> { + toml::from_str::(content) + .map(|f| { + f.filters + .into_iter() + .map(|(name, def)| (name, def.match_command)) + .collect() + }) + .unwrap_or_default() +} + +pub fn filter_parse_error(content: &str) -> Option { + toml::from_str::(content) + .err() + .map(|e| e.to_string()) +} + +static MATCH_SET: LazyLock = LazyLock::new(build_match_set); + +pub fn command_matches_filter(command: &str) -> bool { + MATCH_SET.is_match(command) +} + +fn build_match_set() -> RegexSet { + let patterns = collect_match_patterns(); + RegexSet::new(&patterns).unwrap_or_else(|_| { + let valid: Vec = patterns + .into_iter() + .filter(|p| Regex::new(p).is_ok()) + .collect(); + RegexSet::new(&valid).unwrap_or_else(|_| RegexSet::empty()) + }) +} + +fn collect_match_patterns() -> Vec { + let mut patterns: Vec = Vec::new(); + for path in crate::hooks::trust::gated_filter_paths() { + if !path.exists() { + continue; + } + let (status, content) = crate::hooks::trust::check_trust_with_content(&path) + .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); + if matches!( + status, + crate::hooks::trust::TrustStatus::Trusted + | crate::hooks::trust::TrustStatus::EnvOverride + ) && let Some(content) = content + { + patterns.extend(match_patterns_in(&content)); + } + } + patterns.extend(match_patterns_in(BUILTIN_TOML)); + patterns +} + +/// Strip a leading inline-flag group such as `(?i)` so the anchor check sees the +/// pattern body. `(?:` opens a non-capturing group and is left alone. +fn strip_inline_flags(branch: &str) -> &str { + let Some(rest) = branch.strip_prefix("(?") else { + return branch; + }; + let Some(end) = rest.find(')') else { + return branch; + }; + if rest[..end].chars().all(|c| "imsuxU-".contains(c)) { + &rest[end + 1..] + } else { + branch + } +} + +/// Split `pattern` on top-level `|`, ignoring alternations nested inside groups +/// or character classes. +fn top_level_branches(pattern: &str) -> Vec<&str> { + let mut branches = Vec::new(); + let mut depth = 0usize; + let mut escaped = false; + let mut in_class = false; + let mut start = 0usize; + + for (idx, character) in pattern.char_indices() { + if escaped { + escaped = false; + continue; + } + match character { + '\\' => escaped = true, + '[' if !in_class => in_class = true, + ']' if in_class => in_class = false, + '(' if !in_class => depth += 1, + ')' if !in_class => depth = depth.saturating_sub(1), + '|' if !in_class && depth == 0 => { + branches.push(&pattern[start..idx]); + start = idx + 1; + } + _ => {} + } + } + branches.push(&pattern[start..]); + branches +} + +/// A filter selects on argv[0], but its regex runs against the whole command +/// line. A top-level branch that does not start with `^` therefore matches a +/// path component or wrapper argument mid-line, so `timeout 5 /usr/bin/liquibase +/// update` activates the liquibase filter and rewrites the wrapper instead. +fn is_fully_anchored(pattern: &str) -> bool { + top_level_branches(pattern) + .iter() + .all(|branch| strip_inline_flags(branch).starts_with('^')) +} + +fn match_patterns_in(content: &str) -> Vec { + match toml::from_str::(content) { + Ok(file) if file.schema_version == 1 => file + .filters + .into_values() + .map(|def| def.match_command) + .filter(|pattern| is_fully_anchored(pattern)) + .collect(), + _ => Vec::new(), + } +} + +// --------------------------------------------------------------------------- +// Public API — pure functions (testable without global state) +// --------------------------------------------------------------------------- + +/// Find the first matching filter in a slice. O(N) on the number of filters. +/// Tests should call this directly with a local filter list. +pub fn find_filter_in<'a>( + command: &str, + filters: &'a [CompiledFilter], +) -> Option<&'a CompiledFilter> { + filters.iter().find(|f| f.match_regex.is_match(command)) +} + +/// Apply a compiled filter pipeline to raw stdout. Pure String -> String. +/// +/// Pipeline stages (in order): +/// 1. strip_ansi — remove ANSI escape codes +/// 2. replace — regex substitutions, line-by-line, chainable +/// 3. match_output — short-circuit if blob matches a pattern +/// 4. strip/keep_lines — filter lines by regex +/// 5. truncate_lines_at — truncate each line to N chars +/// 6. head/tail_lines — keep first/last N lines +/// 7. max_lines — absolute line cap +/// 8. on_empty — message if result is empty +pub fn apply_filter(filter: &CompiledFilter, stdout: &str) -> String { + apply_filter_with_info(filter, stdout).0 +} + +#[derive(Debug, PartialEq)] +pub enum Lossiness { + None, + /// `tail -n +{tail_offset}` over `tee_payload` reproduces the dropped lines, + /// up to the tee `max_file_size` cap (larger payloads are truncated in the tee file). + Tail { + tee_payload: String, + tail_offset: usize, + }, + Whole, +} + +pub fn apply_filter_with_info(filter: &CompiledFilter, stdout: &str) -> (String, Lossiness) { + let mut lines: Vec = stdout.lines().map(String::from).collect(); + + // 1. strip_ansi + if filter.strip_ansi { + lines = lines + .into_iter() + .map(|l| crate::core::utils::strip_ansi(&l)) + .collect(); + } + + // 2. replace — line-by-line, rules chained sequentially + if !filter.replace.is_empty() { + lines = lines + .into_iter() + .map(|mut line| { + for rule in &filter.replace { + line = rule + .pattern + .replace_all(&line, rule.replacement.as_str()) + .into_owned(); + } + line + }) + .collect(); + } + + // 3. match_output — short-circuit on full blob match (first rule wins) + // If `unless` is set and also matches the blob, the rule is skipped. + if !filter.match_output.is_empty() { + let blob = lines.join("\n"); + for rule in &filter.match_output { + if rule.pattern.is_match(&blob) { + if let Some(ref unless_re) = rule.unless + && unless_re.is_match(&blob) + { + continue; // errors/warnings present — skip this rule + } + return (rule.message.clone(), Lossiness::Whole); + } + } + } + + // 4. strip OR keep (mutually exclusive) + match &filter.line_filter { + LineFilter::Strip(set) => lines.retain(|l| !set.is_match(l)), + LineFilter::Keep(set) => lines.retain(|l| set.is_match(l)), + LineFilter::None => {} + } + + // 5. truncate_lines_at — uses utils::truncate (unicode-safe) + let mut intra_line_loss = false; + if let Some(max_chars) = filter.truncate_lines_at { + lines = lines + .into_iter() + .map(|line| { + let truncated = crate::core::utils::truncate(&line, max_chars); + if truncated != line { + intra_line_loss = true; + } + truncated + }) + .collect(); + } + + let snapshot_for_tail = !intra_line_loss + && filter.tail_lines.is_none() + && (filter.head_lines.is_some() || filter.max_lines.is_some()); + let pre_cut = snapshot_for_tail.then(|| lines.clone()); + + // 6. head + tail + let total = lines.len(); + let mut noncontiguous_drop = false; + let mut head_cut: Option = None; + if let (Some(head), Some(tail)) = (filter.head_lines, filter.tail_lines) { + if total > head + tail { + let mut result = lines[..head].to_vec(); + result.push(format!("... ({} lines omitted)", total - head - tail)); + result.extend_from_slice(&lines[total - tail..]); + lines = result; + noncontiguous_drop = true; + } + } else if let Some(head) = filter.head_lines { + if total > head { + lines.truncate(head); + lines.push(format!("... ({} lines omitted)", total - head)); + head_cut = Some(head); + } + } else if let Some(tail) = filter.tail_lines + && total > tail + { + let omitted = total - tail; + lines = lines[omitted..].to_vec(); + lines.insert(0, format!("... ({} lines omitted)", omitted)); + noncontiguous_drop = true; + } + + // 7. max_lines — absolute cap applied after head/tail (includes omit messages) + let mut max_cut: Option = None; + if let Some(max) = filter.max_lines + && lines.len() > max + { + let dropped = lines.len() - max; + lines.truncate(max); + lines.push(format!("... ({} lines truncated)", dropped)); + max_cut = Some(max); + } + + // 8. on_empty + let result = lines.join("\n"); + if result.trim().is_empty() + && let Some(ref msg) = filter.on_empty + { + return (msg.clone(), Lossiness::None); + } + + let loss = if let Some(snapshot) = pre_cut { + match (head_cut, max_cut) { + (Some(_), Some(_)) => Lossiness::Whole, + (Some(head), None) => Lossiness::Tail { + tee_payload: snapshot.join("\n"), + tail_offset: head + 1, + }, + (None, Some(max)) => Lossiness::Tail { + tee_payload: snapshot.join("\n"), + tail_offset: max + 1, + }, + (None, None) => Lossiness::None, + } + } else if noncontiguous_drop || intra_line_loss || head_cut.is_some() || max_cut.is_some() { + Lossiness::Whole + } else { + Lossiness::None + }; + + (result, loss) +} + +// --------------------------------------------------------------------------- +// rtk verify — inline test execution +// --------------------------------------------------------------------------- + +/// Run inline tests from loaded TOML files (builtin + project-local). +/// +/// - `filter_name_opt`: if `Some`, only run tests for that filter name. +/// - Returns `VerifyResults` with all outcomes and filters that have no tests. +pub fn run_filter_tests(filter_name_opt: Option<&str>) -> VerifyResults { + let mut outcomes = Vec::new(); + let mut all_filter_names: Vec = Vec::new(); + let mut tested_filter_names: std::collections::HashSet = + std::collections::HashSet::new(); + + let builtin = BUILTIN_TOML; + collect_test_outcomes( + builtin, + filter_name_opt, + &mut outcomes, + &mut all_filter_names, + &mut tested_filter_names, + ); + + // Trust-gated: only verify project-local filters if trusted (SA-2025-RTK-002) + let project_path = std::path::Path::new(".rtk/filters.toml"); + if project_path.exists() { + let (trust_status, verified_content) = + crate::hooks::trust::check_trust_with_content(project_path) + .unwrap_or((crate::hooks::trust::TrustStatus::Untrusted, None)); + match trust_status { + crate::hooks::trust::TrustStatus::Trusted + | crate::hooks::trust::TrustStatus::EnvOverride => { + if let Some(content) = verified_content { + collect_test_outcomes( + &content, + filter_name_opt, + &mut outcomes, + &mut all_filter_names, + &mut tested_filter_names, + ); + } + } + _ => { + eprintln!("[rtk] WARNING: untrusted project filters skipped in verify"); + } + } + } + + let filters_without_tests = all_filter_names + .into_iter() + .filter(|name| { + // When a specific filter is requested, only report that one as missing tests + filter_name_opt.is_none_or(|f| name == f) + }) + .filter(|name| !tested_filter_names.contains(name)) + .collect(); + + VerifyResults { + outcomes, + filters_without_tests, + } +} + +fn collect_test_outcomes( + content: &str, + filter_name_opt: Option<&str>, + outcomes: &mut Vec, + all_filter_names: &mut Vec, + tested_filter_names: &mut std::collections::HashSet, +) { + let file: TomlFilterFile = match toml::from_str(content) { + Ok(f) => f, + Err(e) => { + eprintln!("[rtk] warning: TOML parse error during verify: {}", e); + return; + } + }; + + // Compile all filters and track their names + let mut compiled_filters: BTreeMap = BTreeMap::new(); + for (name, def) in file.filters { + all_filter_names.push(name.clone()); + match compile_filter(name.clone(), def) { + Ok(f) => { + compiled_filters.insert(name, f); + } + Err(e) => eprintln!("[rtk] warning: filter '{}' compilation error: {}", name, e), + } + } + + // Run tests + for (filter_name, tests) in file.tests { + if let Some(name) = filter_name_opt + && filter_name != name + { + continue; + } + + tested_filter_names.insert(filter_name.clone()); + + let compiled = match compiled_filters.get(&filter_name) { + Some(f) => f, + None => { + eprintln!( + "[rtk] warning: [[tests.{}]] references unknown filter", + filter_name + ); + continue; + } + }; + + for test in tests { + let actual = apply_filter(compiled, &test.input); + // Trim trailing newlines: TOML multiline strings end with a newline + let actual_cmp = actual.trim_end_matches('\n').to_string(); + let expected_cmp = test.expected.trim_end_matches('\n').to_string(); + outcomes.push(TestOutcome { + filter_name: filter_name.clone(), + test_name: test.name, + passed: actual_cmp == expected_cmp, + actual: actual_cmp, + expected: expected_cmp, + }); + } + } +} + +// --------------------------------------------------------------------------- +// Convenience wrapper (uses singleton — for run_fallback) +// --------------------------------------------------------------------------- + +/// Find a matching filter from the global registry. Initialises the registry +/// lazily on first call. Returns `None` if no filter matches. +pub fn find_matching_filter(command: &str) -> Option<&'static CompiledFilter> { + if std::env::var("RTK_TOML_DEBUG").is_ok() { + eprintln!( + "[rtk:toml] looking up filter for: {:?} ({} filters loaded)", + command, + REGISTRY.filters.len() + ); + } + let result = find_filter_in(command, ®ISTRY.filters); + if std::env::var("RTK_TOML_DEBUG").is_ok() { + match result { + Some(f) => eprintln!("[rtk:toml] matched filter: '{}'", f.name), + None => eprintln!("[rtk:toml] no filter matched — passthrough"), + } + } + result +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // Helper: build a CompiledFilter from inline TOML for tests. + // Never touches the lazy registry. + fn make_filters(toml: &str) -> Vec { + TomlFilterRegistry::parse_and_compile(toml, "test").expect("test TOML should be valid") + } + + fn first_filter(toml: &str) -> CompiledFilter { + make_filters(toml) + .into_iter() + .next() + .expect("expected at least one filter") + } + + #[test] + fn command_matches_filter_agrees_with_find_matching_filter() { + for cmd in ["jj log", "jq .", "frobnicate xyz", "cd /tmp"] { + assert_eq!( + command_matches_filter(cmd), + find_matching_filter(cmd).is_some(), + "match-set disagreed with registry for {cmd:?}" + ); + } + } + + #[test] + fn test_loss_tail_configured_unfired_max_fires_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntail_lines = 100\nmax_lines = 2\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc\nd\ne"); + assert!(out.contains("lines truncated")); + assert_ne!(loss, Lossiness::None); + } + + #[test] + fn test_match_patterns_in_honors_schema_version() { + let v1 = "schema_version = 1\n[filters.a]\nmatch_command = \"^aaa\"\n"; + assert_eq!(match_patterns_in(v1), vec!["^aaa".to_string()]); + + let v2 = "schema_version = 2\n[filters.a]\nmatch_command = \"^aaa\"\n"; + assert!(match_patterns_in(v2).is_empty()); + + assert!(match_patterns_in("not valid toml {{{").is_empty()); + } + + #[test] + fn test_filter_parse_error() { + assert!( + filter_parse_error("schema_version = 1\n[filters.a]\nmatch_command = \"^a\"\n") + .is_none() + ); + assert!(filter_parse_error("[filters.a]\nmatch_command = \"^a\"\n").is_some()); + assert!(filter_parse_error("this is { not toml").is_some()); + } + + #[test] + fn test_toml_parse_tolerates_utf8_bom() { + // Hand-edited filter files on Windows often carry a BOM. The toml + // crate tolerates it natively — this pin is why the TOML file-read + // sites (config.rs, toml_filter.rs) deliberately skip + // strip_leading_bom; if a crate upgrade regresses this, add it there. + let bom = "\u{feff}schema_version = 1\n[filters.a]\nmatch_command = \"^a\"\n"; + assert!(filter_parse_error(bom).is_none()); + assert_eq!(match_patterns_in(bom), vec!["^a".to_string()]); + } + + #[test] + fn test_is_rtk_reserved_command() { + assert!(is_rtk_reserved_command("git")); + assert!(is_rtk_reserved_command("cargo")); + assert!(is_rtk_reserved_command("json")); + assert!(is_rtk_reserved_command("rewrite")); + assert!(!is_rtk_reserved_command("jj")); + assert!(!is_rtk_reserved_command("jq")); + assert!(!is_rtk_reserved_command("just")); + assert!(!is_rtk_reserved_command("frobnicate")); + } + + fn loss_of(toml: &str, input: &str) -> Lossiness { + apply_filter_with_info(&first_filter(toml), input).1 + } + + #[test] + fn test_loss_head_lines_is_tail() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc\nd\ne"); + assert!(out.starts_with("a\nb\n")); + match loss { + Lossiness::Tail { + tee_payload, + tail_offset, + } => { + assert_eq!(tail_offset, 3); + let recovered: Vec<&str> = tee_payload.lines().skip(tail_offset - 1).collect(); + assert_eq!(recovered, vec!["c", "d", "e"]); + } + other => panic!("expected Tail, got {:?}", other), + } + } + + #[test] + fn test_loss_max_lines_is_tail() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nmax_lines = 2\n"; + match loss_of(toml, "a\nb\nc\nd\ne") { + Lossiness::Tail { + tee_payload, + tail_offset, + } => { + assert_eq!(tail_offset, 3); + let recovered: Vec<&str> = tee_payload.lines().skip(tail_offset - 1).collect(); + assert_eq!(recovered, vec!["c", "d", "e"]); + } + other => panic!("expected Tail, got {:?}", other), + } + } + + #[test] + fn test_loss_tail_lines_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntail_lines = 2\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_head_then_max_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\nmax_lines = 2\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_head_plus_tail_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 1\ntail_lines = 1\n"; + assert_eq!(loss_of(toml, "a\nb\nc\nd\ne"), Lossiness::Whole); + } + + #[test] + fn test_loss_truncate_lines_at_is_whole() { + let toml = + "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ntruncate_lines_at = 3\n"; + assert_eq!(loss_of(toml, "abcdefgh\nshort"), Lossiness::Whole); + } + + #[test] + fn test_loss_match_output_is_whole() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\n[[filters.f.match_output]]\npattern = \"ok\"\nmessage = \"all good\"\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "everything ok here\nmore"); + assert_eq!(out, "all good"); + assert_eq!(loss, Lossiness::Whole); + } + + #[test] + fn test_loss_strip_only_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nstrip_lines_matching = [\"^noise\"]\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "keep\nnoise line\nkeep2"); + assert_eq!(out, "keep\nkeep2"); + assert_eq!(loss, Lossiness::None); + } + + #[test] + fn test_loss_on_empty_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nstrip_lines_matching = [\".\"]\non_empty = \"nothing\"\n"; + let (out, loss) = apply_filter_with_info(&first_filter(toml), "a\nb\nc"); + assert_eq!(out, "nothing"); + assert_eq!(loss, Lossiness::None); + } + + #[test] + fn test_loss_no_truncation_is_none() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 10\n"; + assert_eq!(loss_of(toml, "a\nb\nc"), Lossiness::None); + } + + #[test] + fn test_apply_filter_wrapper_matches_with_info() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\nhead_lines = 2\n"; + let f = first_filter(toml); + let input = "a\nb\nc\nd"; + assert_eq!(apply_filter(&f, input), apply_filter_with_info(&f, input).0); + } + + // --- Pipeline primitives (existing) --- + + #[test] + fn test_strip_ansi_removes_codes() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +"#, + ); + let out = apply_filter(&f, "\x1b[31mError\x1b[0m\nnormal"); + assert_eq!(out, "Error\nnormal"); + } + + #[test] + fn test_strip_lines_matching_basic() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["^noise", "^verbose"] +"#, + ); + let input = "noise line\nkeep this\nverbose stuff\nalso keep"; + let out = apply_filter(&f, input); + assert_eq!(out, "keep this\nalso keep"); + } + + #[test] + fn test_keep_lines_matching_basic() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +keep_lines_matching = ["^PASS", "^FAIL"] +"#, + ); + let input = "PASS test_a\nsome noise\nFAIL test_b\nmore noise"; + let out = apply_filter(&f, input); + assert_eq!(out, "PASS test_a\nFAIL test_b"); + } + + #[test] + fn test_truncate_lines_at_unicode_safe() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +truncate_lines_at = 5 +"#, + ); + // utils::truncate(s, 5) takes 2 chars + "..." when len > 5 + // "hello" = 5 chars exactly, stays unchanged + // "日本語xyz" = 6 chars, truncated to "日本..." (take 2 + "...") + let out = apply_filter(&f, "hello\n日本語xyz"); + assert_eq!(out, "hello\n日本..."); + } + + #[test] + fn test_head_lines() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +head_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + assert!(out.starts_with("a\nb\n")); + assert!(out.contains("3 lines omitted")); + } + + #[test] + fn test_tail_lines() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +tail_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + assert!(out.contains("3 lines omitted")); + assert!(out.ends_with("d\ne")); + } + + #[test] + fn test_head_and_tail_combined() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +head_lines = 2 +tail_lines = 2 +"#, + ); + let input = "a\nb\nc\nd\ne\nf"; + let out = apply_filter(&f, input); + assert!(out.starts_with("a\nb\n")); + assert!(out.contains("2 lines omitted")); + assert!(out.ends_with("e\nf")); + } + + #[test] + fn test_max_lines_counts_omit_message() { + // max_lines applied AFTER head — the "omitted" message counts as a line + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +max_lines = 3 +"#, + ); + let input = "a\nb\nc\nd\ne"; + let out = apply_filter(&f, input); + let line_count = out.lines().count(); + // 3 content lines + 1 truncated message = 4 lines output + assert_eq!(line_count, 4); + assert!(out.contains("lines truncated")); + } + + #[test] + fn test_on_empty_when_all_filtered() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = [".*"] +on_empty = "nothing left" +"#, + ); + let out = apply_filter(&f, "line1\nline2"); + assert_eq!(out, "nothing left"); + } + + #[test] + fn test_on_empty_not_triggered_when_output_remains() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +keep_lines_matching = ["keep"] +on_empty = "nothing left" +"#, + ); + let out = apply_filter(&f, "keep this\nnoise"); + assert_eq!(out, "keep this"); + } + + #[test] + fn test_full_pipeline_order() { + // Verify all 8 stages fire in order on a single input + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +strip_lines_matching = ["^noise"] +truncate_lines_at = 10 +head_lines = 3 +max_lines = 4 +on_empty = "empty" +"#, + ); + let input = + "\x1b[31mred line\x1b[0m\nnoise skip\nkeep one\nkeep two\nkeep three\nkeep four"; + let out = apply_filter(&f, input); + // After strip_ansi: "red line", strip noise: removed, head 3 from remaining 4 lines + assert!(out.contains("red line")); + assert!(!out.contains("noise skip")); + assert!(out.contains("lines omitted") || out.contains("lines truncated")); + } + + // --- Validation --- + + #[test] + fn test_mutual_exclusion_strip_keep_errors() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["a"] +keep_lines_matching = ["b"] +"#, + ); + // The filter should be skipped (warning emitted), resulting in empty list + assert!(result.is_empty()); + } + + #[test] + fn test_invalid_regex_returns_err() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "[" +"#, + ); + assert!(result.is_empty()); + } + + #[test] + fn test_schema_version_mismatch_errors() { + let result = TomlFilterRegistry::parse_and_compile( + r#"schema_version = 99 +[filters.f] +match_command = "^cmd" +"#, + "test", + ); + assert!(result.is_err()); + } + + #[test] + fn test_unknown_field_typo_errors() { + // deny_unknown_fields should catch this + let result = TomlFilterRegistry::parse_and_compile( + r#"schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi_typo = true +"#, + "test", + ); + assert!(result.is_err()); + } + + #[test] + fn test_empty_filter_passthrough() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +"#, + ); + let input = "line1\nline2\nline3"; + let out = apply_filter(&f, input); + assert_eq!(out, input); + } + + // --- Registry / find --- + + #[test] + fn test_builtin_filters_compile() { + // Compile-time safety: panics if any src/filters/*.toml is broken + let builtin = BUILTIN_TOML; + let result = TomlFilterRegistry::parse_and_compile(builtin, "builtin"); + assert!( + result.is_ok(), + "builtin filters failed to compile: {:?}", + result + ); + assert!(!result.unwrap().is_empty()); + } + + #[test] + fn test_find_filter_matches_terraform() { + let filters = make_filters( + r#" +schema_version = 1 +[filters.terraform-plan] +match_command = "^terraform\\s+plan" +strip_ansi = true +"#, + ); + let found = find_filter_in("terraform plan -out=tfplan", &filters); + assert!(found.is_some()); + assert_eq!(found.unwrap().name, "terraform-plan"); + } + + #[test] + fn test_find_filter_no_match_returns_none() { + let filters = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^terraform" +"#, + ); + let found = find_filter_in("kubectl get pods", &filters); + assert!(found.is_none()); + } + + #[test] + fn test_project_filters_priority_over_builtin() { + // Project filter has same name but different max_lines — project wins + let project = make_filters( + r#" +schema_version = 1 +[filters.make] +match_command = "^make\\b" +max_lines = 999 +"#, + ); + let builtin = make_filters(BUILTIN_TOML); + + // Simulate the registry: project first + let mut all = project; + all.extend(builtin); + + let found = find_filter_in("make all", &all).expect("should match"); + assert_eq!(found.name, "make"); + // The first (project) match has max_lines=999 + assert_eq!(found.max_lines, Some(999)); + } + + // --- Token savings --- + + #[test] + fn test_terraform_savings_above_60pct() { + let filters = make_filters(BUILTIN_TOML); + let filter = find_filter_in("terraform plan", &filters).expect("terraform-plan built-in"); + + // Inline fixture: realistic terraform plan with many Refreshing state lines (noise). + // Real infra refreshes 30+ resources; the plan section is small. + // All Refreshing/lock/blank/unchanged lines are stripped -> >60% savings. + let input = concat!( + "Acquiring state lock. This may take a few moments...\n", + "Refreshing state... [id=vpc-0a1b2c3d]\n", + "Refreshing state... [id=subnet-11111111]\n", + "Refreshing state... [id=subnet-22222222]\n", + "Refreshing state... [id=subnet-33333333]\n", + "Refreshing state... [id=subnet-44444444]\n", + "Refreshing state... [id=igw-aabbccdd]\n", + "Refreshing state... [id=rtb-aabbccdd]\n", + "Refreshing state... [id=rtb-11223344]\n", + "Refreshing state... [id=sg-00112233]\n", + "Refreshing state... [id=sg-44556677]\n", + "Refreshing state... [id=sg-88990011]\n", + "Refreshing state... [id=nacl-00aabbcc]\n", + "Refreshing state... [id=acm-arn:us-east-1:cert/abc]\n", + "Refreshing state... [id=Z1234567890ABC]\n", + "Refreshing state... [id=alb-arn:my-alb]\n", + "Refreshing state... [id=tg-arn:my-tg]\n", + "Refreshing state... [id=db-ABCDEFGHIJKLMNO]\n", + "Refreshing state... [id=rds-cluster:my-aurora]\n", + "Refreshing state... [id=elasticache:my-cluster]\n", + "Refreshing state... [id=lambda:my-api-function]\n", + "Refreshing state... [id=lambda:my-worker]\n", + "Refreshing state... [id=iam-role:my-lambda-role]\n", + "Refreshing state... [id=iam-role:my-ecs-role]\n", + "Refreshing state... [id=s3:::my-app-assets]\n", + "Refreshing state... [id=s3:::my-app-logs]\n", + "Refreshing state... [id=cloudfront:ABCDEFGHIJK]\n", + "Refreshing state... [id=ssm:/my/app/db-url]\n", + "Refreshing state... [id=ssm:/my/app/api-key]\n", + "Refreshing state... [id=secretsmanager:my-secret]\n", + "Releasing state lock. This may take a few moments...\n", + "\n", + "Terraform will perform the following actions:\n", + "\n", + " # aws_instance.web will be created\n", + " + resource \"aws_instance\" \"web\" {\n", + " + ami = \"ami-0c55b159cbfafe1f0\"\n", + " + instance_type = \"t3.micro\"\n", + " }\n", + "\n", + "Plan: 1 to add, 0 to change, 0 to destroy.\n", + ); + let out = apply_filter(filter, input); + let input_words = input.split_whitespace().count(); + let out_words = out.split_whitespace().count(); + let savings = 100.0 - (out_words as f64 / input_words as f64 * 100.0); + assert!( + savings >= 60.0, + "terraform-plan filter: expected >=60% savings, got {:.1}% (in={} out={})", + savings, + input_words, + out_words + ); + } + + #[test] + fn test_make_savings_above_60pct() { + let filters = make_filters(BUILTIN_TOML); + let filter = find_filter_in("make all", &filters).expect("make built-in"); + + let input = r#"make[1]: Entering directory '/home/user/project' +make[2]: Entering directory '/home/user/project/src' +gcc -O2 -Wall -c foo.c -o foo.o + +make[2]: Nothing to be done for 'install'. +make[3]: Entering directory '/home/user/project/src/lib' +ar rcs libfoo.a foo.o bar.o baz.o +make[3]: Leaving directory '/home/user/project/src/lib' +make[2]: Leaving directory '/home/user/project/src' + +make[1]: Leaving directory '/home/user/project' +gcc -O2 -Wall -c bar.c -o bar.o + +gcc -O2 -Wall -c baz.c -o baz.o + +make[1]: Entering directory '/home/user/project/test' +make[2]: Entering directory '/home/user/project/test/unit' +./run_tests --verbose +make[2]: Nothing to be done for 'check'. +make[2]: Leaving directory '/home/user/project/test/unit' +make[1]: Leaving directory '/home/user/project/test' + +ld -o myapp foo.o bar.o baz.o -lfoo + +make[1]: Entering directory '/home/user/project/docs' +doxygen Doxyfile +make[1]: Leaving directory '/home/user/project/docs' +"#; + let out = apply_filter(filter, input); + let input_words = input.split_whitespace().count(); + let out_words = out.split_whitespace().count(); + let savings = 100.0 - (out_words as f64 / input_words as f64 * 100.0); + assert!( + savings >= 60.0, + "make filter: expected >=60% savings, got {:.1}% (in={} out={})", + savings, + input_words, + out_words + ); + } + + #[test] + fn test_spring_boot_match_command_requires_spring_named_jar() { + let filters = make_filters(BUILTIN_TOML); + let spring_boot = filters + .iter() + .find(|f| f.name == "spring-boot") + .expect("spring-boot filter must exist"); + + assert!( + !spring_boot + .match_regex + .is_match("java -jar build/libs/my-other-tool.jar"), + "a non-Spring jar must not activate the spring-boot filter" + ); + + let spring_jar = find_filter_in("java -jar build/libs/my-spring-app.jar", &filters) + .expect("a jar with 'spring' in its filename must still match"); + assert_eq!(spring_jar.name, "spring-boot"); + + let mvn_run = find_filter_in("mvn spring-boot:run", &filters) + .expect("mvn spring-boot:run must still match"); + assert_eq!(mvn_run.name, "spring-boot"); + + let capitalized = find_filter_in("java -jar build/libs/MySpringApp.jar", &filters) + .expect("a jar with 'Spring' capitalized in its filename must still match"); + assert_eq!(capitalized.name, "spring-boot"); + + assert!( + !spring_boot + .match_regex + .is_match("java -jar /opt/spring-cache/other-tool.jar"), + "'spring' appearing only in a directory segment (not the jar filename itself) must not activate the spring-boot filter" + ); + + // Windows paths: '\' must be treated as a path separator too, not swallowed + // into the filename-only match (the exact bug this guard exists to catch). + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\spring-cache\other.jar"), + "'spring' appearing only in a Windows directory segment must not activate the spring-boot filter" + ); + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\dev\spring-workspace\build\other-tool.jar"), + "'spring' appearing only in a nested Windows directory segment must not activate the spring-boot filter" + ); + assert!( + spring_boot + .match_regex + .is_match(r"java -jar C:\dev\my-spring-app.jar"), + "a Windows-path jar with 'spring' in its own filename must still match" + ); + + // argv reaches this regex as one space-joined string, so a path containing spaces + // is indistinguishable from a following argument. Widening the prefix across + // whitespace would let a later 'spring'-named jar argument re-trigger the filter, + // so jars under such a path stay on full passthrough instead. + assert!( + !spring_boot + .match_regex + .is_match(r"java -jar C:\Program Files\app\my-spring-app.jar"), + "a jar under a path containing spaces is deliberately left to full passthrough" + ); + } + + #[test] + fn test_liquibase_match_command_ignores_path_substring() { + let filters = make_filters(BUILTIN_TOML); + let liquibase = filters + .iter() + .find(|f| f.name == "liquibase") + .expect("liquibase filter must exist"); + + assert!( + !liquibase.match_regex.is_match("rm -rf /opt/liquibase"), + "'liquibase' appearing only as a path argument must not activate the liquibase filter" + ); + + let bare = find_filter_in("liquibase status", &filters) + .expect("bare liquibase invocation must still match"); + assert_eq!(bare.name, "liquibase"); + + // Production callers (run_fallback in src/main.rs, strip_absolute_path in + // src/discover/registry.rs) always basename argv[0] before this regex runs, + // so a raw path-qualified string must NOT match on its own — the regex has + // no path-prefix branch to fall back on. + assert!( + !liquibase + .match_regex + .is_match("/usr/local/bin/liquibase update"), + "a raw path-qualified invocation must not match — callers basename argv[0] before matching" + ); + + for wrapped in [ + "timeout 5 /usr/bin/liquibase update", + "nohup /opt/tools/liquibase update", + ] { + assert!( + !liquibase.match_regex.is_match(wrapped), + "a wrapper command must not activate the liquibase filter: {wrapped}" + ); + } + } + + #[test] + fn test_ssh_match_command_excludes_ssh_dash_utilities() { + let filters = make_filters(BUILTIN_TOML); + let ssh = filters + .iter() + .find(|f| f.name == "ssh") + .expect("ssh filter must exist"); + + assert!( + !ssh.match_regex.is_match("ssh-keygen -t ed25519"), + "ssh-keygen must not activate the plain ssh connection filter" + ); + assert!( + !ssh.match_regex.is_match("ssh-add id_ed25519"), + "ssh-add must not activate the plain ssh connection filter" + ); + + let plain = find_filter_in("ssh user@host", &filters) + .expect("plain ssh invocation must still match"); + assert_eq!(plain.name, "ssh"); + + let bare = find_filter_in("ssh", &filters).expect("bare ssh with no args must still match"); + assert_eq!(bare.name, "ssh"); + } + + // --- Edge cases --- + + #[test] + fn test_empty_input() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = [".*"] +"#, + ); + let out = apply_filter(&f, ""); + assert_eq!(out, ""); + } + + #[test] + fn test_unicode_preserved() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_lines_matching = ["^noise"] +"#, + ); + let out = apply_filter(&f, "日本語テスト\nnoise\n中文内容"); + assert_eq!(out, "日本語テスト\n中文内容"); + } + + // --- match_output tests (PR1) --- + + #[test] + fn test_match_output_basic_short_circuit() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +"#, + ); + let out = apply_filter(&f, "Switched to branch 'main'"); + assert_eq!(out, "ok"); + } + + #[test] + fn test_match_output_second_rule_matches() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "switched" }, + { pattern = "Already on", message = "already" }, +] +"#, + ); + let out = apply_filter(&f, "Already on 'main'"); + assert_eq!(out, "already"); + } + + #[test] + fn test_match_output_no_match_pipeline_continues() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +strip_lines_matching = ["^noise"] +"#, + ); + let out = apply_filter(&f, "noise\nkeep this"); + // No match_output match, pipeline continues and strips noise + assert_eq!(out, "keep this"); + } + + #[test] + fn test_match_output_strip_ansi_before_match() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +strip_ansi = true +match_output = [ + { pattern = "Switched to branch", message = "ok" }, +] +"#, + ); + // ANSI stripped before match_output check (stage 1 before stage 3) + let out = apply_filter(&f, "\x1b[32mSwitched to branch\x1b[0m 'main'"); + assert_eq!(out, "ok"); + } + + #[test] + fn test_match_output_no_match_then_on_empty() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Switched", message = "ok" }, +] +strip_lines_matching = [".*"] +on_empty = "nothing" +"#, + ); + // No match_output match; pipeline strips all lines; on_empty fires + let out = apply_filter(&f, "foo bar baz"); + assert_eq!(out, "nothing"); + } + + #[test] + fn test_match_output_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "[invalid", message = "ok" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- match_output unless tests (PR3) --- + + #[test] + fn test_match_output_unless_blocks_short_circuit_when_errors_present() { + // "total size is" matches, but "error" also matches — unless fires, rule is skipped. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^rsync" +match_output = [ + { pattern = "total size is", message = "ok (synced)", unless = "error|failed" }, +] +"#, + ); + let input = "rsync: [sender] error\ntotal size is 1000 speedup is 3.33\n"; + let out = apply_filter(&f, input); + // Should NOT return "ok (synced)" because "error" matches the unless pattern + assert_ne!( + out.trim(), + "ok (synced)", + "unless should have blocked short-circuit when errors are present" + ); + // The raw lines should pass through (no further strip rules in this filter) + assert!(out.contains("error")); + } + + #[test] + fn test_match_output_unless_allows_short_circuit_when_no_errors() { + // "total size is" matches and "error" does NOT appear — unless does not fire, rule wins. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^rsync" +match_output = [ + { pattern = "total size is", message = "ok (synced)", unless = "error|failed" }, +] +"#, + ); + let input = "file.txt\ntotal size is 98765 speedup is 77.31\n"; + let out = apply_filter(&f, input); + assert_eq!(out.trim(), "ok (synced)"); + } + + #[test] + fn test_match_output_unless_falls_through_to_next_rule() { + // First rule blocked by unless; second rule (no unless) should match. + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "success", message = "ok", unless = "error" }, + { pattern = "success", message = "ok with warnings" }, +] +"#, + ); + let input = "success\nerror: something went wrong\n"; + let out = apply_filter(&f, input); + // First rule skipped (unless matched), second rule (no unless) fires + assert_eq!(out.trim(), "ok with warnings"); + } + + #[test] + fn test_match_output_unless_no_field_behaves_like_before() { + // When unless is absent, behaviour is identical to original (no regression). + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "Build complete", message = "ok (build complete)" }, +] +"#, + ); + let out = apply_filter(&f, "Build complete!\n"); + assert_eq!(out.trim(), "ok (build complete)"); + } + + #[test] + fn test_match_output_unless_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +match_output = [ + { pattern = "success", message = "ok", unless = "[invalid" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- replace tests (PR1) --- + + #[test] + fn test_replace_basic_all_occurrences() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "foo", replacement = "bar" }, +] +"#, + ); + let out = apply_filter(&f, "foo baz foo\nfoo"); + assert_eq!(out, "bar baz bar\nbar"); + } + + #[test] + fn test_replace_chaining_sequential() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "aaa", replacement = "bbb" }, + { pattern = "bbb", replacement = "ccc" }, +] +"#, + ); + // Rule 2 operates on the output of rule 1 + let out = apply_filter(&f, "aaa"); + assert_eq!(out, "ccc"); + } + + #[test] + fn test_replace_backreferences() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "(\\w+):(\\w+)", replacement = "$2:$1" }, +] +"#, + ); + let out = apply_filter(&f, "hello:world"); + assert_eq!(out, "world:hello"); + } + + #[test] + fn test_replace_then_strip_interaction() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "noise", replacement = "DROPPED" }, +] +strip_lines_matching = ["^DROPPED"] +"#, + ); + // replace transforms "noise line" -> "DROPPED line", strip removes it + let out = apply_filter(&f, "noise line\nkeep this"); + assert_eq!(out, "keep this"); + } + + #[test] + fn test_replace_empty_input_noop() { + let f = first_filter( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "foo", replacement = "bar" }, +] +"#, + ); + let out = apply_filter(&f, ""); + assert_eq!(out, ""); + } + + #[test] + fn test_replace_invalid_regex_rejected() { + let result = make_filters( + r#" +schema_version = 1 +[filters.f] +match_command = "^cmd" +replace = [ + { pattern = "[invalid", replacement = "bar" }, +] +"#, + ); + assert!(result.is_empty()); + } + + // --- verify (PR2) --- + + #[test] + fn test_run_filter_tests_passes_on_correct_expected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +strip_lines_matching = ["^make\\[\\d+\\]:"] + +[[tests.make]] +name = "strips entering/leaving lines" +input = """ +make[1]: Entering directory '/home/user' +gcc -O2 foo.c +make[1]: Leaving directory '/home/user' +""" +expected = """ +gcc -O2 foo.c +""" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + assert_eq!(outcomes.len(), 1); + assert!( + outcomes[0].passed, + "test should pass: {:?}", + outcomes[0].actual + ); + } + + #[test] + fn test_run_filter_tests_fails_on_wrong_expected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +strip_lines_matching = ["^make\\[\\d+\\]:"] + +[[tests.make]] +name = "wrong expected" +input = "make[1]: Entering\ngcc foo.c" +expected = "wrong output" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + assert_eq!(outcomes.len(), 1); + assert!(!outcomes[0].passed); + } + + #[test] + fn test_filters_without_tests_detected() { + let content = r#" +schema_version = 1 + +[filters.make] +match_command = "^make\\b" +"#; + let mut outcomes = Vec::new(); + let mut all_names = Vec::new(); + let mut tested = std::collections::HashSet::new(); + collect_test_outcomes(content, None, &mut outcomes, &mut all_names, &mut tested); + // No tests defined, but filter exists + assert_eq!(outcomes.len(), 0); + assert!(all_names.contains(&"make".to_string())); + assert!(!tested.contains("make")); + } + + // --- Multi-file architecture tests (build.rs) --- + + /// Verify BUILTIN_TOML was generated with the correct schema_version header. + /// build.rs injects it — if the const is somehow stale this fails immediately. + #[test] + fn test_builtin_toml_has_schema_version() { + assert!( + BUILTIN_TOML.contains("schema_version = 1"), + "BUILTIN_TOML must start with 'schema_version = 1' (injected by build.rs)" + ); + } + + /// Verify every expected filter name is present in BUILTIN_TOML. + /// This is the safeguard against accidentally deleting a filter file. + #[test] + fn test_builtin_all_expected_filters_present() { + let filters = make_filters(BUILTIN_TOML); + let names: std::collections::HashSet<&str> = + filters.iter().map(|f| f.name.as_str()).collect(); + + let expected = [ + "ansible-playbook", + "brew-install", + "composer-install", + "df", + "dotnet-build", + "du", + "fail2ban-client", + "gcloud", + "hadolint", + "helm", + "iptables", + "liquibase", + "make", + "markdownlint", + "mix-compile", + "mix-format", + "ping", + "pio-run", + "poetry-install", + "pre-commit", + "ps", + "pulumi-destroy", + "pulumi-preview", + "pulumi-refresh", + "pulumi-stack", + "pulumi-up", + "quarto-render", + "rsync", + "shellcheck", + "shopify-theme", + "sops", + "swift-build", + "systemctl-status", + "terraform-plan", + "tofu-fmt", + "tofu-init", + "tofu-plan", + "tofu-validate", + "trunk-build", + "uv-sync", + "yamllint", + ]; + + for name in &expected { + assert!( + names.contains(name), + "Built-in filter '{}' is missing — was its .toml file deleted from src/filters/?", + name + ); + } + } + + /// Verify the exact count of built-in filters. + /// Fails if a file is added/removed without updating this test. + #[test] + fn test_builtin_filter_count() { + let filters = make_filters(BUILTIN_TOML); + assert_eq!( + filters.len(), + 62, + "Expected exactly 62 built-in filters, got {}. \ + Update this count when adding/removing filters in src/filters/.", + filters.len() + ); + } + + /// Every built-in filter must be anchored on every top-level branch: the + /// match runs against the whole command line, so an unanchored branch + /// activates the filter on a path component or wrapper argument. + #[test] + fn test_builtin_all_filters_match_command_anchored() { + // Read the raw patterns rather than the compiled registry: both loaders + // drop unanchored filters, so a compiled view cannot observe one. + let file: TomlFilterFile = + toml::from_str(BUILTIN_TOML).expect("built-in filters should parse"); + assert!(!file.filters.is_empty()); + for (name, def) in &file.filters { + assert!( + is_fully_anchored(&def.match_command), + "Filter '{}' has match_command '{}' with a top-level branch that does not start with '^'", + name, + def.match_command + ); + } + + assert!(is_fully_anchored(r"^liquibase(?:\s|$)")); + assert!(is_fully_anchored(r"^(liquibase\b|/liquibase\b)")); + assert!(is_fully_anchored(r"^pnpm\b|^npm\b")); + assert!(is_fully_anchored(r"(?i)^liquibase\b")); + assert!(!is_fully_anchored(r"^liquibase\b|/liquibase\b")); + assert!(!is_fully_anchored(r"(?:^|/)liquibase(?:\s|$)")); + } + + /// The invariant above covers `BUILTIN_TOML`; project-local and global + /// filters are user-authored, so both loaders drop unanchored patterns + /// rather than letting them match mid-command. + #[test] + fn test_unanchored_user_filter_is_rejected_by_both_loaders() { + let unanchored = + "schema_version = 1\n[filters.mytool]\nmatch_command = \"(?:^|/)mytool\\\\b\"\n"; + assert!(match_patterns_in(unanchored).is_empty()); + assert!( + TomlFilterRegistry::parse_and_compile(unanchored, "test") + .expect("schema is valid") + .is_empty() + ); + + let anchored = "schema_version = 1\n[filters.mytool]\nmatch_command = \"^mytool\\\\b\"\n"; + assert_eq!(match_patterns_in(anchored), vec!["^mytool\\b".to_string()]); + assert_eq!( + TomlFilterRegistry::parse_and_compile(anchored, "test") + .expect("schema is valid") + .len(), + 1 + ); + } + + /// Verify that every built-in filter has at least one inline test. + /// Prevents shipping filters with zero test coverage. + #[test] + fn test_builtin_all_filters_have_inline_tests() { + let mut all_names: Vec = Vec::new(); + let mut tested: std::collections::HashSet = std::collections::HashSet::new(); + let mut outcomes = Vec::new(); + collect_test_outcomes( + BUILTIN_TOML, + None, + &mut outcomes, + &mut all_names, + &mut tested, + ); + + let untested: Vec<&str> = all_names + .iter() + .filter(|name| !tested.contains(name.as_str())) + .map(|s| s.as_str()) + .collect(); + + assert!( + untested.is_empty(), + "The following built-in filters have no inline tests: {:?}\n\ + Add [[tests.]] entries to the corresponding src/filters/.toml file.", + untested + ); + } + + /// Verify that adding a new filter entry to any TOML content makes it + /// immediately discoverable via find_filter_in — simulating how a new + /// src/filters/my-tool.toml would work after cargo build. + #[test] + fn test_new_filter_discoverable_after_concat() { + // Simulate build.rs: concat BUILTIN_TOML with a brand-new filter block + let new_filter = r#" +[filters.my-new-tool] +description = "Compact my-new-tool output" +match_command = "^my-new-tool\\b" +strip_lines_matching = ["^\\s*$"] +max_lines = 30 +on_empty = "my-new-tool: ok" + +[[tests.my-new-tool]] +name = "strips blank lines" +input = "output line 1\n\noutput line 2" +expected = "output line 1\noutput line 2" +"#; + let combined = format!("{}\n\n{}", BUILTIN_TOML, new_filter); + let filters = make_filters(&combined); + + // All 62 existing filters still present + 1 new = 63 + assert_eq!( + filters.len(), + 63, + "Expected 63 filters after concat (62 built-in + 1 new)" + ); + + // New filter is discoverable + let found = find_filter_in("my-new-tool --verbose", &filters); + assert!( + found.is_some(), + "Newly added filter must be discoverable via find_filter_in" + ); + assert_eq!(found.unwrap().name, "my-new-tool"); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/core/truncate.rs b/crates/sparkplane-rtk/upstream/src/core/truncate.rs new file mode 100644 index 0000000..58285f6 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/truncate.rs @@ -0,0 +1,60 @@ +//! Global truncation caps shared by every filter. See `src/core/README.md` +//! ("Truncation Caps") for the cap classes, config policy, and deviation rules. + +/// Errors: most actionable, shown the most. +pub const CAP_ERRORS: usize = 20; +/// Warnings and test failures: lower signal density than errors. +pub const CAP_WARNINGS: usize = 10; +/// Flat lists (PRs, services, packages): one line per item. +pub const CAP_LIST: usize = 20; +/// Inventories (`pip list`, `docker images`): exhaustive lookups. +pub const CAP_INVENTORY: usize = 50; + +/// A cap reduced for a verbose data class. Falls back to `cap` when `by >= cap` +/// so a deviation can never empty the list; `0` stays `0`. `const fn`, underflow-safe. +pub const fn reduced(cap: usize, by: usize) -> usize { + if by < cap { cap - by } else { cap } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reduced_preserves_current_values() { + assert_eq!(reduced(CAP_WARNINGS, 5), 5); + assert_eq!(reduced(CAP_LIST, 5), 15); + } + + #[test] + fn reduced_falls_back_to_cap_when_offset_too_large() { + assert_eq!(reduced(4, 5), 4); + assert_eq!(reduced(5, 5), 5); + } + + #[test] + fn reduced_honors_zero_cap() { + assert_eq!(reduced(0, 5), 0); + } + + // Sweep every plausible (cap, by) a future config could produce and assert + // the invariants that make caps safe: the result never wraps past `cap`, and + // the offset never empties a non-zero cap. `usize::MAX` covers a wraparound bug. + #[test] + fn reduced_is_underflow_safe_across_all_inputs() { + for cap in 0..=64usize { + for by in [0usize, 1, 5, 10, 64, usize::MAX] { + let r = reduced(cap, by); + assert!(r <= cap, "reduced({cap}, {by}) = {r} exceeds cap (wrapped)"); + if cap == 0 { + assert_eq!(r, 0, "zero cap must stay zero"); + } else { + assert!(r >= 1, "reduced({cap}, {by}) = {r} emptied a non-zero cap"); + } + if by < cap { + assert_eq!(r, cap - by, "exact deviation must be preserved"); + } + } + } + } +} diff --git a/crates/sparkplane-rtk/upstream/src/core/utils.rs b/crates/sparkplane-rtk/upstream/src/core/utils.rs new file mode 100644 index 0000000..809f5bb --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/core/utils.rs @@ -0,0 +1,1842 @@ +//! Utility functions for text processing and command execution. +//! +//! Provides common helpers used across rtk commands: +//! - ANSI color code stripping +//! - Text truncation +//! - Command execution with error context + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use regex::Regex; +use serde_json::Value; +use std::ffi::OsStr; +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use std::sync::LazyLock; +use std::sync::OnceLock; + +/// Compute `days` ago from now, clamped instead of panicking on overflow. +/// +/// `days` is typically user-supplied (`--since `) and unbounded. Naively +/// building `chrono::Duration::days(days as i64)` and subtracting it from +/// `Utc::now()` panics for a large enough value ("DateTime - TimeDelta +/// overflowed") — this happened identically in `rtk discover --since` and `rtk +/// hook audit --since` before both were routed through this helper. `days` is +/// capped to `i64::MAX` before the `as i64` cast (a `u64` past `i64::MAX` would +/// otherwise reinterpret as negative, turning "look back" into "look forward"), +/// and any remaining overflow building or applying the `Duration` falls back to +/// `DateTime::::MIN_UTC` — an arbitrarily distant cutoff means "no lower +/// bound", which is exactly what an absurdly large `--since` should behave like. +pub fn days_ago_cutoff(days: u64) -> DateTime { + let days = days.min(i64::MAX as u64) as i64; + chrono::Duration::try_days(days) + .and_then(|d| Utc::now().checked_sub_signed(d)) + .unwrap_or(DateTime::::MIN_UTC) +} + +/// Truncates a string to `max_len` characters, appending `...` if needed. +/// +/// # Arguments +/// * `s` - The string to truncate +/// * `max_len` - Maximum length before truncation (minimum 3 to include "...") +/// +/// # Examples +/// ``` +/// use rtk::utils::truncate; +/// assert_eq!(truncate("hello world", 8), "hello..."); +/// assert_eq!(truncate("hi", 10), "hi"); +/// ``` +pub fn truncate(s: &str, max_len: usize) -> String { + let char_count = s.chars().count(); + if char_count <= max_len { + s.to_string() + } else if max_len < 3 { + // If max_len is too small, just return "..." + "...".to_string() + } else { + format!("{}...", s.chars().take(max_len - 3).collect::()) + } +} + +/// Strip ANSI escape codes (colors, styles) from a string. +/// +/// # Arguments +/// * `text` - Text potentially containing ANSI escape codes +/// +/// # Examples +/// ``` +/// use rtk::utils::strip_ansi; +/// let colored = "\x1b[31mError\x1b[0m"; +/// assert_eq!(strip_ansi(colored), "Error"); +/// ``` +pub fn strip_ansi(text: &str) -> String { + static ANSI_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\x1b\[[0-9;]*[a-zA-Z]").unwrap()); + + ANSI_RE.replace_all(text, "").to_string() +} + +/// Strip one or more leading UTF-8 BOMs (`EF BB BF`, sometimes doubled), +/// which serde_json refuses to parse. Windows editors (Notepad, PowerShell +/// 5.1 `Out-File -Encoding utf8`) prepend one to hand-edited config files, +/// and Cursor on Windows ships hook payloads with them. Strip defensively +/// wherever RTK parses JSON a human or another tool may have written. +pub fn strip_leading_bom(input: &str) -> &str { + let mut s = input; + while let Some(rest) = s.strip_prefix('\u{feff}') { + s = rest; + } + s +} + +/// BOM-tolerant `serde_json::from_str`. Prefer this over `serde_json::from_str` +/// anywhere the JSON may have been written by a human, an editor, or another tool. +pub fn from_json_str<'a, T: serde::Deserialize<'a>>(s: &'a str) -> serde_json::Result { + serde_json::from_str(strip_leading_bom(s)) +} + +/// Executes a command and returns cleaned stdout/stderr. +/// +/// # Arguments +/// * `cmd` - Command to execute (e.g., "eslint") +/// * `args` - Command arguments +/// +/// # Returns +/// `(stdout: String, stderr: String, exit_code: i32)` +/// Formats a token count with K/M suffixes for readability. +/// +/// # Arguments +/// * `n` - Number of tokens +/// +/// # Returns +/// Formatted string (e.g., "1.2M", "59.2K", "694") +/// +/// # Examples +/// ``` +/// use rtk::utils::format_tokens; +/// assert_eq!(format_tokens(1_234_567), "1.2M"); +/// assert_eq!(format_tokens(59_234), "59.2K"); +/// assert_eq!(format_tokens(694), "694"); +/// ``` +pub fn format_tokens(n: usize) -> String { + if n >= 1_000_000 { + format!("{:.1}M", n as f64 / 1_000_000.0) + } else if n >= 1_000 { + format!("{:.1}K", n as f64 / 1_000.0) + } else { + format!("{}", n) + } +} + +/// Formats a USD amount with adaptive precision. +/// +/// # Arguments +/// * `amount` - Amount in dollars +/// +/// # Returns +/// Formatted string with $ prefix +/// +/// # Examples +/// ``` +/// use rtk::utils::format_usd; +/// assert_eq!(format_usd(1234.567), "$1234.57"); +/// assert_eq!(format_usd(12.345), "$12.35"); +/// assert_eq!(format_usd(0.123), "$0.12"); +/// assert_eq!(format_usd(0.0096), "$0.0096"); +/// ``` +pub fn format_usd(amount: f64) -> String { + if !amount.is_finite() { + return "$0.00".to_string(); + } + if amount >= 0.01 { + format!("${:.2}", amount) + } else { + format!("${:.4}", amount) + } +} + +/// Format cost-per-token as $/MTok (e.g., "$3.86/MTok") +/// +/// # Arguments +/// * `cpt` - Cost per token (not per million tokens) +/// +/// # Returns +/// Formatted string like "$3.86/MTok" +/// +/// # Examples +/// ``` +/// use rtk::utils::format_cpt; +/// assert_eq!(format_cpt(0.000003), "$3.00/MTok"); +/// assert_eq!(format_cpt(0.0000038), "$3.80/MTok"); +/// assert_eq!(format_cpt(0.00000386), "$3.86/MTok"); +/// ``` +pub fn format_cpt(cpt: f64) -> String { + if !cpt.is_finite() || cpt <= 0.0 { + return "$0.00/MTok".to_string(); + } + let cpt_per_million = cpt * 1_000_000.0; + format!("${:.2}/MTok", cpt_per_million) +} + +/// Join items into a newline-separated string, appending an overflow hint when total > max. +/// +/// # Examples +/// ``` +/// use rtk::utils::join_with_overflow; +/// let items = vec!["a".to_string(), "b".to_string()]; +/// assert_eq!(join_with_overflow(&items, 5, 3, "items"), "a\nb\n... +2 more items"); +/// assert_eq!(join_with_overflow(&items, 2, 3, "items"), "a\nb"); +/// ``` +pub fn join_with_overflow(items: &[String], total: usize, max: usize, label: &str) -> String { + let mut out = items.join("\n"); + if total > max { + out.push_str(&format!("\n… +{} more {}", total - max, label)); + } + out +} + +/// Truncate an ISO 8601 datetime string to just the date portion (first 10 chars). +/// +/// # Examples +/// ``` +/// use rtk::utils::truncate_iso_date; +/// assert_eq!(truncate_iso_date("2024-01-15T10:30:00Z"), "2024-01-15"); +/// assert_eq!(truncate_iso_date("2024-01-15"), "2024-01-15"); +/// assert_eq!(truncate_iso_date("short"), "short"); +/// ``` +pub fn truncate_iso_date(date: &str) -> &str { + if date.len() >= 10 { &date[..10] } else { date } +} + +/// Format a confirmation message: "ok \ \" +/// Used for write operations (merge, create, comment, edit, etc.) +/// +/// # Examples +/// ``` +/// use rtk::utils::ok_confirmation; +/// assert_eq!(ok_confirmation("merged", "#42"), "ok merged #42"); +/// assert_eq!(ok_confirmation("created", "PR #5 https://..."), "ok created PR #5 https://..."); +/// ``` +pub fn ok_confirmation(action: &str, detail: &str) -> String { + if detail.is_empty() { + format!("ok {}", action) + } else { + format!("ok {} {}", action, detail) + } +} + +/// Extract exit code from a process output. Returns the actual exit code, or +/// `128 + signal` per Unix convention when terminated by a signal (no exit code +/// available). Falls back to 1 on non-Unix platforms. +pub fn exit_code_from_output(output: &std::process::Output, label: &str) -> i32 { + match output.status.code() { + Some(code) => code, + None => { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if let Some(sig) = output.status.signal() { + eprintln!("[rtk] {}: process terminated by signal {}", label, sig); + return 128 + sig; + } + } + eprintln!("[rtk] {}: process terminated by signal", label); + 1 + } + } +} + +/// Extract exit code from an ExitStatus (for `.status()` calls, not `.output()`). +/// Returns the actual exit code, or `128 + signal` per Unix convention when +/// terminated by a signal. Falls back to 1 on non-Unix platforms. +pub fn exit_code_from_status(status: &std::process::ExitStatus, label: &str) -> i32 { + match status.code() { + Some(code) => code, + None => { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if let Some(sig) = status.signal() { + eprintln!("[rtk] {}: process terminated by signal {}", label, sig); + return 128 + sig; + } + } + eprintln!("[rtk] {}: process terminated by signal", label); + 1 + } + } +} + +/// Return the last `n` lines of output with a label, for use as a fallback +/// when filter parsing fails. Logs a diagnostic to stderr. +pub fn fallback_tail(output: &str, label: &str, n: usize) -> String { + eprintln!( + "[rtk] {}: output format not recognized, showing last {} lines", + label, n + ); + let lines: Vec<&str> = output.lines().collect(); + let start = lines.len().saturating_sub(n); + lines[start..].join("\n") +} + +/// Create a directory owner-only (0700 on Unix), tightening one that already exists. +/// Serializes tests across modules that mutate process-global recall env vars +/// (`RTK_RECALL`, `RTK_TEE`): a static declared inside one test module is not +/// shared with other modules, so those tests would not actually serialize. +#[cfg(test)] +pub(crate) static TEST_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +pub fn create_private_dir(path: &std::path::Path) -> std::io::Result<()> { + fs::create_dir_all(path)?; + set_owner_only(path, 0o700); + Ok(()) +} + +/// Restrict an existing file to owner-only access (0600 on Unix). +pub fn restrict_file(path: &std::path::Path) { + set_owner_only(path, 0o600); +} + +/// Open a file owner-only (0600 on Unix), applied at creation so content is +/// never briefly readable under a permissive umask. `mode` is ignored for a +/// file that already exists, so an older one is still tightened afterwards. +pub fn open_private( + opts: &mut fs::OpenOptions, + path: &std::path::Path, +) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let file = opts.open(path)?; + set_owner_only(path, 0o600); + Ok(file) +} + +/// Pure core of `set_owner_only`'s "is a chmod even needed" check, taking the raw +/// `st_mode` reading directly so it's deterministically testable without touching +/// the filesystem (an unprivileged `chmod` setting setuid/setgid isn't reliably +/// honored across every environment — e.g. some sandboxed/containerized +/// filesystems silently drop it — so a test that round-trips through a real file +/// would be flaky rather than actually exercising this logic). +/// +/// Masks with `0o7777`, NOT `0o777`: setuid/setgid/sticky live in the `0o7000` +/// range, above the rwx bits but below the file-type bits `st_mode` also carries. +/// A `0o777` mask would strip those dangerous bits out of the comparison too, so +/// a file with e.g. setuid set (`0o4600`) would read as "already correct" against +/// a `target` of `0o600` and permanently skip the chmod that would clear it — +/// defeating the self-heal `set_owner_only` exists to provide, for exactly the +/// files (external tampering, restored backups) it's meant to catch. +#[cfg(unix)] +fn mode_already_correct(actual: u32, target: u32) -> bool { + actual & 0o7777 == target +} + +#[cfg(unix)] +fn set_owner_only(path: &std::path::Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + // Skip the chmod syscall once permissions already match. This runs on every + // `Tracker::new()` call, which is now on the PreToolUse hook's hot path for + // *every* Bash tool call (not just RTK-covered ones — see rtk-ai/rtk#3206's + // hook_decisions ground-truth logging), under the project's <10ms latency + // budget, so a redundant chmod on the already-correct common case adds up. + // Falls through to chmod on any metadata-read failure, erring toward + // enforcing the permission rather than silently skipping it. + // + if let Ok(meta) = fs::metadata(path) + && mode_already_correct(meta.permissions().mode(), mode) + { + return; + } + let _ = fs::set_permissions(path, fs::Permissions::from_mode(mode)); +} + +#[cfg(not(unix))] +fn set_owner_only(_path: &std::path::Path, _mode: u32) {} + +/// Build a Command for Ruby tools, auto-detecting bundle exec. +/// Uses `bundle exec ` when a Gemfile exists (transitive deps like rake +/// won't appear in the Gemfile but still need bundler for version isolation). +pub fn ruby_exec(tool: &str) -> Command { + if std::path::Path::new("Gemfile").exists() { + let mut c = Command::new("bundle"); + c.arg("exec").arg(tool); + return c; + } + Command::new(tool) +} + +/// Count whitespace-delimited tokens in text. Used by filter tests to verify +/// token savings claims. +#[cfg(test)] +pub fn count_tokens(text: &str) -> usize { + text.split_whitespace().count() +} + +/// Detect the package manager used in the current directory. +/// Returns "pnpm", "yarn", "bun", or "npm" based on lockfile presence. +/// +/// # Examples +/// ```no_run +/// use rtk::utils::detect_package_manager; +/// let pm = detect_package_manager(); +/// // "pnpm" for pnpm-lock.yaml, "yarn" for yarn.lock, "bun" for bun.lock(b), else "npm" +/// ``` +#[allow(dead_code)] +pub fn detect_package_manager() -> &'static str { + detect_package_manager_in(std::path::Path::new(".")) +} + +/// Lockfile detection against an explicit directory, so callers (and tests) do +/// not have to move the process's current directory to ask the question. +pub fn detect_package_manager_in(dir: &std::path::Path) -> &'static str { + if dir.join("pnpm-lock.yaml").exists() { + "pnpm" + } else if dir.join("yarn.lock").exists() { + "yarn" + } else if dir.join("bun.lockb").exists() || dir.join("bun.lock").exists() { + "bun" + } else { + "npm" + } +} + +/// Build a Command using the detected package manager's exec mechanism. +/// Returns a Command ready to have tool-specific args appended. +pub fn package_manager_exec(tool: &str) -> Command { + tool_exec(None, tool, MissingTool::Fail) +} + +/// What the npm arm does when `tool` is not installed. `npx` fetches on demand +/// by default, which is right for a tool the user named themselves and +/// surprising for one rtk picked on their behalf. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum MissingTool { + Fetch, + Fail, +} + +/// The runner used to resolve a tool that is not on PATH. A runner the user +/// named wins outright. Otherwise lockfile detection picks one, except that a +/// detected bun resolves through npx: bunx always fetches a missing tool into +/// its cache and cannot be told not to, so `MissingTool` could not be honoured +/// and the presence of a lockfile alone would decide what gets downloaded. +pub fn exec_runner(runner: Option<&str>, missing: MissingTool) -> &str { + match runner { + Some(named) => named, + // Nothing was named and rtk may fetch: npx is the only runner that can, + // and lockfile detection would hand `pnpm exec` a tool the project + // does not have. + None if missing == MissingTool::Fetch => "npx", + None => match detect_package_manager() { + // bunx always fetches; npm's runner is npx. + "bun" | "npm" => "npx", + other => other, + }, + } +} + +/// Build a Command that runs `tool`, preferring the package runner the user +/// actually named over lockfile detection. +/// +/// RTK forwards what was typed rather than substituting an engine for it: a +/// user who types `bunx tsc` must not get `pnpm` because a lockfile is present. +/// Detection applies only when nothing was named, as with a bare `rtk tsc`. +/// +/// A tool already on PATH is run directly only when no runner was named. +pub fn tool_exec(runner: Option<&str>, tool: &str, missing: MissingTool) -> Command { + // Only when nothing was named: `bunx tsc` must resolve the project's tsc, + // not a global one that happens to be on PATH. Both bunx and npx prefer + // node_modules/.bin before fetching, so naming one is a real choice. + if runner.is_none() && tool_exists(tool) { + resolved_command(tool) + } else { + match exec_runner(runner, missing) { + "pnpm" => { + let mut c = resolved_command("pnpm"); + c.arg("exec").arg("--").arg(tool); + c + } + "yarn" => { + let mut c = resolved_command("yarn"); + c.arg("exec").arg("--").arg(tool); + c + } + "bun" | "bunx" => { + let mut c = resolved_command("bunx"); + c.arg(tool); + c + } + _ => { + let mut c = resolved_command("npx"); + if missing == MissingTool::Fail { + c.arg("--no-install"); + } + c.arg("--").arg(tool); + c + } + } + } +} + +/// Encode one argument for a child's raw command line the way libuv's +/// `quote_cmd_arg` does: wrap it in `"`, escape an inner `"` as `\"`, and double +/// every backslash run that ends up in front of a quote so it stays literal. +/// +/// std's encoder emits the same bytes but wraps only for a space or a tab, which +/// is what leaves MSYS children with a mangled command line (#3727). +// Windows-only in production; the rules stay unit-tested on every platform. +#[cfg_attr(not(windows), allow(dead_code))] +pub fn quote_arg_for_child(arg: &str) -> String { + let mut out = String::with_capacity(arg.len() + 2); + out.push('"'); + let mut backslashes = 0usize; + for ch in arg.chars() { + match ch { + '\\' => backslashes += 1, + '"' => { + for _ in 0..backslashes * 2 + 1 { + out.push('\\'); + } + backslashes = 0; + out.push('"'); + } + _ => { + for _ in 0..backslashes { + out.push('\\'); + } + backslashes = 0; + out.push(ch); + } + } + } + // The closing quote is a quote too, so a trailing run is doubled as well. + for _ in 0..backslashes * 2 { + out.push('\\'); + } + out.push('"'); + out +} + +/// Pass caller-supplied arguments to a child so that MSYS/Cygwin children on +/// Windows receive them intact. +/// +/// Use instead of `Command::arg`/`args` for anything that arrived on rtk's own +/// command line — a pattern, a path, a flag value. +pub trait ChildArgExt { + fn child_arg>(&mut self, arg: S) -> &mut Command; + + fn child_args(&mut self, args: I) -> &mut Command + where + I: IntoIterator, + S: AsRef; +} + +impl ChildArgExt for Command { + fn child_arg>(&mut self, arg: S) -> &mut Command { + push_child_arg(self, arg.as_ref()); + self + } + + fn child_args(&mut self, args: I) -> &mut Command + where + I: IntoIterator, + S: AsRef, + { + for arg in args { + push_child_arg(self, arg.as_ref()); + } + self + } +} + +/// Windows: `"` is the only character std encodes differently from libuv, so +/// re-encode just those arguments and leave the rest on std's path. `.bat`/`.cmd` +/// shims stay on it too — cmd.exe parses by its own rules and `raw_arg` would +/// bypass the escaping std applies for them. +#[cfg(windows)] +fn push_child_arg(cmd: &mut Command, arg: &OsStr) { + match arg.to_str() { + Some(s) if s.contains('"') && !is_batch_program(cmd) => { + std::os::windows::process::CommandExt::raw_arg(cmd, quote_arg_for_child(s)); + } + _ => { + cmd.arg(arg); + } + } +} + +#[cfg(windows)] +fn is_batch_program(cmd: &Command) -> bool { + std::path::Path::new(cmd.get_program()) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("bat") || ext.eq_ignore_ascii_case("cmd")) +} + +/// Unix: the argument vector reaches `execvp` verbatim, so there is nothing to +/// encode. +#[cfg(not(windows))] +fn push_child_arg(cmd: &mut Command, arg: &OsStr) { + cmd.arg(arg); +} + +/// Resolve a binary name to its full path, honoring PATHEXT on Windows. +/// +/// On Windows, Node.js tools are installed as `.CMD`/`.BAT`/`.PS1` shims. +/// Rust's `std::process::Command::new()` does NOT honor PATHEXT, so +/// `Command::new("vitest")` fails even when `vitest.CMD` is on PATH. +/// +/// This function uses the `which` crate to perform proper PATH+PATHEXT resolution. +/// +/// # Arguments +/// * `name` - Binary name (e.g., "vitest", "eslint", "tsc") +/// +/// # Returns +/// Full path to the resolved binary, or error if not found. +pub fn resolve_binary(name: &str) -> Result { + which::which(name).context(format!("Binary '{}' not found on PATH", name)) +} + +/// Create a `Command` with PATHEXT-aware binary resolution. +/// +/// Drop-in replacement for `Command::new(name)` that works on Windows +/// with `.CMD`/`.BAT`/`.PS1` wrappers. +/// +/// Falls back to `Command::new(name)` if resolution fails, so native +/// commands (git, cargo) still work even if `which` can't find them. +/// +/// # Arguments +/// * `name` - Binary name (e.g., "vitest", "eslint") +/// +/// # Returns +/// A `Command` configured with the resolved binary path. +pub fn resolved_command(name: &str) -> Command { + match resolve_binary(name) { + Ok(path) => Command::new(path), + Err(e) => { + // On Windows, resolution failure likely means a .CMD/.BAT wrapper + // wasn't found — always warn so users have a signal. + // On Unix, this is less common; only log in debug builds. + if cfg!(any(target_os = "windows", debug_assertions)) { + eprintln!( + "rtk: Failed to resolve '{}' via PATH, falling back to direct exec: {}", + name, e + ); + } + + Command::new(name) + } + } +} + +/// Return Composer bin directories in precedence order. +/// +/// Composer allows overriding the default `vendor/bin` via `COMPOSER_BIN_DIR` +/// or `composer.json` `config.bin-dir`. Keep the default as a fallback so we +/// continue recognizing the common layout even when the repo is not configured. +pub fn composer_bin_dirs() -> Vec { + // Resolution depends only on the process's env + cwd composer.json, both + // constant for a single rtk invocation. The rewrite hot path queries this + // several times per command segment, so read the file once and cache. + static CACHE: OnceLock> = OnceLock::new(); + CACHE + .get_or_init(|| { + let env_bin_dir = std::env::var("COMPOSER_BIN_DIR").ok(); + let composer_json = fs::read_to_string("composer.json").ok(); + composer_bin_dirs_from(env_bin_dir.as_deref(), composer_json.as_deref()) + }) + .clone() +} + +pub fn composer_tool_paths(tool: &str) -> Vec { + composer_bin_dirs() + .into_iter() + .map(|dir| dir.join(tool)) + .collect() +} + +fn composer_bin_dirs_from(env_bin_dir: Option<&str>, composer_json: Option<&str>) -> Vec { + let mut dirs = Vec::new(); + + if let Some(dir) = env_bin_dir + .map(str::trim) + .filter(|dir| !dir.is_empty()) + .map(PathBuf::from) + .or_else(|| composer_json.and_then(read_composer_bin_dir)) + { + dirs.push(dir); + } + + let default_dir = PathBuf::from("vendor/bin"); + if !dirs.iter().any(|dir| dir == &default_dir) { + dirs.push(default_dir); + } + + dirs +} + +fn read_composer_bin_dir(composer_json: &str) -> Option { + let parsed: Value = from_json_str(composer_json).ok()?; + let bin_dir = parsed.get("config")?.get("bin-dir")?.as_str()?.trim(); + if bin_dir.is_empty() { + None + } else { + Some(PathBuf::from(bin_dir)) + } +} + +/// Join lines with newlines, or return "ok" when there is nothing left to show. +/// +/// Shared by output filters that collapse fully to a success marker. +pub fn join_or_ok(lines: &[&str]) -> String { + if lines.is_empty() { + "ok".to_string() + } else { + lines.join("\n") + } +} + +/// Check if a tool exists on PATH (PATHEXT-aware on Windows). +/// +/// Replaces manual `Command::new("which").arg(tool)` checks that fail on Windows. +pub fn tool_exists(name: &str) -> bool { + which::which(name).is_ok() +} + +/// Check if a compile-time environment variable was set to a non-empty value. +/// +/// `option_env!` yields `Some("")` when the build environment exports the +/// variable with an empty value, which a CI `env:` block fed by an unset +/// repository variable does. `.is_some()` alone then reports a feature as +/// configured when it is not, so pair every `option_env!` gate with this. +/// +/// # Examples +/// ``` +/// use rtk::utils::env_is_some; +/// assert!(env_is_some(Some("https://example.com"))); +/// assert!(!env_is_some(Some(""))); +/// assert!(!env_is_some(None)); +/// ``` +pub fn env_is_some(value: Option<&str>) -> bool { + value.is_some_and(|v| !v.is_empty()) +} + +/// Extract short name from AWS ARN. +/// Example: `arn:aws:ecs:region:acct:service/cluster/name` -> `name` +/// For simple ARNs like `arn:aws:iam::123:user/alice`, returns `alice`. +pub fn shorten_arn(arn: &str) -> &str { + // ARNs use "/" or ":" as separators. Try "/" first (service/cluster/name pattern), + // then fall back to ":" for Lambda/IAM ARNs. + let slash_result = arn.rsplit('/').next().unwrap_or(arn); + // If rsplit('/') returned the whole string (no '/' found), try ':' + if slash_result == arn { + arn.rsplit(':').next().unwrap_or(arn) + } else { + slash_result + } +} + +/// Convert bytes to human-readable format (KB, MB, GB, TB). +/// Used for S3 object sizes. +pub fn human_bytes(bytes: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = KB * 1024; + const GB: u64 = MB * 1024; + const TB: u64 = GB * 1024; + + if bytes >= TB { + format!("{:.1} TB", bytes as f64 / TB as f64) + } else if bytes >= GB { + format!("{:.1} GB", bytes as f64 / GB as f64) + } else if bytes >= MB { + format!("{:.1} MB", bytes as f64 / MB as f64) + } else if bytes >= KB { + format!("{:.1} KB", bytes as f64 / KB as f64) + } else { + format!("{} B", bytes) + } +} + +/// Decode child process output bytes, respecting the Windows console code page. +/// +/// Valid UTF-8 is returned untouched — the overwhelmingly common case, and the +/// only one on Unix. Otherwise the buffer is decoded a line at a time: lines +/// that are valid UTF-8 keep their bytes, and only the lines that are not get +/// re-decoded through the console code page (GBK, Big5, CP850, …). Decoding +/// the whole buffer as a legacy code page at the first bad byte would mangle +/// output that was almost entirely valid UTF-8. +/// +/// Falls back to lossy UTF-8 (`U+FFFD`) when no code page applies, matching the +/// previous behavior. +pub fn decode_process_output(bytes: &[u8]) -> String { + // Fast path: fully valid UTF-8 needs no scanning and no allocation beyond + // the copy. This is every Unix run and every UTF-8 console on Windows. + match std::str::from_utf8(bytes) { + Ok(s) => s.to_owned(), + Err(_) => decode_mixed(bytes, output_codepage()), + } +} + +/// Decode `bytes` line by line, keeping valid UTF-8 lines verbatim and passing +/// the rest through code page `cp` (lossy UTF-8 when `cp` is `None`). +/// +/// The line is the decoding unit because a byte run is not one: GB18030's +/// four-byte sequences embed bytes in the ASCII digit range, so any rule that +/// stops a run at the first byte under `0x80` splits them. `\n` is unambiguous +/// in every encoding handled here — none of UTF-8, GBK, gb18030, Big5, +/// Shift_JIS, EUC-KR or the single-byte pages can produce it as a trail byte — +/// and a process does not switch encoding mid-line, so the whole line can be +/// handed to one decoder. +/// +/// Takes the code page as a parameter so the walk and the mapping are +/// unit-testable on every platform, not only Windows. +fn decode_mixed(bytes: &[u8], cp: Option) -> String { + let mut out = String::with_capacity(bytes.len()); + // `split_inclusive` keeps the terminator, so line endings survive intact + // and an empty input yields no chunks at all. + for line in bytes.split_inclusive(|&b| b == b'\n') { + match std::str::from_utf8(line) { + Ok(valid) => out.push_str(valid), + Err(_) => out.push_str(&decode_line(line, cp)), + } + } + out +} + +/// Decode one line that failed UTF-8 validation using code page `cp`. +/// +/// A code page result is only accepted when it decodes cleanly. A line that is +/// really UTF-8 with a corrupt byte usually fails the code page decoder too, +/// and lossy UTF-8 preserves its valid characters where the code page would +/// turn all of them into mojibake. +fn decode_line(line: &[u8], cp: Option) -> String { + if let Some(cp) = cp { + // encoding_rs covers the ANSI and DBCS pages (1252, GBK, gb18030, + // Shift_JIS, Big5, …); `codepage` maps the Windows page number onto it. + if let Some(encoding) = codepage::to_encoding(cp) { + let (decoded, _, had_errors) = encoding.decode(line); + if !had_errors { + return decoded.into_owned(); + } + // encoding_rs implements only WHATWG encodings, which exclude the + // legacy OEM/DOS pages (437, 850, 852, …) that plain cmd.exe still + // defaults to in many locales. `oem_cp` supplies those tables. + } else if let Some(table) = oem_cp::code_table::DECODING_TABLE_CP_MAP.get(&cp) { + if let Some(decoded) = table.decode_string_checked(line) { + return decoded; + } + } else { + warn_unmapped_codepage(cp); + } + } + String::from_utf8_lossy(line).into_owned() +} + +/// Warn once that output is being decoded lossily because the console code +/// page has no known table — previously this fell back silently. +fn warn_unmapped_codepage(cp: u16) { + static WARNED: std::sync::Once = std::sync::Once::new(); + WARNED.call_once(|| { + eprintln!( + "[rtk] warning: no decoder for console code page {}; \ + non-UTF-8 output will be shown with replacement characters", + cp + ); + }); +} + +/// The code page child output should be decoded with, or `None` when the +/// platform has no such concept (every Unix) and lossy UTF-8 should be used. +#[cfg(not(windows))] +fn output_codepage() -> Option { + None +} + +/// Windows: the console output code page, cached after the first lookup. +/// +/// `GetConsoleOutputCP` describes the console rtk is attached to, which the +/// child inherits. It returns 0 when there is no console — rtk running under a +/// hook with its output piped, the common case flagged in review — and a +/// console program writing to a pipe uses the ANSI code page instead, so +/// `GetACP` is the fallback rather than giving up and decoding lossily. +/// +/// This remains a best guess: a child is free to emit any encoding regardless +/// of either code page. It is only ever consulted for bytes that already +/// failed UTF-8 validation, so a wrong guess degrades to the same replacement +/// characters that the previous lossy conversion produced. +#[cfg(windows)] +fn output_codepage() -> Option { + static CODEPAGE: OnceLock> = OnceLock::new(); + *CODEPAGE.get_or_init(|| { + #[allow(unsafe_code)] + // nosemgrep: unsafe-block — read-only Win32 APIs, no memory or thread safety risk + let cp = unsafe { + let console = windows_sys::Win32::System::Console::GetConsoleOutputCP(); + if console != 0 { + console + } else { + windows_sys::Win32::Globalization::GetACP() + } + }; + u16::try_from(cp).ok().filter(|&cp| cp != 0) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_strip_leading_bom_helper() { + // Direct unit test on the helper so future refactors can't + // regress the loop semantics without a clear failure signal. + assert_eq!(strip_leading_bom(""), ""); + assert_eq!(strip_leading_bom("hello"), "hello"); + assert_eq!(strip_leading_bom("\u{feff}hello"), "hello"); + assert_eq!(strip_leading_bom("\u{feff}\u{feff}hello"), "hello"); + assert_eq!(strip_leading_bom("\u{feff}\u{feff}\u{feff}hello"), "hello"); + // BOM in the middle is preserved (not "leading"). + assert_eq!(strip_leading_bom("a\u{feff}b"), "a\u{feff}b"); + } + + #[test] + fn test_from_json_str_strips_bom() { + let v: Value = from_json_str("\u{feff}{\"foo\": 1}").unwrap(); + assert_eq!(v["foo"], 1); + } + + #[test] + fn test_from_json_str_no_bom() { + let v: Value = from_json_str("{\"foo\": 1}").unwrap(); + assert_eq!(v["foo"], 1); + } + + #[test] + fn test_from_json_str_borrowed_type() { + // Deserialize<'a> (not DeserializeOwned) must keep zero-copy + // borrowing working through the wrapper. + #[derive(serde::Deserialize)] + struct Borrowed<'a> { + name: &'a str, + } + let b: Borrowed = from_json_str("\u{feff}{\"name\": \"rtk\"}").unwrap(); + assert_eq!(b.name, "rtk"); + } + + #[test] + fn env_is_some_rejects_unset_and_empty() { + assert!(env_is_some(Some("https://telemetry.example"))); + // An unset CI repository variable still exports the env var, so the + // empty string reaches `option_env!` as `Some("")`. + assert!(!env_is_some(Some(""))); + assert!(!env_is_some(None)); + } + + #[test] + fn test_truncate_short_string() { + assert_eq!(truncate("hello", 10), "hello"); + } + + #[test] + fn test_days_ago_cutoff_normal_value_is_in_the_past() { + let cutoff = days_ago_cutoff(30); + assert!(cutoff < Utc::now()); + } + + #[test] + fn test_days_ago_cutoff_does_not_panic_on_huge_since_days() { + // rtk-ai/rtk#3206 review: `rtk discover --since 100000000` and `rtk hook + // audit --since 100000000` both panicked with "DateTime - TimeDelta + // overflowed". Must clamp instead of crashing. + assert_eq!(days_ago_cutoff(100_000_000), DateTime::::MIN_UTC); + } + + #[test] + fn test_days_ago_cutoff_does_not_panic_on_u64_max() { + assert_eq!(days_ago_cutoff(u64::MAX), DateTime::::MIN_UTC); + } + + #[test] + fn test_truncate_long_string() { + let result = truncate("hello world", 8); + assert_eq!(result, "hello..."); + } + + #[test] + fn test_truncate_exact_length() { + assert_eq!(truncate("hello", 5), "hello"); + } + + #[test] + fn test_truncate_edge_case() { + // max_len < 3 returns just "..." + assert_eq!(truncate("hello", 2), "..."); + // When string length equals max_len, return as is + assert_eq!(truncate("abc", 3), "abc"); + // When string is longer and max_len is exactly 3, return "..." + assert_eq!(truncate("hello world", 3), "..."); + } + + #[test] + fn test_composer_bin_dirs_use_default_when_unconfigured() { + assert_eq!( + composer_bin_dirs_from(None, None), + vec![PathBuf::from("vendor/bin")] + ); + } + + #[test] + fn test_composer_bin_dirs_prefer_env_override() { + let composer_json = r#"{"config":{"bin-dir":"tools/bin"}}"#; + assert_eq!( + composer_bin_dirs_from(Some("custom/bin"), Some(composer_json)), + vec![PathBuf::from("custom/bin"), PathBuf::from("vendor/bin")] + ); + } + + #[test] + fn test_composer_bin_dirs_read_composer_config() { + let composer_json = r#"{"config":{"bin-dir":"tools/bin"}}"#; + assert_eq!( + composer_bin_dirs_from(None, Some(composer_json)), + vec![PathBuf::from("tools/bin"), PathBuf::from("vendor/bin")] + ); + } + + #[test] + fn test_strip_ansi_simple() { + let input = "\x1b[31mError\x1b[0m"; + assert_eq!(strip_ansi(input), "Error"); + } + + #[test] + fn test_strip_ansi_multiple() { + let input = "\x1b[1m\x1b[32mSuccess\x1b[0m\x1b[0m"; + assert_eq!(strip_ansi(input), "Success"); + } + + #[test] + fn test_strip_ansi_no_codes() { + assert_eq!(strip_ansi("plain text"), "plain text"); + } + + #[test] + fn test_strip_ansi_complex() { + let input = "\x1b[32mGreen\x1b[0m normal \x1b[31mRed\x1b[0m"; + assert_eq!(strip_ansi(input), "Green normal Red"); + } + + #[test] + fn test_format_tokens_millions() { + assert_eq!(format_tokens(1_234_567), "1.2M"); + assert_eq!(format_tokens(12_345_678), "12.3M"); + } + + #[test] + fn test_format_tokens_thousands() { + assert_eq!(format_tokens(59_234), "59.2K"); + assert_eq!(format_tokens(1_000), "1.0K"); + } + + #[test] + fn test_format_tokens_small() { + assert_eq!(format_tokens(694), "694"); + assert_eq!(format_tokens(0), "0"); + } + + #[test] + fn test_format_usd_large() { + assert_eq!(format_usd(1234.567), "$1234.57"); + assert_eq!(format_usd(1000.0), "$1000.00"); + } + + #[test] + fn test_format_usd_medium() { + assert_eq!(format_usd(12.345), "$12.35"); + assert_eq!(format_usd(0.99), "$0.99"); + } + + #[test] + fn test_format_usd_small() { + assert_eq!(format_usd(0.0096), "$0.0096"); + assert_eq!(format_usd(0.0001), "$0.0001"); + } + + #[test] + fn test_format_usd_edge() { + assert_eq!(format_usd(0.01), "$0.01"); + assert_eq!(format_usd(0.009), "$0.0090"); + } + + #[test] + fn test_ok_confirmation_with_detail() { + assert_eq!(ok_confirmation("merged", "#42"), "ok merged #42"); + assert_eq!( + ok_confirmation("created", "PR #5 https://github.com/foo/bar/pull/5"), + "ok created PR #5 https://github.com/foo/bar/pull/5" + ); + } + + #[test] + fn test_ok_confirmation_no_detail() { + assert_eq!(ok_confirmation("commented", ""), "ok commented"); + } + + #[test] + fn test_format_cpt_normal() { + assert_eq!(format_cpt(0.000003), "$3.00/MTok"); + assert_eq!(format_cpt(0.0000038), "$3.80/MTok"); + assert_eq!(format_cpt(0.00000386), "$3.86/MTok"); + } + + #[test] + fn test_format_cpt_edge_cases() { + assert_eq!(format_cpt(0.0), "$0.00/MTok"); // zero + assert_eq!(format_cpt(-0.000001), "$0.00/MTok"); // negative + assert_eq!(format_cpt(f64::INFINITY), "$0.00/MTok"); // infinite + assert_eq!(format_cpt(f64::NAN), "$0.00/MTok"); // NaN + } + + #[test] + fn test_detect_package_manager_default() { + // In the test environment (rtk repo), there's no JS lockfile + // so it should default to "npm" + let pm = detect_package_manager(); + assert!(["pnpm", "yarn", "bun", "npm"].contains(&pm)); + } + + #[test] + fn test_truncate_multibyte_thai() { + // Thai characters are 3 bytes each + let thai = "สวัสดีครับ"; + let result = truncate(thai, 5); + // Should not panic, should produce valid UTF-8 + assert!(result.len() <= thai.len()); + assert!(result.ends_with("...")); + } + + #[test] + fn test_truncate_multibyte_emoji() { + let emoji = "🎉🎊🎈🎁🎂🎄🎃🎆🎇✨"; + let result = truncate(emoji, 5); + assert!(result.ends_with("...")); + } + + #[test] + fn test_truncate_multibyte_cjk() { + let cjk = "你好世界测试字符串"; + let result = truncate(cjk, 6); + assert!(result.ends_with("...")); + } + + #[test] + fn test_truncate_multibyte_cyrillic_issue_2787() { + // Regression: `rtk gain --history` panicked slicing this at byte 22, + // which falls inside 'н' (Cyrillic chars are 2 bytes each) + let cmd = "rtk ls -la Ародинамический расчёт Новый"; + let result = truncate(cmd, 25); + assert_eq!(result.chars().count(), 25); + assert!(result.ends_with("...")); + assert_eq!(result, "rtk ls -la Ародинамиче..."); + } + + // ===== resolve_binary tests (issue #212) ===== + + #[test] + fn test_resolve_binary_finds_known_command() { + // "cargo" must be on PATH in any Rust dev environment + let result = resolve_binary("cargo"); + assert!( + result.is_ok(), + "resolve_binary('cargo') should succeed, got: {:?}", + result.err() + ); + } + + #[test] + fn test_resolve_binary_returns_absolute_path() { + let path = resolve_binary("cargo").expect("cargo should be resolvable"); + assert!( + path.is_absolute(), + "resolve_binary should return absolute path, got: {:?}", + path + ); + } + + #[test] + fn test_resolve_binary_fails_for_unknown() { + let result = resolve_binary("nonexistent_binary_xyz_99999"); + assert!( + result.is_err(), + "resolve_binary should fail for nonexistent binary" + ); + } + + #[test] + fn test_resolve_binary_path_contains_binary_name() { + let path = resolve_binary("cargo").expect("cargo should be resolvable"); + let filename = path + .file_name() + .expect("should have filename") + .to_string_lossy(); + // On Windows this could be "cargo.exe", on Unix just "cargo" + assert!( + filename.starts_with("cargo"), + "resolved path filename should start with 'cargo', got: {}", + filename + ); + } + + // ===== resolved_command tests (issue #212) ===== + + #[test] + fn test_resolved_command_executes_known_command() { + let output = resolved_command("cargo") + .arg("--version") + .output() + .expect("resolved_command('cargo') should execute"); + assert!( + output.status.success(), + "cargo --version should succeed via resolved_command" + ); + } + + // ===== tool_exists tests (issue #212) ===== + + #[test] + fn test_tool_exists_finds_cargo() { + assert!( + tool_exists("cargo"), + "tool_exists('cargo') should return true" + ); + } + + #[test] + fn test_tool_exists_rejects_unknown() { + assert!( + !tool_exists("nonexistent_binary_xyz_99999"), + "tool_exists should return false for nonexistent binary" + ); + } + + #[test] + fn test_tool_exists_finds_git() { + assert!(tool_exists("git"), "tool_exists('git') should return true"); + } + + // ===== Child-process argument quoting (issue #3727) ===== + + #[test] + fn test_quote_arg_wraps_a_quote_with_no_space_around_it() { + assert_eq!(quote_arg_for_child(r#"a"b"#), r#""a\"b""#); + } + + #[test] + fn test_quote_arg_wraps_the_reported_json_key_pattern() { + // `rtk grep -c '"type"' file`, the shape reported in #3727. + assert_eq!(quote_arg_for_child(r#""type""#), r#""\"type\"""#); + } + + #[test] + fn test_quote_arg_wraps_repeated_quotes() { + assert_eq!(quote_arg_for_child(r#"a""b"#), r#""a\"\"b""#); + } + + #[test] + fn test_quote_arg_doubles_backslash_runs_before_a_quote() { + assert_eq!(quote_arg_for_child(r#"a\"b"#), r#""a\\\"b""#); + assert_eq!(quote_arg_for_child(r#"a\\"b"#), r#""a\\\\\"b""#); + } + + #[test] + fn test_quote_arg_doubles_a_trailing_backslash_run() { + assert_eq!(quote_arg_for_child(r#"a"b\"#), r#""a\"b\\""#); + } + + #[test] + fn test_quote_arg_keeps_backslashes_that_precede_ordinary_text() { + assert_eq!(quote_arg_for_child(r#"C:\a\b "x""#), r#""C:\a\b \"x\"""#); + } + + #[test] + fn test_quote_arg_keeps_a_space_inside_the_wrapping() { + assert_eq!(quote_arg_for_child(r#"a b "c""#), r#""a b \"c\"""#); + } + + #[cfg(target_os = "windows")] + #[test] + fn test_child_arg_re_encodes_only_arguments_holding_a_quote() { + let mut cmd = Command::new("grep"); + cmd.child_args(["-c", r#""type""#, "q.jsonl"]); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!(args, ["-c", r#""\"type\"""#, "q.jsonl"]); + } + + #[cfg(target_os = "windows")] + #[test] + fn test_child_arg_leaves_batch_shims_on_stds_encoding() { + // cmd.exe parses by its own rules, so `raw_arg` must not be used there. + let mut cmd = Command::new("gradlew.bat"); + cmd.child_arg(r#""type""#); + let args: Vec<_> = cmd.get_args().collect(); + assert_eq!(args, [r#""type""#]); + } + + // ===== Windows-specific PATHEXT resolution tests (issue #212) ===== + + #[cfg(target_os = "windows")] + mod windows_tests { + use super::super::*; + use std::fs; + + /// Create a temporary .cmd wrapper to simulate Node.js tool installation + fn create_temp_cmd_wrapper(dir: &std::path::Path, name: &str) -> std::path::PathBuf { + let cmd_path = dir.join(format!("{}.cmd", name)); + fs::write(&cmd_path, "@echo off\r\necho fake-tool-output\r\n") + .expect("failed to create .cmd wrapper"); + cmd_path + } + + /// Build a PATH string that includes the temp dir + fn path_with_dir(dir: &std::path::Path) -> std::ffi::OsString { + let original = std::env::var_os("PATH").unwrap_or_default(); + let mut new_path = std::ffi::OsString::from(dir.as_os_str()); + new_path.push(";"); + new_path.push(&original); + new_path + } + + #[test] + fn test_resolve_binary_finds_cmd_wrapper() { + let temp_dir = tempfile::tempdir().expect("failed to create temp dir"); + create_temp_cmd_wrapper(temp_dir.path(), "fake-tool-test"); + + // Use which::which_in to avoid mutating global PATH (thread-safe) + let search_path = path_with_dir(temp_dir.path()); + let result = which::which_in( + "fake-tool-test", + Some(search_path), + std::env::current_dir().unwrap(), + ); + + assert!( + result.is_ok(), + "which_in should find .cmd wrapper on Windows, got: {:?}", + result.err() + ); + + let path = result.unwrap(); + let ext = path + .extension() + .unwrap_or_default() + .to_string_lossy() + .to_lowercase(); + assert!( + ext == "cmd" || ext == "bat", + "resolved path should have .cmd/.bat extension, got: {:?}", + path + ); + } + + #[test] + fn test_resolve_binary_finds_bat_wrapper() { + let temp_dir = tempfile::tempdir().expect("failed to create temp dir"); + let bat_path = temp_dir.path().join("fake-bat-tool.bat"); + fs::write(&bat_path, "@echo off\r\necho bat-output\r\n") + .expect("failed to create .bat wrapper"); + + let search_path = path_with_dir(temp_dir.path()); + let result = which::which_in( + "fake-bat-tool", + Some(search_path), + std::env::current_dir().unwrap(), + ); + + assert!( + result.is_ok(), + "which_in should find .bat wrapper on Windows, got: {:?}", + result.err() + ); + } + + #[test] + fn test_resolved_command_executes_cmd_wrapper() { + let temp_dir = tempfile::tempdir().expect("failed to create temp dir"); + create_temp_cmd_wrapper(temp_dir.path(), "fake-exec-test"); + + // Resolve the full path, then execute it directly (no PATH mutation) + let search_path = path_with_dir(temp_dir.path()); + let resolved = which::which_in( + "fake-exec-test", + Some(search_path), + std::env::current_dir().unwrap(), + ) + .expect("should resolve fake-exec-test"); + + let output = Command::new(&resolved).output(); + + assert!( + output.is_ok(), + "Command with resolved path should execute .cmd wrapper on Windows" + ); + let output = output.unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("fake-tool-output"), + "should get output from .cmd wrapper, got: {}", + stdout + ); + } + + #[test] + fn test_resolved_command_fallback_on_unknown_binary() { + // When resolve_binary fails, resolved_command should fall back to + // Command::new(name) instead of panicking. On Windows this also + // prints a warning to stderr. + let mut cmd = resolved_command("nonexistent_binary_xyz_99999"); + // The Command should be created (not panic). Attempting to run it + // will fail, but that's expected — we just verify the fallback path + // produces a usable Command. + let result = cmd.output(); + assert!( + result.is_err() || !result.unwrap().status.success(), + "nonexistent binary should fail to execute, but resolved_command must not panic" + ); + } + + #[test] + fn test_tool_exists_finds_cmd_wrapper() { + let temp_dir = tempfile::tempdir().expect("failed to create temp dir"); + create_temp_cmd_wrapper(temp_dir.path(), "fake-exists-test"); + + let search_path = path_with_dir(temp_dir.path()); + let result = which::which_in( + "fake-exists-test", + Some(search_path), + std::env::current_dir().unwrap(), + ); + + assert!( + result.is_ok(), + "which_in should find .cmd wrapper on Windows" + ); + } + } + + // ===== AWS helper function tests ===== + + #[test] + fn test_shorten_arn_ecs_service() { + assert_eq!( + shorten_arn("arn:aws:ecs:us-east-1:123:service/cluster/api-service"), + "api-service" + ); + } + + #[test] + fn test_shorten_arn_iam_user() { + assert_eq!(shorten_arn("arn:aws:iam::123456789012:user/alice"), "alice"); + } + + #[test] + fn test_shorten_arn_lambda() { + assert_eq!( + shorten_arn("arn:aws:lambda:us-west-2:123:function:my-function"), + "my-function" + ); + } + + #[test] + fn test_shorten_arn_fallback() { + // Non-ARN string - return as-is + assert_eq!(shorten_arn("simple-name"), "simple-name"); + } + + #[test] + fn test_human_bytes_bytes() { + assert_eq!(human_bytes(0), "0 B"); + assert_eq!(human_bytes(512), "512 B"); + assert_eq!(human_bytes(1023), "1023 B"); + } + + #[test] + fn test_human_bytes_kb() { + assert_eq!(human_bytes(1024), "1.0 KB"); + assert_eq!(human_bytes(2048), "2.0 KB"); + assert_eq!(human_bytes(1536), "1.5 KB"); + } + + #[test] + fn test_human_bytes_mb() { + assert_eq!(human_bytes(1_048_576), "1.0 MB"); + assert_eq!(human_bytes(5_242_880), "5.0 MB"); + } + + #[test] + fn test_human_bytes_gb() { + assert_eq!(human_bytes(1_073_741_824), "1.0 GB"); + assert_eq!(human_bytes(2_147_483_648), "2.0 GB"); + } + + #[test] + fn test_human_bytes_tb() { + assert_eq!(human_bytes(1_099_511_627_776), "1.0 TB"); + } + + #[test] + fn test_count_tokens_basic() { + assert_eq!(count_tokens("hello world"), 2); + assert_eq!(count_tokens("one two three four"), 4); + } + + #[test] + fn test_count_tokens_empty() { + assert_eq!(count_tokens(""), 0); + assert_eq!(count_tokens(" "), 0); + } + + #[test] + fn test_count_tokens_multiple_spaces() { + assert_eq!(count_tokens("hello world"), 2); + assert_eq!(count_tokens(" hello world "), 2); + } + + #[cfg(unix)] + fn mode_of(path: &std::path::Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + fs::metadata(path).unwrap().permissions().mode() & 0o777 + } + + #[test] + #[cfg(unix)] + fn test_create_private_dir_is_owner_only() { + let tmp = tempfile::tempdir().unwrap(); + let nested = tmp.path().join("rtk").join("tee"); + create_private_dir(&nested).unwrap(); + assert_eq!(mode_of(&nested), 0o700); + } + + #[test] + #[cfg(unix)] + fn test_create_private_dir_tightens_existing_dir() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("legacy"); + fs::create_dir_all(&dir).unwrap(); + set_owner_only(&dir, 0o755); + + create_private_dir(&dir).unwrap(); + assert_eq!(mode_of(&dir), 0o700); + } + + #[test] + fn test_create_private_dir_is_idempotent() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("a").join("b"); + create_private_dir(&dir).unwrap(); + create_private_dir(&dir).expect("second call must succeed"); + assert!(dir.is_dir()); + } + + #[test] + #[cfg(unix)] + fn test_restrict_file_is_owner_only() { + let tmp = tempfile::tempdir().unwrap(); + let file = tmp.path().join("history.db"); + fs::write(&file, b"x").unwrap(); + set_owner_only(&file, 0o644); + + restrict_file(&file); + assert_eq!(mode_of(&file), 0o600); + } + + #[test] + fn test_restrict_file_ignores_missing_path() { + let tmp = tempfile::tempdir().unwrap(); + restrict_file(&tmp.path().join("absent.db-wal")); + } + + #[test] + fn test_decode_process_output_valid_utf8() { + assert_eq!(decode_process_output(b"hello world"), "hello world"); + } + + #[test] + fn test_decode_process_output_chinese_utf8() { + let input = "测试中文".as_bytes(); + assert_eq!(decode_process_output(input), "测试中文"); + } + + #[test] + fn test_decode_process_output_empty() { + assert_eq!(decode_process_output(b""), ""); + } + + #[test] + fn test_decode_process_output_invalid_utf8_no_panic() { + let bytes: &[u8] = &[0xFF, 0xFE, 0x41, 0x42]; + let result = decode_process_output(bytes); + assert!(!result.is_empty()); + } + + // `decode_mixed` takes the code page as a parameter so these run on every + // platform, not only Windows — the CI runners that would exercise the + // Windows-gated versions do not exist. + + /// The bug this fix targets: GBK output from a Chinese-locale console. + #[test] + fn test_decode_mixed_gbk_run() { + assert_eq!(decode_mixed(&[0xB2, 0xE2, 0xCA, 0xD4], Some(936)), "测试"); + } + + /// GB18030 (54936) must not be treated as plain GBK: it has to keep its + /// own table so 4-byte sequences decode. + #[test] + fn test_decode_mixed_gb18030_is_not_gbk() { + // 0x81 0x35 0xF4 0x37 is a 4-byte GB18030 sequence. + let decoded = decode_mixed(&[0x81, 0x35, 0xF4, 0x37], Some(54936)); + assert_eq!( + decoded.chars().count(), + 1, + "expected one char: {:?}", + decoded + ); + assert!( + !decoded.contains('\u{FFFD}'), + "got replacement: {:?}", + decoded + ); + assert_eq!( + codepage::to_encoding(54936).map(|e| e.name()), + Some("gb18030") + ); + } + + /// Legacy OEM/DOS pages are cmd.exe's default in many locales and are not + /// WHATWG encodings, so they come from `oem_cp` rather than encoding_rs. + #[test] + fn test_decode_mixed_oem_codepages() { + assert_eq!(decode_mixed(&[0xB0, 0xDB], Some(437)), "░█"); + // 850 and 852 are likewise absent from encoding_rs. + assert!(codepage::to_encoding(437).is_none()); + assert!(!decode_mixed(&[0xE1], Some(850)).contains('\u{FFFD}')); + } + + /// The regression the review flagged: a buffer that is almost entirely + /// valid UTF-8 must not be reinterpreted wholesale because of one bad + /// line. Only the GBK line is re-decoded; the UTF-8 lines keep their bytes. + #[test] + fn test_decode_mixed_only_redecodes_invalid_lines() { + let mut bytes = "héllo wörld\n".as_bytes().to_vec(); + bytes.extend_from_slice(&[0xB2, 0xE2, 0xCA, 0xD4, b'\n']); // GBK 测试 + bytes.extend_from_slice("grüße\n".as_bytes()); + + assert_eq!( + decode_mixed(&bytes, Some(936)), + "héllo wörld\n测试\ngrüße\n" + ); + } + + /// A corrupt byte inside an otherwise-UTF-8 line falls back to lossy UTF-8 + /// rather than mojibake, because the code page decode does not come out + /// clean. The surrounding lines are untouched either way. + #[test] + fn test_decode_mixed_corrupt_byte_prefers_lossy_utf8() { + let mut bytes = "first\n".as_bytes().to_vec(); + bytes.extend_from_slice("naïve".as_bytes()); + bytes.push(0xC3); // dangling lead byte: invalid UTF-8 and invalid GBK + bytes.extend_from_slice(b"\nlast\n"); + + let decoded = decode_mixed(&bytes, Some(936)); + assert!(decoded.starts_with("first\n"), "{:?}", decoded); + assert!(decoded.contains("naïve"), "{:?}", decoded); + assert!(decoded.ends_with("\nlast\n"), "{:?}", decoded); + } + + /// Line endings and the final line without a terminator must survive. + #[test] + fn test_decode_mixed_preserves_line_structure() { + let mut bytes = b"a\r\n".to_vec(); + bytes.extend_from_slice(&[0xB2, 0xE2, b'\n']); // GBK 测 + bytes.extend_from_slice(b"tail"); // no trailing newline + + assert_eq!(decode_mixed(&bytes, Some(936)), "a\r\n测\ntail"); + } + + /// No code page (every Unix run) keeps the old lossy behavior. + #[test] + fn test_decode_mixed_without_codepage_is_lossy() { + let mut bytes = b"ok ".to_vec(); + bytes.push(0xFF); + assert_eq!(decode_mixed(&bytes, None), "ok \u{FFFD}"); + } + + /// An unmapped code page must degrade to lossy rather than panic. + #[test] + fn test_decode_mixed_unknown_codepage_is_lossy() { + assert_eq!(decode_mixed(&[0xFF], Some(60000)), "\u{FFFD}"); + } + + /// Truncated trailing UTF-8 must terminate rather than loop forever. + #[test] + fn test_decode_mixed_truncated_utf8_terminates() { + // First two bytes of a three-byte character, cut short. + let decoded = decode_mixed(&[b'a', 0xE6, 0xB5], None); + assert!(decoded.starts_with('a'), "{:?}", decoded); + } + + /// Every byte value must survive both paths without panicking. + #[test] + fn test_decode_mixed_all_byte_values_no_panic() { + let all: Vec = (0u8..=255).collect(); + for cp in [None, Some(936), Some(437), Some(65001), Some(1252)] { + let _ = decode_mixed(&all, cp); + } + } + + /// Without a code page — every Unix run — the result must stay identical to + /// the `String::from_utf8_lossy` this replaced. Splitting on `\n` cannot + /// change it: a newline is valid ASCII, so no ill-formed sequence spans one. + #[test] + fn test_decode_process_output_unchanged_without_codepage() { + let cases: [&[u8]; 6] = [ + b"", + b"plain ascii\n", + "utf8 ünïcödé\nsecond ライン\n".as_bytes(), + &[0xFF, 0xFE, b'\n', b'o', b'k'], + &[b'a', b'\n', 0xE6, 0xB5, b'\n', b'b'], + &[0xB2, 0xE2, 0xCA, 0xD4], + ]; + for bytes in cases { + assert_eq!( + decode_mixed(bytes, None), + String::from_utf8_lossy(bytes), + "lossy mismatch for {:?}", + bytes + ); + } + } + + // rtk-ai/rtk#3206 review: set_owner_only now skips the chmod syscall once + // permissions already match, since Tracker::new() (which calls this via + // create_private_dir/open_private/restrict_db_files) is on the PreToolUse + // hook's hot path for every Bash tool call. These don't observe the skipped + // syscall directly, but confirm the already-correct case stays correct + // (idempotent) across repeated calls, on both a directory and a file. + + #[test] + #[cfg(unix)] + fn test_set_owner_only_idempotent_when_already_correct_dir() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("already-private"); + create_private_dir(&dir).unwrap(); + assert_eq!(mode_of(&dir), 0o700); + + // Second call hits the already-correct fast path — must stay 0o700. + create_private_dir(&dir).unwrap(); + assert_eq!(mode_of(&dir), 0o700); + } + + #[test] + #[cfg(unix)] + fn test_restrict_file_idempotent_when_already_correct() { + let tmp = tempfile::tempdir().unwrap(); + let file = tmp.path().join("history.db"); + fs::write(&file, b"x").unwrap(); + restrict_file(&file); + assert_eq!(mode_of(&file), 0o600); + + // Second call hits the already-correct fast path — must stay 0o600. + restrict_file(&file); + assert_eq!(mode_of(&file), 0o600); + } + + // Regression: the fast-path comparison must mask with 0o7777 (including + // setuid/setgid/sticky), not 0o777 — a 0o777 mask would make a file with e.g. + // setuid set (0o4600) read as "already 0o600" and permanently skip the chmod + // that clears the dangerous bit. Tested against the pure mask logic directly + // (not round-tripped through a real chmod) since an unprivileged chmod setting + // setuid isn't reliably honored across every environment — some sandboxed/ + // containerized filesystems silently drop it, which would make a + // filesystem-based version of this test flaky rather than exercise the logic. + #[test] + #[cfg(unix)] + fn test_mode_already_correct_rejects_setuid_bit() { + assert!( + !mode_already_correct(0o4600, 0o600), + "setuid set (0o4600) must not read as already-correct against target 0o600" + ); + assert!( + !mode_already_correct(0o2600, 0o600), + "setgid set (0o2600) must not read as already-correct against target 0o600" + ); + assert!( + !mode_already_correct(0o1600, 0o600), + "sticky bit set (0o1600) must not read as already-correct against target 0o600" + ); + } + + #[test] + #[cfg(unix)] + fn test_mode_already_correct_accepts_exact_match() { + assert!(mode_already_correct(0o600, 0o600)); + assert!(mode_already_correct(0o700, 0o700)); + assert!(!mode_already_correct(0o644, 0o600)); + } + + #[test] + #[cfg(unix)] + fn test_mode_already_correct_ignores_file_type_bits() { + // `st_mode` carries file-type bits (e.g. S_IFREG = 0o100000) above the + // 0o7777 permission range this function masks to — those must not affect + // the comparison either way. + assert!(mode_already_correct(0o100600, 0o600)); + } + + #[test] + fn test_detect_package_manager_recognizes_bun() { + // Asks about an explicit directory: chdir is process-global and would + // race the other tests in this binary that assert on the real cwd. + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(dir.path().join("bun.lockb"), "").expect("write lockfile"); + assert_eq!(detect_package_manager_in(dir.path()), "bun"); + } + + #[test] + fn test_missing_tool_policy_controls_npx_no_install() { + // Semantics are per caller: a tool the user named may be fetched, one + // rtk chose may not. Changing either would change what runs, not what + // is printed, which is not rtk's job. + let args: Vec = + tool_exec(Some("npm"), "definitely-not-installed", MissingTool::Fetch) + .get_args() + .map(|a| a.to_string_lossy().into_owned()) + .collect(); + assert!(!args.contains(&"--no-install".to_string()), "{args:?}"); + + let args: Vec = + tool_exec(Some("npm"), "definitely-not-installed", MissingTool::Fail) + .get_args() + .map(|a| a.to_string_lossy().into_owned()) + .collect(); + assert!(args.contains(&"--no-install".to_string()), "{args:?}"); + } + + #[test] + fn test_exec_runner_prefers_the_named_runner_and_never_detects_bunx() { + // A named runner is used as given, whatever the fetch policy. + for missing in [MissingTool::Fetch, MissingTool::Fail] { + assert_eq!(exec_runner(Some("bunx"), missing), "bunx"); + assert_eq!(exec_runner(Some("pnpm"), missing), "pnpm"); + } + + // Detection never resolves through bunx: it always fetches a missing + // tool, so MissingTool::Fail could not be honoured. + assert_ne!(exec_runner(None, MissingTool::Fail), "bun"); + assert_ne!(exec_runner(None, MissingTool::Fail), "bunx"); + + // Nothing named and rtk may fetch: npx is the only runner that can. + assert_eq!(exec_runner(None, MissingTool::Fetch), "npx"); + } +} diff --git a/crates/sparkplane-rtk/upstream/src/filters/README.md b/crates/sparkplane-rtk/upstream/src/filters/README.md new file mode 100644 index 0000000..5b9bbb9 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/README.md @@ -0,0 +1,142 @@ +# Built-in Filters + +> See also [docs/contributing/TECHNICAL.md](../../docs/contributing/TECHNICAL.md) for the full architecture overview + +Each `.toml` file in this directory defines one filter and its inline tests. +Files are concatenated alphabetically by `build.rs` into a single TOML blob embedded in the binary. + +## When to Use a TOML Filter + +TOML filters strip noise lines — they don't reformat output. The filtered result must still look like real command output (see [Design Philosophy](../../CONTRIBUTING.md#design-philosophy)). For the full TOML-vs-Rust decision criteria, see [CONTRIBUTING.md](../../CONTRIBUTING.md#toml-vs-rust-which-one). + +TOML works well for commands with **predictable, line-by-line text output** where regex filtering achieves 60%+ savings: +- Install/update logs (brew, composer, poetry) — strip `Using ...` / `Already installed` lines +- System monitoring (df, ps, systemctl) — keep essential rows, drop headers/decorations +- Simple linters (shellcheck, yamllint, hadolint) — strip context, keep findings +- Infra tools (terraform plan, helm, rsync) — strip progress, keep summary + +For the full contribution checklist (including `discover/rules.rs` registration), see [src/cmds/README.md — Adding a New Command Filter](../cmds/README.md#adding-a-new-command-filter). + +## Adding a filter + +1. Copy any existing `.toml` file and rename it (e.g. `my-tool.toml`) +2. Update the three required fields: `description`, `match_command`, and at least one action field +3. Add `[[tests.my-tool]]` entries to verify the filter behaves correctly +4. Run `cargo test` — the build step validates TOML syntax and runs inline tests + +## File format + +```toml +[filters.my-tool] +description = "Short description of what this filter does" +match_command = "^my-tool\\b" # regex matched against the full command string +strip_ansi = true # optional: strip ANSI escape codes first +strip_lines_matching = [ # optional: drop lines matching any of these regexes + "^\\s*$", + "^noise pattern", +] +max_lines = 40 # optional: keep only the first N lines after filtering +on_empty = "my-tool: ok" # optional: message to emit when output is empty after filtering + +[[tests.my-tool]] +name = "descriptive test name" +input = "raw command output here" +expected = "expected filtered output" +``` + +## Available filter fields + +| Field | Type | Description | +|-------|------|-------------| +| `description` | string | Human-readable description | +| `match_command` | regex | Matches the command string (e.g. `"^docker\\s+inspect"`) | +| `strip_ansi` | bool | Strip ANSI escape codes before processing | +| `filter_stderr` | bool | Capture and merge stderr into stdout before filtering (use for tools like liquibase that emit banners to stderr) | +| `strip_lines_matching` | regex[] | Drop lines matching any regex | +| `keep_lines_matching` | regex[] | Keep only lines matching at least one regex | +| `replace` | array | Regex substitutions (`{ pattern, replacement }`) | +| `match_output` | array | Short-circuit rules (`{ pattern, message }`) | +| `truncate_lines_at` | int | Truncate lines longer than N characters | +| `max_lines` | int | Keep only the first N lines | +| `tail_lines` | int | Keep only the last N lines (applied after other filters) | +| `on_empty` | string | Fallback message when filtered output is empty | + +## Naming convention + +Use the command name as the filename: `terraform-plan.toml`, `docker-inspect.toml`, `mix-compile.toml`. +For commands with subcommands, prefer `-.toml` over grouping multiple filters in one file. + +## Build and runtime pipeline + +How a `.toml` file goes from contributor → binary → filtered output. + +```mermaid +flowchart TD + A[["src/filters/my-tool.toml\n(new file)"]] --> B + + subgraph BUILD ["cargo build"] + B["build.rs\n1. ls src/filters/*.toml\n2. sort alphabetically\n3. concat → BUILTIN_TOML"] --> C + C{"TOML valid?\nDuplicate names?"} -->|"fail"| D[["Build fails\nerror points to bad file"]] + C -->|"ok"| E[["OUT_DIR/builtin_filters.toml\n(generated)"]] + E --> F["rustc embeds via include_str!"] + F --> G[["rtk binary\nBUILTIN_TOML embedded"]] + end + + subgraph TESTS ["cargo test"] + H["test_builtin_filter_count\nassert_eq!(filters.len(), N)"] -->|"wrong count"| I[["FAIL"]] + J["test_builtin_all_filters_present\nassert!(names.contains('my-tool'))"] -->|"name missing"| K[["FAIL"]] + L["test_builtin_all_filters_have_inline_tests\nassert!(tested.contains(name))"] -->|"no tests"| M[["FAIL"]] + end + + subgraph RUNTIME ["rtk my-tool args"] + R["TomlFilterRegistry::load()\n1. .rtk/filters.toml\n2. ~/.config/rtk/filters.toml\n3. BUILTIN_TOML\n4. passthrough"] --> S + S{"match_command\nmatches?"} -->|"no match"| T[["exec raw (passthrough)"]] + S -->|"match"| U["exec command\ncapture stdout"] + U --> V["8-stage pipeline\nstrip_ansi → replace → match_output\n→ strip/keep_lines → truncate\n→ tail_lines → max_lines → on_empty"] + V --> W[["print filtered output + exit code"]] + end + + G --> H & J & L & R +``` + +## Filter lookup priority + +```mermaid +flowchart LR + CMD["rtk my-tool args"] --> P1 + P1{"1. .rtk/filters.toml\n(project-local)"} + P1 -->|"match"| WIN["apply filter"] + P1 -->|"no match"| P2 + P2{"2. ~/.config/rtk/filters.toml\n(user-global)"} + P2 -->|"match"| WIN + P2 -->|"no match"| P3 + P3{"3. BUILTIN_TOML\n(binary)"} + P3 -->|"match"| WIN + P3 -->|"no match"| P4[["exec raw (passthrough)"]] +``` + +First match wins. A project filter with the same name as a built-in shadows the built-in and triggers a warning: + +``` +[rtk] warning: filter 'make' is shadowing a built-in filter +``` + +## Custom filters and trust + +You can add your own filters in two places (both use the format above): + +- **Project-local** — `.rtk/filters.toml` (committed with a repo, applies in that project) +- **User-global** — `~/.config/rtk/filters.toml` (applies in every project) + +Because a filter can rewrite or hide the command output an agent sees, **custom filter files are not applied until you trust them**. An untrusted (or edited) filter file is skipped **silently** on the command path — RTK never prints a warning around a rewritten command. You discover and enable untrusted filters through commands you run deliberately: + +```bash +rtk trust # lists each detected filter (labelled project/global) + a risk summary, then asks to confirm ([y/N], or --yes) +rtk untrust # revokes trust +``` + +- Trust is recorded as a SHA-256 of the file's contents, so **editing a trusted file requires re-running `rtk trust`** — a content change invalidates trust. +- `rtk init` detects existing custom filters and lets you enable them — an interactive `[y/N]`, or `--trust-filters` / `--no-trust-filters` for scripts. It stays silent for an empty template (comments only) and, when run non-interactively, leaves filters disabled. +- Built-in filters (this directory) are compiled into the binary and always trusted; only the on-disk project and user-global files are gated. + +> **Honest limitation:** this is consent + tamper-evidence, not a sandbox. An attacker who can write your filter file can usually also write the trust store (`~/.local/share/rtk/trusted_filters.json`) and bypass the gate. It defends the common case — a filter dropped in by a script, a dotfile sync, or an untrusted repo — not a same-user attacker who specifically targets RTK. diff --git a/crates/sparkplane-rtk/upstream/src/filters/ansible-playbook.toml b/crates/sparkplane-rtk/upstream/src/filters/ansible-playbook.toml new file mode 100644 index 0000000..6294379 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ansible-playbook.toml @@ -0,0 +1,34 @@ +[filters.ansible-playbook] +description = "Compact ansible-playbook output" +match_command = "^ansible-playbook\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ok: \\[", + "^skipping: \\[", +] +max_lines = 60 + +[[tests.ansible-playbook]] +name = "strips ok and skipping lines, keeps changed and failures" +input = """ +PLAY [all] ********************************************************************* + +TASK [Gathering Facts] ********************************************************* +ok: [web01] +ok: [web02] + +TASK [Install nginx] *********************************************************** +changed: [web01] +skipping: [web02] + +PLAY RECAP ********************************************************************* +web01 : ok=2 changed=1 unreachable=0 failed=0 +web02 : ok=1 changed=0 unreachable=0 failed=0 +""" +expected = "PLAY [all] *********************************************************************\nTASK [Gathering Facts] *********************************************************\nTASK [Install nginx] ***********************************************************\nchanged: [web01]\nPLAY RECAP *********************************************************************\nweb01 : ok=2 changed=1 unreachable=0 failed=0\nweb02 : ok=1 changed=0 unreachable=0 failed=0" + +[[tests.ansible-playbook]] +name = "failed task preserved" +input = "TASK [Start service] ***\nfailed: [web01] => {\"msg\": \"Service not found\"}\nPLAY RECAP ***\nweb01 : ok=1 failed=1" +expected = "TASK [Start service] ***\nfailed: [web01] => {\"msg\": \"Service not found\"}\nPLAY RECAP ***\nweb01 : ok=1 failed=1" diff --git a/crates/sparkplane-rtk/upstream/src/filters/basedpyright.toml b/crates/sparkplane-rtk/upstream/src/filters/basedpyright.toml new file mode 100644 index 0000000..0826cdb --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/basedpyright.toml @@ -0,0 +1,47 @@ +[filters.basedpyright] +description = "Compact basedpyright type checker output — strip blank lines, keep errors" +match_command = "^basedpyright\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Searching for source files", + "^Found \\d+ source file", + "^Pyright \\d+\\.\\d+", + "^basedpyright \\d+\\.\\d+", +] +max_lines = 50 +on_empty = "basedpyright: ok" + +[[tests.basedpyright]] +name = "strips noise, keeps errors and summary" +input = """ +basedpyright 1.22.0 +Searching for source files +Found 42 source files + +/home/user/app/main.py + /home/user/app/main.py:10:5 - error: "foo" is not defined (reportUndefinedVariable) + /home/user/app/main.py:25:1 - error: Type "str" is not assignable to type "int" (reportAssignmentType) + +/home/user/app/utils.py + /home/user/app/utils.py:8:9 - warning: Variable "x" is not accessed (reportUnusedVariable) + +3 errors, 1 warning, 0 informations +""" +expected = "/home/user/app/main.py\n /home/user/app/main.py:10:5 - error: \"foo\" is not defined (reportUndefinedVariable)\n /home/user/app/main.py:25:1 - error: Type \"str\" is not assignable to type \"int\" (reportAssignmentType)\n/home/user/app/utils.py\n /home/user/app/utils.py:8:9 - warning: Variable \"x\" is not accessed (reportUnusedVariable)\n3 errors, 1 warning, 0 informations" + +[[tests.basedpyright]] +name = "clean output" +input = """ +basedpyright 1.22.0 +Searching for source files +Found 10 source files + +0 errors, 0 warnings, 0 informations +""" +expected = "0 errors, 0 warnings, 0 informations" + +[[tests.basedpyright]] +name = "empty input returns on_empty message" +input = "" +expected = "basedpyright: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/biome.toml b/crates/sparkplane-rtk/upstream/src/filters/biome.toml new file mode 100644 index 0000000..800f778 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/biome.toml @@ -0,0 +1,45 @@ +[filters.biome] +description = "Compact Biome lint/format output — strip blank lines, keep diagnostics" +match_command = "^biome\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Checked \\d+ file", + "^Fixed \\d+ file", + "^The following command", + "^Run it with", +] +max_lines = 50 +on_empty = "biome: ok" + +[[tests.biome]] +name = "lint strips noise, keeps diagnostics" +input = """ +Checked 42 files in 0.5s + +src/app.tsx:5:3 lint/suspicious/noExplicitAny ━━━━━━━━━━━━━━━━━━━━ + × Unexpected any. Specify a different type. + 3 │ interface Props { + 4 │ data: any; + 5 │ ^^^ + +src/utils.ts:12:1 lint/complexity/noForEach ━━━━━━━━━━━━━━━━━━━━ + × Prefer for...of instead of forEach. + 12 │ items.forEach(item => process(item)); + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Found 2 errors. +""" +expected = "src/app.tsx:5:3 lint/suspicious/noExplicitAny ━━━━━━━━━━━━━━━━━━━━\n × Unexpected any. Specify a different type.\n 3 │ interface Props {\n 4 │ data: any;\n 5 │ ^^^\nsrc/utils.ts:12:1 lint/complexity/noForEach ━━━━━━━━━━━━━━━━━━━━\n × Prefer for...of instead of forEach.\n 12 │ items.forEach(item => process(item));\n │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nFound 2 errors." + +[[tests.biome]] +name = "clean check" +input = """ +Checked 42 files in 0.3s +""" +expected = "biome: ok" + +[[tests.biome]] +name = "empty input returns on_empty message" +input = "" +expected = "biome: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/brew-install.toml b/crates/sparkplane-rtk/upstream/src/filters/brew-install.toml new file mode 100644 index 0000000..3c8893b --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/brew-install.toml @@ -0,0 +1,37 @@ +[filters.brew-install] +description = "Compact brew install/upgrade output — strip downloads, short-circuit when already installed" +match_command = "^brew\\s+(install|upgrade)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^==> Downloading", + "^==> Pouring", + "^Already downloaded:", + "^###", + "^==> Fetching", +] +match_output = [ + { pattern = "already installed", message = "ok (already installed)" }, +] +max_lines = 20 + +[[tests.brew-install]] +name = "already installed short-circuits" +input = """ +Warning: rtk 0.27.1 is already installed and up-to-date. +To reinstall 0.27.1, run: + brew reinstall rtk +""" +expected = "ok (already installed)" + +[[tests.brew-install]] +name = "install strips download lines" +input = """ +==> Fetching jq +==> Downloading https://homebrew.bintray.com/bottles/jq-1.7.1.arm64_sonoma.bottle.tar.gz +######################################################################## 100.0% +==> Pouring jq-1.7.1.arm64_sonoma.bottle.tar.gz +==> Summary +/opt/homebrew/Cellar/jq/1.7.1: 18 files, 1.2MB +""" +expected = "==> Summary\n/opt/homebrew/Cellar/jq/1.7.1: 18 files, 1.2MB" diff --git a/crates/sparkplane-rtk/upstream/src/filters/bundle-install.toml b/crates/sparkplane-rtk/upstream/src/filters/bundle-install.toml new file mode 100644 index 0000000..80e0748 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/bundle-install.toml @@ -0,0 +1,61 @@ +[filters.bundle-install] +description = "Compact bundle install/update — strip 'Using' lines, keep installs and errors" +match_command = "^bundle\\s+(install|update)\\b" +strip_ansi = true +strip_lines_matching = [ + "^Using ", + "^\\s*$", + "^Fetching gem metadata", + "^Resolving dependencies", +] +match_output = [ + { pattern = "Bundle complete!", message = "ok bundle: complete" }, + { pattern = "Bundle updated!", message = "ok bundle: updated" }, +] +max_lines = 30 + +[[tests.bundle-install]] +name = "all cached short-circuits" +input = """ +Using bundler 2.5.6 +Using rake 13.1.0 +Using ast 2.4.2 +Using base64 0.2.0 +Using minitest 5.22.2 +Bundle complete! 85 Gemfile dependencies, 200 gems now installed. +Use `bundle info [gemname]` to see where a bundled gem is installed. +""" +expected = "ok bundle: complete" + +[[tests.bundle-install]] +name = "mixed install keeps Fetching and Installing lines" +input = """ +Fetching gem metadata from https://rubygems.org/......... +Resolving dependencies... +Using rake 13.1.0 +Using ast 2.4.2 +Fetching rspec 3.13.0 +Installing rspec 3.13.0 +Using rubocop 1.62.0 +Fetching simplecov 0.22.0 +Installing simplecov 0.22.0 +Bundle complete! 85 Gemfile dependencies, 202 gems now installed. +""" +expected = "ok bundle: complete" + +[[tests.bundle-install]] +name = "update output" +input = """ +Fetching gem metadata from https://rubygems.org/......... +Resolving dependencies... +Using rake 13.1.0 +Fetching rspec 3.14.0 (was 3.13.0) +Installing rspec 3.14.0 (was 3.13.0) +Bundle updated! +""" +expected = "ok bundle: updated" + +[[tests.bundle-install]] +name = "empty output" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/composer-install.toml b/crates/sparkplane-rtk/upstream/src/filters/composer-install.toml new file mode 100644 index 0000000..84d42de --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/composer-install.toml @@ -0,0 +1,40 @@ +[filters.composer-install] +description = "Compact composer install/update/require output — strip downloads, short-circuit when up-to-date" +match_command = "^composer\\s+(install|update|require)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ - Downloading ", + "^ - Installing ", + "^Loading composer", + "^Updating dependencies", +] +match_output = [ + { pattern = "Nothing to install, update or remove", message = "ok (up to date)" }, +] +max_lines = 30 + +[[tests.composer-install]] +name = "nothing to do short-circuits" +input = """ +Loading composer repositories with package information +Updating dependencies +Lock file operations: 0 installs, 0 updates, 0 removals +Nothing to install, update or remove +Generating autoload files +""" +expected = "ok (up to date)" + +[[tests.composer-install]] +name = "install strips download lines" +input = """ +Loading composer repositories with package information +Updating dependencies + - Downloading symfony/console (v6.4.0) + - Installing symfony/console (v6.4.0): Extracting archive + - Downloading psr/log (3.0.0) + - Installing psr/log (3.0.0): Extracting archive +Writing lock file +Generating autoload files +""" +expected = "Writing lock file\nGenerating autoload files" diff --git a/crates/sparkplane-rtk/upstream/src/filters/df.toml b/crates/sparkplane-rtk/upstream/src/filters/df.toml new file mode 100644 index 0000000..f2edf1e --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/df.toml @@ -0,0 +1,16 @@ +[filters.df] +description = "Compact df output — truncate wide columns, limit rows" +match_command = "^df(\\s|$)" +strip_ansi = true +truncate_lines_at = 80 +max_lines = 20 + +[[tests.df]] +name = "short output passes through unchanged" +input = "Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/sda1 4096000 123456 3972544 4% /" +expected = "Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/sda1 4096000 123456 3972544 4% /" + +[[tests.df]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/dotnet-build.toml b/crates/sparkplane-rtk/upstream/src/filters/dotnet-build.toml new file mode 100644 index 0000000..46d9beb --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/dotnet-build.toml @@ -0,0 +1,64 @@ +[filters.dotnet-build] +description = "Compact dotnet build output — short-circuit on success, strip banners" +match_command = "^dotnet\\s+build\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Microsoft \\(R\\)", + "^Copyright \\(C\\)", + "^ Determining projects", +] +match_output = [ + { pattern = "0 Warning\\(s\\)\\n\\s+0 Error\\(s\\)", message = "ok (build succeeded)" }, +] +max_lines = 40 + +[[tests.dotnet-build]] +name = "successful build short-circuits to ok" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... + All projects are up-to-date for restore. + MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll + +Build succeeded. + 0 Warning(s) + 0 Error(s) + +Time Elapsed 00:00:02.34 +""" +expected = "ok (build succeeded)" + +[[tests.dotnet-build]] +name = "build with warnings not short-circuited" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... + MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll + +Build succeeded. + 3 Warning(s) + 0 Error(s) + +Time Elapsed 00:00:01.87 +""" +expected = " MyApp -> /home/user/MyApp/bin/Debug/net8.0/MyApp.dll\nBuild succeeded.\n 3 Warning(s)\n 0 Error(s)\nTime Elapsed 00:00:01.87" + +[[tests.dotnet-build]] +name = "build errors pass through" +input = """ +Microsoft (R) Build Engine version 17.8.3+195e7f5a3 +Copyright (C) Microsoft Corporation. All rights reserved. + + Determining projects to restore... +src/Program.cs(10,5): error CS1002: ; expected [/home/user/MyApp/MyApp.csproj] + +Build FAILED. + 0 Warning(s) + 1 Error(s) +""" +expected = "src/Program.cs(10,5): error CS1002: ; expected [/home/user/MyApp/MyApp.csproj]\nBuild FAILED.\n 0 Warning(s)\n 1 Error(s)" diff --git a/crates/sparkplane-rtk/upstream/src/filters/du.toml b/crates/sparkplane-rtk/upstream/src/filters/du.toml new file mode 100644 index 0000000..378927d --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/du.toml @@ -0,0 +1,16 @@ +[filters.du] +description = "Compact du output" +match_command = "^du\\b" +strip_lines_matching = ["^\\s*$"] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.du]] +name = "preserves sizes, strips blank lines" +input = "4.0K\t./src\n\n8.0K\t./tests\n16K\t." +expected = "4.0K\t./src\n8.0K\t./tests\n16K\t." + +[[tests.du]] +name = "single line passthrough" +input = "128K\t." +expected = "128K\t." diff --git a/crates/sparkplane-rtk/upstream/src/filters/fail2ban-client.toml b/crates/sparkplane-rtk/upstream/src/filters/fail2ban-client.toml new file mode 100644 index 0000000..f2107cf --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/fail2ban-client.toml @@ -0,0 +1,15 @@ +[filters.fail2ban-client] +description = "Compact fail2ban-client output" +match_command = "^fail2ban-client\\b" +strip_lines_matching = ["^\\s*$"] +max_lines = 30 + +[[tests.fail2ban-client]] +name = "strips blank lines" +input = "Status for the jail: sshd\n|- Filter\n| |- Currently failed: 3\n\n|- Actions\n `- Total banned: 42" +expected = "Status for the jail: sshd\n|- Filter\n| |- Currently failed: 3\n|- Actions\n `- Total banned: 42" + +[[tests.fail2ban-client]] +name = "single line passthrough" +input = "Shutdown successful" +expected = "Shutdown successful" diff --git a/crates/sparkplane-rtk/upstream/src/filters/gcc.toml b/crates/sparkplane-rtk/upstream/src/filters/gcc.toml new file mode 100644 index 0000000..cf5f983 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/gcc.toml @@ -0,0 +1,49 @@ +[filters.gcc] +description = "Compact gcc/g++ compiler output — strip notes, keep errors and warnings" +match_command = "^g(cc|\\+\\+)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s+\\|\\s*$", + "^In file included from", + "^\\s+from\\s", + "^\\d+ warnings? generated", + "^\\d+ errors? generated", +] +max_lines = 50 +on_empty = "gcc: ok" + +[[tests.gcc]] +name = "strips include chain, keeps errors and warnings" +input = """ +In file included from /usr/include/stdio.h:42: + from main.c:1: +main.c:10:5: error: use of undeclared identifier 'foo' + foo(); + ^ +main.c:15:12: warning: unused variable 'x' [-Wunused-variable] + int x = 42; + ^ +2 warnings generated. +1 error generated. +""" +expected = "main.c:10:5: error: use of undeclared identifier 'foo'\n foo();\n ^\nmain.c:15:12: warning: unused variable 'x' [-Wunused-variable]\n int x = 42;\n ^" + +[[tests.gcc]] +name = "clean compilation" +input = """ +""" +expected = "gcc: ok" + +[[tests.gcc]] +name = "linker error kept" +input = """ +/usr/bin/ld: /tmp/main.o: undefined reference to 'missing_func' +collect2: error: ld returned 1 exit status +""" +expected = "/usr/bin/ld: /tmp/main.o: undefined reference to 'missing_func'\ncollect2: error: ld returned 1 exit status" + +[[tests.gcc]] +name = "empty input returns on_empty message" +input = "" +expected = "gcc: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/gcloud.toml b/crates/sparkplane-rtk/upstream/src/filters/gcloud.toml new file mode 100644 index 0000000..275d7c3 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/gcloud.toml @@ -0,0 +1,22 @@ +[filters.gcloud] +description = "Compact gcloud output" +match_command = "^gcloud\\b" +strip_ansi = true +strip_lines_matching = ["^\\s*$"] +truncate_lines_at = 120 +max_lines = 30 + +[[tests.gcloud]] +name = "strips blank lines, preserves output" +input = """ +Updated property [core/project]. + +NAME REGION STATUS +my-cluster us-central1 RUNNING +""" +expected = "Updated property [core/project].\nNAME REGION STATUS\nmy-cluster us-central1 RUNNING" + +[[tests.gcloud]] +name = "single line passthrough" +input = "Listed 0 items." +expected = "Listed 0 items." diff --git a/crates/sparkplane-rtk/upstream/src/filters/hadolint.toml b/crates/sparkplane-rtk/upstream/src/filters/hadolint.toml new file mode 100644 index 0000000..141a9be --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/hadolint.toml @@ -0,0 +1,24 @@ +[filters.hadolint] +description = "Compact hadolint Dockerfile linting output" +match_command = "^hadolint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.hadolint]] +name = "Dockerfile warnings kept, blank lines stripped" +input = """ +Dockerfile:3 DL3008 warning: Pin versions in apt-get install +Dockerfile:5 DL3009 info: Delete apt-get lists after installing + +Dockerfile:8 DL4006 warning: Set SHELL option -o pipefail before RUN with pipe +""" +expected = "Dockerfile:3 DL3008 warning: Pin versions in apt-get install\nDockerfile:5 DL3009 info: Delete apt-get lists after installing\nDockerfile:8 DL4006 warning: Set SHELL option -o pipefail before RUN with pipe" + +[[tests.hadolint]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/helm.toml b/crates/sparkplane-rtk/upstream/src/filters/helm.toml new file mode 100644 index 0000000..790ddd3 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/helm.toml @@ -0,0 +1,103 @@ +[filters.helm] +description = "Compact helm output" +match_command = "^helm\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^W\\d{4}", +] +truncate_lines_at = 120 + +[[tests.helm]] +name = "strips blank lines, preserves release info" +input = """ +NAME: my-release +LAST DEPLOYED: Mon Jan 15 10:30:00 2024 +NAMESPACE: default +STATUS: deployed +REVISION: 3 + +NOTES: +Application is running. +""" +expected = "NAME: my-release\nLAST DEPLOYED: Mon Jan 15 10:30:00 2024\nNAMESPACE: default\nSTATUS: deployed\nREVISION: 3\nNOTES:\nApplication is running." + +[[tests.helm]] +name = "strips glog W-prefix warnings" +input = "W0115 10:30:00 warning message from internal\nNAME: my-chart\nSTATUS: deployed" +expected = "NAME: my-chart\nSTATUS: deployed" + +[[tests.helm]] +name = "helm template output is not truncated" +input = """ +# Source: my-chart/templates/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: my-ns +--- +# Source: my-chart/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-secret +--- +# Source: my-chart/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-cm +--- +# Source: my-chart/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc +--- +# Source: my-chart/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: my-svc +--- +# Source: my-chart/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-deploy +""" +expected = """# Source: my-chart/templates/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: my-ns +--- +# Source: my-chart/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-secret +--- +# Source: my-chart/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-cm +--- +# Source: my-chart/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc +--- +# Source: my-chart/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: my-svc +--- +# Source: my-chart/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-deploy""" diff --git a/crates/sparkplane-rtk/upstream/src/filters/iptables.toml b/crates/sparkplane-rtk/upstream/src/filters/iptables.toml new file mode 100644 index 0000000..95f417d --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/iptables.toml @@ -0,0 +1,27 @@ +[filters.iptables] +description = "Compact iptables output" +match_command = "^iptables\\b" +strip_lines_matching = [ + "^\\s*$", + "^Chain DOCKER", + "^Chain BR-", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.iptables]] +name = "strips Docker chains, preserves real rules" +input = """ +Chain INPUT (policy ACCEPT) +num target prot opt source destination +1 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 +Chain DOCKER (1 references) + DOCKER all -- 0.0.0.0/0 0.0.0.0/0 +Chain BR-abcdef (0 references) +""" +expected = "Chain INPUT (policy ACCEPT)\nnum target prot opt source destination\n1 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0\n DOCKER all -- 0.0.0.0/0 0.0.0.0/0" + +[[tests.iptables]] +name = "preserves FORWARD and OUTPUT chains" +input = "Chain FORWARD (policy DROP)\n1 ACCEPT tcp\nChain OUTPUT (policy ACCEPT)\n1 ACCEPT all" +expected = "Chain FORWARD (policy DROP)\n1 ACCEPT tcp\nChain OUTPUT (policy ACCEPT)\n1 ACCEPT all" diff --git a/crates/sparkplane-rtk/upstream/src/filters/jira.toml b/crates/sparkplane-rtk/upstream/src/filters/jira.toml new file mode 100644 index 0000000..9de5ad3 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/jira.toml @@ -0,0 +1,20 @@ +[filters.jira] +description = "Compact Jira CLI output — strip verbose metadata, keep essentials" +match_command = "^jira\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*--", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.jira]] +name = "strips blank lines from issue list" +input = "TYPE\tKEY\tSUMMARY\tSTATUS\n\nStory\tPROJ-123\tAdd login feature\tIn Progress\n\nBug\tPROJ-456\tFix crash on startup\tOpen" +expected = "TYPE\tKEY\tSUMMARY\tSTATUS\nStory\tPROJ-123\tAdd login feature\tIn Progress\nBug\tPROJ-456\tFix crash on startup\tOpen" + +[[tests.jira]] +name = "single issue view" +input = "KEY: PROJ-123\nSummary: Add login feature\nStatus: In Progress\nAssignee: john@example.com" +expected = "KEY: PROJ-123\nSummary: Add login feature\nStatus: In Progress\nAssignee: john@example.com" diff --git a/crates/sparkplane-rtk/upstream/src/filters/jj.toml b/crates/sparkplane-rtk/upstream/src/filters/jj.toml new file mode 100644 index 0000000..6b330a3 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/jj.toml @@ -0,0 +1,28 @@ +[filters.jj] +description = "Compact Jujutsu (jj) output — strip blank lines, truncate" +match_command = "^jj\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Hint:", + "^Working copy now at:", +] +max_lines = 30 +truncate_lines_at = 120 + +[[tests.jj]] +name = "log output stripped of hints" +input = """ +@ qpvuntsm patrick@example.com 2026-03-10 12:00 abc123 +│ feat: add new feature +◉ zzzzzzzz root() + +Working copy now at: qpvuntsm abc123 feat: add new feature +Hint: use `jj log` to see the full history +""" +expected = "@ qpvuntsm patrick@example.com 2026-03-10 12:00 abc123\n│ feat: add new feature\n◉ zzzzzzzz root()" + +[[tests.jj]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/jq.toml b/crates/sparkplane-rtk/upstream/src/filters/jq.toml new file mode 100644 index 0000000..49f68ab --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/jq.toml @@ -0,0 +1,24 @@ +[filters.jq] +description = "Compact jq output — truncate large JSON results" +match_command = "^jq\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 40 +truncate_lines_at = 120 + +[[tests.jq]] +name = "short output passes through" +input = """ +{ + "name": "test", + "version": "1.0" +} +""" +expected = "{\n \"name\": \"test\",\n \"version\": \"1.0\"\n}" + +[[tests.jq]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/just.toml b/crates/sparkplane-rtk/upstream/src/filters/just.toml new file mode 100644 index 0000000..31e58a5 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/just.toml @@ -0,0 +1,26 @@ +[filters.just] +description = "Compact just task runner output — strip recipe headers, keep command output" +match_command = "^just\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Available recipes:", + "^\\s*just --list", +] +truncate_lines_at = 150 +max_lines = 50 + +[[tests.just]] +name = "preserves command output" +input = "cargo test\n\ntest result: ok. 42 passed; 0 failed\n" +expected = "cargo test\ntest result: ok. 42 passed; 0 failed" + +[[tests.just]] +name = "preserves error output" +input = "error: Compilation failed\nsrc/main.rs:10: expected `;`" +expected = "error: Compilation failed\nsrc/main.rs:10: expected `;`" + +[[tests.just]] +name = "empty input" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/liquibase.toml b/crates/sparkplane-rtk/upstream/src/filters/liquibase.toml new file mode 100644 index 0000000..12d068f --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/liquibase.toml @@ -0,0 +1,84 @@ +[filters.liquibase] +description = "Compact liquibase output — strip headers and generic info" +match_command = "^liquibase(?:\\s|$)" +strip_ansi = true +filter_stderr = true +strip_lines_matching = [ + "^\\s*$", + "^Starting Liquibase at", + "^Liquibase (?:Community|Open Source)", + "^Liquibase Home:", + "^Java Home", + "^Libraries:", + "^\\s*-\\s+\\S+\\.jar", + "^INFO \\[liquibase\\.integration\\]", + "^INFO \\[liquibase\\.core\\] Reading resource", + "^INFO \\[liquibase\\.core\\] Parsing", + "^(?:\\[?INFO\\]?\\s*)?#+$", + "^\\s*##" +] +on_empty = "liquibase: ok" +max_lines = 200 + +[[tests.liquibase]] +name = "strip ascii banner and info logs from subcommand" +input = ''' +#################################################### +## _ _ _ _ ## +## | | (_) (_) | ## +#################################################### +Starting Liquibase at 10:12:11 (version 4.29.1) +Liquibase Version: 4.29.1 +Liquibase Open Source 4.29.1 by Liquibase +INFO [liquibase.integration] Starting command +INFO [liquibase.core] Reading resource db/changelog.xml +INFO [liquibase.core] Parsing db/changelog.xml +Running Changeset: filepath::id::author +Changeset filepath::id::author ran successfully +''' +expected = ''' +Liquibase Version: 4.29.1 +Running Changeset: filepath::id::author +Changeset filepath::id::author ran successfully''' + +[[tests.liquibase]] +name = "strip --version noise, keep only version line" +input = ''' +#################################################### +## _ _ _ _ ## +#################################################### +Starting Liquibase at 13:45:24 using Java 17.0.15 (version 4.30.0 #4943 built at 2024-10-31 17:00+0000) +Liquibase Home: D:\mcp\bash\lbr\third-party +Java Home C:\Program Files\Java\jdk-17.0.15 (Version 17.0.15) +Libraries: + - internal\lib\commons-io.jar: Apache Commons IO 2.17.0 By The Apache Software Foundation + - internal\lib\picocli.jar: picocli 4.7.6 By Remko Popma + - lib\ojdbc10-19.30.0.0.jar: JDBC 19.30.0.0.0 By Oracle Corporation + +Liquibase Version: 4.30.0 +Liquibase Open Source 4.30.0 by Liquibase +''' +expected = ''' +Liquibase Version: 4.30.0''' + +[[tests.liquibase]] +name = "keep status and error lines" +input = ''' +#################################################### +## _ _ _ _ ## +#################################################### +Starting Liquibase at 10:00:00 (version 4.30.0) +Liquibase Version: 4.30.0 +Liquibase Open Source 4.30.0 by Liquibase +HR@jdbc:oracle:thin:@localhost:1523:XE is up to date +Liquibase command 'status' was executed successfully. +''' +expected = ''' +Liquibase Version: 4.30.0 +HR@jdbc:oracle:thin:@localhost:1523:XE is up to date +Liquibase command 'status' was executed successfully.''' + +[[tests.liquibase]] +name = "empty input" +input = "" +expected = "liquibase: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/make.toml b/crates/sparkplane-rtk/upstream/src/filters/make.toml new file mode 100644 index 0000000..63925d4 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/make.toml @@ -0,0 +1,41 @@ +[filters.make] +description = "Compact make output" +match_command = "^make\\b" +strip_lines_matching = [ + "^make\\[\\d+\\]:", + "^\\s*$", + "^Nothing to be done", +] +max_lines = 50 +on_empty = "make: ok" + +[[tests.make]] +name = "strips entering/leaving lines" +input = """ +make[1]: Entering directory '/home/user' +gcc -O2 foo.c +make[1]: Leaving directory '/home/user' +""" +expected = """ +gcc -O2 foo.c +""" + +[[tests.make]] +name = "strips blank lines" +input = """ +gcc -O2 foo.c + +gcc -O2 bar.c +""" +expected = """ +gcc -O2 foo.c +gcc -O2 bar.c +""" + +[[tests.make]] +name = "on_empty when all stripped" +input = """ +make[1]: Entering directory '/home/user' +make[1]: Leaving directory '/home/user' +""" +expected = "make: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/markdownlint.toml b/crates/sparkplane-rtk/upstream/src/filters/markdownlint.toml new file mode 100644 index 0000000..2b25d8c --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/markdownlint.toml @@ -0,0 +1,24 @@ +[filters.markdownlint] +description = "Compact markdownlint output — strip blank lines, limit rows" +match_command = "^markdownlint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.markdownlint]] +name = "linting errors stripped of blank lines" +input = """ +README.md:1:1 MD041/first-line-heading/first-line-h1 First line in file should be a top level heading +README.md:10:1 MD022/blanks-around-headings Headings should be surrounded by blank lines + +README.md:15:80 MD013/line-length Line length [Expected: 80; Actual: 95] +""" +expected = "README.md:1:1 MD041/first-line-heading/first-line-h1 First line in file should be a top level heading\nREADME.md:10:1 MD022/blanks-around-headings Headings should be surrounded by blank lines\nREADME.md:15:80 MD013/line-length Line length [Expected: 80; Actual: 95]" + +[[tests.markdownlint]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/mise.toml b/crates/sparkplane-rtk/upstream/src/filters/mise.toml new file mode 100644 index 0000000..7223d12 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/mise.toml @@ -0,0 +1,30 @@ +[filters.mise] +description = "Compact mise task runner output — strip status lines, keep task results" +match_command = "^mise\\s+(run|exec|install|upgrade)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^mise\\s+(trust|install|upgrade).*✓", + "^mise\\s+Installing\\s", + "^mise\\s+Downloading\\s", + "^mise\\s+Extracting\\s", + "^mise\\s+\\w+@[\\d.]+ installed", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "mise: ok" + +[[tests.mise]] +name = "strips install noise, keeps task output" +input = "mise Installing node@20.0.0\nmise Downloading node@20.0.0\nmise Extracting node@20.0.0\nmise node@20.0.0 installed\n\nlint check passed\n2 warnings found" +expected = "lint check passed\n2 warnings found" + +[[tests.mise]] +name = "preserves error output" +input = "mise run lint\nError: biome check failed\nsrc/index.ts:5 — unused variable" +expected = "mise run lint\nError: biome check failed\nsrc/index.ts:5 — unused variable" + +[[tests.mise]] +name = "empty after stripping" +input = "mise trust ~/dev/.mise.toml ✓\nmise install node@20 ✓\n" +expected = "mise: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/mix-compile.toml b/crates/sparkplane-rtk/upstream/src/filters/mix-compile.toml new file mode 100644 index 0000000..14072f6 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/mix-compile.toml @@ -0,0 +1,27 @@ +[filters.mix-compile] +description = "Compact mix compile output" +match_command = "^mix\\s+compile(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^Compiling \\d+ file", + "^\\s*$", + "^Generated\\s", +] +max_lines = 40 +on_empty = "mix compile: ok" + +[[tests.mix-compile]] +name = "strips compile noise, preserves warnings" +input = """ +Compiling 12 files (.ex) +Generated my_app app + +warning: variable "conn" is unused + lib/router.ex:42 +""" +expected = "warning: variable \"conn\" is unused\n lib/router.ex:42" + +[[tests.mix-compile]] +name = "on_empty when only noise" +input = "Compiling 3 files (.ex)\nGenerated my_app app\n" +expected = "mix compile: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/mix-format.toml b/crates/sparkplane-rtk/upstream/src/filters/mix-format.toml new file mode 100644 index 0000000..460e23a --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/mix-format.toml @@ -0,0 +1,15 @@ +[filters.mix-format] +description = "Compact mix format output" +match_command = "^mix\\s+format(\\s|$)" +on_empty = "mix format: ok" +max_lines = 20 + +[[tests.mix-format]] +name = "empty output returns ok" +input = "" +expected = "mix format: ok" + +[[tests.mix-format]] +name = "changed files pass through" +input = "lib/my_app.ex\ntest/my_app_test.exs" +expected = "lib/my_app.ex\ntest/my_app_test.exs" diff --git a/crates/sparkplane-rtk/upstream/src/filters/nx.toml b/crates/sparkplane-rtk/upstream/src/filters/nx.toml new file mode 100644 index 0000000..d42dfb7 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/nx.toml @@ -0,0 +1,25 @@ +[filters.nx] +description = "Compact Nx monorepo output — strip task graph noise, keep results" +match_command = "^(pnpm\\s+)?nx\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*>\\s*NX\\s+Running target", + "^\\s*>\\s*NX\\s+Nx read the output", + "^\\s*>\\s*NX\\s+View logs", + "^———————", + "^—————————", + "^\\s+Nx \\(powered by", +] +truncate_lines_at = 150 +max_lines = 60 + +[[tests.nx]] +name = "strips Nx noise, keeps build output" +input = "\n > NX Running target build for project myapp\n\n———————————————————————————————————————\nCompiled successfully.\nOutput: dist/apps/myapp\n\n > NX View logs at /tmp/.nx/runs/abc123\n\n Nx (powered by computation caching)\n" +expected = "Compiled successfully.\nOutput: dist/apps/myapp" + +[[tests.nx]] +name = "preserves error output" +input = "ERROR: Cannot find module '@myapp/shared'\n\n > NX Running target build for project myapp\n\nFailed at step: build" +expected = "ERROR: Cannot find module '@myapp/shared'\nFailed at step: build" diff --git a/crates/sparkplane-rtk/upstream/src/filters/ollama.toml b/crates/sparkplane-rtk/upstream/src/filters/ollama.toml new file mode 100644 index 0000000..e325ec9 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ollama.toml @@ -0,0 +1,23 @@ +[filters.ollama] +description = "Strip ANSI spinners and cursor control from ollama output, keep final text" +match_command = "^ollama\\s+run\\b" +strip_ansi = true +strip_lines_matching = [ + "^[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏\\s]*$", + "^\\s*$", +] + +[[tests.ollama]] +name = "strips spinner lines, keeps response" +input = "⠋ \n⠙ \n⠹ \nHello! How can I help you today?" +expected = "Hello! How can I help you today?" + +[[tests.ollama]] +name = "preserves multi-line response" +input = "⠋ \n⠙ \nLine one of the response.\nLine two of the response." +expected = "Line one of the response.\nLine two of the response." + +[[tests.ollama]] +name = "empty input" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/oxlint.toml b/crates/sparkplane-rtk/upstream/src/filters/oxlint.toml new file mode 100644 index 0000000..74326ca --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/oxlint.toml @@ -0,0 +1,43 @@ +[filters.oxlint] +description = "Compact oxlint output — strip blank lines, keep diagnostics" +match_command = "^oxlint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Finished in \\d+", + "^Found \\d+ warning", +] +max_lines = 50 +on_empty = "oxlint: ok" + +[[tests.oxlint]] +name = "strips noise, keeps diagnostics" +input = """ + × eslint(no-console): Unexpected console statement. + ╭─[src/app.ts:5:3] + 5 │ console.log("debug"); + │ ^^^^^^^^^^^ + ╰──── + + × eslint(no-unused-vars): 'x' is defined but never used. + ╭─[src/utils.ts:2:7] + 2 │ let x = 42; + │ ^ + ╰──── + +Found 2 warnings on 2 files. +Finished in 12ms on 100 files. +""" +expected = " × eslint(no-console): Unexpected console statement.\n ╭─[src/app.ts:5:3]\n 5 │ console.log(\"debug\");\n │ ^^^^^^^^^^^\n ╰────\n × eslint(no-unused-vars): 'x' is defined but never used.\n ╭─[src/utils.ts:2:7]\n 2 │ let x = 42;\n │ ^\n ╰────" + +[[tests.oxlint]] +name = "clean output" +input = """ +Finished in 5ms on 100 files. +""" +expected = "oxlint: ok" + +[[tests.oxlint]] +name = "empty input returns on_empty message" +input = "" +expected = "oxlint: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/ping.toml b/crates/sparkplane-rtk/upstream/src/filters/ping.toml new file mode 100644 index 0000000..793b121 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ping.toml @@ -0,0 +1,63 @@ +[filters.ping] +description = "Compact ping output — strip per-packet lines, keep summary" +match_command = "^ping\\b" +strip_ansi = true +strip_lines_matching = [ + "^PING ", + "^Pinging ", + "^\\d+ bytes from ", + "^Reply from .+: bytes=", + "^\\s*$", +] +tail_lines = 4 + +[[tests.ping]] +name = "success keeps summary only" +input = """ +PING example.com (93.184.216.34): 56 data bytes +64 bytes from 93.184.216.34: icmp_seq=0 ttl=56 time=14.2 ms +64 bytes from 93.184.216.34: icmp_seq=1 ttl=56 time=13.8 ms +64 bytes from 93.184.216.34: icmp_seq=2 ttl=56 time=14.1 ms +64 bytes from 93.184.216.34: icmp_seq=3 ttl=56 time=13.9 ms + +--- example.com ping statistics --- +4 packets transmitted, 4 packets received, 0.0% packet loss +round-trip min/avg/max/stddev = 13.8/14.0/14.2/0.2 ms +""" +expected = """--- example.com ping statistics --- +4 packets transmitted, 4 packets received, 0.0% packet loss +round-trip min/avg/max/stddev = 13.8/14.0/14.2/0.2 ms""" + +[[tests.ping]] +name = "windows format keeps stats block only" +input = """ +Pinging 192.0.2.1 with 32 bytes of data: +Reply from 192.0.2.1: bytes=32 time=14ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=13ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=14ms TTL=56 +Reply from 192.0.2.1: bytes=32 time=13ms TTL=56 + +Ping statistics for 192.0.2.1: + Packets: Sent = 4, Received = 4, Lost = 0 (0% loss), +Approximate round trip times in milli-seconds: + Minimum = 13ms, Maximum = 14ms, Average = 13ms +""" +expected = """Ping statistics for 192.0.2.1: + Packets: Sent = 4, Received = 4, Lost = 0 (0% loss), +Approximate round trip times in milli-seconds: + Minimum = 13ms, Maximum = 14ms, Average = 13ms""" + +[[tests.ping]] +name = "unreachable host passes error through" +input = """ +PING unreachable.example.com (192.0.2.1): 56 data bytes +Request timeout for icmp_seq 0 +Request timeout for icmp_seq 1 + +--- unreachable.example.com ping statistics --- +2 packets transmitted, 0 packets received, 100.0% packet loss +""" +expected = """Request timeout for icmp_seq 0 +Request timeout for icmp_seq 1 +--- unreachable.example.com ping statistics --- +2 packets transmitted, 0 packets received, 100.0% packet loss""" diff --git a/crates/sparkplane-rtk/upstream/src/filters/pio-run.toml b/crates/sparkplane-rtk/upstream/src/filters/pio-run.toml new file mode 100644 index 0000000..54d2471 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/pio-run.toml @@ -0,0 +1,40 @@ +[filters.pio-run] +description = "Compact PlatformIO build output" +match_command = "^pio\\s+run(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Verbose mode", + "^CONFIGURATION:", + "^LDF:", + "^Library Manager:", + "^Compiling\\s", + "^Linking\\s", + "^Building\\s", + "^Checking size", +] +max_lines = 30 +on_empty = "pio run: ok" + +[[tests.pio-run]] +name = "strips build noise, preserves errors" +input = """ +Verbose mode can be enabled via `-v, --verbose` option +CONFIGURATION: https://docs.platformio.org/page/boards/espressif32/esp32dev.html +LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf +Compiling .pio/build/esp32dev/src/main.cpp.o +Building .pio/build/esp32dev/firmware.elf +Linking .pio/build/esp32dev/firmware.elf +Checking size .pio/build/esp32dev/firmware.elf +src/main.cpp:10:3: error: 'LED_BUILTINN' was not declared +""" +expected = "src/main.cpp:10:3: error: 'LED_BUILTINN' was not declared" + +[[tests.pio-run]] +name = "on_empty when clean build with only noise" +input = """ +Verbose mode can be enabled via `-v, --verbose` option +Compiling .pio/build/esp32dev/src/main.cpp.o +Linking .pio/build/esp32dev/firmware.elf +""" +expected = "pio run: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/poetry-install.toml b/crates/sparkplane-rtk/upstream/src/filters/poetry-install.toml new file mode 100644 index 0000000..bceb4dd --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/poetry-install.toml @@ -0,0 +1,50 @@ +[filters.poetry-install] +description = "Compact poetry install/lock/update output — strip downloads, short-circuit when up-to-date" +match_command = "^poetry\\s+(install|lock|update)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^ [-•] Downloading ", + "^ [-•] Installing .* \\(", + "^Creating virtualenv", + "^Using virtualenv", +] +match_output = [ + { pattern = "No dependencies to install or update|No changes\\.", message = "ok (up to date)" }, +] +max_lines = 30 + +[[tests.poetry-install]] +name = "up to date short-circuits" +input = """ +Installing dependencies from lock file + +No dependencies to install or update +""" +expected = "ok (up to date)" + +[[tests.poetry-install]] +name = "poetry 2.x bullet syntax short-circuits to ok" +input = """ +• Installing requests (2.31.0) +• Installing certifi (2023.11.17) + +No changes. +""" +expected = "ok (up to date)" + +[[tests.poetry-install]] +name = "install strips download lines" +input = """ +Installing dependencies from lock file + + - Downloading requests-2.31.0-py3-none-any.whl (62.6 kB) + - Installing certifi (2023.11.17) + - Installing charset-normalizer (3.3.2) + - Installing idna (3.6) + - Installing urllib3 (2.1.0) + - Installing requests (2.31.0) + +Writing lock file +""" +expected = "Installing dependencies from lock file\nWriting lock file" diff --git a/crates/sparkplane-rtk/upstream/src/filters/pre-commit.toml b/crates/sparkplane-rtk/upstream/src/filters/pre-commit.toml new file mode 100644 index 0000000..04fcc33 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/pre-commit.toml @@ -0,0 +1,35 @@ +[filters.pre-commit] +description = "Compact pre-commit output" +match_command = "^pre-commit\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\[INFO\\] Installing environment", + "^\\[INFO\\] Once installed this environment will be reused", + "^\\[INFO\\] This may take a few minutes", + "^\\s*$", +] +max_lines = 40 + +[[tests.pre-commit]] +name = "strips INFO install noise, keeps hook results" +input = """ +[INFO] Installing environment for https://github.com/psf/black. +[INFO] Once installed this environment will be reused. +[INFO] This may take a few minutes... +Trim Trailing Whitespace.................................................Passed +Fix End of Files.........................................................Passed +Check Yaml...............................................................Failed +- hook id: check-yaml +- exit code: 1 +""" +expected = "Trim Trailing Whitespace.................................................Passed\nFix End of Files.........................................................Passed\nCheck Yaml...............................................................Failed\n- hook id: check-yaml\n- exit code: 1" + +[[tests.pre-commit]] +name = "all passed — no INFO noise" +input = """ +[INFO] Installing environment for https://github.com/pre-commit/mirrors-isort. +[INFO] Once installed this environment will be reused. +isort....................................................................Passed +black....................................................................Passed +""" +expected = "isort....................................................................Passed\nblack....................................................................Passed" diff --git a/crates/sparkplane-rtk/upstream/src/filters/ps.toml b/crates/sparkplane-rtk/upstream/src/filters/ps.toml new file mode 100644 index 0000000..685ef93 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ps.toml @@ -0,0 +1,16 @@ +[filters.ps] +description = "Compact ps output — truncate wide lines, limit rows" +match_command = "^ps(\\s|$)" +strip_ansi = true +truncate_lines_at = 120 +max_lines = 30 + +[[tests.ps]] +name = "short process list passes through unchanged" +input = "USER PID %CPU %MEM COMMAND\nroot 1 0.0 0.0 /sbin/launchd\nflorian 42 0.1 0.2 bash" +expected = "USER PID %CPU %MEM COMMAND\nroot 1 0.0 0.0 /sbin/launchd\nflorian 42 0.1 0.2 bash" + +[[tests.ps]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/pulumi-destroy.toml b/crates/sparkplane-rtk/upstream/src/filters/pulumi-destroy.toml new file mode 100644 index 0000000..1052c0f --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/pulumi-destroy.toml @@ -0,0 +1,60 @@ +[filters.pulumi-destroy] +description = "Compact Pulumi destroy output" +match_command = "^pulumi\\s+destroy(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Destroying \\(", + "^Previewing (update|destroy)", + "^View in Browser", + "^View Live:", + "^Duration:", + "^Permalink:", + "^\\s*Type\\s+Name\\s+", + "^Loading policy packs", + "^@ (Previewing (update|destroy)|Destroying)", + "^More information at:", + "^Use `pulumi ", + "^The resources in the stack have been deleted", + "^If you want to remove the stack completely", + "^\\s+-\\s+.*\\bdeleting\\s+\\(", + "^\\s{4,}at\\s+\\S+\\s*\\(", + "^\\s{4,}at\\s+/", + "^\\s+at\\s+processTicksAndRejections", + "^\\s+promise:\\s+Promise", + "^\\s+\\[Circular", + "^\\s*-.jar name, and jars under a path containing spaces, are deliberately left to full passthrough rather than risk misclassifying an unrelated tool." +match_command = "^(mvn\\s+spring-boot:run|java\\s+-jar\\s+(?:\\S*[/\\\\])?[^/\\\\\\s]*(?i:spring)[^/\\\\\\s]*\\.jar|gradle\\s+.*bootRun)" +strip_ansi = true +keep_lines_matching = [ + "Started\\s.*\\sin\\s", + "Tomcat started on port", + "ERROR", + "WARN", + "Exception", + "Caused by:", + "Application run failed", + "BUILD\\s", + "Tests run:", + "FAILURE", + "listening on port", +] +max_lines = 30 + +[[tests.spring-boot]] +name = "keeps startup summary and errors" +input = " . ____ _ \n /\\\\ / ___'_ __ _ _(_)_ __ \n( ( )\\___ | '_ | '_| | '_ \\ \n \\/ ___)| |_)| | | | | || )\n ' |____| .__|_| |_|_| |_\\__|\n :: Spring Boot :: (v3.2.0)\n2024-01-01 INFO Initializing Spring\n2024-01-01 INFO Bean 'dataSource' created\n2024-01-01 INFO Tomcat started on port 8080\n2024-01-01 INFO Started MyApp in 3.2 seconds" +expected = "2024-01-01 INFO Tomcat started on port 8080\n2024-01-01 INFO Started MyApp in 3.2 seconds" + +[[tests.spring-boot]] +name = "preserves errors" +input = " :: Spring Boot :: (v3.2.0)\n2024-01-01 INFO Initializing Spring\n2024-01-01 ERROR Application run failed\nCaused by: java.lang.NullPointerException" +expected = "2024-01-01 ERROR Application run failed\nCaused by: java.lang.NullPointerException" diff --git a/crates/sparkplane-rtk/upstream/src/filters/ssh.toml b/crates/sparkplane-rtk/upstream/src/filters/ssh.toml new file mode 100644 index 0000000..71c953f --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ssh.toml @@ -0,0 +1,44 @@ +[filters.ssh] +description = "Compact ssh output — strip connection banners, keep command output" +match_command = "^ssh(?:\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Warning: Permanently added", + "^Connection to .+ closed", + "^Authenticated to", + "^debug1:", + "^OpenSSH_", + "^Pseudo-terminal", +] +max_lines = 200 +truncate_lines_at = 120 + +[[tests.ssh]] +name = "strips connection banners, keeps command output" +input = """ +Warning: Permanently added '192.168.1.10' (ED25519) to the list of known hosts. + +total 32 +drwxr-xr-x 4 user user 4096 Mar 10 12:00 app +-rw-r--r-- 1 user user 1234 Mar 10 11:00 config.yaml + +Connection to 192.168.1.10 closed. +""" +expected = "total 32\ndrwxr-xr-x 4 user user 4096 Mar 10 12:00 app\n-rw-r--r-- 1 user user 1234 Mar 10 11:00 config.yaml" + +[[tests.ssh]] +name = "verbose debug lines stripped" +input = """ +debug1: Connecting to host.example.com port 22. +debug1: Connection established. +Authenticated to host.example.com ([1.2.3.4]:22). +uptime: 12:00:00 up 42 days, load average: 0.10, 0.15, 0.12 +Connection to host.example.com closed. +""" +expected = "uptime: 12:00:00 up 42 days, load average: 0.10, 0.15, 0.12" + +[[tests.ssh]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/stat.toml b/crates/sparkplane-rtk/upstream/src/filters/stat.toml new file mode 100644 index 0000000..8c240c0 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/stat.toml @@ -0,0 +1,44 @@ +[filters.stat] +description = "Compact stat output — strip device/inode/birth noise" +match_command = "^stat\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Device:", + "^\\s*Birth:", +] +truncate_lines_at = 120 +max_lines = 20 + +[[tests.stat]] +name = "linux stat output strips device and birth" +input = """ + File: main.rs + Size: 12345 Blocks: 24 IO Block: 4096 regular file +Device: 801h/2049d Inode: 1234567 Links: 1 +Access: (0644/-rw-r--r--) Uid: ( 1000/ patrick) Gid: ( 1000/ patrick) +Access: 2026-03-10 12:00:00.000000000 +0100 +Modify: 2026-03-10 11:00:00.000000000 +0100 +Change: 2026-03-10 11:00:00.000000000 +0100 + Birth: 2026-03-09 10:00:00.000000000 +0100 +""" +expected = " File: main.rs\n Size: 12345 Blocks: 24 IO Block: 4096 regular file\nAccess: (0644/-rw-r--r--) Uid: ( 1000/ patrick) Gid: ( 1000/ patrick)\nAccess: 2026-03-10 12:00:00.000000000 +0100\nModify: 2026-03-10 11:00:00.000000000 +0100\nChange: 2026-03-10 11:00:00.000000000 +0100" + +[[tests.stat]] +name = "macOS stat -x strips device and birth" +input = """ + File: "main.rs" + Size: 82848 FileType: Regular File + Mode: (0644/-rw-r--r--) Uid: ( 501/ patrick) Gid: ( 20/ staff) +Device: 1,15 Inode: 66302332 Links: 1 +Access: Wed Mar 18 21:21:15 2026 +Modify: Wed Mar 18 20:56:11 2026 +Change: Wed Mar 18 20:56:11 2026 + Birth: Wed Mar 18 20:56:11 2026 +""" +expected = " File: \"main.rs\"\n Size: 82848 FileType: Regular File\n Mode: (0644/-rw-r--r--) Uid: ( 501/ patrick) Gid: ( 20/ staff)\nAccess: Wed Mar 18 21:21:15 2026\nModify: Wed Mar 18 20:56:11 2026\nChange: Wed Mar 18 20:56:11 2026" + +[[tests.stat]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/swift-build.toml b/crates/sparkplane-rtk/upstream/src/filters/swift-build.toml new file mode 100644 index 0000000..fb866e5 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/swift-build.toml @@ -0,0 +1,41 @@ +[filters.swift-build] +description = "Compact swift build output — short-circuit on success, strip Compiling/Linking" +match_command = "^swift\\s+build\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Compiling ", + "^Linking ", +] +match_output = [ + { pattern = "Build complete!", message = "ok (build complete)", unless = "warning:|error:" }, +] +max_lines = 40 + +[[tests.swift-build]] +name = "successful build short-circuits to ok" +input = """ +Build complete! +""" +expected = "ok (build complete)" + +[[tests.swift-build]] +name = "build errors pass through after stripping noise" +input = """ +Compiling MyApp MyApp.swift +/home/user/MyApp/Sources/MyApp/main.swift:5:1: error: use of unresolved identifier 'foo' +foo() +^~~ +Linking MyApp +error: build had 1 command failure +""" +expected = "/home/user/MyApp/Sources/MyApp/main.swift:5:1: error: use of unresolved identifier 'foo'\nfoo()\n^~~\nerror: build had 1 command failure" + +[[tests.swift-build]] +name = "warnings not swallowed when Build complete present" +input = """ +CompileSwift normal x86_64 MyFile.swift +/path/to/MyFile.swift:42:10: warning: unused variable 'x' +Build complete! (with warnings) +""" +expected = "CompileSwift normal x86_64 MyFile.swift\n/path/to/MyFile.swift:42:10: warning: unused variable 'x'\nBuild complete! (with warnings)" diff --git a/crates/sparkplane-rtk/upstream/src/filters/systemctl-status.toml b/crates/sparkplane-rtk/upstream/src/filters/systemctl-status.toml new file mode 100644 index 0000000..4a8a084 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/systemctl-status.toml @@ -0,0 +1,33 @@ +[filters.systemctl-status] +description = "Compact systemctl status output — strip blank lines, limit to 20 lines" +match_command = "^systemctl\\s+status\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 20 + +[[tests.systemctl-status]] +name = "verbose unit status stripped of blank lines" +input = """ +● nginx.service - A high performance web server + Loaded: loaded (/lib/systemd/system/nginx.service; enabled) + Active: active (running) since Mon 2024-01-15 10:30:00 UTC; 2h ago + Docs: man:nginx(8) + Main PID: 1234 (nginx) + Tasks: 3 (limit: 4915) + Memory: 8.5M + + CGroup: /system.slice/nginx.service + ├─1234 nginx: master process /usr/sbin/nginx + └─1235 nginx: worker process + +Jan 15 10:30:00 host nginx[1234]: nginx/1.24.0 +Jan 15 10:30:00 host systemd[1]: Started nginx.service +""" +expected = "● nginx.service - A high performance web server\n Loaded: loaded (/lib/systemd/system/nginx.service; enabled)\n Active: active (running) since Mon 2024-01-15 10:30:00 UTC; 2h ago\n Docs: man:nginx(8)\n Main PID: 1234 (nginx)\n Tasks: 3 (limit: 4915)\n Memory: 8.5M\n CGroup: /system.slice/nginx.service\n ├─1234 nginx: master process /usr/sbin/nginx\n └─1235 nginx: worker process\nJan 15 10:30:00 host nginx[1234]: nginx/1.24.0\nJan 15 10:30:00 host systemd[1]: Started nginx.service" + +[[tests.systemctl-status]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/crates/sparkplane-rtk/upstream/src/filters/task.toml b/crates/sparkplane-rtk/upstream/src/filters/task.toml new file mode 100644 index 0000000..31868fc --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/task.toml @@ -0,0 +1,27 @@ +[filters.task] +description = "Compact go-task output — strip task headers, keep command results" +match_command = "^task\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^task: \\[.*\\] ", + "^task: Task .* is up to date", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "task: ok" + +[[tests.task]] +name = "strips task headers, keeps output" +input = "task: [build] go build ./...\n\ntask: [test] go test ./...\nok myapp 0.5s\n\ntask: Task \"lint\" is up to date" +expected = "ok myapp 0.5s" + +[[tests.task]] +name = "preserves error output" +input = "task: [build] go build ./...\n./main.go:10: undefined: foo\ntask: Failed to run task \"build\": exit status 1" +expected = "./main.go:10: undefined: foo\ntask: Failed to run task \"build\": exit status 1" + +[[tests.task]] +name = "all up to date" +input = "task: Task \"build\" is up to date\ntask: Task \"lint\" is up to date\n" +expected = "task: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/terraform-plan.toml b/crates/sparkplane-rtk/upstream/src/filters/terraform-plan.toml new file mode 100644 index 0000000..7ee22f3 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/terraform-plan.toml @@ -0,0 +1,35 @@ +[filters.terraform-plan] +description = "Compact Terraform plan output" +match_command = "^terraform\\s+plan(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^Refreshing state", + "^\\s*#.*unchanged", + "^\\s*$", + "^Acquiring state lock", + "^Releasing state lock", +] +max_lines = 80 +on_empty = "terraform plan: no changes detected" + +[[tests.terraform-plan]] +name = "strips Refreshing state lines and blank lines" +input = """ +Acquiring state lock. This may take a few moments... +Refreshing state... [id=vpc-abc] +Refreshing state... [id=sg-123] +Releasing state lock. This may take a few moments... + +Terraform will perform the following actions: + + # aws_instance.web will be created + + resource "aws_instance" "web" {} + +Plan: 1 to add, 0 to change, 0 to destroy. +""" +expected = "Terraform will perform the following actions:\n # aws_instance.web will be created\n + resource \"aws_instance\" \"web\" {}\nPlan: 1 to add, 0 to change, 0 to destroy." + +[[tests.terraform-plan]] +name = "strips noise, preserves non-blank content" +input = "Refreshing state... [id=vpc-abc]\nNo changes. Your infrastructure matches the configuration." +expected = "No changes. Your infrastructure matches the configuration." diff --git a/crates/sparkplane-rtk/upstream/src/filters/tofu-fmt.toml b/crates/sparkplane-rtk/upstream/src/filters/tofu-fmt.toml new file mode 100644 index 0000000..bfb78d9 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/tofu-fmt.toml @@ -0,0 +1,16 @@ +[filters.tofu-fmt] +description = "Compact OpenTofu fmt output" +match_command = "^tofu\\s+fmt(\\s|$)" +strip_ansi = true +on_empty = "tofu fmt: ok (no changes)" +max_lines = 30 + +[[tests.tofu-fmt]] +name = "empty output returns on_empty message" +input = "" +expected = "tofu fmt: ok (no changes)" + +[[tests.tofu-fmt]] +name = "changed files pass through" +input = "main.tf\nvariables.tf" +expected = "main.tf\nvariables.tf" diff --git a/crates/sparkplane-rtk/upstream/src/filters/tofu-init.toml b/crates/sparkplane-rtk/upstream/src/filters/tofu-init.toml new file mode 100644 index 0000000..d983d94 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/tofu-init.toml @@ -0,0 +1,38 @@ +[filters.tofu-init] +description = "Compact OpenTofu init output" +match_command = "^tofu\\s+init(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^- Downloading", + "^- Installing", + "^- Using previously-installed", + "^\\s*$", + "^Initializing provider", + "^Initializing the backend", + "^Initializing modules", +] +max_lines = 20 +on_empty = "tofu init: ok" + +[[tests.tofu-init]] +name = "strips downloading/installing lines" +input = """ +Initializing the backend... +Initializing provider plugins... +- Downloading hashicorp/aws 5.0.0... +- Installing hashicorp/aws 5.0.0... +- Using previously-installed hashicorp/random 3.5.1 + +OpenTofu has been successfully initialized! +""" +expected = "OpenTofu has been successfully initialized!" + +[[tests.tofu-init]] +name = "on_empty when all noise stripped" +input = """ +Initializing the backend... +Initializing provider plugins... +- Using previously-installed hashicorp/aws 5.0.0 + +""" +expected = "tofu init: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/tofu-plan.toml b/crates/sparkplane-rtk/upstream/src/filters/tofu-plan.toml new file mode 100644 index 0000000..72310e7 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/tofu-plan.toml @@ -0,0 +1,35 @@ +[filters.tofu-plan] +description = "Compact OpenTofu plan output" +match_command = "^tofu\\s+plan(\\s|$)" +strip_ansi = true +strip_lines_matching = [ + "^Refreshing state", + "^\\s*#.*unchanged", + "^\\s*$", + "^Acquiring state lock", + "^Releasing state lock", +] +max_lines = 80 +on_empty = "tofu plan: no changes detected" + +[[tests.tofu-plan]] +name = "strips Refreshing state and lock lines" +input = """ +Acquiring state lock. This may take a few moments... +Refreshing state... [id=vpc-abc123] +Refreshing state... [id=sg-def456] +Releasing state lock. This may take a few moments... + +OpenTofu will perform the following actions: + + # aws_instance.web will be created + + resource "aws_instance" "web" {} + +Plan: 1 to add, 0 to change, 0 to destroy. +""" +expected = "OpenTofu will perform the following actions:\n # aws_instance.web will be created\n + resource \"aws_instance\" \"web\" {}\nPlan: 1 to add, 0 to change, 0 to destroy." + +[[tests.tofu-plan]] +name = "on_empty when all noise stripped" +input = "Refreshing state... [id=vpc-abc]\nAcquiring state lock. This may take a few moments...\nReleasing state lock. This may take a few moments..." +expected = "tofu plan: no changes detected" diff --git a/crates/sparkplane-rtk/upstream/src/filters/tofu-validate.toml b/crates/sparkplane-rtk/upstream/src/filters/tofu-validate.toml new file mode 100644 index 0000000..f6aaed8 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/tofu-validate.toml @@ -0,0 +1,17 @@ +[filters.tofu-validate] +description = "Compact OpenTofu validate output" +match_command = "^tofu\\s+validate(\\s|$)" +strip_ansi = true +match_output = [ + { pattern = "Success! The configuration is valid", message = "ok (valid)" }, +] + +[[tests.tofu-validate]] +name = "success short-circuits to ok" +input = "Success! The configuration is valid." +expected = "ok (valid)" + +[[tests.tofu-validate]] +name = "error passes through unchanged" +input = "Error: Invalid resource type\n on main.tf line 3: resource \"aws_instancee\" \"web\"" +expected = "Error: Invalid resource type\n on main.tf line 3: resource \"aws_instancee\" \"web\"" diff --git a/crates/sparkplane-rtk/upstream/src/filters/trunk-build.toml b/crates/sparkplane-rtk/upstream/src/filters/trunk-build.toml new file mode 100644 index 0000000..2baf5b4 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/trunk-build.toml @@ -0,0 +1,39 @@ +[filters.trunk-build] +description = "Compact trunk build output" +match_command = "^trunk\\s+build(\\s|$|[;|&()<>])" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*Compiling\\s", + "^\\s*Downloading\\s", + "^\\s*Fetching\\s", + "^\\s*Fresh\\s", + "^\\s*Checking\\s", +] +tail_lines = 10 +max_lines = 30 +on_empty = "trunk build: ok" + +[[tests.trunk-build]] +name = "strips compile noise, keeps tail summary" +input = """ + Compiling tokio v1.35.0 + Compiling hyper v0.14.28 + Compiling my-crate v0.1.0 + Downloading serde v1.0.195 + Fresh regex v1.10.2 + + Finished release [optimized] target(s) in 45.23s + Binary: target/release/my-crate (5.2MB) +""" +expected = " Finished release [optimized] target(s) in 45.23s\n Binary: target/release/my-crate (5.2MB)" + +[[tests.trunk-build]] +name = "on_empty when all noise stripped" +input = """ + Compiling my-crate v0.1.0 + Fresh serde v1.0 + Checking tokio v1.35.0 + +""" +expected = "trunk build: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/turbo.toml b/crates/sparkplane-rtk/upstream/src/filters/turbo.toml new file mode 100644 index 0000000..c5a09ac --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/turbo.toml @@ -0,0 +1,30 @@ +[filters.turbo] +description = "Compact Turborepo output — strip cache status noise, keep task results" +match_command = "^turbo\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*cache (hit|miss|bypass)", + "^\\s*\\d+ packages in scope", + "^\\s*Tasks:\\s+\\d+", + "^\\s*Duration:\\s+", + "^\\s*Remote caching (enabled|disabled)", +] +truncate_lines_at = 150 +max_lines = 50 +on_empty = "turbo: ok" + +[[tests.turbo]] +name = "strips cache noise, keeps task output" +input = " cache hit, replaying logs abc123\n cache miss, executing abc456\n\n3 packages in scope\n\n> myapp:build\n\nCompiled successfully.\n\nTasks: 2 successful, 2 total (1 cached)\nDuration: 3.2s" +expected = "> myapp:build\nCompiled successfully." + +[[tests.turbo]] +name = "preserves error output" +input = "> myapp:lint\n\nError: src/index.ts(5,1): error TS2304\n\nTasks: 0 successful, 1 total\nDuration: 1.1s" +expected = "> myapp:lint\nError: src/index.ts(5,1): error TS2304" + +[[tests.turbo]] +name = "empty after stripping" +input = " cache hit, replaying logs abc\n\n" +expected = "turbo: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/ty.toml b/crates/sparkplane-rtk/upstream/src/filters/ty.toml new file mode 100644 index 0000000..689fde9 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/ty.toml @@ -0,0 +1,50 @@ +[filters.ty] +description = "Compact ty type checker output — strip blank lines, keep errors" +match_command = "^ty\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^Checking \\d+ file", + "^ty \\d+\\.\\d+", +] +max_lines = 50 +on_empty = "ty: ok" + +[[tests.ty]] +name = "strips noise, keeps diagnostics" +input = """ +ty 0.1.0 +Checking 15 files + +error[unresolved-reference]: Name `foo` used when not defined + --> app/main.py:10:5 + | +10 | foo() + | ^^^ + | + +warning[unused-variable]: Variable `x` is not used + --> app/utils.py:8:9 + | + 8 | x = 42 + | ^ + | + +Found 1 error, 1 warning +""" +expected = "error[unresolved-reference]: Name `foo` used when not defined\n --> app/main.py:10:5\n |\n10 | foo()\n | ^^^\n |\nwarning[unused-variable]: Variable `x` is not used\n --> app/utils.py:8:9\n |\n 8 | x = 42\n | ^\n |\nFound 1 error, 1 warning" + +[[tests.ty]] +name = "clean output" +input = """ +ty 0.1.0 +Checking 10 files + +All checks passed! +""" +expected = "All checks passed!" + +[[tests.ty]] +name = "empty input returns on_empty message" +input = "" +expected = "ty: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/uv-sync.toml b/crates/sparkplane-rtk/upstream/src/filters/uv-sync.toml new file mode 100644 index 0000000..b9eee32 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/uv-sync.toml @@ -0,0 +1,37 @@ +[filters.uv-sync] +description = "Compact uv sync/pip install output — strip downloads, short-circuit when up-to-date" +match_command = "^uv\\s+(sync|pip\\s+install)\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s+Downloading ", + "^\\s+Using cached ", + "^\\s+Preparing ", +] +match_output = [ + { pattern = "Audited \\d+ package", message = "ok (up to date)" }, +] +max_lines = 20 + +[[tests.uv-sync]] +name = "audited packages short-circuits to ok" +input = """ +Resolved 42 packages in 123ms +Audited 42 packages in 0.05ms +""" +expected = "ok (up to date)" + +[[tests.uv-sync]] +name = "install strips download and cached lines" +input = """ + Downloading requests-2.31.0-py3-none-any.whl (62.6 kB) + Using cached certifi-2023.11.17-py3-none-any.whl (162 kB) + Preparing packages... +Installed 5 packages in 23ms + + certifi==2023.11.17 + + charset-normalizer==3.3.2 + + idna==3.6 + + requests==2.31.0 + + urllib3==2.1.0 +""" +expected = "Installed 5 packages in 23ms\n + certifi==2023.11.17\n + charset-normalizer==3.3.2\n + idna==3.6\n + requests==2.31.0\n + urllib3==2.1.0" diff --git a/crates/sparkplane-rtk/upstream/src/filters/xcodebuild.toml b/crates/sparkplane-rtk/upstream/src/filters/xcodebuild.toml new file mode 100644 index 0000000..f959de9 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/xcodebuild.toml @@ -0,0 +1,99 @@ +[filters.xcodebuild] +description = "Compact xcodebuild output — strip build phases, keep errors/warnings/summary" +match_command = "^xcodebuild\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^CompileC\\s", + "^CompileSwift\\s", + "^Ld\\s", + "^CreateBuildDirectory\\s", + "^MkDir\\s", + "^ProcessInfoPlistFile\\s", + "^CopySwiftLibs\\s", + "^CodeSign\\s", + "^Signing Identity:", + "^RegisterWithLaunchServices", + "^Validate\\s", + "^ProcessProductPackaging", + "^Touch\\s", + "^LinkStoryboards", + "^CompileStoryboard", + "^CompileAssetCatalog", + "^GenerateDSYMFile", + "^PhaseScriptExecution", + "^PBXCp\\s", + "^SetMode\\s", + "^SetOwnerAndGroup\\s", + "^Ditto\\s", + "^CpResource\\s", + "^CpHeader\\s", + "^\\s+cd\\s+/", + "^\\s+export\\s", + "^\\s+/Applications/Xcode", + "^\\s+/usr/bin/", + "^\\s+builtin-", + "^note: Using new build system", +] +max_lines = 60 +on_empty = "xcodebuild: ok" + +[[tests.xcodebuild]] +name = "strips build phases, keeps errors and summary" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/App/ViewController.swift + cd /Users/dev/App + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-frontend -c +CompileSwift normal arm64 /Users/dev/App/AppDelegate.swift + cd /Users/dev/App + export DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer +Ld /Users/dev/Build/Products/Debug/App normal arm64 + cd /Users/dev/App + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang +CodeSign /Users/dev/Build/Products/Debug/App.app + cd /Users/dev/App + builtin-codesign --force --sign + +/Users/dev/App/ViewController.swift:42:9: error: use of unresolved identifier 'foo' +/Users/dev/App/Model.swift:18:5: warning: variable 'x' was never used + +** BUILD FAILED ** +""" +expected = "/Users/dev/App/ViewController.swift:42:9: error: use of unresolved identifier 'foo'\n/Users/dev/App/Model.swift:18:5: warning: variable 'x' was never used\n** BUILD FAILED **" + +[[tests.xcodebuild]] +name = "clean build success" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/App/Main.swift + cd /Users/dev/App +Ld /Users/dev/Build/Products/Debug/App normal arm64 + cd /Users/dev/App +CodeSign /Users/dev/Build/Products/Debug/App.app + cd /Users/dev/App + builtin-codesign --force --sign + +** BUILD SUCCEEDED ** +""" +expected = "** BUILD SUCCEEDED **" + +[[tests.xcodebuild]] +name = "test output keeps test results" +input = """ +note: Using new build system +CompileSwift normal arm64 /Users/dev/AppTests/Tests.swift + cd /Users/dev/App +Test Suite 'All tests' started at 2026-03-10 12:00:00 +Test Suite 'AppTests' started at 2026-03-10 12:00:00 +Test Case '-[AppTests testExample]' passed (0.001 seconds). +Test Case '-[AppTests testFailing]' failed (0.002 seconds). +Test Suite 'AppTests' passed at 2026-03-10 12:00:01 +Executed 2 tests, with 1 failure in 0.003 seconds +""" +expected = "Test Suite 'All tests' started at 2026-03-10 12:00:00\nTest Suite 'AppTests' started at 2026-03-10 12:00:00\nTest Case '-[AppTests testExample]' passed (0.001 seconds).\nTest Case '-[AppTests testFailing]' failed (0.002 seconds).\nTest Suite 'AppTests' passed at 2026-03-10 12:00:01\nExecuted 2 tests, with 1 failure in 0.003 seconds" + +[[tests.xcodebuild]] +name = "empty input returns on_empty message" +input = "" +expected = "xcodebuild: ok" diff --git a/crates/sparkplane-rtk/upstream/src/filters/yadm.toml b/crates/sparkplane-rtk/upstream/src/filters/yadm.toml new file mode 100644 index 0000000..f2cd3d1 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/yadm.toml @@ -0,0 +1,21 @@ +[filters.yadm] +description = "Compact yadm (git wrapper) output — same filtering as git" +match_command = "^yadm\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", + "^\\s*\\(use \"git ", + "^\\s*\\(use \"yadm ", +] +truncate_lines_at = 120 +max_lines = 40 + +[[tests.yadm]] +name = "strips hint lines" +input = "On branch main\nYour branch is up to date with 'origin/main'.\n\n (use \"yadm add\" to update what will be committed)\n\nChanges not staged for commit:\n modified: .bashrc" +expected = "On branch main\nYour branch is up to date with 'origin/main'.\nChanges not staged for commit:\n modified: .bashrc" + +[[tests.yadm]] +name = "short output preserved" +input = "Already up to date." +expected = "Already up to date." diff --git a/crates/sparkplane-rtk/upstream/src/filters/yamllint.toml b/crates/sparkplane-rtk/upstream/src/filters/yamllint.toml new file mode 100644 index 0000000..8a33d51 --- /dev/null +++ b/crates/sparkplane-rtk/upstream/src/filters/yamllint.toml @@ -0,0 +1,25 @@ +[filters.yamllint] +description = "Compact yamllint output — strip blank lines, limit rows" +match_command = "^yamllint\\b" +strip_ansi = true +strip_lines_matching = [ + "^\\s*$", +] +max_lines = 50 +truncate_lines_at = 120 + +[[tests.yamllint]] +name = "multi-warning output stripped of blank lines" +input = """ +config.yml + 3:1 warning missing document start "---" (document-start) + 5:12 error too many spaces inside braces (braces) + + 8:1 error wrong indentation: expected 2 but found 4 (indentation) +""" +expected = "config.yml\n 3:1 warning missing document start \"---\" (document-start)\n 5:12 error too many spaces inside braces (braces)\n 8:1 error wrong indentation: expected 2 but found 4 (indentation)" + +[[tests.yamllint]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/data/economics-prices.json b/data/economics-prices.json new file mode 100644 index 0000000..fa97ceb --- /dev/null +++ b/data/economics-prices.json @@ -0,0 +1,40 @@ +{ + "version": "2026-09-30.1", + "currency": "USD", + "tariff": "standard_uncached", + "models": [ + { + "model": "gpt-6.1-sol", + "source": "https://developers.openai.com/api/docs/models/gpt-6.1-sol", + "verified_at": "2026-09-30", + "effective_from": "2026-09-30", + "input_microusd_per_million": 2000000, + "output_microusd_per_million": 10000000, + "long_context_threshold": 272000, + "long_input_microusd_per_million": 4000000, + "long_output_microusd_per_million": 15000000 + }, + { + "model": "claude-sonnet-4-6", + "source": "https://platform.claude.com/docs/en/models/sonnet-4-6/overview", + "verified_at": "2026-09-30", + "effective_from": "2026-09-30", + "input_microusd_per_million": 3000000, + "output_microusd_per_million": 15000000, + "long_context_threshold": null, + "long_input_microusd_per_million": null, + "long_output_microusd_per_million": null + }, + { + "model": "gemini-3.5-flash", + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "verified_at": "2026-09-30", + "effective_from": "2026-09-30", + "input_microusd_per_million": 1500000, + "output_microusd_per_million": 9000000, + "long_context_threshold": null, + "long_input_microusd_per_million": null, + "long_output_microusd_per_million": null + } + ] +} diff --git a/docs/how-to/develop-spark.md b/docs/how-to/develop-spark.md index fd72d0a..ddc5101 100644 --- a/docs/how-to/develop-spark.md +++ b/docs/how-to/develop-spark.md @@ -418,6 +418,15 @@ The generated `sparkplane-launch-models.json` must include both `text` and `image`. Relaunch Codex so it reads that catalog. Do not hand-edit the catalog or its ownership receipt to make an old session accept attachments. +Updating the Qwen image-count policy from one to 16 also requires a managed +stop and re-serve: the count belongs to the signed engine fingerprint. The +model artifacts are unchanged, so a model download refresh is unnecessary. +Keep the same instance name, context, launch arguments and resources. Verify +successive `view_image` results and a 16-image request through the public +gateway after activation. The shared 512 KiB engine image budget remains; +normalization divides it among the frames, and the gateway enforces the +combined source-pixel limit before resizing. + If regeneration reports an ownership conflict, preserve a private backup of the affected profile and its receipt, move the indicated profile aside, and rerun `launch ... --config --json`. Additional tables such as `[tui]` in an earlier diff --git a/docs/reference/spark.md b/docs/reference/spark.md index f30a07a..b2c8430 100644 --- a/docs/reference/spark.md +++ b/docs/reference/spark.md @@ -37,7 +37,8 @@ sparkplane status --json sparkplane doctor --json sparkplane qualify --manifest --signature (--dry-run | --yes) [--detach] [--json] sparkplane serve [--name ] [--detach] [--dry-run] [--json] -sparkplane launch [--mode auto|inherit] [--model ] [--allow-network] [--config] [--restore] [-y] [-- ...] +sparkplane launch [--mode auto|inherit] [--eco-mode max|none] [--model ] [--allow-network] [--config] [--restore] [-y] [-- ...] +sparkplane economics [--session ] [--json] sparkplane ls [--json] sparkplane ps [--json] sparkplane logs [--limit N] @@ -204,8 +205,10 @@ committed only after exact-container cleanup is acknowledged. ## Engine profiles and modalities - Ornith accepts bounded inline JPEG, PNG, or WebP through OpenAI - Responses and Anthropic Messages. The adapters validate media - type, file magic, decoded bytes, image count, and dimensions + Responses and Anthropic Messages. The gateway resizes and re-encodes oversized + images to the selected qualified engine profile, preserving aspect ratio and + transparency. Images already within the profile's limits keep their original + bytes. The adapters validate media type, file magic, decoded bytes, image count, and dimensions before contacting vLLM. Remote URLs, local paths, traversal, unsupported media, and images sent to text-only instances are rejected. @@ -265,11 +268,145 @@ Existing model records retain their verified artifact traits across upgrades. When a catalog adds vision support, refresh the record through the managed `download --update-alias` operation and re-serve it with the updated engine policy before relaunching Codex. -The Qwen3.8 vLLM policy accepts one inline PNG, JPEG, or WebP image per request, -at most 512 KiB of image data, with width and height at most 4096 pixels. +### Image history and upload limits + +The Qwen3.8 vLLM policy accepts up to 16 inline PNG, JPEG, or WebP images per request, +sharing at most 512 KiB of image data, with width and height at most 4096 pixels. +The gateway accepts larger source uploads and normalizes them before either +native or translated forwarding; the model's image-count limit still applies. +Each frame receives an equal share of the total image byte budget so earlier +screenshots cannot consume the space needed by subsequent `view_image` results. +Long sessions retain a rolling window of images. At 12 images, older images +already followed by a model response are replaced with explicit text references, +leaving six recent frames. Large source payloads trigger earlier eviction of +processed frames to keep their combined inline payload near 16 MiB; this may +retain fewer than six frames. Frames awaiting their first inspection are +protected. A batch of 16 new images remains supported, and larger new batches +must be split. +Tool calls, call IDs, accompanying text and written observations are retained. +The same policy handles Responses, Chat Completions and nested Anthropic tool +results. Text-only profiles continue rejecting image input. + +Vision-enabled `launch` invocations run a private, authenticated loopback adapter. +It streams the historical JSON one item at a time and removes old image payloads +before uploading to the appliance, so repeatedly viewing images does not make +the appliance upload grow with every old screenshot. Source histories may exceed +32 MiB; individual items and the projected request remain capped at 32 MiB, and +history text is capped at 8 MiB. Upload and projection have 120-second deadlines, +with two workers. The adapter uses the pinned appliance CA, refuses redirects, +and exposes only inference routes for the selected instance. Its listener and +invocation credential disappear when the child exits. + +Evicted originals are retained privately in +`/spark/image-archive/`, under stable content-derived filenames. +The model receives a path it can reopen with `view_image`; inherited agent +filesystem permissions still apply. The archive uses 0700 directories and 0600 +files, deduplicates copies, and caps storage at 512 MiB and 4096 images. Writes +remove the least recently used originals when necessary and expire unused +originals after 90 days. Older references can therefore expire; existing source +files remain another way to inspect those images. Direct gateway clients get +the retention policy, but must prune or compact their own uploads before ingress. + +No caption inference is added: descriptions come from the agent's existing +observations. Codex launches compact the conversation at 75% of the qualified +context window, preserving room for new tool results and compaction. Image +retention remains active with `--eco-mode=none`; that switch controls RTK command +compression. These bounds do not change engine context, sampling or resources. +Codex `view_image` results and other typed function/custom tool outputs preserve +inline images as multimodal tool content, including resizing and the shared +request image budget. Their base64 is never converted into prompt text. Regular +string and JSON tool outputs retain their text representation. +The root-owned agent configuration has a separate `[gateway]` upload policy: + +```toml +[gateway] +request_body_bytes = 33554432 # 32 MiB, including JSON and base64 +image_bytes = 16777216 # 16 MiB per decoded source file +image_pixels = 32000000 # all source images combined, before resizing +image_dimension = 16384 # maximum source width or height +max_parallel_requests = 2 # concurrent uploads and image workers +``` + +These defaults also apply to older configurations without a `[gateway]` section. +Configuration ceilings are 64 MiB requests, 32 MiB source files, 32 million +total source pixels, 16384 pixels per dimension, and four concurrent workers. Uploads have a +30-second deadline; image decoding has a 128 MiB allocation limit. The normalized +inference body and other inference POST bodies are capped at 8 MiB; control API +bodies retain their 1 MiB limit. Saturated upload workers return JSON 503 with +`Retry-After: 1`; oversized uploads return protocol-specific JSON 413. Gateway +rejections do not forward images to the engine. Resizing may reduce fine detail +to fit the qualified profile's byte budget. + +Engine request rejections use safe protocol-specific errors: invalid requests +return 400, body limits return 413, and engine capacity limits return 429 with +`Retry-After: 1`. Context overflow returns OpenAI code `context_length_exceeded` +so clients can compact the conversation. Engine error bodies and private request +contents are not forwarded. Unavailable engines and transport failures return +503. + See the [live capability activation procedure](../how-to/develop-spark.md#activate-a-capability-change-on-an-existing-instance) for signing, cached-model refresh, managed restart, and attachment verification. +### Eco mode and session economics + +`--eco-mode=max` is the default. Sparkplane embeds an Apache-2.0 RTK library, +pinned to upstream revision `6d4b77eadee1c66dc1f68466ad77e96d1b6e4989`. +There is no RTK executable download or installer. Invocation-local Codex and +Claude hooks, and an OpenCode plugin, compact supported shell output. The +initial profiles cover Git status, Cargo build/check/clippy/test, pytest and +RTK's bundled declarative filters. Custom RTK configuration is not loaded. +Unknown commands, shell scripts, pipes, redirects, explicit structured output, +binary output and unsuccessful commands pass through unchanged. Captures over +8 MiB per stream pass through; stdout, stderr and the command exit code remain +separate. For an explicit raw escape, use `sparkplane __eco raw -- `. + +`--eco-mode=none` disables Sparkplane's compression hooks while retaining +inference accounting. Compression stays inactive with `--mode inherit`, so +rewriting cannot circumvent the agent's own command permissions. Agent hook +support and trust still apply: for Codex, review the Sparkplane hooks in +`/hooks`. Sparkplane does not grant hook trust. Disabled, unsupported or +untrusted hooks leave the session usable and are reported as inactive. +Persistent agent hook settings are not modified. + +On a supporting appliance, every launch gets a unique inference-only +credential, valid for at most seven days. It is passed only in the child's +environment and revoked when the coding agent exits. Usage comes from engine +responses before protocol translation, including streamed and native Responses +usage; repeated final usage events count once. Other clients' credentials do +not contribute. The session API is administrator-only. Missing engine usage, +interrupted streams or unavailable accounting produce an explicit incomplete +or unavailable result. Older appliances can still launch agents with usage +shown as unavailable. + +The report appears on stderr after normal exit, failure or a handled interrupt. +Finalization has a five-second client timeout. The managed model keeps running; +`stop ` continues to stop only that model. A session report includes +input/output tokens, duration, compressed command counts, estimated output +reduction and two separate USD columns: + +- **Cloud equivalent:** known local inference tokens valued at published + standard uncached input/output rates for GPT-6.1 Sol, Claude Sonnet 4.6 and + Gemini 3.5 Flash. The GPT comparison applies its longer-context tier per + request above 272,000 input tokens. +- **Estimated RTK savings:** removed tool-output bytes divided by four, + valued once at standard input rates. This estimate does not assume repeated + context reuse and is unavailable when compression metrics are incomplete. + +These columns are not added together. Different tokenizers, prompt caching, +hardware and electricity can change actual costs; these are comparisons, not +bills or measured cash savings. The bundled price catalog records official +source links, verification dates, rates and a version. Launch never fetches +prices from the network; each report retains the catalog it used. + +`sparkplane economics` displays the most recent report; +`economics --session --json` emits the structured report on stdout. +Reports are scoped by host under the Sparkplane config directory, with +mode-0700 directories and mode-0600 files. They contain only aggregate usage +and comparison data, never credentials, prompts, command text or output. +Reports older than 90 days are pruned on save or read; the appliance prunes +session/request ledgers when creating a session. Invocation-local compression +metrics are removed when the launcher exits. + ### Internet access for development Launches default to `--mode auto` (`SPARKPLANE_LAUNCH_MODE=auto`): full diff --git a/specs/openapi/sparkplane-control-v1.json b/specs/openapi/sparkplane-control-v1.json index 7aac4cc..ba97ea8 100644 --- a/specs/openapi/sparkplane-control-v1.json +++ b/specs/openapi/sparkplane-control-v1.json @@ -10,6 +10,9 @@ "/api/sparkplane/v1/instances", "/api/sparkplane/v1/instances/{id}", "/api/sparkplane/v1/instances/{id}/logs", + "/api/sparkplane/v1/launch-sessions", + "/api/sparkplane/v1/launch-sessions/{id}", + "/api/sparkplane/v1/launch-sessions/{id}/finish", "/api/sparkplane/v1/metrics", "/api/sparkplane/v1/models", "/api/sparkplane/v1/models/{id}", @@ -48,6 +51,9 @@ "InstanceDocument", "InstanceListDocument", "InstanceObservedState", + "LaunchSessionCreated", + "LaunchSessionDocument", + "LaunchSessionRequest", "ModelArtifactFileDocument", "ModelArtifactFormat", "ModelArtifactRole", @@ -83,6 +89,7 @@ "ResourcePolicy", "ServeAdmissionRequest", "ServeRequest", + "SessionUsage", "StatusDocument", "StopRequest", "TokenCreateRequest", diff --git a/src/main.rs b/src/main.rs index b20e915..09d095b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,6 +14,16 @@ struct Cli { } fn main() { + if std::env::args_os().nth(1).is_some_and(|a| a == "__eco") { + let code = match sparkplane::spark::eco::run(std::env::args_os().skip(2).collect()) { + Ok(code) => code, + Err(error) => { + eprintln!("eco: {error}"); + 1 + } + }; + std::process::exit(code); + } if std::env::args_os().skip(1).eq(["--bridge-protocol"]) { println!("sparkplane.bridge/v1"); return; diff --git a/src/spark/agent.rs b/src/spark/agent.rs index 8a1840e..f498b1b 100644 --- a/src/spark/agent.rs +++ b/src/spark/agent.rs @@ -89,6 +89,8 @@ pub struct AgentConfig { pub resources: ResourcePolicyConfig, pub retention: RetentionConfig, pub models: ModelsConfig, + #[serde(default)] + pub gateway: gateway::IngressLimits, } #[derive(Debug, Clone, Deserialize)] @@ -130,6 +132,8 @@ pub struct AgentState { download_slots: Arc, start_slots: Arc, inference_slots: Arc>, + gateway_limits: gateway::IngressLimits, + ingress_slots: Arc, admission: TransitionCoordinator, routes: RouteRegistry, qualification_authority: Option>, @@ -198,6 +202,10 @@ impl AgentState { download_slots: Arc::new(tokio::sync::Semaphore::new(1)), start_slots: Arc::new(tokio::sync::Semaphore::new(1)), inference_slots: Arc::new(std::sync::Mutex::new(BTreeMap::new())), + gateway_limits: gateway::IngressLimits::default(), + ingress_slots: Arc::new(tokio::sync::Semaphore::new( + gateway::IngressLimits::default().max_parallel_requests, + )), admission: TransitionCoordinator::new(), routes: RouteRegistry::default(), qualification_authority: None, @@ -270,6 +278,12 @@ impl AgentState { self } + fn with_gateway_limits(mut self, limits: gateway::IngressLimits) -> Self { + self.ingress_slots = Arc::new(tokio::sync::Semaphore::new(limits.max_parallel_requests)); + self.gateway_limits = limits; + self + } + fn with_models(mut self, models: HubAcquirer, max_parallel: usize) -> Self { self.models = Some(Arc::new(models)); self.download_slots = Arc::new(tokio::sync::Semaphore::new(max_parallel)); @@ -341,6 +355,9 @@ impl Cidr { operation_events, cancel_operation, create_token, + create_launch_session, + get_launch_session, + finish_launch_session, list_tokens, revoke_token, list_models, @@ -371,6 +388,10 @@ impl Cidr { OperationEvent, TokenCreateRequest, TokenCreatedDocument, + super::wire::LaunchSessionRequest, + super::wire::LaunchSessionCreated, + super::wire::LaunchSessionDocument, + super::wire::SessionUsage, TokenListDocument, DownloadRequest, DownloadPlanDocument, @@ -403,6 +424,18 @@ pub fn router(state: AgentState) -> Router { let authenticated = Router::new() .route(&format!("{API_BASE}/status"), get(status)) .route(&format!("{API_BASE}/doctor"), get(doctor)) + .route( + &format!("{API_BASE}/launch-sessions"), + axum::routing::post(create_launch_session), + ) + .route( + &format!("{API_BASE}/launch-sessions/{{id}}"), + get(get_launch_session), + ) + .route( + &format!("{API_BASE}/launch-sessions/{{id}}/finish"), + axum::routing::post(finish_launch_session), + ) .route(&format!("{API_BASE}/metrics"), get(metrics)) .route( &format!("{API_BASE}/certificates/status"), @@ -451,6 +484,8 @@ pub fn router(state: AgentState) -> Router { get(list_tokens).post(create_token), ) .route(&format!("{API_BASE}/tokens/{{id}}"), delete(revoke_token)) + .layer(DefaultBodyLimit::max(1024 * 1024)); + let inference = Router::new() .route("/openai/{instance}/v1/models", get(gateway_models)) .route( "/openai/{instance}/v1/completions", @@ -478,11 +513,160 @@ pub fn router(state: AgentState) -> Router { axum::routing::post(gateway_anthropic_count_tokens), ) .route("/anthropic/{instance}/{*path}", any(anthropic_not_found)) - .layer(DefaultBodyLimit::max(gateway::MAX_COMPLETION_BODY_BYTES)) + .layer(DefaultBodyLimit::max(gateway::HARD_REQUEST_BYTES)) + .route_layer(middleware::from_fn_with_state( + state.clone(), + inference_ingress, + )); + let authenticated = authenticated + .merge(inference) .route_layer(middleware::from_fn_with_state(state.clone(), authenticate)); authenticated.fallback(not_found).with_state(state) } +// The worker owns a clone so cancellation cannot free its CPU/memory slot early. +#[derive(Clone)] +struct IngressLease(Arc); + +fn ingress_error( + path: &str, + status: StatusCode, + code: &'static str, + message: &'static str, +) -> Response { + if path.starts_with("/anthropic/") { + anthropic_error( + status, + gateway::AnthropicError { + error_type: if status == StatusCode::SERVICE_UNAVAILABLE { + "overloaded_error" + } else { + "invalid_request_error" + }, + message, + }, + ) + } else { + openai_error(status, gateway::OpenAiError { code, message }) + } +} + +async fn inference_ingress( + State(state): State, + request: Request, + next: Next, +) -> Response { + if request.method() != Method::POST { + return next.run(request).await; + } + let path = request.uri().path().to_owned(); + let limit = if path.ends_with("/v1/responses") + || path.ends_with("/v1/messages") + || path.ends_with("/v1/chat/completions") + { + state.gateway_limits.request_body_bytes + } else { + gateway::MAX_COMPLETION_BODY_BYTES.min(state.gateway_limits.request_body_bytes) + }; + if request + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|h| h.to_str().ok()) + .and_then(|h| h.parse::().ok()) + .is_some_and(|size| size > limit as u64) + { + return ingress_error( + &path, + StatusCode::PAYLOAD_TOO_LARGE, + "request_too_large", + "request exceeds the gateway upload byte limit", + ); + } + let permit = match state.ingress_slots.clone().try_acquire_owned() { + Ok(permit) => IngressLease(Arc::new(permit)), + Err(_) => { + let mut response = ingress_error( + &path, + StatusCode::SERVICE_UNAVAILABLE, + "gateway_busy", + "gateway upload and image workers are busy; retry shortly", + ); + response + .headers_mut() + .insert(header::RETRY_AFTER, HeaderValue::from_static("1")); + return response; + } + }; + let (mut parts, body) = request.into_parts(); + let body = match tokio::time::timeout( + Duration::from_secs(30), + axum::body::to_bytes(body, limit), + ) + .await + { + Ok(Ok(body)) => body, + Ok(Err(error)) => { + if !std::error::Error::source(&error) + .is_some_and(|source| source.is::()) + { + return ingress_error( + &path, + StatusCode::BAD_REQUEST, + "invalid_request_error", + "gateway upload body cannot be read", + ); + } + return ingress_error( + &path, + StatusCode::PAYLOAD_TOO_LARGE, + "request_too_large", + "request exceeds the gateway upload byte limit", + ); + } + Err(_) => { + return ingress_error( + &path, + StatusCode::REQUEST_TIMEOUT, + "request_timeout", + "gateway upload deadline exceeded", + ); + } + }; + parts.extensions.insert(permit); + next.run(Request::from_parts(parts, Body::from(body))).await +} + +async fn normalize_gateway_images( + body: Bytes, + protocol: gateway::ImageProtocol, + profile: gateway::GatewayProfile, + limits: gateway::IngressLimits, + lease: IngressLease, +) -> Result, gateway::OpenAiError> { + tokio::task::spawn_blocking(move || { + let _permit = lease.0; + gateway::normalize_image_request(&body, protocol, &profile, &limits) + }) + .await + .unwrap_or(Err(gateway::OpenAiError { + code: "invalid_request_error", + message: "image preprocessing failed", + })) +} + +async fn meter_inference_route( + state: &AgentState, + auth: &AuthenticatedToken, + instance: &str, + route: Arc, +) -> Result, StateError> { + let mut metered_route = (*route).clone(); + metered_route.upstream = metered_route.upstream.with_meter(Arc::new( + super::sessions::UsageMeter::begin(state.database.as_ref(), &auth.id, instance).await?, + )); + Ok(Arc::new(metered_route)) +} + async fn openai_not_found() -> Response { openai_error( StatusCode::NOT_FOUND, @@ -551,6 +735,7 @@ async fn gateway_anthropic_messages( AxumPath(instance): AxumPath, RawQuery(query): RawQuery, Extension(auth): Extension, + Extension(lease): Extension, headers: HeaderMap, body: Result, ) -> Response { @@ -583,6 +768,30 @@ async fn gateway_anthropic_messages( if !route.profile.allows(PublicAction::Responses) { return anthropic_not_found().await; } + let body = match normalize_gateway_images( + body, + gateway::ImageProtocol::Anthropic, + route.profile.clone(), + state.gateway_limits.clone(), + lease, + ) + .await + { + Ok(body) => body, + Err(error) => { + return anthropic_error( + if error.code == "request_too_large" { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_REQUEST + }, + gateway::AnthropicError { + error_type: "invalid_request_error", + message: error.message, + }, + ); + } + }; let request = match gateway::rewrite_anthropic_request_with_profile( &body, &route.served_model, @@ -591,6 +800,10 @@ async fn gateway_anthropic_messages( Ok(request) => request, Err(error) => return anthropic_error(StatusCode::BAD_REQUEST, error), }; + let route = match meter_inference_route(&state, &auth, &instance, route).await { + Ok(route) => route, + Err(error) => return state_problem(error), + }; let token_permit = match auth.acquire_inference().await { Ok(permit) => permit, Err(()) => return anthropic_upstream_unavailable(), @@ -605,7 +818,7 @@ async fn gateway_anthropic_messages( .await { Ok(upstream) => upstream, - Err(_) => return anthropic_upstream_unavailable(), + Err(error) => return anthropic_generation_error(error), }; let encoder = if request.omit_reasoning { gateway::AnthropicEncoder::with_omitted_reasoning(route.public_model.clone()) @@ -834,6 +1047,15 @@ async fn gateway_completions( RouteLookup::Warming => return gateway_warming(), RouteLookup::Missing => return not_found().await, }; + let mut metered_route = (*route).clone(); + metered_route.upstream = metered_route.upstream.with_meter(Arc::new( + match super::sessions::UsageMeter::begin(state.database.as_ref(), &auth.id, &instance).await + { + Ok(meter) => meter, + Err(error) => return state_problem(error), + }, + )); + let route = Arc::new(metered_route); if !route.profile.allows(PublicAction::Completions) { return not_found().await; } @@ -855,7 +1077,7 @@ async fn gateway_completions( if streaming { let upstream = match route.upstream.completion_stream(&upstream_body).await { Ok(stream) => stream, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; let public_model = route.public_model.clone(); let events = futures_util::stream::unfold( @@ -913,7 +1135,7 @@ async fn gateway_completions( .request("POST", "/v1/completions", upstream_body.len()) { Ok(request) => request, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; match route.upstream.send(&request, &upstream_body).await { Ok(response) if (200..300).contains(&response.status) => { @@ -922,21 +1144,74 @@ async fn gateway_completions( Err(()) => gateway_upstream_unavailable(), } } - _ => gateway_upstream_unavailable(), + Ok(response) => gateway_upstream_response_error(response), + Err(error) => gateway_generation_error(error), } } fn openai_error(status: StatusCode, error: gateway::OpenAiError) -> Response { - let error_type = if error.code == "server_error" { - "server_error" - } else { - "invalid_request_error" + let error_type = match error.code { + "server_error" => "server_error", + "rate_limit_error" => "rate_limit_error", + _ => "invalid_request_error", }; let body = serde_json::json!({"error": {"message": error.message, "type": error_type, "param": null, "code": error.code}}); (status, Json(body)).into_response() } +fn gateway_request_rejected(rejection: super::upstream::RequestRejection) -> Response { + let mut response = openai_error( + StatusCode::from_u16(rejection.status()).expect("finite engine rejection status"), + gateway::OpenAiError { + code: rejection.code(), + message: rejection.message(), + }, + ); + if rejection == super::upstream::RequestRejection::RateLimited { + response + .headers_mut() + .insert(header::RETRY_AFTER, HeaderValue::from_static("1")); + } + response +} + +fn gateway_generation_error(error: super::upstream::UpstreamError) -> Response { + error + .request_rejection() + .map(gateway_request_rejected) + .unwrap_or_else(gateway_upstream_unavailable) +} + +fn gateway_upstream_response_error(response: super::upstream::UpstreamResponse) -> Response { + super::upstream::RequestRejection::from_response(response.status, &response.bytes) + .map(gateway_request_rejected) + .unwrap_or_else(gateway_upstream_unavailable) +} + +fn anthropic_generation_error(error: super::upstream::UpstreamError) -> Response { + let Some(rejection) = error.request_rejection() else { + return anthropic_upstream_unavailable(); + }; + let mut response = anthropic_error( + StatusCode::from_u16(rejection.status()).expect("finite engine rejection status"), + gateway::AnthropicError { + error_type: if rejection == super::upstream::RequestRejection::RateLimited { + "rate_limit_error" + } else { + "invalid_request_error" + }, + message: rejection.message(), + }, + ); + if rejection == super::upstream::RequestRejection::RateLimited { + response + .headers_mut() + .insert(header::RETRY_AFTER, HeaderValue::from_static("1")); + } + response +} + fn valid_inference_headers(headers: &HeaderMap) -> bool { headers.len() <= 64 && headers @@ -962,6 +1237,7 @@ async fn gateway_responses( AxumPath(instance): AxumPath, RawQuery(query): RawQuery, Extension(auth): Extension, + Extension(lease): Extension, headers: HeaderMap, body: Bytes, ) -> Response { @@ -985,6 +1261,27 @@ async fn gateway_responses( if !route.profile.allows(PublicAction::Responses) { return openai_not_found().await; } + let body = match normalize_gateway_images( + body, + gateway::ImageProtocol::Responses, + route.profile.clone(), + state.gateway_limits.clone(), + lease, + ) + .await + { + Ok(body) => body, + Err(error) => { + return openai_error( + if error.code == "request_too_large" { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_REQUEST + }, + error, + ); + } + }; if route.profile.native_responses { let request = match gateway::rewrite_native_responses_request_with_profile( &body, @@ -994,6 +1291,10 @@ async fn gateway_responses( Ok(request) => request, Err(error) => return openai_error(StatusCode::BAD_REQUEST, error), }; + let route = match meter_inference_route(&state, &auth, &instance, route).await { + Ok(route) => route, + Err(error) => return state_problem(error), + }; let token_permit = match auth.acquire_inference().await { Ok(permit) => permit, Err(()) => return gateway_upstream_unavailable(), @@ -1008,6 +1309,10 @@ async fn gateway_responses( Ok(request) => request, Err(error) => return openai_error(StatusCode::BAD_REQUEST, error), }; + let route = match meter_inference_route(&state, &auth, &instance, route).await { + Ok(route) => route, + Err(error) => return state_problem(error), + }; let token_permit = match auth.acquire_inference().await { Ok(permit) => permit, Err(()) => return gateway_upstream_unavailable(), @@ -1031,7 +1336,7 @@ async fn gateway_native_responses( .await { Ok(upstream) => upstream, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; let stream = futures_util::stream::unfold( (upstream, permit, token_permit), @@ -1054,7 +1359,7 @@ async fn gateway_native_responses( .request("POST", "/v1/responses", request.body.len()) { Ok(request) => request, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; let response = route .upstream @@ -1067,6 +1372,9 @@ async fn gateway_native_responses( drop(permit); drop(token_permit); match response { + Ok(response) if !(200..300).contains(&response.status) => { + gateway_upstream_response_error(response) + } Ok(response) => Response::builder() .status( StatusCode::from_u16(response.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), @@ -1150,8 +1458,9 @@ async fn gateway_chat_completions( AxumPath(instance): AxumPath, RawQuery(query): RawQuery, Extension(auth): Extension, + Extension(lease): Extension, headers: HeaderMap, - body: Bytes, + body: Result, ) -> Response { if reject_query(query).is_some() || !valid_inference_headers(&headers) @@ -1170,9 +1479,50 @@ async fn gateway_chat_completions( RouteLookup::Warming => return gateway_warming(), RouteLookup::Missing => return not_found().await, }; + let mut metered_route = (*route).clone(); + metered_route.upstream = metered_route.upstream.with_meter(Arc::new( + match super::sessions::UsageMeter::begin(state.database.as_ref(), &auth.id, &instance).await + { + Ok(meter) => meter, + Err(error) => return state_problem(error), + }, + )); + let route = Arc::new(metered_route); if !route.profile.allows(PublicAction::Chat) { return openai_not_found().await; } + let body = match body { + Ok(body) => body, + Err(_) => { + return ingress_error( + "/openai/v1/chat/completions", + StatusCode::PAYLOAD_TOO_LARGE, + "request_too_large", + "request body is too large", + ); + } + }; + let body = match normalize_gateway_images( + body, + gateway::ImageProtocol::Chat, + route.profile.clone(), + state.gateway_limits.clone(), + lease, + ) + .await + { + Ok(body) => body, + Err(error) => { + return openai_error( + if error.code == "request_too_large" { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_REQUEST + }, + error, + ); + } + }; let request = match gateway::rewrite_chat_request_with_profile( &body, &route.served_model, @@ -1204,7 +1554,7 @@ async fn gateway_chat( .await { Ok(upstream) => upstream, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; return chat_sse(upstream, route.public_model.clone(), permit, token_permit); } @@ -1214,7 +1564,7 @@ async fn gateway_chat( .request("POST", "/v1/chat/completions", request.body.len()) { Ok(request) => request, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; let response = route.upstream.send(&upstream_request, &request.body).await; drop(permit); @@ -1225,7 +1575,8 @@ async fn gateway_chat( .map(IntoResponse::into_response) .unwrap_or_else(|error| openai_error(StatusCode::BAD_GATEWAY, error)) } - _ => gateway_upstream_unavailable(), + Ok(response) => gateway_upstream_response_error(response), + Err(error) => gateway_generation_error(error), } } @@ -1272,7 +1623,7 @@ async fn gateway_responses_stream( .await { Ok(upstream) => upstream, - Err(_) => return gateway_upstream_unavailable(), + Err(error) => return gateway_generation_error(error), }; let encoder = gateway::ResponsesEncoder::new(route.public_model.clone(), request.custom_tools); if request.stream { @@ -2323,6 +2674,14 @@ async fn reconcile_once( .await; continue; } + if let Some(routes) = routes { + routes.mark_warming(&instance.name, instance.generation); + } + // Admission can be temporarily closed while startup telemetry settles. + // Only an admitted restart attempt consumes the engine failure budget. + if !persistent_restart_allowed(database, executor).await? { + continue; + } let failed = database .record_restart_failure(&instance.id, instance.generation, unix_millis() / 1_000) .await @@ -2340,9 +2699,6 @@ async fn reconcile_once( else { continue; }; - if !persistent_restart_allowed(database, executor).await? { - continue; - } let Ok(prepared) = executor .prepare_instance(StartInstanceInput { instance_id: instance.id.clone(), @@ -2489,7 +2845,7 @@ async fn reconcile_running_engine( if let Some(routes) = routes { routes.mark_warming(&instance.name, instance.generation); } - if !observed.running || !persistent_restart_allowed(database, executor).await? { + if !observed.running { let _ = executor .disable_restart(StopInstanceInput { instance_id: instance.id.clone(), @@ -2503,6 +2859,20 @@ async fn reconcile_running_engine( .map_err(|_| ())?; return Ok(()); } + if !persistent_restart_allowed(database, executor).await? { + // Keep the route unpublished and Docker restart disabled, but do not + // classify an admission deferral as a failure of this running engine. + if observed.restart_policy != "no" { + let _ = executor + .disable_restart(StopInstanceInput { + instance_id: instance.id.clone(), + generation: instance.generation, + grace_seconds: 0, + }) + .await; + } + return Ok(()); + } let address = observed.address.parse().map_err(|_| ())?; let route = ObservedRoute::new( &observed.instance_id, @@ -4216,6 +4586,95 @@ fn qualification_executor_problem(error: ExecutorClientError) -> Response { problem(status, code, detail) } +#[utoipa::path(post, path = "/api/sparkplane/v1/launch-sessions", request_body = super::wire::LaunchSessionRequest, responses((status = 200, body = super::wire::LaunchSessionCreated)))] +async fn create_launch_session( + State(state): State, + Extension(auth): Extension, + RawQuery(query): RawQuery, + Json(body): Json, +) -> Response { + if let Some(response) = reject_query(query) { + return response; + } + if let Err(response) = require_executor_for_mutation(&state).await { + return response; + } + let Some(db) = &state.database else { + return database_unavailable(); + }; + match db.create_launch_session(&auth.id, body).await { + Ok(created) => { + match db.auth_snapshot().await { + Ok(snapshot) => state.store_auth(snapshot), + Err(error) => return state_problem(error), + } + let mut response = Json(created).into_response(); + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + response + } + Err(error) => state_problem(error), + } +} + +#[utoipa::path(get, path = "/api/sparkplane/v1/launch-sessions/{id}", params(("id" = String, Path)), responses((status = 200, body = super::wire::LaunchSessionDocument)))] +async fn get_launch_session( + State(state): State, + AxumPath(id): AxumPath, + RawQuery(query): RawQuery, +) -> Response { + if let Some(response) = reject_query(query) { + return response; + } + if id.parse::().is_err() { + return state_problem(StateError::Invalid("invalid session id".into())); + } + let Some(db) = &state.database else { + return database_unavailable(); + }; + match db.launch_session(&id, false, "").await { + Ok(session) => Json(session).into_response(), + Err(error) => state_problem(error), + } +} + +#[utoipa::path(post, path = "/api/sparkplane/v1/launch-sessions/{id}/finish", params(("id" = String, Path)), responses((status = 200, body = super::wire::LaunchSessionDocument)))] +async fn finish_launch_session( + State(state): State, + AxumPath(id): AxumPath, + Extension(auth): Extension, + RawQuery(query): RawQuery, +) -> Response { + if let Some(response) = reject_query(query) { + return response; + } + if id.parse::().is_err() { + return state_problem(StateError::Invalid("invalid session id".into())); + } + let Some(db) = &state.database else { + return database_unavailable(); + }; + match db.launch_session(&id, true, &auth.id).await { + Ok(mut session) => { + match db.auth_snapshot().await { + Ok(snapshot) => state.store_auth(snapshot), + Err(error) => return state_problem(error), + } + let deadline = tokio::time::Instant::now() + Duration::from_secs(4); + while session.usage.pending_requests > 0 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(20)).await; + match db.launch_session(&id, false, "").await { + Ok(value) => session = value, + Err(_) => break, + } + } + Json(session).into_response() + } + Err(error) => state_problem(error), + } +} + #[utoipa::path(post, path = "/api/sparkplane/v1/tokens", request_body = TokenCreateRequest, responses((status = 202, body = TokenCreatedDocument)))] async fn create_token( State(state): State, @@ -4520,6 +4979,14 @@ async fn authenticate( let Some(verifier) = snapshot.tokens.get(id) else { return auth_failed(&request_path); }; + if verifier.token.revoked_at.is_some() + || verifier.token.expires_at.as_ref().is_some_and(|expiry| { + chrono::DateTime::parse_from_rfc3339(expiry) + .map_or(true, |expiry| expiry <= chrono::Utc::now()) + }) + { + return auth_failed(&request_path); + } if !verifier.verify( &state .database @@ -4615,7 +5082,8 @@ fn required_scope(method: &Method, path: &str) -> Option { TokenScope::InstancesWrite }); } - if path.starts_with(&format!("{API_BASE}/tokens")) + if path.starts_with(&format!("{API_BASE}/launch-sessions")) + || path.starts_with(&format!("{API_BASE}/tokens")) || path == format!("{API_BASE}/doctor") || path.starts_with(&format!("{API_BASE}/certificates")) { @@ -4877,6 +5345,7 @@ pub async fn serve( config.schema == "sparkplane.agent/v1", "unsupported Spark agent configuration schema" ); + config.gateway.validate().map_err(anyhow::Error::msg)?; anyhow::ensure!( config.engine_catalog == Path::new("/etc/sparkplane/engines"), "engine catalog must use the fixed root-owned path" @@ -4989,7 +5458,8 @@ pub async fn serve( )?, config.operations.max_parallel_downloads, ) - .with_start_slots(config.operations.max_parallel_starts); + .with_start_slots(config.operations.max_parallel_starts) + .with_gateway_limits(config.gateway); state = state.with_qualification_authority(Arc::::from(qualification_authority)); if let Some(executor) = executor { state = state.with_executor(executor); @@ -5191,6 +5661,44 @@ mod tests { const TOKEN: &str = "test-bootstrap-token-with-at-least-256-bits-of-random-material"; + #[test] + fn gateway_limits_default_for_existing_configs_and_have_hard_ceilings() { + let mut config: toml::Value = + toml::from_str(include_str!("../../configs/sparkplane/agent.toml")).unwrap(); + let shipped: super::AgentConfig = config.clone().try_into().unwrap(); + shipped.gateway.validate().unwrap(); + config.as_table_mut().unwrap().remove("gateway"); + let legacy: super::AgentConfig = config.try_into().unwrap(); + assert_eq!( + serde_json::to_value(shipped.gateway).unwrap(), + serde_json::to_value(legacy.gateway).unwrap() + ); + for field in [ + "request_body_bytes", + "image_bytes", + "image_pixels", + "image_dimension", + "max_parallel_requests", + ] { + let mut limits = + serde_json::to_value(crate::spark::gateway::IngressLimits::default()).unwrap(); + limits[field] = 0.into(); + assert!( + serde_json::from_value::(limits.clone()) + .unwrap() + .validate() + .is_err() + ); + limits[field] = u32::MAX.into(); + assert!( + serde_json::from_value::(limits) + .unwrap() + .validate() + .is_err() + ); + } + } + fn llama_chat_sse() -> String { [ r#"{"choices":[{"delta":{"reasoning_content":"inspect"},"finish_reason":null}]}"#, @@ -5435,14 +5943,550 @@ mod tests { } #[tokio::test] - async fn native_responses_route_preserves_tool_history_and_engine_sse() { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.unwrap(); - let mut request = vec![0; 16_384]; - let count = socket.read(&mut request).await.unwrap(); + async fn inference_upload_rejections_are_json_and_authentication_runs_first() { + let state = AgentState::new(TOKEN, Vec::new(), Vec::new()); + for path in [ + "/openai/fixture/v1/responses", + "/anthropic/fixture/v1/messages", + ] { + let request = || { + Request::post(path) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(vec![b' '; 33 * 1024 * 1024])) + .unwrap() + }; + let denied = router(state.clone()).oneshot(request()).await.unwrap(); + assert_eq!(denied.status(), axum::http::StatusCode::UNAUTHORIZED); + let mut request = request(); + request.headers_mut().insert( + header::AUTHORIZATION, + format!("Bearer {TOKEN}").parse().unwrap(), + ); + let response = router(state.clone()).oneshot(request).await.unwrap(); + assert_eq!(response.status(), axum::http::StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(response.headers()[header::CONTENT_TYPE], "application/json"); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + let document: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert!( + document["error"]["message"] + .as_str() + .unwrap() + .contains("gateway") + ); + } + } + + #[tokio::test] + async fn large_image_uploads_fit_the_qualified_profile_over_https() { + use base64::Engine as _; + use sha2::{Digest, Sha256}; + let (sent, mut received) = tokio::sync::mpsc::unbounded_channel(); + let engine = axum::Router::new().route("/v1/{*path}", axum::routing::post( + move |uri: axum::http::Uri, axum::Json(body): axum::Json| { + let sent = sent.clone(); + async move { + let native = uri.path() == "/v1/responses"; + sent.send(body).unwrap(); + let body = if native { + "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":7,\"output_tokens\":3}}}\n\n".to_owned() + } else {llama_chat_sse()}; + ([(header::CONTENT_TYPE, "text/event-stream")], body) + } + } + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let engine_address = listener.local_addr().unwrap(); + let engine_server = + tokio::spawn(async move { axum::serve(listener, engine).await.unwrap() }); + let state = AgentState::new(TOKEN, Vec::new(), Vec::new()); + let profile = crate::spark::engine::EnginePolicy::parse(include_str!( + "../../configs/sparkplane/engines/vllm-qwen38-mmap.toml" + )) + .unwrap() + .gateway_profile(None); + for (name, native) in [ + ("native", true), + ("translated", false), + ("anthropic", false), + ("tool-native", true), + ("tool-translated", false), + ] { + let mut profile = profile.clone(); + profile.native_responses = native; + let upstream = crate::spark::upstream::ObservedRoute::new( + "i_11111111111111111111111111111111", + 1, + engine_address.ip(), + engine_address.port(), + [("POST", "/v1/responses"), ("POST", "/v1/chat/completions")], + ) + .unwrap(); + state.routes.publish_with_profile( + name, + "public-model".into(), + "served-model".into(), + profile, + upstream, + ); + } + let rcgen::CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let certificate = cert.pem(); + let tls = super::tls13_config( + certificate.as_bytes().to_vec(), + signing_key.serialize_pem().into_bytes(), + ) + .await + .unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let handle = axum_server::Handle::new(); + let server_handle = handle.clone(); + let server = tokio::spawn(async move { + axum_server::from_tcp_rustls(listener, tls) + .unwrap() + .handle(server_handle) + .serve(router(state).into_make_service_with_connect_info::()) + .await + .unwrap(); + }); + let client = reqwest::Client::builder() + .tls_built_in_root_certs(false) + .add_root_certificate(reqwest::Certificate::from_pem(certificate.as_bytes()).unwrap()) + .build() + .unwrap(); + let mut seed = 0x12345678_u32; + let pixels = (0..1024 * 512 * 3) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed as u8 + }) + .collect(); + let image = + image::DynamicImage::ImageRgb8(image::RgbImage::from_raw(1024, 512, pixels).unwrap()); + let mut png = std::io::Cursor::new(Vec::new()); + image.write_to(&mut png, image::ImageFormat::Png).unwrap(); + let encoded = base64::engine::general_purpose::STANDARD.encode(png.into_inner()); + for name in [ + "native", + "translated", + "anthropic", + "tool-native", + "tool-translated", + ] { + let mut request = if name == "anthropic" { + serde_json::json!({"model":"public-model","max_tokens":8,"stream":true,"messages":[{"role":"user","content":[{"type":"text","text":"describe"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":encoded}}]}]}) + } else { + serde_json::json!({"input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"describe"},{"type":"input_image","image_url":format!("data:image/png;base64,{encoded}"),"detail":"high"}]}],"stream":true,"store":false}) + }; + if name.starts_with("tool-") { + let image = request["input"][0]["content"][1].clone(); + request["input"] = serde_json::json!([ + {"type":"message","role":"user","content":"inspect the screenshot"}, + {"type":"function_call","call_id":"call_view","name":"view_image","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_view","output":[{"type":"input_text","text":"Screenshot from the tool."},image]} + ]); + } + assert!(serde_json::to_vec(&request).unwrap().len() > 1024 * 1024); + let path = if name == "anthropic" { + format!("/anthropic/{name}/v1/messages") + } else { + format!("/openai/{name}/v1/responses") + }; + let response = client + .post(format!("https://localhost:{}{path}", address.port())) + .bearer_auth(TOKEN) + .header("anthropic-version", "2023-06-01") + .json(&request) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.bytes().await.unwrap(); + assert_eq!( + status, + reqwest::StatusCode::OK, + "{}", + String::from_utf8_lossy(&body) + ); + let forwarded = received.recv().await.unwrap(); + let url = if name == "tool-native" { + forwarded["input"][2]["output"][1]["image_url"] + .as_str() + .unwrap() + } else if name == "tool-translated" { + assert_eq!(forwarded["messages"][2]["role"], "tool"); + assert_eq!(forwarded["messages"][2]["tool_call_id"], "call_view"); + forwarded["messages"][2]["content"][1]["image_url"]["url"] + .as_str() + .unwrap() + } else if name == "native" { + forwarded["input"][0]["content"][1]["image_url"] + .as_str() + .unwrap() + } else { + forwarded["messages"][0]["content"][1]["image_url"]["url"] + .as_str() + .unwrap() + }; + let (media, data) = url + .strip_prefix("data:") + .unwrap() + .split_once(";base64,") + .unwrap(); + let bytes = base64::engine::general_purpose::STANDARD + .decode(data) + .unwrap(); + let vision = profile.vision.as_ref().unwrap(); + assert!(vision.media_types.contains(media)); + assert!(bytes.len() <= vision.max_bytes); + let decoded = image::ImageReader::new(std::io::Cursor::new(bytes)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + assert!(decoded.width() <= vision.max_width && decoded.height() <= vision.max_height); + assert!(decoded.width() <= 1024 && decoded.height() <= 512); + assert_eq!(decoded.width(), decoded.height() * 2); + } + let mut small_png = std::io::Cursor::new(Vec::new()); + image + .resize(256, 128, image::imageops::FilterType::Lanczos3) + .write_to(&mut small_png, image::ImageFormat::Png) + .unwrap(); + let small_encoded = + base64::engine::general_purpose::STANDARD.encode(small_png.into_inner()); + for count in [2, 3, 16] { + for name in ["tool-native", "tool-translated", "anthropic"] { + let encoded = if count == 16 { + &small_encoded + } else { + &encoded + }; + let request = if name == "anthropic" { + let image = serde_json::json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":encoded}}); + serde_json::json!({"model":"public-model","max_tokens":8,"stream":true,"messages":[{"role":"user","content":vec![image;count]}]}) + } else { + let image = serde_json::json!({"type":"input_image","image_url":format!("data:image/png;base64,{encoded}")}); + let mut input = vec![ + serde_json::json!({"type":"message","role":"user","content":"compare the screenshots"}), + ]; + for index in 0..count { + let id = format!("view_{index}"); + input.push(serde_json::json!({"type":"function_call","name":"view_image","call_id":id,"arguments":"{}"})); + input.push(serde_json::json!({"type":"function_call_output","call_id":id,"output":[{"type":"input_text","text":format!("Frame {index}")},image]})); + if index + 1 < count { + input.push(serde_json::json!({"type":"message","role":"assistant","content":"Inspect the next screenshot."})); + } + } + serde_json::json!({"input":input,"stream":true}) + }; + let path = if name == "anthropic" { + format!("/anthropic/{name}/v1/messages") + } else { + format!("/openai/{name}/v1/responses") + }; + let response = client + .post(format!("https://localhost:{}{path}", address.port())) + .bearer_auth(TOKEN) + .header("anthropic-version", "2023-06-01") + .json(&request) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.bytes().await.unwrap(); + assert_eq!( + status, + reqwest::StatusCode::OK, + "{}", + String::from_utf8_lossy(&body) + ); + let forwarded = received.recv().await.unwrap(); + let messages = forwarded[if name == "tool-native" { + "input" + } else { + "messages" + }] + .as_array() + .unwrap(); + let mut images = 0; + let mut total = 0; + let retained = if count == 16 && name != "anthropic" { + 10 + } else { + count + }; + for message in messages { + let parts = message[if message["type"] == "function_call_output" { + "output" + } else { + "content" + }] + .as_array(); + for part in parts.into_iter().flatten() { + let url = if part["type"] == "input_image" { + part["image_url"].as_str() + } else { + part["image_url"]["url"].as_str() + }; + if let Some(url) = url { + let bytes = base64::engine::general_purpose::STANDARD + .decode(url.split_once(";base64,").unwrap().1) + .unwrap(); + assert!( + bytes.len() + <= profile.vision.as_ref().unwrap().max_total_bytes / retained + ); + total += bytes.len(); + images += 1; + } + } + } + assert_eq!(images, retained); + assert!(total <= profile.vision.as_ref().unwrap().max_total_bytes); + } + } + // The launch adapter streams a source history larger than ingress allows, + // then forwards only its recent window over the same pinned HTTPS route. + let config = tempfile::tempdir().unwrap(); + std::fs::create_dir(config.path().join("spark")).unwrap(); + std::fs::write(config.path().join("spark/fixture.ca.pem"), &certificate).unwrap(); + let pin = format!("sha256:{:x}", Sha256::digest(certificate.as_bytes())); + std::fs::write(config.path().join("spark.toml"), format!("[hosts.fixture]\nurl='https://localhost:{}'\nca_cert_sha256='{pin}'\ncredential='spark/unused'\nrequest_timeout_seconds=30\n", address.port())).unwrap(); + let adapter = crate::spark::inference_adapter::Adapter::start( + config.path(), + "fixture", + "tool-native", + TOKEN, + ) + .unwrap(); + let image = serde_json::json!({"type":"input_image","image_url":format!("data:image/png;base64,{encoded}")}); + let mut input = vec![ + serde_json::json!({"type":"message","role":"user","content":"Inspect the screenshots."}), + ]; + for index in 0..25 { + input.push(serde_json::json!({"type":"function_call","name":"view_image","call_id":format!("long_{index}"),"arguments":"{}"})); + input.push(serde_json::json!({"type":"function_call_output","call_id":format!("long_{index}"),"output":[{"type":"input_text","text":format!("Frame {index}")},image]})); + if index < 24 { + input.push(serde_json::json!({"type":"message","role":"assistant","content":"Observed the button."})); + } + } + let payload = + serde_json::to_vec(&serde_json::json!({"input":input,"stream":true})).unwrap(); + assert!(payload.len() > crate::spark::gateway::IngressLimits::default().request_body_bytes); + let response = client + .post(format!("{}/openai/v1/responses", adapter.base_url)) + .bearer_auth(adapter.token()) + .header("content-type", "application/json") + .body(payload) + .send() + .await + .unwrap(); + let status = response.status(); + let text = response.text().await.unwrap(); + assert_eq!(status, reqwest::StatusCode::OK, "{text}"); + assert!(text.contains("response.completed")); + let mut forwarded = received.recv().await.unwrap(); + let mut count = 0; + crate::spark::image_history::visit_parts( + &mut forwarded["input"], + crate::spark::image_history::Protocol::Responses, + &mut |_| count += 1, + ); + assert!(count <= 12); + assert_eq!(forwarded["input"][74]["call_id"], "long_24"); + assert_eq!(forwarded["input"][74]["output"][0]["text"], "Frame 24"); + drop(adapter); + handle.shutdown(); + server.await.unwrap(); + engine_server.abort(); + } + + #[tokio::test] + async fn upstream_client_rejections_keep_safe_status_codes_and_context_errors() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for (status, body, expected, code) in [ + ( + 400, + r#"{"error":{"code":"context_length_exceeded","message":"private prompt SECRET"}}"#, + 400, + "context_length_exceeded", + ), + ( + 400, + r#"{"error":{"message":"This model's maximum context length is 262144 tokens. private prompt SECRET"}}"#, + 400, + "context_length_exceeded", + ), + ( + 400, + r#"{"error":{"message":"private prompt SECRET"}}"#, + 400, + "invalid_request_error", + ), + (413, "private prompt SECRET", 413, "request_too_large"), + (429, "private prompt SECRET", 429, "rate_limit_error"), + (500, "private prompt SECRET", 503, "server_error"), + ] { + for kind in [ + "responses", + "native_stream", + "native_json", + "chat_json", + "anthropic", + ] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 16384]; + assert!(socket.read(&mut request).await.unwrap() > 0); + socket.write_all(format!("HTTP/1.1 {status} Error\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",body.len()).as_bytes()).await.unwrap(); + }); + let state = AgentState::new(TOKEN, Vec::new(), Vec::new()); + let upstream = crate::spark::upstream::ObservedRoute::new( + "i_11111111111111111111111111111111", + 1, + address.ip(), + address.port(), + [("POST", "/v1/chat/completions"), ("POST", "/v1/responses")], + ) + .unwrap(); + let mut profile = crate::spark::gateway::GatewayProfile::text(); + profile.native_responses = kind.starts_with("native"); + profile.native_response_timeout_seconds = 5; + state.routes.publish_with_profile( + "fixture", + "public-model".into(), + "served-model".into(), + profile, + upstream, + ); + let (path, body) = match kind { + "anthropic" => ( + "/anthropic/fixture/v1/messages", + r#"{"model":"public-model","max_tokens":8,"messages":[{"role":"user","content":"work"}]}"#, + ), + "chat_json" => ( + "/openai/fixture/v1/chat/completions", + r#"{"messages":[{"role":"user","content":"work"}],"stream":false}"#, + ), + "native_json" => ( + "/openai/fixture/v1/responses", + r#"{"input":"work","stream":false}"#, + ), + _ => ( + "/openai/fixture/v1/responses", + r#"{"input":"work","stream":true}"#, + ), + }; + let request = Request::post(path) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .header("anthropic-version", "2023-06-01") + .body(Body::from(body)) + .unwrap(); + let response = router(state).oneshot(request).await.unwrap(); + assert_eq!( + response.status().as_u16(), + expected, + "{kind}, engine status {status}" + ); + if expected == 429 { + assert_eq!(response.headers()[header::RETRY_AFTER], "1"); + } + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("SECRET")); + assert!(!String::from_utf8_lossy(&bytes).contains("private prompt")); + let response: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + if kind == "anthropic" { + assert_eq!(response["type"], "error"); + assert_eq!( + response["error"]["type"], + if expected == 503 { + "api_error" + } else if expected == 429 { + "rate_limit_error" + } else { + "invalid_request_error" + } + ); + } else { + assert_eq!(response["error"]["code"], code); + } + server.await.unwrap(); + } + } + } + + #[tokio::test] + async fn gateway_busy_rejections_are_bounded_and_release_after_cancellation() { + let state = AgentState::new(TOKEN, Vec::new(), Vec::new()); + let first = state.ingress_slots.clone().try_acquire_owned().unwrap(); + let second = state.ingress_slots.clone().try_acquire_owned().unwrap(); + for path in [ + "/openai/fixture/v1/responses", + "/anthropic/fixture/v1/messages", + ] { + let response = router(state.clone()) + .oneshot( + Request::post(path) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .body(Body::from("{}")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + response.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE + ); + assert_eq!(response.headers()[header::RETRY_AFTER], "1"); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + assert!( + serde_json::from_slice::(&bytes).unwrap()["error"]["message"] + .as_str() + .unwrap() + .contains("busy") + ); + } + drop((first, second)); + // A cancelled upload frees its slot without waiting for the 30-second deadline. + let request = Request::post("/openai/fixture/v1/responses") + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .body(Body::from_stream(futures_util::stream::pending::< + Result, + >())) + .unwrap(); + assert!( + tokio::time::timeout( + Duration::from_millis(50), + router(state.clone()).oneshot(request) + ) + .await + .is_err() + ); + assert_eq!(state.ingress_slots.available_permits(), 2); + } + + #[tokio::test] + async fn native_responses_route_preserves_tool_history_and_engine_sse() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 16_384]; + let count = socket.read(&mut request).await.unwrap(); let request = String::from_utf8_lossy(&request[..count]); assert!(request.starts_with("POST /v1/responses ")); assert!(request.contains(r#""model":"public-model""#)); @@ -6317,6 +7361,62 @@ mod tests { mutations: Arc, } + struct DeferredReconcileExecutor { + matched: bool, + swap_in_pages_delta: Option, + guard_heartbeat: bool, + disables: Arc, + unexpected_mutations: Arc, + } + + impl sparkplane_ipc::Handler for DeferredReconcileExecutor { + async fn handle(&self, request: sparkplane_ipc::Request) -> sparkplane_ipc::Response { + let action = &request.params["action"]; + if let Some(expected) = action.get("reconcile_scan") { + let identity = &expected[0]; + let matched = if self.matched { + vec![serde_json::json!({ + "instance_id":identity["instance_id"],"generation":identity["generation"], + "container_id":"container-g1","network_id":"network","address":"172.30.0.2","port":8000, + "running":true,"restart_policy":"unless-stopped","health_method":"GET","health_path":"/health", + "allowed_routes":[["GET","/health"]],"gateway_profile":crate::spark::gateway::GatewayProfile::text(), + "served_model":"Ornith-1.5-9B","semantic_prompt":"health","semantic_max_tokens":1, + "startup_deadline_seconds":900,"init_pid":1,"pid_start_time_ticks":1, + "cgroup_path":"/system.slice/docker-container-g1.scope" + })] + } else { + Vec::new() + }; + return sparkplane_ipc::Response::Ok { + schema_version: sparkplane_ipc::SCHEMA_VERSION, + request_id: request.request_id, + result: serde_json::json!({"action":"reconcile_scan","scan":{ + "matched":matched,"missing":if self.matched { serde_json::json!([]) } else { expected.clone() },"quarantined":[]}}), + blob: None, + }; + } + if action.get("disable_restart_policy").is_some() { + self.disables.fetch_add(1, Ordering::SeqCst); + } else if action.get("prepare_instance").is_some() + || action.get("start_instance").is_some() + || action.get("promote_restart_policy").is_some() + || action.get("stop_instance").is_some() + { + self.unexpected_mutations.fetch_add(1, Ordering::SeqCst); + } + let mut response = ResourceExecutor.handle(request).await; + if let sparkplane_ipc::Response::Ok { result, .. } = &mut response { + if result["action"] == "inspect_resources" { + result["snapshot"]["swap_in_pages_delta"] = + serde_json::json!(self.swap_in_pages_delta); + } else if result["action"] == "health" { + result["health"]["guard_heartbeat"] = serde_json::json!(self.guard_heartbeat); + } + } + response + } + } + struct StopRaceExecutor { stops: Arc, removed: Arc, @@ -6659,6 +7759,84 @@ mod tests { database.shutdown().unwrap(); } + #[tokio::test] + async fn deferred_reconciliation_keeps_admission_closed_without_consuming_restart_budget() { + for (matched, swap_in_pages_delta, guard_heartbeat) in [ + (true, None, true), + (true, Some(1), true), + (true, Some(0), false), + (false, Some(1), true), + ] { + let (state, database, root) = durable_state().await; + let socket = root.join("executor.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let disables = Arc::new(AtomicUsize::new(0)); + let unexpected_mutations = Arc::new(AtomicUsize::new(0)); + let server = tokio::spawn( + sparkplane_ipc::Server::new(DeferredReconcileExecutor { + matched, + swap_in_pages_delta, + guard_heartbeat, + disables: Arc::clone(&disables), + unexpected_mutations: Arc::clone(&unexpected_mutations), + }) + .serve(listener), + ); + let model = ornith_model(); + database.promote_model(model.clone(), false).await.unwrap(); + database + .accept_operation( + "bootstrap", + "instance.serve", + "01K00000000000000000000000", + &"a".repeat(64), + Some("ornith".into()), + ) + .await + .unwrap(); + let instance = database + .begin_serve(creating_instance(&model)) + .await + .unwrap() + .instance; + if !matched { + let route = crate::spark::upstream::ObservedRoute::new( + &instance.id, + instance.generation, + "127.0.0.1".parse().unwrap(), + 9, + [("GET", "/health")], + ) + .unwrap(); + state.routes.publish( + &instance.name, + instance.model.clone(), + "served-model".into(), + route, + ); + } + let executor = crate::spark::executor::ExecutorClient::new(socket); + for _ in 0..6 { + super::reconcile_once(&database, &executor, Some(&state.routes), &state.admission) + .await + .unwrap(); + } + let current = database.instance(&instance.id).await.unwrap(); + assert_eq!(current.restart_failures, 0); + assert!(!current.restart_suppressed); + assert_eq!(current.engine_fingerprint, instance.engine_fingerprint); + assert_eq!(current.generation, instance.generation); + assert!(!matches!( + state.routes.lookup(&instance.name), + crate::spark::gateway::RouteLookup::Healthy(_) + )); + assert_eq!(disables.load(Ordering::SeqCst), if matched { 6 } else { 0 }); + assert_eq!(unexpected_mutations.load(Ordering::SeqCst), 0); + database.shutdown().unwrap(); + server.abort(); + } + } + #[tokio::test] async fn reconcile_restarts_the_same_engine_and_artifact_generation() { let (state, database, root) = durable_state().await; @@ -7201,6 +8379,313 @@ mod tests { server.abort(); } + #[tokio::test] + async fn launch_session_usage_and_revocation_round_trip_over_https() { + use crate::spark::wire::{LaunchSessionCreated, LaunchSessionDocument}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let (state, database, root) = durable_state().await; + let model = ornith_model(); + database.promote_model(model.clone(), false).await.unwrap(); + let instance = database + .begin_serve(creating_instance(&model)) + .await + .unwrap() + .instance; + let rcgen::CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let certificate = cert.pem(); + let tls = super::tls13_config( + certificate.as_bytes().to_vec(), + signing_key.serialize_pem().into_bytes(), + ) + .await + .unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let handle = axum_server::Handle::new(); + let server_handle = handle.clone(); + let server_state = state.clone(); + let server = tokio::spawn(async move { + axum_server::from_tcp_rustls(listener, tls) + .unwrap() + .handle(server_handle) + .serve(router(server_state).into_make_service_with_connect_info::()) + .await + .unwrap(); + }); + let client = reqwest::Client::builder() + .tls_built_in_root_certs(false) + .add_root_certificate(reqwest::Certificate::from_pem(certificate.as_bytes()).unwrap()) + .build() + .unwrap(); + let base = format!("https://localhost:{}", address.port()); + let id = ulid::Ulid::new().to_string(); + let create = + serde_json::json!({"id":id,"instance":"ornith","integration":"codex","eco_mode":"max"}); + let response = client + .post(format!("{base}{API_BASE}/launch-sessions")) + .bearer_auth(TOKEN) + .json(&create) + .send() + .await + .unwrap() + .error_for_status() + .unwrap(); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + let created: LaunchSessionCreated = response.json().await.unwrap(); + let bearer = created.bearer_token.unwrap(); + let repeated: LaunchSessionCreated = client + .post(format!("{base}{API_BASE}/launch-sessions")) + .bearer_auth(TOKEN) + .json(&create) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert!(repeated.bearer_token.is_none()); + assert_eq!( + client + .get(format!("{base}{API_BASE}/launch-sessions/{id}")) + .bearer_auth(&bearer) + .send() + .await + .unwrap() + .status(), + reqwest::StatusCode::UNAUTHORIZED + ); + for (path, body, missing, shared) in [ + ( + "/openai/ornith/v1/responses", + serde_json::json!({"input":"private fixture prompt","stream":false}), + false, + false, + ), + ( + "/anthropic/ornith/v1/messages", + serde_json::json!({"model":"sparkplane","messages":[{"role":"user","content":"fixture"}],"max_tokens":10,"stream":true}), + false, + false, + ), + ( + "/openai/ornith/v1/chat/completions", + serde_json::json!({"messages":[{"role":"user","content":"fixture"}],"stream":true}), + false, + false, + ), + ( + "/openai/ornith/v1/responses", + serde_json::json!({"input":"fixture","stream":true}), + true, + false, + ), + ( + "/openai/ornith/v1/responses", + serde_json::json!({"input":"another client","stream":false}), + false, + true, + ), + ] { + let fixture = if missing { + llama_chat_sse() + .lines() + .filter(|line| !line.contains("usage")) + .collect::>() + .join("\n") + + "\n" + } else { + llama_chat_sse() + }; + let (engine, task) = llama_chat_server_with(fixture).await; + let upstream = crate::spark::upstream::ObservedRoute::new( + &instance.id, + instance.generation, + engine.ip(), + engine.port(), + [("POST", "/v1/chat/completions")], + ) + .unwrap(); + state.routes.publish( + "ornith", + model.canonical.clone(), + "Ornith-1.5-9B".into(), + upstream, + ); + let response = client + .post(format!("{base}{path}")) + .header("anthropic-version", "2023-06-01") + .bearer_auth(if shared { TOKEN } else { &bearer }) + .json(&body) + .send() + .await + .unwrap(); + let status = response.status(); + let output = response.bytes().await.unwrap(); + assert!( + status.is_success(), + "{path}: {status}: {}", + String::from_utf8_lossy(&output) + ); + assert!(!output.is_empty()); + task.await.unwrap(); + } + for stream in [false, true] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let engine = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = [0; 16384]; + let count = socket.read(&mut request).await.unwrap(); + assert!( + String::from_utf8_lossy(&request[..count]).starts_with("POST /v1/responses ") + ); + let document = r#"{"id":"resp_fixture","object":"response","status":"completed","output":[],"usage":{"input_tokens":11,"output_tokens":5,"total_tokens":16}}"#; + let body = if stream { + format!( + "data: {{\"type\":\"response.completed\",\"response\":{document}}}\r\n\r\n" + ) + } else { + document.into() + }; + let content = if stream { + "text/event-stream" + } else { + "application/json" + }; + socket.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: {content}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",body.len()).as_bytes()).await.unwrap(); + for chunk in body.as_bytes().chunks(3) { + socket.write_all(chunk).await.unwrap(); + tokio::task::yield_now().await; + } + }); + let upstream = crate::spark::upstream::ObservedRoute::new( + &instance.id, + instance.generation, + engine.ip(), + engine.port(), + [("POST", "/v1/responses")], + ) + .unwrap(); + let mut profile = crate::spark::gateway::GatewayProfile::text(); + profile.native_responses = true; + profile.native_response_timeout_seconds = 5; + state.routes.publish_with_profile( + "ornith", + model.canonical.clone(), + "Ornith-1.5-9B".into(), + profile, + upstream, + ); + let output = client + .post(format!("{base}/openai/ornith/v1/responses")) + .bearer_auth(&bearer) + .json(&serde_json::json!({"input":"fixture","stream":stream})) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .bytes() + .await + .unwrap(); + assert!(String::from_utf8_lossy(&output).contains("resp_fixture")); + task.await.unwrap(); + } + let finish = format!("{base}{API_BASE}/launch-sessions/{id}/finish"); + let finished: LaunchSessionDocument = client + .post(&finish) + .bearer_auth(TOKEN) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(finished.usage.requests, 6); + assert_eq!(finished.usage.input_tokens, 43); + assert_eq!(finished.usage.output_tokens, 19); + assert_eq!(finished.usage.unknown_usage_requests, 1); + assert_eq!(finished.usage.pending_requests, 0); + let repeated: LaunchSessionDocument = client + .post(&finish) + .bearer_auth(TOKEN) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(finished.finished_at, repeated.finished_at); + assert_eq!(repeated.usage.input_tokens, 43); + assert_eq!( + client + .get(format!("{base}/openai/ornith/v1/models")) + .bearer_auth(&bearer) + .send() + .await + .unwrap() + .status(), + reqwest::StatusCode::UNAUTHORIZED + ); + let expired_create = serde_json::json!({"id":ulid::Ulid::new().to_string(),"instance":"ornith","integration":"claude","eco_mode":"none"}); + let expiring: LaunchSessionCreated = client + .post(format!("{base}{API_BASE}/launch-sessions")) + .bearer_auth(TOKEN) + .json(&expired_create) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + let expired_bearer = expiring.bearer_token.unwrap(); + let expired_id = expired_bearer + .strip_prefix("sparkplane_") + .unwrap() + .split('_') + .next() + .unwrap(); + let mut snapshot = database.auth_snapshot().await.unwrap(); + snapshot + .tokens + .get_mut(expired_id) + .unwrap() + .token + .expires_at = Some((chrono::Utc::now() - chrono::Duration::seconds(1)).to_rfc3339()); + state.store_auth(snapshot); + assert_eq!( + client + .get(format!("{base}/openai/ornith/v1/models")) + .bearer_auth(expired_bearer) + .send() + .await + .unwrap() + .status(), + reqwest::StatusCode::UNAUTHORIZED + ); + let document = serde_json::to_string(&finished).unwrap(); + assert!(!document.contains("private fixture prompt")); + assert!(!document.contains(&bearer)); + assert_eq!( + database.list_instances().await.unwrap()[0].desired, + crate::spark::wire::InstanceDesiredState::Running + ); + handle.graceful_shutdown(None); + server.await.unwrap(); + database.shutdown().unwrap(); + std::fs::remove_dir_all(root).unwrap(); + } + async fn durable_state() -> (super::AgentState, crate::spark::state::DbActor, PathBuf) { let root = tempfile::tempdir().unwrap().keep(); let database = crate::spark::state::DbActor::open( diff --git a/src/spark/cli.rs b/src/spark/cli.rs index b5d1247..45a9f02 100644 --- a/src/spark/cli.rs +++ b/src/spark/cli.rs @@ -126,6 +126,8 @@ pub enum SparkCommand { after_help = "Examples:\n sparkplane dgx-spark launch codex --model ornith-1.5:35b\n sparkplane dgx-spark launch codex --mode inherit --allow-network -- --sandbox workspace-write\n sparkplane dgx-spark launch claude --mode inherit --model ornith-1.5:35b -- --permission-mode plan\n sparkplane dgx-spark launch opencode --config --model ornith-1.5:35b\n sparkplane dgx-spark launch codex --model ornith-1.5:35b --dry-run --json\n\nArguments after `--` are passed directly to the selected local agent without a shell. The Spark administrator credential is never given to the child process.\n\nAuto mode (default) gives agents full access without action approvals. Use --mode inherit to retain agent permissions. --allow-network requests invocation-only network access: Codex workspace-write networking, or Claude sandbox domains. OpenCode already has network access; its tool permissions remain unchanged. Filesystem sandboxing and approval policies are not disabled. Managed restrictions still apply. Claude --settings cannot be forwarded with this flag.\n\nEnvironment:\n SPARKPLANE_LAUNCH_MODE, SPARKPLANE_LAUNCH_MODEL, SPARKPLANE_LAUNCH_ALLOW_NETWORK, SPARKPLANE_LAUNCH_CONFIG, SPARKPLANE_LAUNCH_RESTORE, SPARKPLANE_YES, SPARKPLANE_DRY_RUN, SPARKPLANE_JSON, SPARKPLANE_CONFIG_DIR" )] Launch(LaunchArgs), + /// View a retained private session economics report. + Economics(EconomicsArgs), /// List currently active managed model processes. #[command( after_help = "Examples:\n sparkplane dgx-spark ps\n sparkplane dgx-spark ps --json\n\nThe default output is a compact table. Absent and failed historical instances are omitted.\n\nEnvironment:\n SPARKPLANE_JSON, SPARKPLANE_CONFIG_DIR" @@ -430,6 +432,9 @@ pub struct LaunchArgs { /// Agent permissions: auto grants full access without action approvals; inherit keeps agent settings. #[arg(long, default_value = "auto", value_parser = ["auto", "inherit"], env = "SPARKPLANE_LAUNCH_MODE")] pub mode: String, + /// Embedded RTK compression: max enables supported filters; none disables it. + #[arg(long, default_value = "max", value_parser = ["max", "none"], env = "SPARKPLANE_LAUNCH_ECO_MODE")] + pub eco_mode: String, /// Allow this agent session's sandbox network access; keep filesystem and approval policies. #[arg(long, env = "SPARKPLANE_LAUNCH_ALLOW_NETWORK", conflicts_with_all = ["configure", "restore"])] pub allow_network: bool, @@ -451,6 +456,16 @@ pub struct LaunchArgs { pub extra_args: Vec, } +#[derive(Debug, Args)] +pub struct EconomicsArgs { + #[arg(long)] + pub session: Option, + #[arg(long, env = "SPARKPLANE_JSON")] + pub json: bool, + #[arg(long, env = "SPARKPLANE_CONFIG_DIR")] + pub config_dir: Option, +} + #[derive(Debug, Args)] pub struct StopArgs { pub instance: String, @@ -747,6 +762,19 @@ pub fn dispatch(cli: SparkCli) -> anyhow::Result<()> { SparkCommand::Download(args) => dispatch_download(&cli.host, args), SparkCommand::Serve(args) => dispatch_serve(&cli.host, args), SparkCommand::Launch(args) => dispatch_launch(&cli.host, args), + SparkCommand::Economics(args) => { + let report = super::economics::load( + &args.config_dir.unwrap_or_else(default_config_dir), + &cli.host, + args.session.as_deref(), + )?; + if args.json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + super::economics::display(&report); + } + Ok(()) + } SparkCommand::Ps(args) => dispatch_instances(&cli.host, args), SparkCommand::Logs(args) => dispatch_logs(&cli.host, args), SparkCommand::Stop(args) => dispatch_stop(&cli.host, args), diff --git a/src/spark/client.rs b/src/spark/client.rs index 9b870cc..1a3169e 100644 --- a/src/spark/client.rs +++ b/src/spark/client.rs @@ -73,6 +73,39 @@ pub struct ClaudeCodeClientConfig { pub shell: String, } +/// Inference launched through the local adapter keeps the appliance's pinned TLS +/// authority and never follows redirects carrying its private session credential. +pub(super) fn inference_transport( + config_dir: &Path, + host: &str, +) -> Result<(reqwest::Client, Url), ClientError> { + let profiles: Profiles = toml::from_str(&read_text(&config_dir.join("spark.toml"))?) + .map_err(|_| usage("invalid Spark host profiles"))?; + let profile = profiles + .hosts + .get(host) + .ok_or_else(|| usage("Spark host is not configured"))?; + let base = Url::parse(&profile.url).map_err(|_| usage("invalid Spark URL"))?; + reject_plaintext_lan(&base)?; + let ca = fs::read(config_dir.join("spark").join(format!("{host}.ca.pem"))) + .map_err(|_| unreachable("pinned Spark CA certificate is unavailable"))?; + if format!("sha256:{:x}", Sha256::digest(&ca)) != profile.ca_cert_sha256 { + return Err(unreachable("pinned Spark CA fingerprint does not match")); + } + let certificate = reqwest::Certificate::from_pem(&ca) + .map_err(|_| unreachable("pinned Spark CA certificate is invalid"))?; + let http = reqwest::Client::builder() + .tls_built_in_root_certs(false) + .add_root_certificate(certificate) + .https_only(base.scheme() == "https") + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(1800)) + .build() + .map_err(|_| unreachable("could not construct pinned inference transport"))?; + Ok((http, base)) +} + pub fn codex_client_config( config_dir: &Path, host: &str, @@ -230,6 +263,54 @@ impl Drop for SparkClient { } impl SparkClient { + pub fn create_launch_session( + &self, + request: &super::wire::LaunchSessionRequest, + ) -> Result, ClientError> { + let url = self + .base + .join("api/sparkplane/v1/launch-sessions") + .map_err(|_| usage("invalid session route"))?; + let response = self + .http + .post(url) + .bearer_auth(&self.token) + .timeout(Duration::from_secs(5)) + .json(request) + .send() + .map_err(|_| unreachable("session accounting unavailable"))?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + if !response.status().is_success() { + return Err(map_problem(response.status(), response.json().ok())); + } + decode_response(response).map(Some) + } + + pub fn finish_launch_session( + &self, + id: &str, + ) -> Result { + if id.parse::().is_err() { + return Err(usage("invalid session id")); + } + let url = self + .base + .join(&format!("api/sparkplane/v1/launch-sessions/{id}/finish")) + .map_err(|_| usage("invalid session route"))?; + let response = self + .http + .post(url) + .bearer_auth(&self.token) + .timeout(Duration::from_secs(5)) + .send() + .map_err(|_| unreachable("session finalization unavailable"))?; + if !response.status().is_success() { + return Err(map_problem(response.status(), response.json().ok())); + } + decode_response(response) + } pub fn load(config_dir: &Path, host: &str) -> Result { let profiles: Profiles = toml::from_str(&read_text(&config_dir.join("spark.toml"))?) .map_err(|error| usage(format!("invalid Spark host profiles: {error}")))?; @@ -658,7 +739,7 @@ fn valid_model_id(id: &str) -> bool { .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) } -fn validate_instance_reference(value: &str) -> Result<(), ClientError> { +pub(super) fn validate_instance_reference(value: &str) -> Result<(), ClientError> { if !value.is_empty() && value.len() <= 96 && value diff --git a/src/spark/eco.rs b/src/spark/eco.rs new file mode 100644 index 0000000..8c0d06d --- /dev/null +++ b/src/spark/eco.rs @@ -0,0 +1,568 @@ +//! Invocation-local agent hooks backed by the embedded RTK library. +use super::{ + cli::LaunchIntegration, + economics::{self, Compression}, +}; +use anyhow::{Result, ensure}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::{ + ffi::OsString, + fs, + io::{self, Read, Write}, + path::{Path, PathBuf}, + process::{Command, ExitStatus, Stdio}, +}; +use tokio::io::AsyncReadExt; + +const MAX_CAPTURE: usize = 8 * 1024 * 1024; + +#[derive(Serialize, Deserialize)] +struct Metric { + category: String, + raw_bytes: u64, + filtered_bytes: u64, + error: bool, +} + +pub struct EcoSession { + path: PathBuf, + enabled: bool, + policy_enabled: bool, +} +impl EcoSession { + pub fn create(id: &str, enabled: bool, mode: &str) -> Result { + ensure!(id.parse::().is_ok(), "invalid session id"); + let path = std::env::temp_dir().join(format!("sparkplane-eco-{id}")); + ensure!(!path.exists(), "eco session directory already exists"); + economics::private_dir(&path)?; + Ok(Self { + path, + enabled, + policy_enabled: mode == "auto", + }) + } + pub fn configure(&self, command: &mut Command, integration: LaunchIntegration) -> Result<()> { + command.env("SPARKPLANE_ECO_SESSION_DIR", &self.path).env( + "SPARKPLANE_ECO_REWRITE_ENABLED", + if self.enabled && self.policy_enabled { + "1" + } else { + "0" + }, + ); + if !self.enabled || !self.policy_enabled { + return Ok(()); + } + let binary = std::env::current_exe()?; + let hook = format!( + "{} __eco hook {}", + shell_quote(&binary.to_string_lossy()), + integration.as_str() + ); + let hooks = json!({"PreToolUse":[{"matcher":"Bash","hooks":[{"type":"command","command":hook,"timeout":3}]}],"SessionStart":[{"hooks":[{"type":"command","command":hook,"timeout":3}]}]}); + match integration { + LaunchIntegration::Codex => { + for event in ["PreToolUse", "SessionStart"] { + let table: toml::Value = serde_json::from_value(hooks[event].clone())?; + command.arg("-c").arg(format!("hooks.{event}={}", table)); + } + } + LaunchIntegration::Claude => { + // Auto mode owns invocation settings. Carry its permission settings into the + // combined object; user settings files remain loaded by Claude itself. + let args = command + .get_args() + .map(|v| v.to_string_lossy().into_owned()) + .collect::>(); + let mut settings = args + .windows(2) + .rfind(|pair| pair[0] == "--settings") + .and_then(|pair| serde_json::from_str::(&pair[1]).ok()) + .unwrap_or_else(|| json!({})); + settings["hooks"] = hooks; + let retained = command.get_args().map(OsString::from).collect::>(); + // Command has no remove-argument API. Rebuild with the same program, + // environment and inherited stdio, replacing only owned settings. + let mut replacement = Command::new(command.get_program()); + for (key, value) in command.get_envs() { + if let Some(value) = value { + replacement.env(key, value); + } else { + replacement.env_remove(key); + } + } + if let Some(dir) = command.get_current_dir() { + replacement.current_dir(dir); + } + let mut index = 0; + while index < retained.len() { + if retained[index] == "--settings" { + index += 2; + } else { + replacement.arg(&retained[index]); + index += 1; + } + } + replacement + .arg("--settings") + .arg(serde_json::to_string(&settings)?); + *command = replacement; + } + LaunchIntegration::Opencode => { + let plugin = self.path.join("eco.ts"); + fs::write(&plugin, opencode_plugin(&binary))?; + let content = command + .get_envs() + .find(|(k, _)| *k == "OPENCODE_CONFIG_CONTENT") + .and_then(|(_, v)| v) + .ok_or_else(|| anyhow::anyhow!("missing OpenCode session config"))?; + let mut settings: Value = serde_json::from_str(&content.to_string_lossy())?; + let plugins = settings + .as_object_mut() + .ok_or_else(|| anyhow::anyhow!("invalid OpenCode config"))? + .entry("plugin") + .or_insert_with(|| json!([])); + plugins + .as_array_mut() + .ok_or_else(|| anyhow::anyhow!("invalid OpenCode plugins"))? + .push(json!(format!("file://{}", plugin.display()))); + command.env("OPENCODE_CONFIG_CONTENT", serde_json::to_string(&settings)?); + } + } + Ok(()) + } + pub fn summary(&self) -> Compression { + let mut result = Compression { + status: if !self.enabled { + "disabled" + } else if !self.policy_enabled { + "inactive (inherited permissions)" + } else { + "inactive (hook unavailable or awaiting trust)" + } + .into(), + ..Compression::default() + }; + if !self.enabled || !self.policy_enabled { + return result; + } + let Ok(entries) = fs::read_dir(&self.path) else { + result.metrics_errors += 1; + return result; + }; + for entry in entries { + let Ok(entry) = entry else { + result.metrics_errors += 1; + continue; + }; + let path = entry.path(); + if path.extension().is_none_or(|v| v != "json") { + continue; + } + match economics::read_record::(&path) { + Ok(metric) if metric.category == "hook" => result.status = "active".into(), + Ok(metric) => { + result.commands += 1; + result.status = "active".into(); + result.raw_bytes = result.raw_bytes.saturating_add(metric.raw_bytes); + result.filtered_bytes = + result.filtered_bytes.saturating_add(metric.filtered_bytes); + if metric.error { + result.metrics_errors += 1; + } + } + Err(_) => result.metrics_errors += 1, + } + } + result + } +} +impl Drop for EcoSession { + fn drop(&mut self) { + if fs::symlink_metadata(&self.path).is_ok() && economics::private_dir(&self.path).is_ok() { + let _ = fs::remove_dir_all(&self.path); + } + } +} + +fn opencode_plugin(binary: &Path) -> String { + format!( + r#"export const SparkplaneEco = async () => {{ + const binary = {}; + const activate = Bun.spawn([binary, "__eco", "activate"], {{stdout:"ignore",stderr:"ignore"}}); + await activate.exited; + return {{"tool.execute.before": async (input, output) => {{ + if (!["bash", "shell"].includes(String(input.tool).toLowerCase())) return; + if (typeof output.args?.command !== "string") return; + const child = Bun.spawn([binary, "__eco", "rewrite", output.args.command], {{stdout:"pipe",stderr:"ignore"}}); + const command = (await new Response(child.stdout).text()).trim(); + if (await child.exited === 0 && command) output.args.command = command; + }}}}; +}}; +"#, + serde_json::to_string(&binary.to_string_lossy()).expect("path JSON") + ) +} + +pub fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) +} + +pub fn rewrite(command: &str) -> Result> { + if command.len() > 64 * 1024 + || command.chars().any(|c| { + matches!( + c, + '\n' | '\r' + | '\0' + | '$' + | '`' + | '|' + | '&' + | ';' + | '<' + | '>' + | '*' + | '?' + | '(' + | ')' + ) + }) + { + return Ok(None); + } + let Some(args) = shlex::split(command) else { + return Ok(None); + }; + if sparkplane_rtk::select(&args).is_none() { + return Ok(None); + } + let binary = std::env::current_exe()?; + Ok(Some(format!( + "{} __eco exec -- {}", + shell_quote(&binary.to_string_lossy()), + args.iter() + .map(|a| shell_quote(a)) + .collect::>() + .join(" ") + ))) +} + +fn metric(metric: &Metric) { + metric_named(&uuid::Uuid::new_v4().to_string(), metric); +} + +fn metric_named(id: &str, metric: &Metric) { + let Some(path) = std::env::var_os("SPARKPLANE_ECO_SESSION_DIR").map(PathBuf::from) else { + return; + }; + if economics::private_dir(&path).is_err() { + eprintln!("Sparkplane eco metrics unavailable"); + return; + } + if economics::write_record(&path.join(format!("{id}.json")), metric).is_err() { + eprintln!("Sparkplane eco metrics unavailable"); + } +} + +fn activate() { + metric(&Metric { + category: "hook".into(), + raw_bytes: 0, + filtered_bytes: 0, + error: false, + }); +} + +pub fn run(args: Vec) -> Result { + let Some(action) = args.first().and_then(|a| a.to_str()) else { + anyhow::bail!("missing eco action"); + }; + match action { + "activate" => { + activate(); + Ok(0) + } + "rewrite" => { + ensure!(args.len() == 2, "rewrite expects one command"); + if std::env::var("SPARKPLANE_ECO_REWRITE_ENABLED") + .ok() + .as_deref() + != Some("0") + && let Some(command) = rewrite(&args[1].to_string_lossy())? + { + println!("{command}"); + } + Ok(0) + } + "hook" => { + let mut input = String::new(); + io::stdin() + .take(1024 * 1024 + 1) + .read_to_string(&mut input)?; + if input.len() > 1024 * 1024 { + return Ok(0); + } + let Ok(payload) = serde_json::from_str::(&input) else { + return Ok(0); + }; + if std::env::var("SPARKPLANE_ECO_REWRITE_ENABLED") + .ok() + .as_deref() + != Some("1") + { + return Ok(0); + } + activate(); + if payload.get("hook_event_name").and_then(Value::as_str) == Some("SessionStart") { + return Ok(0); + } + if payload.get("tool_name").and_then(Value::as_str) != Some("Bash") { + return Ok(0); + } + let Some(command) = payload + .pointer("/tool_input/command") + .and_then(Value::as_str) + else { + return Ok(0); + }; + if let Some(rewritten) = rewrite(command)? { + println!( + "{}", + json!({"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"allow","updatedInput":{"command":rewritten}}}) + ); + } + Ok(0) + } + "exec" | "raw" => { + let operands = &args[1..]; + let operands = operands + .strip_prefix(&[OsString::from("--")]) + .unwrap_or(operands); + ensure!(!operands.is_empty(), "missing command"); + let mut command = Command::new(&operands[0]); + command.args(&operands[1..]); + if action == "raw" { + return Ok(exit_code(wait(command)?)); + } + execute(command, operands) + } + _ => anyhow::bail!("unknown eco action"), + } +} + +pub fn exit_code(status: ExitStatus) -> i32 { + use std::os::unix::process::ExitStatusExt; + status + .code() + .unwrap_or_else(|| 128 + status.signal().unwrap_or(1)) +} + +/// Keep the parent alive long enough to finish accounting when its child exits. +pub fn wait(command: Command) -> Result { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(async { + let mut child = tokio::process::Command::from(command).spawn()?; + wait_child(&mut child, false).await + }) +} + +async fn wait_child(child: &mut tokio::process::Child, group: bool) -> Result { + use tokio::signal::unix::{SignalKind, signal}; + let mut interrupt = signal(SignalKind::interrupt())?; + let mut terminate = signal(SignalKind::terminate())?; + loop { + let signum = tokio::select! {status=child.wait()=>return Ok(status?),_=interrupt.recv()=>nix::sys::signal::Signal::SIGINT,_=terminate.recv()=>nix::sys::signal::Signal::SIGTERM}; + if let Some(pid) = child.id() { + let pid = pid as i32; + let _ = nix::sys::signal::kill( + nix::unistd::Pid::from_raw(if group { -pid } else { pid }), + signum, + ); + } + } +} + +async fn capture( + mut reader: impl tokio::io::AsyncRead + Unpin, + stderr: bool, +) -> io::Result<(Vec, u64, bool)> { + let mut bytes = Vec::new(); + let mut count = 0u64; + let mut streamed = false; + let mut chunk = [0u8; 8192]; + loop { + let n = reader.read(&mut chunk).await?; + if n == 0 { + break; + } + count = count.saturating_add(n as u64); + if !streamed && bytes.len() + n <= MAX_CAPTURE { + bytes.extend_from_slice(&chunk[..n]); + } else { + if stderr { + let mut out = io::stderr().lock(); + if !streamed { + out.write_all(&bytes)?; + } + out.write_all(&chunk[..n])?; + } else { + let mut out = io::stdout().lock(); + if !streamed { + out.write_all(&bytes)?; + } + out.write_all(&chunk[..n])?; + } + bytes.clear(); + streamed = true; + } + } + Ok((bytes, count, streamed)) +} + +fn execute(mut command: Command, args: &[OsString]) -> Result { + use std::os::unix::process::CommandExt; + let args = args + .iter() + .map(|v| v.to_string_lossy().into_owned()) + .collect::>(); + let filter = sparkplane_rtk::select(&args); + let metric_id = uuid::Uuid::new_v4().to_string(); + // An interrupted wrapper or failed final write leaves an incomplete record. + metric_named( + &metric_id, + &Metric { + category: "command".into(), + raw_bytes: 0, + filtered_bytes: 0, + error: true, + }, + ); + command + .stdin(Stdio::inherit()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(async { + let mut child = match tokio::process::Command::from(command).spawn() { + Ok(child) => child, + Err(error) => { + eprintln!("could not execute command: {error}"); + return Ok(if error.kind() == io::ErrorKind::NotFound { + 127 + } else { + 126 + }); + } + }; + let stdout = child.stdout.take().expect("stdout pipe"); + let stderr = child.stderr.take().expect("stderr pipe"); + let (status, out, err) = tokio::join!( + wait_child(&mut child, true), + capture(stdout, false), + capture(stderr, true) + ); + let status = exit_code(status?); + let (out, out_count, out_streamed) = out?; + let (err, err_count, err_streamed) = err?; + let filtered_out = filter + .as_ref() + .map(|f| sparkplane_rtk::filter(f, &out, status)) + .unwrap_or_else(|| out.clone()); + let filtered_err = match &filter { + Some(sparkplane_rtk::Filter::Cargo(label)) => sparkplane_rtk::filter( + &sparkplane_rtk::Filter::Cargo(if *label == "test" { "build" } else { label }), + &err, + status, + ), + _ => err.clone(), + }; + if !out_streamed { + io::stdout().write_all(&filtered_out)?; + } + if !err_streamed { + io::stderr().write_all(&filtered_err)?; + } + let filtered_count = if out_streamed { + out_count + } else { + filtered_out.len() as u64 + } + if err_streamed { + err_count + } else { + filtered_err.len() as u64 + }; + metric_named( + &metric_id, + &Metric { + category: "command".into(), + raw_bytes: out_count.saturating_add(err_count), + filtered_bytes: filtered_count, + error: false, + }, + ); + Ok(status) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hooks_preserve_permissions_and_stay_invocation_local() { + let session = EcoSession::create(&ulid::Ulid::new().to_string(), true, "auto").unwrap(); + let mut claude = Command::new("claude"); + claude + .args([ + "--permission-mode", + "bypassPermissions", + "--settings", + r#"{"sandbox":{"enabled":false},"skipDangerousModePermissionPrompt":true}"#, + ]) + .env("FIXTURE", "kept"); + session + .configure(&mut claude, LaunchIntegration::Claude) + .unwrap(); + let args = claude + .get_args() + .map(|s| s.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + args.iter().filter(|s| s.as_str() == "--settings").count(), + 1 + ); + let settings: Value = serde_json::from_str(&args[3]).unwrap(); + assert_eq!(settings["sandbox"]["enabled"], false); + assert_eq!(settings["skipDangerousModePermissionPrompt"], true); + assert!(settings["hooks"]["PreToolUse"].is_array()); + assert!(claude.get_envs().any(|(key,value)|key=="FIXTURE"&&value==Some(std::ffi::OsStr::new("kept")))); + let mut codex = Command::new("codex"); + session + .configure(&mut codex, LaunchIntegration::Codex) + .unwrap(); + let args = codex + .get_args() + .map(|s| s.to_string_lossy().into_owned()) + .collect::>(); + for arg in args.iter().skip(1).step_by(2) { + let _: toml::Value = toml::from_str(arg).unwrap(); + } + assert!(!args.iter().any(|s| s.contains("bypass-hook-trust"))); + let inherited = + EcoSession::create(&ulid::Ulid::new().to_string(), true, "inherit").unwrap(); + let mut native = Command::new("codex"); + native.args(["-c", "approval_policy=\"on-request\""]); + inherited + .configure(&mut native, LaunchIntegration::Codex) + .unwrap(); + assert_eq!(native.get_args().count(), 2); + assert!(inherited.summary().status.contains("inherited")); + } +} diff --git a/src/spark/economics.rs b/src/spark/economics.rs new file mode 100644 index 0000000..f2cb813 --- /dev/null +++ b/src/spark/economics.rs @@ -0,0 +1,507 @@ +//! Reproducible public-price comparisons and private aggregate report storage. +use super::wire::LaunchSessionDocument; +use anyhow::{Result, ensure}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + fs, + io::{self, Read, Write}, + os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt}, + path::{Path, PathBuf}, +}; + +pub const SCHEMA: &str = "sparkplane.session-economics/v1"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Price { + pub model: String, + pub source: String, + pub verified_at: String, + pub effective_from: String, + pub input_microusd_per_million: u64, + pub output_microusd_per_million: u64, + pub long_context_threshold: Option, + pub long_input_microusd_per_million: Option, + pub long_output_microusd_per_million: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PriceCatalog { + pub version: String, + pub currency: String, + pub tariff: String, + pub models: Vec, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct Compression { + pub status: String, + pub commands: u64, + pub raw_bytes: u64, + pub filtered_bytes: u64, + pub metrics_errors: u64, +} +impl Compression { + pub fn saved_tokens(&self) -> u64 { + self.raw_bytes + .div_ceil(4) + .saturating_sub(self.filtered_bytes.div_ceil(4)) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Comparison { + pub model: String, + pub inference_usd_nanos: Option, + pub estimated_rtk_usd_nanos: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Report { + pub schema: String, + pub session_id: String, + pub host: String, + pub model: String, + pub integration: String, + pub eco_mode: String, + pub started_at: String, + pub finished_at: Option, + pub exit_code: Option, + pub inference: Option, + pub compression: Compression, + pub catalog: PriceCatalog, + pub comparisons: Vec, +} +impl Report { + pub fn calculate(&mut self) { + self.comparisons = self + .catalog + .models + .iter() + .map(|price| { + let inference = self.inference.as_ref().and_then(|session| { + let usage = &session.usage; + if (session.finished_at.is_none() && usage.requests == 0) + || (usage.requests > 0 && usage.unknown_usage_requests == usage.requests) + { + return None; + } + let short_input = usage + .input_tokens + .checked_sub(usage.long_context_input_tokens)?; + let short_output = usage + .output_tokens + .checked_sub(usage.long_context_output_tokens)?; + let long_in = price + .long_input_microusd_per_million + .unwrap_or(price.input_microusd_per_million); + let long_out = price + .long_output_microusd_per_million + .unwrap_or(price.output_microusd_per_million); + let value = u128::from(short_input) + * u128::from(price.input_microusd_per_million) + + u128::from(short_output) * u128::from(price.output_microusd_per_million) + + u128::from(usage.long_context_input_tokens) * u128::from(long_in) + + u128::from(usage.long_context_output_tokens) * u128::from(long_out); + u64::try_from(value / 1000).ok() + }); + let rtk = (self.compression.metrics_errors == 0 + && self.compression.status == "active") + .then(|| { + u64::try_from( + u128::from(self.compression.saved_tokens()) + * u128::from(price.input_microusd_per_million) + / 1000, + ) + .ok() + }) + .flatten(); + Comparison { + model: price.model.clone(), + inference_usd_nanos: inference, + estimated_rtk_usd_nanos: rtk, + } + }) + .collect(); + } +} + +pub fn catalog() -> Result { + Ok(serde_json::from_str(include_str!( + "../../data/economics-prices.json" + ))?) +} + +pub(crate) fn private_dir(path: &Path) -> Result<()> { + if !path.exists() { + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(path)?; + } + let meta = fs::symlink_metadata(path)?; + ensure!( + meta.is_dir() + && meta.uid() == rustix::process::geteuid().as_raw() + && meta.mode() & 0o077 == 0, + "economics directory has unsafe ownership or permissions" + ); + Ok(()) +} + +pub(crate) fn write_record(path: &Path, value: &impl Serialize) -> Result<()> { + let parent = path + .parent() + .ok_or_else(|| anyhow::anyhow!("record has no parent"))?; + private_dir(parent)?; + let bytes = serde_json::to_vec_pretty(value)?; + let temporary = parent.join(format!(".staging-{}", uuid::Uuid::new_v4())); + let mut file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(&temporary)?; + file.write_all(&bytes)?; + file.sync_all()?; + fs::rename(&temporary, path)?; + fs::File::open(parent)?.sync_all()?; + Ok(()) +} + +pub(crate) fn read_record(path: &Path) -> Result { + let file = fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path)?; + let meta = file.metadata()?; + ensure!( + meta.is_file() + && meta.uid() == rustix::process::geteuid().as_raw() + && meta.mode() & 0o077 == 0 + && meta.len() <= 1024 * 1024, + "invalid economics record" + ); + let mut bytes = Vec::new(); + file.take(1024 * 1024 + 1).read_to_end(&mut bytes)?; + ensure!(bytes.len() <= 1024 * 1024, "economics record too large"); + Ok(serde_json::from_slice(&bytes)?) +} + +fn directory(config: &Path, host: &str) -> PathBuf { + config + .join("economics") + .join(format!("{:x}", Sha256::digest(host.as_bytes()))) +} + +pub fn save(config: &Path, report: &Report) -> Result<()> { + ensure!( + report.session_id.parse::().is_ok(), + "invalid session id" + ); + let dir = directory(config, &report.host); + private_dir(&dir)?; + write_record(&dir.join(format!("{}.json", report.session_id)), report)?; + prune(&dir) +} + +fn prune(dir: &Path) -> Result<()> { + let cutoff = chrono::Utc::now() - chrono::Duration::days(90); + for entry in fs::read_dir(dir)? { + let path = entry?.path(); + if path.extension().is_some_and(|s| s == "json") + && let Ok(old) = read_record::(&path) + && chrono::DateTime::parse_from_rfc3339(&old.started_at).is_ok_and(|t| t < cutoff) + { + fs::remove_file(path)?; + } + } + Ok(()) +} + +pub fn load(config: &Path, host: &str, id: Option<&str>) -> Result { + let dir = directory(config, host); + // Enforce retention even when no further launch writes a report. + ensure!( + fs::symlink_metadata(&dir)?.is_dir(), + "invalid economics directory" + ); + private_dir(&dir)?; + prune(&dir)?; + let path = if let Some(id) = id { + ensure!(id.parse::().is_ok(), "invalid session id"); + dir.join(format!("{id}.json")) + } else { + let mut paths = fs::read_dir(&dir)? + .filter_map(Result::ok) + .map(|e| e.path()) + .filter(|p| { + p.extension().is_some_and(|s| s == "json") + && p.file_stem() + .and_then(|s| s.to_str()) + .is_some_and(|s| s.parse::().is_ok()) + }) + .collect::>(); + paths.sort(); + paths + .pop() + .ok_or_else(|| anyhow::anyhow!("no retained economics reports"))? + }; + let report: Report = read_record(&path)?; + ensure!( + report.schema == SCHEMA && report.host == host, + "economics report identity mismatch" + ); + Ok(report) +} + +fn money(value: Option) -> String { + match value { + None => "unavailable".into(), + Some(0) => "$0.00".into(), + Some(v) if v < 10_000_000 => "<$0.01".into(), + Some(v) => { + let cents = (u128::from(v) + 5_000_000) / 10_000_000; + format!("${}.{:02}", cents / 100, cents % 100) + } + } +} + +pub fn render(report: &Report, fancy: bool, width: usize) -> String { + let width = width.clamp(40, 100); + let title = if fancy { + "\x1b[1;32mSession economics\x1b[0m" + } else { + "Session economics" + }; + let mut text = format!("\n{}\n{title}\n", "─".repeat(width)); + text.push_str(&format!( + "{} · {} · exit {}\n", + report.integration, + report.model, + report + .exit_code + .map(|v| v.to_string()) + .unwrap_or_else(|| "interrupted".into()) + )); + if let (Ok(start), Some(end)) = ( + chrono::DateTime::parse_from_rfc3339(&report.started_at), + report + .finished_at + .as_ref() + .and_then(|t| chrono::DateTime::parse_from_rfc3339(t).ok()), + ) { + let seconds = (end - start).num_seconds().max(0); + text.push_str(&format!("Duration: {}m {}s\n", seconds / 60, seconds % 60)); + } + if let Some(session) = &report.inference { + let usage = &session.usage; + let incomplete = usage.unknown_usage_requests > 0 + || usage.pending_requests > 0 + || usage.accounting_errors > 0 + || session.finished_at.is_none(); + text.push_str(&format!( + "Tokens burned{}: {} input + {} output = {} total · {} requests\n", + if incomplete { " (known)" } else { "" }, + usage.input_tokens, + usage.output_tokens, + usage.input_tokens.saturating_add(usage.output_tokens), + usage.requests + )); + if usage.unknown_usage_requests > 0 + || usage.pending_requests > 0 + || usage.accounting_errors > 0 + || session.finished_at.is_none() + { + text.push_str("Inference accounting incomplete; known usage is a lower bound.\n"); + } + } else { + text.push_str("Inference usage unavailable.\n"); + } + let rtk = &report.compression; + text.push_str(&format!( + "RTK: {} · {} commands\n", + rtk.status, rtk.commands + )); + if rtk.metrics_errors > 0 { + text.push_str("RTK metrics incomplete; dollar savings unavailable.\n"); + } + if rtk.commands > 0 { + let pct = if rtk.raw_bytes > 0 { + 100.0 * (1.0 - rtk.filtered_bytes as f64 / rtk.raw_bytes as f64) + } else { + 0.0 + }; + text.push_str(&format!( + "Tool output: ~{} → ~{} tokens · {:.1}% smaller · ~{} tokens saved\n", + rtk.raw_bytes.div_ceil(4), + rtk.filtered_bytes.div_ceil(4), + pct, + rtk.saved_tokens() + )); + if fancy { + let kept = rtk + .filtered_bytes + .saturating_mul(20) + .checked_div(rtk.raw_bytes) + .unwrap_or(0) + .min(20) as usize; + text.push_str(&format!( + "[{}{}]\n", + "█".repeat(kept), + "░".repeat(20 - kept) + )); + } + } + text.push_str("\nCloud equivalent / estimated RTK savings (USD):\n"); + for row in &report.comparisons { + text.push_str(&format!( + " {:<20} {:>11} / {:>11}\n", + row.model, + money(row.inference_usd_nanos), + money(row.estimated_rtk_usd_nanos) + )); + } + text.push_str(&format!( + "Public standard uncached rates · verified {}\n", + report + .catalog + .models + .first() + .map(|p| p.verified_at.as_str()) + .unwrap_or("unknown") + )); + text.push_str("Estimates use local token counts; RTK is valued once at input rates.\nCloud comparisons exclude electricity/hardware and are not added together.\n"); + text +} + +pub fn display(report: &Report) { + use std::io::IsTerminal; + eprint!( + "{}", + render( + report, + io::stderr().is_terminal() && std::env::var_os("NO_COLOR").is_none(), + std::env::var("COLUMNS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(80) + ) + ); +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn money_rounds_and_tiny_values_are_visible() { + assert_eq!(money(Some(1)), "<$0.01"); + assert_eq!(money(None), "unavailable"); + assert_eq!(money(Some(999_000_000)), "$1.00"); + } + #[test] + fn catalog_has_official_sources_and_known_context_tier() { + let prices = catalog().unwrap(); + assert_eq!(prices.models.len(), 3); + assert_eq!(prices.models[0].long_context_threshold, Some(272_000)); + } + + fn report() -> Report { + let now = chrono::Utc::now().to_rfc3339(); + Report { + schema: SCHEMA.into(), + session_id: ulid::Ulid::new().to_string(), + host: "fixture".into(), + model: "local-model".into(), + integration: "codex".into(), + eco_mode: "max".into(), + started_at: now.clone(), + finished_at: Some(now.clone()), + exit_code: Some(0), + inference: Some(LaunchSessionDocument { + schema: "sparkplane.launch-session/v1".into(), + id: ulid::Ulid::new().to_string(), + instance: "fixture".into(), + model: "local-model".into(), + integration: "codex".into(), + eco_mode: "max".into(), + started_at: now.clone(), + finished_at: Some(now), + usage: super::super::wire::SessionUsage { + requests: 1, + input_tokens: 272001, + output_tokens: 4, + long_context_input_tokens: 272001, + long_context_output_tokens: 4, + ..Default::default() + }, + }), + compression: Compression { + status: "active".into(), + commands: 1, + raw_bytes: 400, + filtered_bytes: 100, + metrics_errors: 0, + }, + catalog: catalog().unwrap(), + comparisons: Vec::new(), + } + } + + #[test] + fn costs_use_each_requests_context_tier_and_keep_rtk_separate() { + let mut report = report(); + report.calculate(); + assert_eq!( + report.comparisons[0].inference_usd_nanos, + Some(1_088_064_000) + ); + assert_eq!(report.comparisons[0].estimated_rtk_usd_nanos, Some(150_000)); + assert_eq!(report.comparisons[1].inference_usd_nanos, Some(816_063_000)); + let usage = &mut report.inference.as_mut().unwrap().usage; + usage.input_tokens = 272000; + usage.long_context_input_tokens = 0; + usage.long_context_output_tokens = 0; + report.calculate(); + assert_eq!(report.comparisons[0].inference_usd_nanos, Some(544_040_000)); + report + .inference + .as_mut() + .unwrap() + .usage + .unknown_usage_requests = 1; + report.compression.metrics_errors = 1; + report.calculate(); + assert_eq!(report.comparisons[0].estimated_rtk_usd_nanos, None); + let text = render(&report, false, 40); + assert!(text.contains("lower bound")); + assert!(!text.contains('\x1b')); + } + + #[test] + fn reports_are_private_scoped_and_expire_on_read() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let root = tempfile::tempdir().unwrap(); + let mut report = report(); + report.calculate(); + save(root.path(), &report).unwrap(); + assert_eq!( + load(root.path(), "fixture", None).unwrap().session_id, + report.session_id + ); + assert!(load(root.path(), "other", Some(&report.session_id)).is_err()); + assert!(load(root.path(), "fixture", Some("../escape")).is_err()); + let path = directory(root.path(), "fixture").join(format!("{}.json", report.session_id)); + assert_eq!(fs::metadata(&path).unwrap().mode() & 0o777, 0o600); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + assert!(load(root.path(), "fixture", None).is_err()); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + let link = path.with_extension("link"); + symlink(&path, &link).unwrap(); + assert!(read_record::(&link).is_err()); + report.started_at = (chrono::Utc::now() - chrono::Duration::days(91)).to_rfc3339(); + write_record(&path, &report).unwrap(); + assert!(load(root.path(), "fixture", None).is_err()); + assert!(!path.exists()); + } +} diff --git a/src/spark/gateway.rs b/src/spark/gateway.rs index 5a8f027..1d9baed 100644 --- a/src/spark/gateway.rs +++ b/src/spark/gateway.rs @@ -17,7 +17,14 @@ use super::upstream::{ decode_embedding_response, }; -pub const MAX_COMPLETION_BODY_BYTES: usize = 1024 * 1024; +pub const MAX_COMPLETION_BODY_BYTES: usize = 8 * 1024 * 1024; + +#[path = "gateway_images.rs"] +mod images; +// Protocol harnesses also include this file as a private module. +#[allow(unused_imports)] +pub use images::{HARD_REQUEST_BYTES, ImageProtocol, IngressLimits, normalize_image_request}; + pub const RETRY_AFTER_SECONDS: &str = "1"; pub const MAX_OUTPUT_TOKENS: u64 = 32_768; const MAX_INSTANCE_CONCURRENCY: usize = 4; @@ -276,17 +283,20 @@ struct AnthropicImageSource { #[serde(untagged)] enum AnthropicToolResultContent { Text(String), - Blocks(Vec), + Blocks(Vec), } #[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct AnthropicToolResultText { - #[serde(rename = "type")] - kind: String, - text: String, - #[serde(default)] - cache_control: Option, +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +enum AnthropicToolResultBlock { + Text { + text: String, + #[serde(default)] + cache_control: Option, + }, + Image { + source: AnthropicImageSource, + }, } #[derive(Debug, Deserialize)] @@ -741,6 +751,7 @@ fn omit_empty_tool_controls(object: &mut serde_json::Map) { struct ImageBudget { count: usize, bytes: usize, + source_pixels: u64, } fn parse_data_uri(value: &str) -> Result<(&str, &str), OpenAiError> { @@ -1549,18 +1560,36 @@ pub fn rewrite_chat_request_with_profile( .get("messages") .and_then(Value::as_array) .ok_or_else(|| invalid_request("messages must be an array"))?; - if messages.iter().any(|message| { - message["content"].as_array().is_some_and(|parts| { - parts.iter().any(|part| { - matches!(part["type"].as_str(), Some("image_url" | "input_image")) - || part.get("image_url").is_some() - || part.get("file_id").is_some() - }) - }) - }) { - return Err(invalid_request( - "chat image inputs are unsupported; use the Responses route", - )); + let mut images = ImageBudget::default(); + for message in messages { + for part in message["content"].as_array().into_iter().flatten() { + if part["type"] == "image_url" { + if !matches!(message["role"].as_str(), Some("user" | "tool")) + || part.as_object().is_none_or(|o| { + o.keys() + .any(|key| !matches!(key.as_str(), "type" | "image_url")) + }) + || part["image_url"].as_object().is_none_or(|o| { + o.keys() + .any(|key| !matches!(key.as_str(), "url" | "detail")) + }) + { + return Err(invalid_request( + "chat images must be inline user or tool-result content", + )); + } + let url = part["image_url"]["url"] + .as_str() + .ok_or_else(|| invalid_request("image_url must be an inline data URI"))?; + let (media, data) = parse_data_uri(url)?; + decode_image(media, data, profile.vision.as_ref(), &mut images)?; + } else if part.get("image_url").is_some() + || part.get("file_id").is_some() + || part["type"] == "input_image" + { + return Err(invalid_request("chat image content type is unsupported")); + } + } } let stream = object .get("stream") @@ -1991,9 +2020,17 @@ fn anthropic_blocks( "tool_result does not match a prior tool_use", )); } - let content = anthropic_tool_result_text(content)?; + let content = anthropic_tool_result_content(content, vision, images)?; let content = if is_error { - format!("Tool error: {content}") + match content { + Value::String(text) => Value::String(format!("Tool error: {text}")), + Value::Array(mut parts) => { + parts + .insert(0, serde_json::json!({"type":"text","text":"Tool error:"})); + Value::Array(parts) + } + value => value, + } } else { content }; @@ -2044,22 +2081,45 @@ fn anthropic_blocks( Ok(()) } -fn anthropic_tool_result_text( +fn anthropic_tool_result_content( content: AnthropicToolResultContent, -) -> Result { + vision: Option<&VisionPolicy>, + images: &mut ImageBudget, +) -> Result { match content { - AnthropicToolResultContent::Text(text) => Ok(text), + AnthropicToolResultContent::Text(text) => Ok(text.into()), AnthropicToolResultContent::Blocks(blocks) => { - blocks - .into_iter() - .try_fold(String::new(), |mut text, block| { - validate_cache_control(block.cache_control.as_ref())?; - if block.kind != "text" { - return Err(anthropic_invalid("tool_result content type is unsupported")); + let mut content = Vec::new(); + let mut text = String::new(); + let mut multimodal = false; + for block in blocks { + match block { + AnthropicToolResultBlock::Text { + text: part, + cache_control, + } => { + validate_cache_control(cache_control.as_ref())?; + text.push_str(&part); + content.push(serde_json::json!({"type":"text","text":part})); } - text.push_str(&block.text); - Ok(text) - }) + AnthropicToolResultBlock::Image { source } => { + if source.kind != "base64" { + return Err(anthropic_invalid( + "image source must be inline base64 tool-result content", + )); + } + let image = decode_image(&source.media_type, &source.data, vision, images) + .map_err(|error| anthropic_invalid(error.message))?; + content.push(canonical_image(&image)); + multimodal = true; + } + } + } + Ok(if multimodal { + Value::Array(content) + } else { + text.into() + }) } } } @@ -2537,7 +2597,9 @@ fn response_input_item( Some("message") => response_message(item, vision, images), Some("function_call") => response_tool_call(item, false), Some("custom_tool_call") => response_tool_call(item, true), - Some("function_call_output" | "custom_tool_call_output") => response_tool_output(item), + Some("function_call_output" | "custom_tool_call_output") => { + response_tool_output(item, vision, images) + } Some("reasoning") => response_reasoning(item), Some("computer_call" | "computer_call_output") => Err(OpenAiError { code: "unsupported_tool", @@ -2602,10 +2664,10 @@ fn response_content( "text":part["text"].as_str().ok_or_else(|| invalid_request("text content is invalid"))? })), Some("input_image") => { - if role != "user" || part.as_object().is_none_or(|object| { + if !matches!(role, "user" | "tool") || part.as_object().is_none_or(|object| { object.keys().any(|key| !matches!(key.as_str(), "type" | "image_url" | "detail")) }) || part.get("file_id").is_some() { - return Err(invalid_request("image input must be bounded inline user content")); + return Err(invalid_request("image input must be bounded inline user or tool-result content")); } let url = part["image_url"] .as_str() @@ -2645,17 +2707,36 @@ fn response_tool_call(item: &Value, custom: bool) -> Result ) } -fn response_tool_output(item: &Value) -> Result { +fn response_tool_output( + item: &Value, + vision: Option<&VisionPolicy>, + images: &mut ImageBudget, +) -> Result { let call_id = item["call_id"] .as_str() .ok_or_else(|| invalid_request("tool output call_id is required"))?; let output = item .get("output") .ok_or_else(|| invalid_request("tool output is required"))?; - let content = output - .as_str() - .map(str::to_owned) - .unwrap_or_else(|| output.to_string()); + // Codex view_image returns typed output blocks. Serializing their base64 as + // text both loses the image and can consume the entire model context. + let content = if output.as_array().is_some_and(|parts| { + parts.iter().any(|part| { + matches!( + part["type"].as_str(), + Some("input_text" | "output_text" | "input_image" | "input_file") + ) + }) + }) { + response_content(output, "tool", vision, images)? + } else { + Value::String( + output + .as_str() + .map(str::to_owned) + .unwrap_or_else(|| output.to_string()), + ) + }; Ok(serde_json::json!({"role": "tool", "tool_call_id": call_id, "content": content})) } diff --git a/src/spark/gateway_images.rs b/src/spark/gateway_images.rs new file mode 100644 index 0000000..3a51bd7 --- /dev/null +++ b/src/spark/gateway_images.rs @@ -0,0 +1,394 @@ +//! Public image uploads have separate limits from qualified engine inputs. +use std::io::{self, Cursor, Write}; + +use base64::Engine as _; +use image::{ + DynamicImage, ImageDecoder, ImageEncoder, ImageFormat, ImageReader, imageops::FilterType, +}; +use serde_json::Value; + +use super::{ + BASE64, GatewayProfile, ImageBudget, MAX_COMPLETION_BODY_BYTES, OpenAiError, VisionPolicy, + decode_image, invalid_request, parse_data_uri, +}; + +pub const HARD_REQUEST_BYTES: usize = 64 * 1024 * 1024; +const DECODE_MEMORY_BYTES: u64 = 128 * 1024 * 1024; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct IngressLimits { + pub request_body_bytes: usize, + pub image_bytes: usize, + pub image_pixels: u64, + pub image_dimension: u32, + pub max_parallel_requests: usize, +} + +impl Default for IngressLimits { + fn default() -> Self { + Self { + request_body_bytes: 32 * 1024 * 1024, + image_bytes: 16 * 1024 * 1024, + image_pixels: 32_000_000, + image_dimension: 16384, + max_parallel_requests: 2, + } + } +} + +impl IngressLimits { + pub fn validate(&self) -> Result<(), &'static str> { + if self.request_body_bytes == 0 + || self.request_body_bytes > HARD_REQUEST_BYTES + || self.image_bytes == 0 + || self.image_bytes > 32 * 1024 * 1024 + || self.image_pixels == 0 + || self.image_pixels > 32_000_000 + || self.image_dimension == 0 + || self.image_dimension > 16384 + || !(1..=4).contains(&self.max_parallel_requests) + { + return Err("gateway ingress limits exceed supported resource bounds"); + } + Ok(()) + } +} + +#[derive(Clone, Copy)] +pub enum ImageProtocol { + Responses, + Anthropic, + Chat, +} + +fn too_large(message: &'static str) -> OpenAiError { + OpenAiError { + code: "request_too_large", + message, + } +} + +pub fn normalize_image_request( + bytes: &[u8], + protocol: ImageProtocol, + profile: &GatewayProfile, + limits: &IngressLimits, +) -> Result, OpenAiError> { + limits.validate().map_err(invalid_request)?; + if bytes.len() > limits.request_body_bytes { + return Err(too_large("request exceeds the gateway upload byte limit")); + } + let mut request: Value = + serde_json::from_slice(bytes).map_err(|_| invalid_request("request JSON is invalid"))?; + let history_protocol = match protocol { + ImageProtocol::Responses => super::super::image_history::Protocol::Responses, + ImageProtocol::Anthropic => super::super::image_history::Protocol::Anthropic, + ImageProtocol::Chat => super::super::image_history::Protocol::Chat, + }; + if let Some(vision) = &profile.vision { + super::super::image_history::project( + &mut request, + history_protocol, + vision.max_count, + None, + ) + .map_err(|_| invalid_request("image count exceeds the model profile limit"))?; + } + let field = match protocol { + ImageProtocol::Responses => "input", + ImageProtocol::Anthropic | ImageProtocol::Chat => "messages", + }; + let mut image_count = 0; + super::super::image_history::visit_parts(&mut request[field], history_protocol, &mut |_| { + image_count += 1 + }); + let mut image_policy = profile.vision.clone(); + if image_count > 0 { + let policy = image_policy + .as_mut() + .ok_or_else(|| invalid_request("image input requires a vision profile"))?; + if image_count > policy.max_count { + return Err(invalid_request( + "image count exceeds the model profile limit", + )); + } + // Reserve an equal share for each frame, including later tool results. + // The strict rewriter still checks the original shared engine budget. + policy.max_bytes = policy.max_bytes.min(policy.max_total_bytes / image_count); + } + let mut budget = ImageBudget::default(); + if let Some(messages) = request.get_mut(field).and_then(Value::as_array_mut) { + for message in messages { + let user = message["role"] == "user"; + let tool = (matches!(protocol, ImageProtocol::Responses) + && matches!( + message["type"].as_str(), + Some("function_call_output" | "custom_tool_call_output") + )) + || (matches!(protocol, ImageProtocol::Chat) && message["role"] == "tool"); + let mut error = None; + super::super::image_history::visit_parts(message, history_protocol, &mut |part| { + if error.is_some() { + return; + } + if !user && !tool { + error = Some(invalid_request( + "image input must be inline user or tool-result content", + )); + return; + } + let prepared = (|| { + let (media, data) = match protocol { + ImageProtocol::Responses | ImageProtocol::Chat => { + let uri = if matches!(protocol, ImageProtocol::Responses) { + part["image_url"].as_str() + } else { + part["image_url"]["url"].as_str() + } + .ok_or_else(|| { + invalid_request("image_url must be an inline data URI") + })?; + parse_data_uri(uri)? + } + ImageProtocol::Anthropic => { + let source = &part["source"]; + if source["type"] != "base64" { + return Err(invalid_request( + "image source must be inline base64 user content", + )); + } + ( + source["media_type"].as_str().ok_or_else(|| { + invalid_request("image media type is required") + })?, + source["data"] + .as_str() + .ok_or_else(|| invalid_request("image base64 is required"))?, + ) + } + }; + prepare_image(media, data, image_policy.as_ref(), &mut budget, limits) + })(); + match prepared { + Ok(image) => match protocol { + ImageProtocol::Responses => { + part["image_url"] = format!( + "data:{};base64,{}", + image.media_type, + BASE64.encode(&image.bytes) + ) + .into() + } + ImageProtocol::Chat => { + part["image_url"]["url"] = format!( + "data:{};base64,{}", + image.media_type, + BASE64.encode(&image.bytes) + ) + .into() + } + ImageProtocol::Anthropic => { + part["source"]["media_type"] = image.media_type.into(); + part["source"]["data"] = BASE64.encode(&image.bytes).into(); + } + }, + Err(problem) => error = Some(problem), + } + }); + if let Some(error) = error { + return Err(error); + } + } + } + let bytes = + serde_json::to_vec(&request).map_err(|_| invalid_request("request cannot be encoded"))?; + if bytes.len() > MAX_COMPLETION_BODY_BYTES { + return Err(too_large( + "normalized request exceeds the 8 MiB inference body limit", + )); + } + Ok(bytes) +} + +fn prepare_image( + media: &str, + encoded: &str, + policy: Option<&VisionPolicy>, + budget: &mut ImageBudget, + limits: &IngressLimits, +) -> Result { + let policy = policy.ok_or_else(|| invalid_request("image input requires a vision profile"))?; + if budget.count >= policy.max_count { + return Err(invalid_request( + "image count exceeds the model profile limit", + )); + } + if !policy.media_types.contains(media) { + return Err(invalid_request( + "image format is unsupported by the model profile", + )); + } + if encoded.len() > limits.image_bytes.div_ceil(3) * 4 { + return Err(too_large("image exceeds the gateway upload byte limit")); + } + let bytes = BASE64 + .decode(encoded) + .map_err(|_| invalid_request("image base64 is invalid"))?; + if bytes.is_empty() { + return Err(invalid_request("image data is empty")); + } + if bytes.len() > limits.image_bytes { + return Err(too_large("image exceeds the gateway upload byte limit")); + } + let format = match media { + "image/png" if bytes.starts_with(b"\x89PNG\r\n\x1a\n") => ImageFormat::Png, + "image/jpeg" if bytes.starts_with(b"\xff\xd8\xff") => ImageFormat::Jpeg, + "image/webp" + if bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP".as_slice()) => + { + ImageFormat::WebP + } + _ => { + return Err(invalid_request( + "image media type does not match its magic bytes", + )); + } + }; + let mut decode_limits = image::Limits::default(); + decode_limits.max_image_width = Some(limits.image_dimension); + decode_limits.max_image_height = Some(limits.image_dimension); + decode_limits.max_alloc = Some(DECODE_MEMORY_BYTES); + let mut reader = ImageReader::with_format(Cursor::new(&bytes), format); + reader.limits(decode_limits); + let mut decoder = reader.into_decoder().map_err(|error| match error { + image::ImageError::Limits(_) => { + too_large("image exceeds the gateway dimension or decode memory limit") + } + _ => invalid_request("image header is invalid"), + })?; + let (width, height) = decoder.dimensions(); + let pixels = u64::from(width) * u64::from(height); + if width == 0 + || height == 0 + || pixels > limits.image_pixels.saturating_sub(budget.source_pixels) + || decoder.total_bytes() > DECODE_MEMORY_BYTES + { + return Err(too_large( + "images exceed the gateway total pixel or decode memory limit", + )); + } + budget.source_pixels += pixels; + let available = policy + .max_total_bytes + .saturating_sub(budget.bytes) + .min(policy.max_bytes); + if available == 0 || policy.max_width == 0 || policy.max_height == 0 { + return Err(invalid_request("image cannot fit the model profile limits")); + } + if bytes.len() <= available && width <= policy.max_width && height <= policy.max_height { + return decode_image(media, encoded, Some(policy), budget); + } + let orientation = decoder + .orientation() + .map_err(|_| invalid_request("image orientation metadata is invalid"))?; + let mut decoded = DynamicImage::from_decoder(decoder) + .map_err(|_| invalid_request("image pixels are invalid"))?; + decoded.apply_orientation(orientation); + let alpha = decoded.color().has_alpha(); + let mut image = decoded.resize( + policy.max_width.min(decoded.width()), + policy.max_height.min(decoded.height()), + FilterType::Lanczos3, + ); + drop(decoded); + image = if alpha { + DynamicImage::ImageRgba8(image.to_rgba8()) + } else { + DynamicImage::ImageRgb8(image.to_rgb8()) + }; + for _ in 0..16 { + let mut formats = Vec::new(); + if alpha || format == ImageFormat::Png { + formats.push(("image/png", ImageFormat::Png, 0)); + } + if !alpha { + formats.extend([ + ("image/jpeg", ImageFormat::Jpeg, 90), + ("image/jpeg", ImageFormat::Jpeg, 80), + ]); + } + formats.push(("image/webp", ImageFormat::WebP, 0)); + for (media, format, quality) in formats { + if !policy.media_types.contains(media) { + continue; + } + if let Some(bytes) = encode(&image, format, quality, available)? { + return decode_image(media, &BASE64.encode(&bytes), Some(policy), budget); + } + } + if image.width() == 1 && image.height() == 1 { + break; + } + let width = (image.width() * 3 / 4).max(1); + let height = (image.height() * 3 / 4).max(1); + image = image.resize(width, height, FilterType::Lanczos3); + } + Err(invalid_request("image cannot fit the model profile limits")) +} + +struct CappedWriter { + bytes: Vec, + limit: usize, + exceeded: bool, +} +impl Write for CappedWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if bytes.len() > self.limit.saturating_sub(self.bytes.len()) { + self.exceeded = true; + return Err(io::Error::other("image encoding byte limit")); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn encode( + image: &DynamicImage, + format: ImageFormat, + quality: u8, + limit: usize, +) -> Result>, OpenAiError> { + let mut output = CappedWriter { + bytes: Vec::new(), + limit, + exceeded: false, + }; + let result = match format { + ImageFormat::Png => image::codecs::png::PngEncoder::new(&mut output).write_image( + image.as_bytes(), + image.width(), + image.height(), + image.color().into(), + ), + ImageFormat::Jpeg => { + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut output, quality) + .encode_image(image) + } + ImageFormat::WebP => image::codecs::webp::WebPEncoder::new_lossless(&mut output).encode( + image.as_bytes(), + image.width(), + image.height(), + image.color().into(), + ), + _ => return Err(invalid_request("image output format is unsupported")), + }; + if output.exceeded { + return Ok(None); + } + result.map_err(|_| invalid_request("image cannot be encoded"))?; + Ok(Some(output.bytes)) +} diff --git a/src/spark/image_history.rs b/src/spark/image_history.rs new file mode 100644 index 0000000..2c071d3 --- /dev/null +++ b/src/spark/image_history.rs @@ -0,0 +1,435 @@ +//! Bounded model-visible image history, shared by the launch adapter and gateway. +//! Original tool calls, observations and instructions are never summarized here. +use std::{cell::Cell, collections::BTreeMap, io::Read, path::PathBuf, rc::Rc}; + +use serde::de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +pub const REQUEST_BYTES: usize = 32 * 1024 * 1024; +const TEXT_BYTES: usize = 8 * 1024 * 1024; +const RECENT: usize = 6; +const HIGH_WATER: usize = 12; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Protocol { + Responses, + Anthropic, + Chat, +} +impl Protocol { + fn field(self) -> &'static str { + if self == Self::Responses { + "input" + } else { + "messages" + } + } + fn image(self, part: &Value) -> bool { + part["type"] + == match self { + Self::Responses => "input_image", + Self::Anthropic => "image", + Self::Chat => "image_url", + } + } + fn text(self, text: String) -> Value { + json!({"type": if self == Self::Responses { "input_text" } else { "text" }, "text":text}) + } +} + +#[derive(Debug, Default, Clone, Copy)] +pub struct Statistics { + pub evicted: usize, + pub removed_bytes: usize, +} +pub type Archive<'a> = dyn FnMut(&str, &Value, Protocol) -> Option + 'a; + +pub fn visit_parts(value: &mut Value, protocol: Protocol, visit: &mut impl FnMut(&mut Value)) { + match value { + Value::Array(parts) => { + for part in parts { + visit_parts(part, protocol, visit); + } + } + Value::Object(_) if protocol.image(value) => visit(value), + Value::Object(object) => { + // Restrict traversal to protocol content; tool arguments and metadata are data. + for field in ["content", "output"] { + if let Some(parts) = object.get_mut(field) { + visit_parts(parts, protocol, visit); + } + } + } + _ => {} + } +} +fn acknowledged(item: &Value, protocol: Protocol) -> bool { + item["role"] == "assistant" + || (protocol == Protocol::Responses + && matches!( + item["type"].as_str(), + Some("function_call" | "custom_tool_call") + )) +} +fn image_payload(part: &Value, protocol: Protocol) -> Option<&str> { + match protocol { + Protocol::Responses => part["image_url"].as_str(), + Protocol::Anthropic => part["source"]["data"].as_str(), + Protocol::Chat => part["image_url"]["url"].as_str(), + } +} +pub fn image_id(part: &Value, protocol: Protocol) -> Option { + image_payload(part, protocol).map(|payload| format!("{:x}", Sha256::digest(payload.as_bytes()))) +} + +fn evictable(part: &Value, protocol: Protocol) -> bool { + let Some(object) = part.as_object() else { + return false; + }; + match protocol { + Protocol::Responses | Protocol::Chat => { + if object + .keys() + .any(|key| !matches!(key.as_str(), "type" | "image_url" | "detail")) + { + return false; + } + let Some(payload) = image_payload(part, protocol) else { + return false; + }; + payload.starts_with("data:image/png;base64,") + || payload.starts_with("data:image/jpeg;base64,") + || payload.starts_with("data:image/webp;base64,") + } + Protocol::Anthropic => { + object + .keys() + .all(|key| matches!(key.as_str(), "type" | "source")) + && part["source"]["type"] == "base64" + && matches!( + part["source"]["media_type"].as_str(), + Some("image/png" | "image/jpeg" | "image/webp") + ) + } + } +} + +fn prune_items( + items: &mut [Value], + protocol: Protocol, + max_count: usize, + archive: &mut Option<&mut Archive<'_>>, + enforce: bool, +) -> Result { + let ack = items.iter().rposition(|item| acknowledged(item, protocol)); + let mut total = 0; + let mut unseen = 0; + let mut payload_bytes = 0; + for (index, item) in items.iter_mut().enumerate() { + visit_parts(item, protocol, &mut |part| { + total += 1; + payload_bytes += image_payload(part, protocol).map(str::len).unwrap_or(0); + if ack.is_none_or(|ack| index >= ack) { + unseen += 1; + } + }); + } + if enforce && unseen > max_count { + return Err("new image batch exceeds the model limit; split the batch"); + } + let high = HIGH_WATER.min(max_count); + if total < high && payload_bytes <= REQUEST_BYTES / 2 { + return Ok(Statistics::default()); + } + let keep = RECENT.min(max_count).max(unseen); + let mut remove = total.saturating_sub(keep); + let mut stats = Statistics::default(); + let mut paths = BTreeMap::new(); + for (index, item) in items.iter_mut().enumerate() { + if matches!( + item["type"].as_str(), + Some("function_call" | "custom_tool_call") + ) && let (Some(id), Some(arguments)) = + (item["call_id"].as_str(), item["arguments"].as_str()) + && let Ok(arguments) = serde_json::from_str::(arguments) + && let Some(path) = arguments["path"].as_str().filter(|path| path.len() <= 4096) + { + paths.insert(id.to_owned(), path.to_owned()); + } + let path = item["call_id"] + .as_str() + .and_then(|id| paths.get(id)) + .cloned(); + if ack.is_none_or(|ack| index >= ack) { + continue; + } + if item["role"] != "user" + && !(protocol == Protocol::Responses + && matches!( + item["type"].as_str(), + Some("function_call_output" | "custom_tool_call_output") + )) + && !(protocol == Protocol::Chat && item["role"] == "tool") + { + continue; + } + visit_parts(item, protocol, &mut |part| { + if (remove == 0 && payload_bytes <= REQUEST_BYTES / 2) || !evictable(part, protocol) { + return; + } + let Some(id) = image_id(part, protocol) else { + return; + }; + let saved = archive + .as_deref_mut() + .and_then(|archive| archive(&id, part, protocol)); + let original = saved + .map(|path| path.to_string_lossy().into_owned()) + .or_else(|| path.clone()); + let reference = match original { + Some(path) => format!("Reopen with view_image using path {}. A source path may have changed; prefer the archived copy when present.", json!(path)), + None => "Original is not available through this gateway; request the image again if details are needed.".into(), + }; + let bytes = image_payload(part, protocol).map(str::len).unwrap_or(0); + stats.removed_bytes += bytes; + payload_bytes = payload_bytes.saturating_sub(bytes); + stats.evicted += 1; + remove = remove.saturating_sub(1); + *part = protocol.text(format!("[Older image {id} omitted from active history. {reference} Written observations are retained.]")); + }); + } + Ok(stats) +} + +/// Project an already buffered gateway request. Never evict images awaiting inspection. +pub fn project( + value: &mut Value, + protocol: Protocol, + max_count: usize, + mut archive: Option<&mut Archive<'_>>, +) -> Result { + let Some(items) = value + .get_mut(protocol.field()) + .and_then(Value::as_array_mut) + else { + return Ok(Statistics::default()); + }; + let ack = items.iter().rposition(|item| acknowledged(item, protocol)); + let mut unseen = 0; + for (index, item) in items.iter_mut().enumerate() { + if ack.is_none_or(|ack| index >= ack) { + visit_parts(item, protocol, &mut |_| unseen += 1); + } + } + if unseen > max_count { + return Err("new image batch exceeds the model limit; split the batch"); + } + let original = std::mem::take(items); + let mut stats = Statistics::default(); + for item in original { + items.push(item); + let next = prune_items(items, protocol, max_count, &mut archive, false)?; + stats.evicted += next.evicted; + stats.removed_bytes += next.removed_bytes; + } + let next = prune_items(items, protocol, max_count, &mut archive, true)?; + stats.evicted += next.evicted; + stats.removed_bytes += next.removed_bytes; + Ok(stats) +} + +struct CountWriter(usize); +impl std::io::Write for CountWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 = self.0.saturating_add(bytes.len()); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} +fn size(value: &Value) -> usize { + let mut writer = CountWriter(0); + let _ = serde_json::to_writer(&mut writer, value); + writer.0 +} +struct ItemReader { + reader: R, + count: Rc>, + deadline: std::time::Instant, +} +impl Read for ItemReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if std::time::Instant::now() >= self.deadline { + return Err(std::io::Error::other("history processing timed out")); + } + let count = self.reader.read(out)?; + let total = self.count.get().saturating_add(count); + self.count.set(total); + if total > REQUEST_BYTES { + return Err(std::io::Error::other( + "individual history item exceeds 32 MiB", + )); + } + Ok(count) + } +} +struct RequestSeed<'a, 'b> { + protocol: Protocol, + max_count: usize, + archive: Option<&'a mut Archive<'b>>, + count: Rc>, + stats: Statistics, +} +impl<'de> DeserializeSeed<'de> for &mut RequestSeed<'_, '_> { + type Value = Value; + fn deserialize>(self, deserializer: D) -> Result { + deserializer.deserialize_map(self) + } +} +impl<'de> Visitor<'de> for &mut RequestSeed<'_, '_> { + type Value = Value; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("an inference request object") + } + fn visit_map>(self, mut map: M) -> Result { + let mut object = serde_json::Map::new(); + let mut other_bytes = 0; + while let Some(key) = map.next_key::()? { + if object.contains_key(&key) { + return Err(de::Error::custom("duplicate request field")); + } + self.count.set(0); + let value = if key == self.protocol.field() { + map.next_value_seed(HistorySeed(&mut *self))? + } else { + map.next_value()? + }; + if key != self.protocol.field() { + other_bytes += key.len() + size(&value); + if other_bytes > TEXT_BYTES { + return Err(de::Error::custom("request text exceeds 8 MiB")); + } + } + object.insert(key, value); + } + Ok(Value::Object(object)) + } + fn visit_seq>(self, mut sequence: S) -> Result { + let mut items = Vec::new(); + let mut text_bytes = 0; + loop { + self.count.set(0); + let Some(mut item) = sequence.next_element::()? else { + break; + }; + let mut image_bytes = 0; + visit_parts(&mut item, self.protocol, &mut |part| { + image_bytes += image_payload(part, self.protocol) + .map(str::len) + .unwrap_or(0); + }); + text_bytes += size(&item).saturating_sub(image_bytes); + if text_bytes > TEXT_BYTES { + return Err(de::Error::custom( + "history text exceeds 8 MiB; compact the conversation", + )); + } + items.push(item); + let stats = prune_items( + &mut items, + self.protocol, + self.max_count, + &mut self.archive, + false, + ) + .map_err(de::Error::custom)?; + self.stats.evicted += stats.evicted; + self.stats.removed_bytes += stats.removed_bytes; + // Bound retained memory independently of the number of discarded wire bytes. + let mut retained = 0; + for item in &mut items { + visit_parts(item, self.protocol, &mut |part| { + retained += image_payload(part, self.protocol) + .map(str::len) + .unwrap_or(0); + }); + } + if retained > REQUEST_BYTES { + return Err(de::Error::custom("active image batch exceeds 32 MiB")); + } + } + let stats = prune_items( + &mut items, + self.protocol, + self.max_count, + &mut self.archive, + true, + ) + .map_err(de::Error::custom)?; + self.stats.evicted += stats.evicted; + self.stats.removed_bytes += stats.removed_bytes; + Ok(Value::Array(items)) + } + fn visit_str(self, text: &str) -> Result { + Ok(text.into()) + } +} +struct HistorySeed<'a, 'b, 'c>(&'a mut RequestSeed<'b, 'c>); +impl<'de> DeserializeSeed<'de> for HistorySeed<'_, '_, '_> { + type Value = Value; + fn deserialize>(self, deserializer: D) -> Result { + deserializer.deserialize_any(&mut *self.0) + } +} + +/// Stream old inline payloads through a bounded item buffer; the full source body +/// is never retained. The adapter also imposes a finite upload/processing deadline. +pub fn project_reader( + reader: R, + protocol: Protocol, + max_count: usize, + archive: Option<&mut Archive<'_>>, +) -> Result<(Value, Statistics), &'static str> { + let count = Rc::new(Cell::new(0)); + let reader = ItemReader { + reader, + count: count.clone(), + deadline: std::time::Instant::now() + std::time::Duration::from_secs(120), + }; + let mut deserializer = serde_json::Deserializer::from_reader(reader); + let mut seed = RequestSeed { + protocol, + max_count, + archive, + count, + stats: Statistics::default(), + }; + let mut value = (&mut seed) + .deserialize(&mut deserializer) + .map_err(|error| { + let diagnostic = error.to_string(); + if diagnostic.starts_with("new image batch exceeds") { + "new image batch exceeds the model limit; split the batch" + } else if diagnostic.starts_with("history text exceeds") + || diagnostic.starts_with("request text exceeds") + { + "history text exceeds the adapter budget; compact the conversation" + } else if error.is_io() || diagnostic.starts_with("active image batch exceeds") { + "image or request exceeds the adapter byte budget" + } else { + "request JSON is invalid" + } + })?; + deserializer + .end() + .map_err(|_| "request has trailing data")?; + let stats = project(&mut value, protocol, max_count, None)?; + seed.stats.evicted += stats.evicted; + seed.stats.removed_bytes += stats.removed_bytes; + if size(&value) > REQUEST_BYTES { + return Err("projected request exceeds 32 MiB; compact the conversation"); + } + Ok((value, seed.stats)) +} diff --git a/src/spark/inference_adapter.rs b/src/spark/inference_adapter.rs new file mode 100644 index 0000000..8dbd367 --- /dev/null +++ b/src/spark/inference_adapter.rs @@ -0,0 +1,670 @@ +//! Invocation-local authenticated loopback adapter. Only fixed inference routes +//! reach the pinned appliance; images are projected before the network upload. +use super::{ + client::{self, ClientError}, + image_history::{self, Protocol}, +}; +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use bytes::Bytes; +use http_body_util::{BodyExt, Full, StreamBody, combinators::UnsyncBoxBody}; +use hyper::{ + Request, Response, + body::{Frame, Incoming}, + server::conn::http1, + service::service_fn, +}; +use hyper_util::rt::TokioIo; +use serde_json::{Value, json}; +use std::{ + fs, + io::{self, Read, Write}, + net::TcpListener, + os::unix::fs::{MetadataExt, OpenOptionsExt}, + path::{Path, PathBuf}, + sync::{Arc, Mutex, mpsc}, + thread, + time::Duration, +}; +use tokio::sync::{Semaphore, oneshot}; + +type Body = UnsyncBoxBody; +const ARCHIVE_BYTES: u64 = 512 * 1024 * 1024; +const IMAGE_BYTES: usize = 16 * 1024 * 1024; + +struct Archive { + root: PathBuf, +} +impl Archive { + fn save(&self, id: &str, part: &Value, protocol: Protocol) -> Option { + let (media, data) = if protocol == Protocol::Anthropic { + if part["source"]["type"] != "base64" { + return None; + } + ( + part["source"]["media_type"].as_str()?, + part["source"]["data"].as_str()?, + ) + } else { + let uri = if protocol == Protocol::Responses { + part["image_url"].as_str()? + } else { + part["image_url"]["url"].as_str()? + }; + uri.strip_prefix("data:")?.split_once(";base64,")? + }; + let extension = match media { + "image/png" => "png", + "image/jpeg" => "jpg", + "image/webp" => "webp", + _ => return None, + }; + if data.len() > IMAGE_BYTES.div_ceil(3) * 4 { + return None; + } + let path = self.root.join(format!("{id}.{extension}")); + let lock = fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(self.root.join("archive.lock")) + .ok()?; + let _lock = + nix::fcntl::Flock::lock(lock, nix::fcntl::FlockArg::LockExclusiveNonblock).ok()?; + if let Ok(meta) = fs::symlink_metadata(&path) { + let expected_bytes = (data.len() / 4 * 3) + .checked_sub(data.bytes().rev().take_while(|byte| *byte == b'=').count())?; + if !meta.is_file() + || meta.mode() & 0o077 != 0 + || meta.uid() != rustix::process::geteuid().as_raw() + || meta.len() != expected_bytes as u64 + { + return None; + } + let file = fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(&path) + .ok()?; + let _ = file.set_modified(std::time::SystemTime::now()); + return Some(path); + } + let bytes = BASE64.decode(data).ok()?; + if bytes.is_empty() || bytes.len() > IMAGE_BYTES { + return None; + } + if !match media { + "image/png" => bytes.starts_with(b"\x89PNG\r\n\x1a\n"), + "image/jpeg" => bytes.starts_with(b"\xff\xd8\xff"), + "image/webp" => bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP"), + _ => false, + } { + return None; + } + let mut entries = Vec::new(); + let mut total = 0; + for (index, entry) in fs::read_dir(&self.root).ok()?.take(4098).enumerate() { + if index > 4096 { + return None; + } + let entry = entry.ok()?; + let name = entry.file_name(); + let name = name.to_str()?; + if name.starts_with(".staging-") && name[9..].parse::().is_ok() { + fs::remove_file(entry.path()).ok()?; + continue; + } + let Some((stem, ext)) = name.split_once('.') else { + continue; + }; + if stem.len() != 64 + || !stem.bytes().all(|b| b.is_ascii_hexdigit()) + || !matches!(ext, "png" | "jpg" | "webp") + { + continue; + } + let meta = fs::symlink_metadata(entry.path()).ok()?; + if !meta.is_file() { + return None; + } + total += meta.len(); + entries.push((meta.modified().ok()?, meta.len(), entry.path())); + } + entries.sort_by_key(|entry| entry.0); + let mut remaining = entries.len(); + for (_, len, old) in &entries { + let expired = fs::symlink_metadata(old) + .ok()? + .modified() + .ok()? + .elapsed() + .ok()? + .as_secs() + > 90 * 86400; + if !expired && total + bytes.len() as u64 <= ARCHIVE_BYTES && remaining < 4096 { + break; + } + fs::remove_file(old).ok()?; + total = total.saturating_sub(*len); + remaining -= 1; + } + if total + bytes.len() as u64 > ARCHIVE_BYTES { + return None; + } + let staging = self.root.join(format!(".staging-{}", uuid::Uuid::new_v4())); + let mut file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(&staging) + .ok()?; + if file + .write_all(&bytes) + .and_then(|_| file.sync_data()) + .and_then(|_| fs::rename(&staging, &path)) + .is_err() + { + let _ = fs::remove_file(&staging); + return None; + } + Some(path) + } +} +struct State { + http: reqwest::Client, + base: reqwest::Url, + instance: String, + remote_token: String, + local_token: String, + archive: Arc>, + slots: Arc, +} +pub struct Adapter { + pub base_url: String, + token: String, + stop: Option>, + worker: Option>, +} +impl Adapter { + pub fn start( + config_dir: &Path, + host: &str, + instance: &str, + remote_token: &str, + ) -> Result { + let (http, base) = client::inference_transport(config_dir, host)?; + let root = config_dir.join("spark").join("image-archive"); + super::economics::private_dir(&root) + .map_err(|_| failure("private image archive is unavailable"))?; + Self::start_with(http, base, instance, remote_token, root) + } + fn start_with( + http: reqwest::Client, + base: reqwest::Url, + instance: &str, + remote_token: &str, + root: PathBuf, + ) -> Result { + client::validate_instance_reference(instance)?; + let listener = TcpListener::bind("127.0.0.1:0") + .map_err(|_| failure("cannot bind the private inference adapter"))?; + listener + .set_nonblocking(true) + .map_err(|_| failure("cannot configure inference adapter"))?; + let address = listener + .local_addr() + .map_err(|_| failure("cannot identify inference adapter"))?; + let token = format!( + "sparkplane_local_{}{}", + uuid::Uuid::new_v4().simple(), + uuid::Uuid::new_v4().simple() + ); + let state = Arc::new(State { + http, + base, + instance: instance.into(), + remote_token: remote_token.into(), + local_token: token.clone(), + archive: Arc::new(Mutex::new(Archive { root })), + slots: Arc::new(Semaphore::new(2)), + }); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .map_err(|_| failure("cannot start inference adapter runtime"))?; + let (stop, mut stopping) = oneshot::channel(); + let worker = thread::spawn(move || { + runtime.block_on(async move { + let Ok(listener) = tokio::net::TcpListener::from_std(listener) else { return; }; + let mut connections = tokio::task::JoinSet::new(); + loop { + tokio::select! { + _ = &mut stopping => break, + Some(_) = connections.join_next(), if !connections.is_empty() => {}, + accepted = listener.accept() => { + let Ok((stream, _)) = accepted else { break; }; + if connections.len() >= 32 { continue; } + let state = state.clone(); + connections.spawn(async move { + let service = service_fn(move |request| { + let state = state.clone(); + async move { Ok::<_, std::convert::Infallible>(handle(request, state).await) } + }); + let _ = http1::Builder::new().max_buf_size(16 * 1024).serve_connection(TokioIo::new(stream), service).await; + }); + } + } + } + connections.abort_all(); + }) + }); + Ok(Self { + base_url: format!("http://{address}"), + token, + stop: Some(stop), + worker: Some(worker), + }) + } + pub fn token(&self) -> &str { + &self.token + } +} +impl Drop for Adapter { + fn drop(&mut self) { + if let Some(stop) = self.stop.take() { + let _ = stop.send(()); + } + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} +fn failure(message: &'static str) -> ClientError { + ClientError { + code: super::EXIT_INTERNAL, + message: message.into(), + } +} +fn error(status: u16, message: &'static str) -> Response { + let bytes = serde_json::to_vec(&json!({"type":"error","error":{"type":"invalid_request_error","code":if status == 413 { "request_too_large" } else if message.starts_with("history text exceeds") { "context_length_exceeded" } else { "invalid_request_error" },"message":message}})).expect("static error"); + let mut builder = Response::builder() + .status(status) + .header("content-type", "application/json"); + if status == 429 { + builder = builder.header("retry-after", "1"); + } + builder + .body( + Full::new(Bytes::from(bytes)) + .map_err(|never| match never {}) + .boxed_unsync(), + ) + .expect("static response") +} +struct ChunkReader { + receiver: mpsc::Receiver, + current: std::io::Cursor, +} +impl Read for ChunkReader { + fn read(&mut self, out: &mut [u8]) -> io::Result { + if out.is_empty() { + return Ok(0); + } + loop { + let n = self.current.read(out)?; + if n != 0 { + return Ok(n); + } + match self.receiver.recv_timeout(Duration::from_secs(30)) { + Ok(bytes) => self.current = std::io::Cursor::new(bytes), + Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(0), + Err(_) => return Err(io::Error::other("upload timed out")), + } + } + } +} +async fn handle(request: Request, state: Arc) -> Response { + if request + .headers() + .get("authorization") + .and_then(|v| v.to_str().ok()) + != Some(&format!("Bearer {}", state.local_token)) + { + return error(401, "private adapter authentication failed"); + } + let path = request.uri().path(); + if (request.uri().query().is_some() + && !(path.starts_with("/anthropic/") && request.uri().query() == Some("beta=true"))) + || request.headers().contains_key("origin") + { + return error(400, "adapter route is unsupported"); + } + let (route, protocol) = match (request.method().as_str(), path) { + ("POST", "/openai/v1/responses") => ( + format!("openai/{}/v1/responses", state.instance), + Some(Protocol::Responses), + ), + ("POST", "/openai/v1/chat/completions") => ( + format!("openai/{}/v1/chat/completions", state.instance), + Some(Protocol::Chat), + ), + ("POST", "/anthropic/v1/messages") => ( + format!("anthropic/{}/v1/messages", state.instance), + Some(Protocol::Anthropic), + ), + ("POST", "/anthropic/v1/messages/count_tokens") => ( + format!("anthropic/{}/v1/messages/count_tokens", state.instance), + Some(Protocol::Anthropic), + ), + ("GET", "/openai/v1/models") => (format!("openai/{}/v1/models", state.instance), None), + _ => return error(404, "adapter route is unsupported"), + }; + if request + .headers() + .get("content-encoding") + .is_some_and(|v| v != "identity") + { + return error(415, "compressed adapter uploads are unsupported"); + } + let permit = match state.slots.clone().try_acquire_owned() { + Ok(p) => p, + Err(_) => return error(429, "private adapter is busy; retry shortly"), + }; + let (parts, mut body) = request.into_parts(); + let payload = if let Some(protocol) = protocol { + let (sender, receiver) = mpsc::sync_channel(2); + let archive = state.archive.clone(); + let parsing = tokio::task::spawn_blocking(move || { + // The permit remains held until blocking parsing exits, including cancellation. + let _permit = permit; + let mut save = + |id: &str, part: &Value, protocol| archive.lock().ok()?.save(id, part, protocol); + image_history::project_reader( + std::io::BufReader::new(ChunkReader { + receiver, + current: std::io::Cursor::new(Bytes::new()), + }), + protocol, + 16, + Some(&mut save), + ) + }); + let feeding = async { + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|_| ())?; + if let Ok(data) = frame.into_data() { + // Backpressure without blocking the asynchronous transport workers. + let sender = sender.clone(); + tokio::task::spawn_blocking(move || sender.send(data)) + .await + .map_err(|_| ())? + .map_err(|_| ())?; + } + } + drop(sender); + Ok::<_, ()>(()) + }; + let fed = tokio::time::timeout(Duration::from_secs(120), feeding).await; + if fed.is_err() { + return error(408, "private adapter upload timed out"); + } + match tokio::time::timeout(Duration::from_secs(120), parsing).await { + Ok(Ok(Ok((value, _)))) if matches!(fed, Ok(Ok(()))) => match serde_json::to_vec(&value) + { + Ok(bytes) => Some(bytes), + Err(_) => return error(400, "request cannot be encoded"), + }, + Ok(Ok(Err(message))) => { + return error( + if message.contains("byte budget") || message.contains("32 MiB") { + 413 + } else { + 400 + }, + message, + ); + } + _ => { + return error( + 413, + "history cannot fit the adapter budgets; compact text or split the new image batch", + ); + } + } + } else { + drop(permit); + None + }; + let Ok(mut url) = state.base.join(&route) else { + return error(502, "invalid pinned upstream route"); + }; + if let Some(query) = parts.uri.query() { + url.set_query(Some(query)); + } + let mut upstream = state + .http + .request(parts.method, url) + .bearer_auth(&state.remote_token); + for name in ["anthropic-version", "anthropic-beta", "accept"] { + if let Some(value) = parts.headers.get(name) { + upstream = upstream.header(name, value); + } + } + if let Some(payload) = payload { + upstream = upstream + .header("content-type", "application/json") + .body(payload); + } + let response = match upstream.send().await { + Ok(r) => r, + Err(_) => return error(502, "pinned Spark inference transport is unavailable"), + }; + let mut builder = Response::builder().status(response.status()); + for name in ["content-type", "retry-after"] { + if let Some(value) = response.headers().get(name) { + builder = builder.header(name, value); + } + } + let stream = futures_util::stream::try_unfold(response, |mut response| async move { + match response.chunk().await { + Ok(Some(chunk)) => Ok(Some((Frame::data(chunk), response))), + Ok(None) => Ok(None), + Err(_) => Err(io::Error::other("inference response stream disconnected")), + } + }); + builder + .body(StreamBody::new(stream).boxed_unsync()) + .expect("upstream status and headers") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct HistoryReader { + next: usize, + current: io::Cursor>, + image: String, + read: Arc, + } + impl Read for HistoryReader { + fn read(&mut self, out: &mut [u8]) -> io::Result { + loop { + let n = self.current.read(out)?; + if n > 0 { + self.read.fetch_add(n, Ordering::Relaxed); + return Ok(n); + } + let bytes = match self.next { + 0 => br#"{"input":["#.to_vec(), + 1..=200 => { + let i = self.next - 1; + format!("{}{}", if i == 0 { "" } else { "," }, + json!({"type":"message","role":"user","content":[{"type":"input_text","text":format!("frame {i}")},{"type":"input_image","image_url":self.image}]})).into_bytes() + } + 201 => b"],\"stream\":true}".to_vec(), + _ => return Ok(0), + }; + // Each inspected frame is followed by an assistant observation. + let bytes = if (1..200).contains(&self.next) { + let mut bytes = bytes; + bytes.extend_from_slice(format!(",{}", json!({"type":"message","role":"assistant","content":"Button remains visible."})).as_bytes()); + bytes + } else { + bytes + }; + self.next += 1; + self.current = io::Cursor::new(bytes); + } + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn streamed_history_exceeds_gateway_upload_limit_but_forwarded_body_stays_small() { + let sent = Arc::new(Mutex::new(Vec::new())); + let observed = sent.clone(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let observed = observed.clone(); + tokio::spawn(async move { + let service = service_fn(move |request: Request| { + let observed = observed.clone(); + async move { + assert_eq!(request.uri().path(), "/openai/fixture/v1/responses"); + assert_eq!(request.headers()["authorization"], "Bearer REMOTE_FIXTURE"); + let bytes = request.into_body().collect().await.unwrap().to_bytes(); + assert!(bytes.len() < image_history::REQUEST_BYTES); + observed + .lock() + .unwrap() + .push(serde_json::from_slice::(&bytes).unwrap()); + Ok::<_, std::convert::Infallible>(Response::new(Full::new( + Bytes::from_static(b"data: {\"type\":\"response.completed\"}\n\n"), + ))) + } + }); + let _ = http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + }); + let root = tempfile::tempdir().unwrap(); + let archive = root.path().join("archive"); + super::super::economics::private_dir(&archive).unwrap(); + let adapter = Adapter::start_with( + reqwest::Client::new(), + format!("http://{address}").parse().unwrap(), + "fixture", + "REMOTE_FIXTURE", + archive.clone(), + ) + .unwrap(); + let http = reqwest::Client::new(); + let unauthorized = http + .post(format!("{}/openai/v1/responses", adapter.base_url)) + .body("bad") + .send() + .await + .unwrap(); + assert_eq!(unauthorized.status(), 401); + let forbidden = http + .post(format!("{}/api/sparkplane/v1/tokens", adapter.base_url)) + .bearer_auth(adapter.token()) + .body("{}") + .send() + .await + .unwrap(); + assert_eq!(forbidden.status(), 404); + assert!(sent.lock().unwrap().is_empty()); + let malformed = http + .post(format!("{}/openai/v1/responses", adapter.base_url)) + .bearer_auth(adapter.token()) + .body("{") + .send() + .await + .unwrap(); + assert_eq!(malformed.status(), 400); + let excessive = json!({"input":[{"type":"message","role":"user","content":(0..17).map(|_| json!({"type":"input_image","image_url":"data:image/png;base64,aW1hZ2U="})).collect::>()}]}); + let excessive = http + .post(format!("{}/openai/v1/responses", adapter.base_url)) + .bearer_auth(adapter.token()) + .json(&excessive) + .send() + .await + .unwrap(); + assert_eq!(excessive.status(), 400); + assert!(excessive.text().await.unwrap().contains("split the batch")); + assert!(sent.lock().unwrap().is_empty()); + let config: toml::Value = toml::from_str(include_str!( + "../../configs/sparkplane/engines/vllm-qwen38-mmap.toml" + )) + .unwrap(); + let mut original = BASE64 + .decode( + config["profiles"][0]["vision"]["health_image_base64"] + .as_str() + .unwrap(), + ) + .unwrap(); + original.resize(200_000, 0); + let image = format!("data:image/png;base64,{}", BASE64.encode(&original)); + let bytes_read = Arc::new(AtomicUsize::new(0)); + let reader = HistoryReader { + next: 0, + current: io::Cursor::new(Vec::new()), + image, + read: bytes_read.clone(), + }; + let endpoint = format!("{}/openai/v1/responses", adapter.base_url); + let token = adapter.token().to_owned(); + let response = tokio::task::spawn_blocking(move || { + reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(120)) + .build() + .unwrap() + .post(endpoint) + .bearer_auth(token) + .body(reqwest::blocking::Body::new(reader)) + .send() + .unwrap() + .text() + .unwrap() + }) + .await + .unwrap(); + assert!(response.contains("response.completed"), "{response}"); + assert!(bytes_read.load(Ordering::Relaxed) > image_history::REQUEST_BYTES); + let value = sent.lock().unwrap().pop().unwrap(); + let mut projected = value.clone(); + let mut images = 0; + image_history::visit_parts(&mut projected["input"], Protocol::Responses, &mut |_| { + images += 1 + }); + assert!(images <= 12); + assert_eq!(value["input"].as_array().unwrap().len(), 399); + assert_eq!(value["input"][1]["content"], "Button remains visible."); + let marker = value["input"][0]["content"][1]["text"].as_str().unwrap(); + let files = fs::read_dir(&archive) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "png")) + .collect::>(); + assert_eq!(files.len(), 1); + assert!(marker.contains(&files[0].path().to_string_lossy().to_string())); + assert_eq!(fs::read(files[0].path()).unwrap(), original); + assert_eq!(fs::metadata(files[0].path()).unwrap().mode() & 0o777, 0o600); + assert_eq!(fs::metadata(&archive).unwrap().mode() & 0o777, 0o700); + let local_address = adapter.base_url.strip_prefix("http://").unwrap().to_owned(); + drop(adapter); + assert!(tokio::net::TcpStream::connect(local_address).await.is_err()); + server.abort(); + } +} diff --git a/src/spark/launch.rs b/src/spark/launch.rs index dfcae32..0f79081 100644 --- a/src/spark/launch.rs +++ b/src/spark/launch.rs @@ -45,6 +45,8 @@ struct LaunchPlan { extra_argument_count: usize, allow_network: bool, mode: String, + eco_mode: String, + eco_capability: &'static str, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -182,6 +184,8 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client extra_argument_count: args.extra_args.len(), allow_network: args.allow_network, mode: args.mode.clone(), + eco_mode: args.eco_mode.clone(), + eco_capability: eco_capability(&args), }, args.json, ); @@ -257,6 +261,8 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client extra_argument_count: args.extra_args.len(), allow_network: args.allow_network, mode: args.mode.clone(), + eco_mode: args.eco_mode.clone(), + eco_capability: eco_capability(&args), }; if args.configure { return render_plan(&plan, args.json); @@ -275,7 +281,51 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client "Network access requested for this invocation; filesystem and approval policies are unchanged." ); } - launch_child( + let session_id = ulid::Ulid::new().to_string(); + let started_at = chrono::Utc::now().to_rfc3339(); + let created = match client.create_launch_session(&super::wire::LaunchSessionRequest { + id: session_id.clone(), + instance: instance.name.clone(), + integration: args.integration.as_str().into(), + eco_mode: args.eco_mode.clone(), + }) { + Ok(value) => value, + Err(_) => { + eprintln!("Session accounting unavailable; inference usage will be marked unknown."); + // Creation may have committed before its response was lost. + let _ = client.finish_launch_session(&session_id); + None + } + }; + let mut inference = created.as_ref().map(|value| value.session.clone()); + let session_token = created + .and_then(|value| value.bearer_token) + .map(LaunchSecret); + if inference.is_some() && session_token.is_none() { + // A lost one-time credential cannot be recovered. Revoke it rather than + // attributing the shared fallback credential to this session. + let _ = client.finish_launch_session(&session_id); + inference = None; + } + let eco = match super::eco::EcoSession::create(&session_id, args.eco_mode == "max", &args.mode) + { + Ok(value) => Some(value), + Err(_) => { + eprintln!("Eco integration unavailable; launch continues without compression."); + None + } + }; + if args.eco_mode == "max" { + eprintln!( + "Eco mode: embedded RTK; {}.", + if args.mode == "inherit" { + "compression inactive to preserve inherited permissions" + } else { + "agent hook trust still applies" + } + ); + } + let result = launch_child( args.integration, args.yes, args.allow_network, @@ -285,10 +335,59 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client host, instance: &instance, model: &model, - token: &token, + token: session_token.as_ref().unwrap_or(&token), }, &args.extra_args, - ) + eco.as_ref(), + ); + if inference.is_some() { + match client.finish_launch_session(&session_id) { + Ok(value) => inference = Some(value), + Err(_) => { + inference = None; + eprintln!("Session finalization unavailable; inference usage is unknown."); + } + } + } + let compression = eco + .as_ref() + .map(super::eco::EcoSession::summary) + .unwrap_or_else(|| super::economics::Compression { + status: "unavailable".into(), + metrics_errors: 1, + ..Default::default() + }); + let mut report = super::economics::Report { + schema: super::economics::SCHEMA.into(), + session_id, + host: host.into(), + model: model.canonical.clone(), + integration: args.integration.as_str().into(), + eco_mode: args.eco_mode.clone(), + started_at, + finished_at: Some(chrono::Utc::now().to_rfc3339()), + exit_code: Some(result.as_ref().err().map(|error| error.code).unwrap_or(0)), + inference, + compression, + catalog: super::economics::catalog().expect("bundled price catalog"), + comparisons: Vec::new(), + }; + report.calculate(); + if super::economics::save(config_dir, &report).is_err() { + eprintln!("Could not retain the private economics report."); + } + super::economics::display(&report); + result +} + +fn eco_capability(args: &LaunchArgs) -> &'static str { + if args.eco_mode == "none" { + "disabled" + } else if args.mode == "inherit" { + "inactive_policy" + } else { + "embedded_hooks_require_agent_support_and_trust" + } } fn validate_args(args: &LaunchArgs) -> Result<(), ClientError> { @@ -732,11 +831,17 @@ fn launch_child( mode: &str, launch: ReadyLaunch<'_>, extra_args: &[String], + eco: Option<&super::eco::EcoSession>, ) -> Result<(), ClientError> { let executable = ensure_executable(integration, yes)?; let mut command = Command::new(executable); if integration != LaunchIntegration::Opencode { configure_launch_permissions(&mut command, integration, mode, allow_network)?; + if let Some(eco) = eco + && eco.configure(&mut command, integration).is_err() + { + eprintln!("Eco hook setup unavailable; launch continues without compression."); + } } command .stdin(Stdio::inherit()) @@ -774,12 +879,88 @@ fn launch_child( if integration == LaunchIntegration::Opencode { configure_launch_permissions(&mut command, integration, mode, allow_network)?; } - let status = command - .status() + if integration == LaunchIntegration::Opencode + && let Some(eco) = eco + && eco.configure(&mut command, integration).is_err() + { + eprintln!("Eco hook setup unavailable; launch continues without compression."); + } + let adapter = if launch + .instance + .input_modalities + .iter() + .any(|modality| modality == "image") + { + let adapter = super::inference_adapter::Adapter::start( + launch.config_dir, + launch.host, + &launch.instance.name, + launch.token.expose(), + )?; + configure_image_adapter(&mut command, integration, &launch.instance.name, &adapter)?; + eprintln!("Image history: recent frames retained; older originals can be reopened."); + Some(adapter) + } else { + None + }; + let status = super::eco::wait(command) .map_err(|_| usage(format!("could not start {}", integration.as_str())))?; + drop(adapter); child_result(integration, status) } +fn configure_image_adapter( + command: &mut Command, + integration: LaunchIntegration, + instance: &str, + adapter: &super::inference_adapter::Adapter, +) -> Result<(), ClientError> { + match integration { + LaunchIntegration::Codex => { + command + .arg("-c") + .arg(format!( + "model_providers.{}.base_url={:?}", + provider_name(instance), + format!("{}/openai/v1", adapter.base_url) + )) + .env("SPARKPLANE_INFERENCE_TOKEN", adapter.token()) + .env("OPENAI_API_KEY", adapter.token()); + } + LaunchIntegration::Claude => { + command + .env( + "ANTHROPIC_BASE_URL", + format!("{}/anthropic", adapter.base_url), + ) + .env("ANTHROPIC_AUTH_TOKEN", adapter.token()); + } + LaunchIntegration::Opencode => { + let content = command + .get_envs() + .find(|(key, _)| *key == "OPENCODE_CONFIG_CONTENT") + .and_then(|(_, value)| value) + .ok_or_else(|| failure(EXIT_INTERNAL, "missing OpenCode session configuration"))?; + let mut value: Value = serde_json::from_str(&content.to_string_lossy()) + .map_err(|_| failure(EXIT_INTERNAL, "invalid OpenCode session configuration"))?; + value["provider"]["sparkplane"]["options"]["baseURL"] = + format!("{}/openai/v1", adapter.base_url).into(); + command + .env( + "OPENCODE_CONFIG_CONTENT", + serde_json::to_string(&value).map_err(|_| { + failure( + EXIT_INTERNAL, + "cannot encode OpenCode session configuration", + ) + })?, + ) + .env("SPARKPLANE_INFERENCE_TOKEN", adapter.token()); + } + } + Ok(()) +} + fn configure_claude_command( command: &mut Command, config_dir: &Path, @@ -849,6 +1030,11 @@ fn configure_codex_command( "responses" ), format!("model_catalog_json={:?}", catalog), + // Leave room for new tool results and a compaction request on local models. + format!( + "model_auto_compact_token_limit={}", + instance.context_window * 3 / 4 + ), ]; if let Some(effort) = &instance.default_reasoning_effort { overrides.push(format!("model_reasoning_effort={effort:?}")); @@ -1072,10 +1258,7 @@ fn child_result(integration: LaunchIntegration, status: ExitStatus) -> Result<() if status.success() { return Ok(()); } - let code = status - .code() - .filter(|code| (1..=125).contains(code)) - .unwrap_or(1); + let code = super::eco::exit_code(status); Err(failure( code, format!("{} exited with status {code}", integration.as_str()), @@ -1838,6 +2021,7 @@ mod tests { let args = LaunchArgs { integration: LaunchIntegration::Codex, mode: "auto".into(), + eco_mode: "max".into(), allow_network: false, model: Some("ornith".into()), configure: false, @@ -1861,4 +2045,18 @@ mod tests { 7 ); } + + #[test] + fn child_signal_exit_status_is_preserved() { + let status = Command::new("sh") + .args(["-c", "kill -TERM $$"]) + .status() + .unwrap(); + assert_eq!( + child_result(LaunchIntegration::Codex, status) + .unwrap_err() + .code, + 143 + ); + } } diff --git a/src/spark/mod.rs b/src/spark/mod.rs index b7c7075..3ee65e9 100644 --- a/src/spark/mod.rs +++ b/src/spark/mod.rs @@ -2,13 +2,19 @@ pub mod cli; pub mod client; +pub mod eco; +pub mod economics; #[cfg(feature = "appliance")] pub mod engine; +pub mod image_history; +pub mod inference_adapter; pub mod install; pub mod launch; pub mod qualification; #[cfg(feature = "appliance")] pub mod reconcile; +#[cfg(feature = "appliance")] +pub mod sessions; pub mod wire; #[cfg(feature = "appliance")] diff --git a/src/spark/sessions.rs b/src/spark/sessions.rs new file mode 100644 index 0000000..ef98676 --- /dev/null +++ b/src/spark/sessions.rs @@ -0,0 +1,460 @@ +//! Private per-launch inference ledger; all SQL runs on the existing database actor. +use std::sync::{Arc, Mutex}; + +use rusqlite::{Connection, OptionalExtension, params}; +use secrecy::SecretString; +use tokio::sync::oneshot; + +use super::{ + state::{DbActor, StateError}, + wire::{LaunchSessionCreated, LaunchSessionDocument, LaunchSessionRequest, SessionUsage}, +}; + +pub(crate) const MIGRATION: &str = r#" +CREATE TABLE launch_sessions ( + id TEXT PRIMARY KEY, token_id TEXT NOT NULL UNIQUE REFERENCES token_metadata(id), + request_hash TEXT NOT NULL, instance TEXT NOT NULL, model TEXT NOT NULL, + integration TEXT NOT NULL, eco_mode TEXT NOT NULL, started_at TEXT NOT NULL, + finished_at TEXT +); +CREATE TABLE launch_requests ( + id TEXT PRIMARY KEY, session_id TEXT NOT NULL REFERENCES launch_sessions(id) ON DELETE CASCADE, + instance TEXT NOT NULL, started_at TEXT NOT NULL, finished_at TEXT, + input_tokens INTEGER, output_tokens INTEGER, outcome TEXT NOT NULL DEFAULT 'pending', accounting_error INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX launch_requests_session_idx ON launch_requests(session_id); +CREATE INDEX launch_sessions_retention_idx ON launch_sessions(started_at); +"#; + +pub(crate) fn migration_needed(db: &Connection) -> Result { + db.query_row("SELECT NOT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='launch_session_migrations')",[],|row|row.get(0)).map_err(sql) +} + +pub(crate) fn migrate(db: &mut Connection) -> Result<(), StateError> { + use sha2::{Digest, Sha256}; + let checksum = format!("{:x}", Sha256::digest(MIGRATION.as_bytes())); + if !migration_needed(db)? { + let versions = db + .prepare("SELECT version,checksum FROM launch_session_migrations ORDER BY version") + .map_err(sql)? + .query_map([], |row| { + Ok((row.get::<_, u32>(0)?, row.get::<_, String>(1)?)) + }) + .map_err(sql)? + .collect::, _>>() + .map_err(sql)?; + if versions != [(1, checksum)] { + return Err(StateError::Unavailable( + "unsupported or modified session ledger migration".into(), + )); + } + return Ok(()); + } + let tx = db.transaction().map_err(sql)?; + tx.execute_batch("CREATE TABLE launch_session_migrations(version INTEGER PRIMARY KEY,checksum TEXT NOT NULL);").map_err(sql)?; + tx.execute_batch(MIGRATION).map_err(sql)?; + tx.execute( + "INSERT INTO launch_session_migrations(version,checksum) VALUES(1,?1)", + [checksum], + ) + .map_err(sql)?; + tx.commit().map_err(sql) +} + +pub(crate) enum Command { + Create { + actor: String, + request: LaunchSessionRequest, + reply: oneshot::Sender>, + }, + Get { + id: String, + finish: bool, + actor: String, + reply: oneshot::Sender>, + }, + Begin { + token: String, + id: String, + instance: String, + reply: oneshot::Sender>, + }, + Record { + id: String, + usage: Option<(u64, u64)>, + outcome: String, + incomplete: bool, + reply: oneshot::Sender>, + }, +} + +impl DbActor { + pub async fn create_launch_session( + &self, + actor: &str, + request: LaunchSessionRequest, + ) -> Result { + let (reply, receive) = oneshot::channel(); + self.session_command(Command::Create { + actor: actor.into(), + request, + reply, + })?; + receive + .await + .map_err(|_| StateError::Unavailable("session database unavailable".into()))? + } + pub async fn launch_session( + &self, + id: &str, + finish: bool, + actor: &str, + ) -> Result { + let (reply, receive) = oneshot::channel(); + self.session_command(Command::Get { + id: id.into(), + finish, + actor: actor.into(), + reply, + })?; + receive + .await + .map_err(|_| StateError::Unavailable("session database unavailable".into()))? + } + async fn begin_usage(&self, token: &str, id: &str, instance: &str) -> Result { + let (reply, receive) = oneshot::channel(); + self.session_command(Command::Begin { + token: token.into(), + id: id.into(), + instance: instance.into(), + reply, + })?; + receive + .await + .map_err(|_| StateError::Unavailable("session database unavailable".into()))? + } + async fn record_usage( + &self, + id: &str, + usage: Option<(u64, u64)>, + outcome: &str, + incomplete: bool, + ) -> Result<(), StateError> { + let (reply, receive) = oneshot::channel(); + self.session_command(Command::Record { + id: id.into(), + usage, + outcome: outcome.into(), + incomplete, + reply, + })?; + receive + .await + .map_err(|_| StateError::Unavailable("session database unavailable".into()))? + } +} + +fn sql(error: rusqlite::Error) -> StateError { + StateError::Unavailable(format!("session database: {error}")) +} + +pub(crate) fn dispatch(db: &mut Connection, pepper: &SecretString, command: Command) { + match command { + Command::Create { + actor, + request, + reply, + } => { + let _ = reply.send(create(db, pepper, &actor, request)); + } + Command::Get { + id, + finish, + actor, + reply, + } => { + let result = if finish { + finish_session(db, &id, &actor).and_then(|()| read(db, &id)) + } else { + read(db, &id) + }; + let _ = reply.send(result); + } + Command::Begin { + token, + id, + instance, + reply, + } => { + let result = (|| { + let expected = db + .query_row( + "SELECT instance,finished_at FROM launch_sessions WHERE token_id=?1", + [&token], + |r| Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?)), + ) + .optional() + .map_err(sql)?; + let Some((expected, finished)) = expected else { + return Ok(false); + }; + if expected != instance || finished.is_some() { + return Err(StateError::Invalid( + "launch session is not active for this instance".into(), + )); + } + db.execute("INSERT INTO launch_requests(id,session_id,instance,started_at) SELECT ?1,id,?2,?3 FROM launch_sessions WHERE token_id=?4 AND finished_at IS NULL", params![id,instance,super::state::now(),token]).map(|n| n == 1).map_err(sql) + })(); + let _ = reply.send(result); + } + Command::Record { + id, + usage, + outcome, + incomplete, + reply, + } => { + let result = usage.map(|(a,b)| (i64::try_from(a),i64::try_from(b))).map(|(a,b)| Ok((a.map_err(|_| StateError::Invalid("usage overflow".into()))?,b.map_err(|_| StateError::Invalid("usage overflow".into()))?))).transpose().and_then(|counts| { + db.execute("UPDATE launch_requests SET input_tokens=?2,output_tokens=?3,outcome=?4,finished_at=?5,accounting_error=?6 WHERE id=?1 AND finished_at IS NULL",params![id,counts.map(|c|c.0),counts.map(|c|c.1),outcome,super::state::now(),incomplete]).map(|_|()).map_err(sql) + }); + let _ = reply.send(result); + } + } +} + +fn create( + db: &mut Connection, + pepper: &SecretString, + actor: &str, + request: LaunchSessionRequest, +) -> Result { + if request.id.parse::().is_err() + || !["codex", "claude", "opencode"].contains(&request.integration.as_str()) + || !["max", "none"].contains(&request.eco_mode.as_str()) + { + return Err(StateError::Invalid("invalid launch session".into())); + } + let hash = super::wire::canonical_request_sha256(&request) + .map_err(|_| StateError::Invalid("invalid session request".into()))?; + if let Some(existing) = db + .query_row( + "SELECT request_hash FROM launch_sessions WHERE id=?1", + [&request.id], + |r| r.get::<_, String>(0), + ) + .optional() + .map_err(sql)? + { + if existing != hash { + return Err(StateError::Conflict("session identity conflict".into())); + } + return Ok(LaunchSessionCreated { + session: read(db, &request.id)?, + bearer_token: None, + }); + } + let instance = super::state::read_instance(db, &request.instance)?; + let active: i64 = db.query_row("SELECT COUNT(*) FROM token_metadata WHERE revoked_at IS NULL AND (expires_at IS NULL OR julianday(expires_at)>julianday('now'))",[],|r|r.get(0)).map_err(sql)?; + if active >= 1024 { + return Err(StateError::Overloaded); + } + let token_id = ulid::Ulid::new().to_string(); + let secret = super::state::random_secret()?; + let verifier = super::state::token_hmac(pepper, &token_id, &secret)?; + let expires = (chrono::Utc::now() + chrono::Duration::days(7)).to_rfc3339(); + let now = super::state::now(); + let tx = db.transaction().map_err(sql)?; + tx.execute( + "DELETE FROM launch_sessions WHERE julianday(started_at) Result<(), StateError> { + let token: String = db + .query_row( + "SELECT token_id FROM launch_sessions WHERE id=?1", + [id], + |r| r.get(0), + ) + .optional() + .map_err(sql)? + .ok_or(StateError::NotFound)?; + let tx = db.transaction().map_err(sql)?; + let now = super::state::now(); + let changed = tx + .execute( + "UPDATE launch_sessions SET finished_at=?2 WHERE id=?1 AND finished_at IS NULL", + params![id, now], + ) + .map_err(sql)?; + tx.execute( + "UPDATE token_metadata SET revoked_at=COALESCE(revoked_at,?2) WHERE id=?1", + params![token, now], + ) + .map_err(sql)?; + if changed > 0 { + tx.execute("INSERT INTO audit(occurred_at,actor_token_id,action,target,outcome,metadata_json) VALUES(?1,?2,'launch-session.finish',?3,'succeeded','{}')",params![now,actor,id]).map_err(sql)?; + } + tx.commit().map_err(sql) +} + +fn read(db: &Connection, id: &str) -> Result { + let mut session = db.query_row("SELECT id,instance,model,integration,eco_mode,started_at,finished_at FROM launch_sessions WHERE id=?1",[id],|r|Ok(LaunchSessionDocument { schema:"sparkplane.launch-session/v1".into(),id:r.get(0)?,instance:r.get(1)?,model:r.get(2)?,integration:r.get(3)?,eco_mode:r.get(4)?,started_at:r.get(5)?,finished_at:r.get(6)?,usage:SessionUsage::default() })).optional().map_err(sql)?.ok_or(StateError::NotFound)?; + session.usage = db.query_row("SELECT COUNT(*),COALESCE(SUM(input_tokens),0),COALESCE(SUM(output_tokens),0),COALESCE(SUM(input_tokens IS NULL),0),COALESCE(SUM(finished_at IS NULL),0),COALESCE(SUM(outcome!='succeeded' AND outcome!='pending'),0),COALESCE(SUM(CASE WHEN input_tokens>272000 THEN input_tokens ELSE 0 END),0),COALESCE(SUM(CASE WHEN input_tokens>272000 THEN output_tokens ELSE 0 END),0),COALESCE(SUM(accounting_error),0) FROM launch_requests WHERE session_id=?1",[id],|r|Ok(SessionUsage {requests:r.get::<_,i64>(0)? as u64,input_tokens:r.get::<_,i64>(1)? as u64,output_tokens:r.get::<_,i64>(2)? as u64,unknown_usage_requests:r.get::<_,i64>(3)? as u64,pending_requests:r.get::<_,i64>(4)? as u64,failed_requests:r.get::<_,i64>(5)? as u64,long_context_input_tokens:r.get::<_,i64>(6)? as u64,long_context_output_tokens:r.get::<_,i64>(7)? as u64,accounting_errors:r.get::<_,i64>(8)? as u64})).map_err(sql)?; + Ok(session) +} + +#[derive(Clone, Default)] +pub(crate) struct UsageMeter(Option>); +impl std::fmt::Debug for UsageMeter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("UsageMeter") + } +} +#[derive(Clone, Copy)] +struct MeterValues { + usage: Option<(u64, u64)>, + outcome: &'static str, + incomplete: bool, +} +struct MeterInner { + db: DbActor, + id: String, + values: Mutex, +} +impl UsageMeter { + pub async fn begin( + db: Option<&DbActor>, + token: &str, + instance: &str, + ) -> Result { + let Some(db) = db else { + return Ok(Self::default()); + }; + let id = ulid::Ulid::new().to_string(); + match db.begin_usage(token, &id, instance).await { + Ok(true) => Ok(Self(Some(Arc::new(MeterInner { + db: db.clone(), + id, + values: Mutex::new(MeterValues { + usage: None, + outcome: "interrupted", + incomplete: false, + }), + })))), + Ok(false) => Ok(Self::default()), + Err(error) => Err(error), + } + } + pub fn observe(&self, event: &super::upstream::GenerationEvent) { + if let Some(inner) = &self.0 + && let Ok(mut values) = inner.values.lock() + { + match event { + super::upstream::GenerationEvent::Usage { + prompt_tokens, + completion_tokens, + } => values.usage = Some((*prompt_tokens, *completion_tokens)), + super::upstream::GenerationEvent::Done => values.outcome = "succeeded", + _ => (), + } + } + } + pub fn json(&self, bytes: &[u8], success: bool) { + if let Some(inner) = &self.0 + && let Ok(mut values) = inner.values.lock() + { + values.outcome = if success { "succeeded" } else { "failed" }; + if let Ok(document) = serde_json::from_slice::(bytes) + && let Some(usage) = document.get("usage") + { + values.usage = usage + .get("prompt_tokens") + .or_else(|| usage.get("input_tokens")) + .and_then(|v| v.as_u64()) + .zip( + usage + .get("completion_tokens") + .or_else(|| usage.get("output_tokens")) + .and_then(|v| v.as_u64()), + ); + } + } + } + pub fn raw(&self, bytes: &[u8]) { + // RawResponseStream frames events; SSE permits data: with no space and + // JSON spread across several data lines, with LF or CRLF separators. + let data = bytes + .split(|b| *b == b'\n') + .filter_map(|line| { + line.strip_suffix(b"\r") + .unwrap_or(line) + .strip_prefix(b"data:") + }) + .map(|line| line.strip_prefix(b" ").unwrap_or(line)) + .collect::>() + .join(&b'\n'); + if let Ok(document) = serde_json::from_slice::(&data) { + let response = document.get("response").unwrap_or(&document); + let terminal = matches!( + document.get("type").and_then(|v| v.as_str()), + Some("response.completed" | "response.incomplete" | "response.failed") + ); + let success = + document.get("type").and_then(|v| v.as_str()) == Some("response.completed"); + if terminal || response.get("usage").is_some() { + self.json(&serde_json::to_vec(response).unwrap_or_default(), success); + } + } + } +} +impl Drop for MeterInner { + fn drop(&mut self) { + let mut values = self.values.lock().map(|v| *v).unwrap_or(MeterValues { + usage: None, + outcome: "interrupted", + incomplete: true, + }); + values.incomplete |= values.outcome != "succeeded"; + let db = self.db.clone(); + let id = self.id.clone(); + tokio::spawn(async move { + if db + .record_usage(&id, values.usage, values.outcome, values.incomplete) + .await + .is_err() + { + tracing::warn!("session usage could not be recorded"); + } + }); + } +} + +impl super::upstream::UsageObserver for UsageMeter { + fn observe(&self, event: &super::upstream::GenerationEvent) { + UsageMeter::observe(self, event); + } + fn json(&self, bytes: &[u8], success: bool) { + UsageMeter::json(self, bytes, success); + } + fn raw(&self, bytes: &[u8]) { + UsageMeter::raw(self, bytes); + } + fn incomplete(&self) { + if let Some(inner) = &self.0 + && let Ok(mut values) = inner.values.lock() + { + values.incomplete = true; + } + } +} diff --git a/src/spark/state.rs b/src/spark/state.rs index daf8e2a..9aefe4d 100644 --- a/src/spark/state.rs +++ b/src/spark/state.rs @@ -297,6 +297,7 @@ enum Command { reply: oneshot::Sender, StateError>>, }, Snapshot(oneshot::Sender>), + Sessions(super::sessions::Command), Shutdown(std_mpsc::SyncSender<()>), } @@ -308,6 +309,15 @@ pub struct DbActor { } impl DbActor { + pub(crate) fn session_command( + &self, + command: super::sessions::Command, + ) -> Result<(), StateError> { + self.sender + .try_send(Command::Sessions(command)) + .map_err(|_| StateError::Overloaded) + } + pub fn open( path: impl AsRef, backup_dir: impl AsRef, @@ -690,7 +700,7 @@ fn open_connection( let version: u32 = connection .pragma_query_value(None, "user_version", |row| row.get(0)) .map_err(state_sql)?; - if version > 0 && version < 4 { + if version > 0 && (version < 4 || super::sessions::migration_needed(&connection)?) { verified_backup(&connection, backup_dir, max_backups)?; } let migrations = Migrations::new(vec![ @@ -705,6 +715,9 @@ fn open_connection( migrations .to_latest(&mut connection) .map_err(state_migration)?; + // Additive launch tables have a separate version ledger. Older releases + // still read the qualified engine state under the unchanged main schema. + super::sessions::migrate(&mut connection)?; verified_backup(&connection, backup_dir, max_backups)?; Ok(connection) } @@ -777,6 +790,9 @@ fn actor_loop( ) { while let Some(command) = receiver.blocking_recv() { match command { + Command::Sessions(command) => { + super::sessions::dispatch(&mut connection, &pepper, command) + } Command::Health(reply) => { let _ = reply.send(database_health(&connection)); } @@ -1588,7 +1604,10 @@ fn list_instances(connection: &Connection) -> Result, Stat .collect() } -fn read_instance(connection: &Connection, reference: &str) -> Result { +pub(crate) fn read_instance( + connection: &Connection, + reference: &str, +) -> Result { let values = { let mut statement = connection .prepare("SELECT metadata_json FROM instances WHERE id=?1 OR name=?1 LIMIT 2") @@ -2341,7 +2360,11 @@ fn valid_cidr(value: &str) -> bool { prefix <= if address.is_ipv4() { 32 } else { 128 } } -fn token_hmac(pepper: &SecretString, id: &str, secret: &str) -> Result<[u8; 32], StateError> { +pub(crate) fn token_hmac( + pepper: &SecretString, + id: &str, + secret: &str, +) -> Result<[u8; 32], StateError> { let mut mac = HmacSha256::new_from_slice(pepper.expose_secret().as_bytes()) .map_err(|_| StateError::Unavailable("token pepper is invalid".into()))?; mac.update(id.as_bytes()); @@ -2349,7 +2372,7 @@ fn token_hmac(pepper: &SecretString, id: &str, secret: &str) -> Result<[u8; 32], Ok(mac.finalize().into_bytes().into()) } -fn random_secret() -> Result { +pub(crate) fn random_secret() -> Result { use std::io::Read; let mut bytes = [0_u8; 32]; fs::File::open("/dev/urandom") @@ -2381,7 +2404,7 @@ fn parse_state(value: &str) -> Result { } } -fn now() -> String { +pub(crate) fn now() -> String { chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true) } fn json(value: &T) -> Result { @@ -2501,6 +2524,123 @@ mod tests { actor.shutdown().unwrap(); } + #[tokio::test] + async fn launch_sessions_isolate_usage_and_finish_idempotently() { + use crate::spark::{ + sessions::UsageMeter, upstream::GenerationEvent, wire::LaunchSessionRequest, + }; + let root = tempfile::tempdir().unwrap(); + let db = actor(root.path()); + let model = model_document("m_0123456789abcdef0123456789abcdef", "ornith:9b"); + db.promote_model(model.clone(), false).await.unwrap(); + db.begin_serve(instance_document(&model, "ornith", "spark-vllm")) + .await + .unwrap(); + let request = LaunchSessionRequest { + id: ulid::Ulid::new().to_string(), + instance: "ornith".into(), + integration: "codex".into(), + eco_mode: "max".into(), + }; + let created = db + .create_launch_session("admin", request.clone()) + .await + .unwrap(); + assert!(created.bearer_token.is_some()); + assert!( + db.create_launch_session("admin", request.clone()) + .await + .unwrap() + .bearer_token + .is_none() + ); + let mut conflict = request.clone(); + conflict.eco_mode = "none".into(); + assert!(matches!( + db.create_launch_session("admin", conflict).await, + Err(StateError::Conflict(_)) + )); + let token = created.bearer_token.unwrap(); + let token_id = token + .strip_prefix("sparkplane_") + .unwrap() + .split('_') + .next() + .unwrap(); + let meter = UsageMeter::begin(Some(&db), token_id, "ornith") + .await + .unwrap(); + meter.observe(&GenerationEvent::Usage { + prompt_tokens: 272001, + completion_tokens: 4, + }); + meter.observe(&GenerationEvent::Usage { + prompt_tokens: 272001, + completion_tokens: 4, + }); + meter.observe(&GenerationEvent::Done); + drop(meter); + let missing = UsageMeter::begin(Some(&db), token_id, "ornith") + .await + .unwrap(); + drop(missing); + for _ in 0..100 { + if db + .launch_session(&request.id, false, "") + .await + .unwrap() + .usage + .pending_requests + == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + } + let finished = db.launch_session(&request.id, true, "admin").await.unwrap(); + assert_eq!(finished.usage.requests, 2); + assert_eq!(finished.usage.input_tokens, 272001); + assert_eq!(finished.usage.output_tokens, 4); + assert_eq!(finished.usage.long_context_input_tokens, 272001); + assert_eq!(finished.usage.unknown_usage_requests, 1); + assert_eq!(finished.usage.pending_requests, 0); + assert_eq!( + finished.finished_at, + db.launch_session(&request.id, true, "admin") + .await + .unwrap() + .finished_at + ); + assert!( + !db.auth_snapshot() + .await + .unwrap() + .tokens + .contains_key(token_id) + ); + db.shutdown().unwrap(); + // The previous release can still open its schema and read the exact + // qualified instance after the additive feature migration. + let mut legacy = Connection::open(root.path().join("state.sqlite3")).unwrap(); + Migrations::new(vec![ + M::up(MIGRATION_1), + M::up(MIGRATION_2), + M::up(MIGRATION_3), + M::up(MIGRATION_4), + M::up(MIGRATION_5), + M::up(MIGRATION_6), + ]) + .to_latest(&mut legacy) + .unwrap(); + let retained = read_instance(&legacy, "ornith").unwrap(); + assert_eq!(retained.context_window, 65536); + assert_eq!(retained.model, model.canonical); + assert_eq!( + retained.engine_fingerprint, + format!("sha256:{}", "b".repeat(64)) + ); + } + #[test] fn migrations_are_valid_checksummed_and_n_minus_one_readable() { assert_eq!( diff --git a/src/spark/upstream.rs b/src/spark/upstream.rs index 0c3bf84..1bbeb5a 100644 --- a/src/spark/upstream.rs +++ b/src/spark/upstream.rs @@ -8,7 +8,7 @@ use http_body_util::{BodyExt, Full, Limited}; use hyper::{Request, client::conn::http1, header}; use hyper_util::rt::TokioIo; -pub const MAX_BODY_BYTES: usize = 1024 * 1024; +pub const MAX_BODY_BYTES: usize = 8 * 1024 * 1024; const MAX_RESPONSE_BYTES: usize = 1024 * 1024; const MAX_ROUTE_COUNT: usize = 32; const MAX_PATH_BYTES: usize = 256; @@ -22,23 +22,107 @@ pub const DEFAULT_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(120); const PROTOCOL_TOOL_MAX_TOKENS: u32 = 256; #[derive(Debug, Clone, PartialEq, Eq)] -pub struct UpstreamError(&'static str); +pub struct UpstreamError { + message: &'static str, + rejection: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RequestRejection { + InvalidRequest, + ContextLength, + PayloadTooLarge, + RateLimited, +} + +impl RequestRejection { + pub const fn status(self) -> u16 { + match self { + Self::InvalidRequest | Self::ContextLength => 400, + Self::PayloadTooLarge => 413, + Self::RateLimited => 429, + } + } + pub const fn code(self) -> &'static str { + match self { + Self::InvalidRequest => "invalid_request_error", + Self::ContextLength => "context_length_exceeded", + Self::PayloadTooLarge => "request_too_large", + Self::RateLimited => "rate_limit_error", + } + } + pub const fn message(self) -> &'static str { + match self { + Self::InvalidRequest => "engine rejected the inference request", + Self::ContextLength => { + "request exceeds the model context window; compact the conversation or reduce requested output" + } + Self::PayloadTooLarge => "request exceeds the engine body limit", + Self::RateLimited => "engine request capacity is exhausted; retry shortly", + } + } + pub fn from_response(status: u16, bytes: &[u8]) -> Option { + match status { + 400 | 422 => { + let value = (bytes.len() <= 16 * 1024) + .then(|| serde_json::from_slice::(bytes).ok()) + .flatten(); + let error = value + .as_ref() + .and_then(|value| value.get("error")) + .or(value.as_ref()); + let context = error.is_some_and(|error| { + error["code"] == "context_length_exceeded" + || error["type"] == "context_length_exceeded" + || error["message"].as_str().is_some_and(|message| { + let message = message.to_ascii_lowercase(); + message.contains("maximum context length") + || message.contains("max_model_len") + }) + }); + Some(if context { + Self::ContextLength + } else { + Self::InvalidRequest + }) + } + 413 => Some(Self::PayloadTooLarge), + 429 => Some(Self::RateLimited), + _ => None, + } + } +} impl std::fmt::Display for UpstreamError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(self.0) + formatter.write_str(self.message) } } impl std::error::Error for UpstreamError {} impl UpstreamError { + const fn new(message: &'static str) -> Self { + Self { + message, + rejection: None, + } + } + fn rejected(rejection: RequestRejection) -> Self { + Self { + message: rejection.message(), + rejection: Some(rejection), + } + } + pub const fn request_rejection(&self) -> Option { + self.rejection + } pub const fn identity_mismatch() -> Self { - Self("engine route identity changed") + Self::new("engine route identity changed") } pub const fn diagnostic(&self) -> &'static str { - self.0 + self.message } } @@ -95,23 +179,23 @@ pub fn decode_embedding_response( tolerance_ppm: u32, ) -> Result { if bytes.len() > MAX_RESPONSE_BYTES { - return Err(UpstreamError("engine embedding response is too large")); + return Err(UpstreamError::new("engine embedding response is too large")); } let value: serde_json::Value = serde_json::from_slice(bytes) - .map_err(|_| UpstreamError("engine embedding response is invalid"))?; + .map_err(|_| UpstreamError::new("engine embedding response is invalid"))?; let data = value["data"] .as_array() .filter(|data| data.len() == expected_count) - .ok_or(UpstreamError("engine embedding count changed"))?; + .ok_or(UpstreamError::new("engine embedding count changed"))?; if value["object"] != "list" || value["model"] != served_model { - return Err(UpstreamError("engine embedding identity changed")); + return Err(UpstreamError::new("engine embedding identity changed")); } let mut vectors = Vec::with_capacity(data.len()); for (expected_index, item) in data.iter().enumerate() { let values = item["embedding"] .as_array() .filter(|values| values.len() == dimensions) - .ok_or(UpstreamError("engine embedding dimension changed"))? + .ok_or(UpstreamError::new("engine embedding dimension changed"))? .iter() .map(|value| { value @@ -119,13 +203,13 @@ pub fn decode_embedding_response( .filter(|value| value.is_finite()) .map(|value| value as f32) .filter(|value| value.is_finite()) - .ok_or(UpstreamError( + .ok_or(UpstreamError::new( "engine embedding contains a non-finite value", )) }) .collect::, _>>()?; if item["object"] != "embedding" || item["index"].as_u64() != Some(expected_index as u64) { - return Err(UpstreamError("engine embedding order changed")); + return Err(UpstreamError::new("engine embedding order changed")); } let norm = values .iter() @@ -134,7 +218,7 @@ pub fn decode_embedding_response( .sqrt(); let tolerance = f64::from(tolerance_ppm) / 1_000_000.0; if normalized && (norm - 1.0).abs() > tolerance { - return Err(UpstreamError("engine embedding normalization changed")); + return Err(UpstreamError::new("engine embedding normalization changed")); } vectors.push(EmbeddingVector { index: expected_index, @@ -143,11 +227,11 @@ pub fn decode_embedding_response( } let prompt_tokens = value["usage"]["prompt_tokens"] .as_u64() - .ok_or(UpstreamError("engine embedding usage is invalid"))?; + .ok_or(UpstreamError::new("engine embedding usage is invalid"))?; let total_tokens = value["usage"]["total_tokens"] .as_u64() .filter(|total| *total == prompt_tokens) - .ok_or(UpstreamError("engine embedding usage is invalid"))?; + .ok_or(UpstreamError::new("engine embedding usage is invalid"))?; Ok(EmbeddingBatch { vectors, prompt_tokens, @@ -182,7 +266,15 @@ pub enum GenerationEvent { Done, } +pub trait UsageObserver: std::fmt::Debug + Send + Sync { + fn observe(&self, event: &GenerationEvent); + fn json(&self, bytes: &[u8], success: bool); + fn raw(&self, bytes: &[u8]); + fn incomplete(&self) {} +} + pub struct CompletionStream { + meter: Option>, receiver: tokio::sync::mpsc::Receiver>, task: tokio::task::JoinHandle<()>, connection_task: tokio::task::JoinHandle<()>, @@ -192,8 +284,22 @@ pub struct CompletionStream { impl CompletionStream { pub async fn next(&mut self) -> Option> { match tokio::time::timeout(self.idle_timeout, self.receiver.recv()).await { - Ok(event) => event, - Err(_) => Some(Err(UpstreamError("engine stream idle timeout"))), + Ok(event) => { + if let Some(meter) = &self.meter { + match &event { + Some(Ok(event)) => meter.observe(event), + Some(Err(_)) => meter.incomplete(), + None => (), + } + } + event + } + Err(_) => { + if let Some(meter) = &self.meter { + meter.incomplete(); + } + Some(Err(UpstreamError::new("engine stream idle timeout"))) + } } } } @@ -206,6 +312,9 @@ impl Drop for CompletionStream { } pub struct RawResponseStream { + meter_failed: bool, + pending_meter: Vec, + meter: Option>, receiver: tokio::sync::mpsc::Receiver>, task: tokio::task::JoinHandle<()>, connection_task: tokio::task::JoinHandle<()>, @@ -214,8 +323,49 @@ pub struct RawResponseStream { impl RawResponseStream { pub async fn next(&mut self) -> Option> { match tokio::time::timeout(DEFAULT_STREAM_IDLE_TIMEOUT, self.receiver.recv()).await { - Ok(chunk) => chunk, - Err(_) => Some(Err(UpstreamError("engine stream idle timeout"))), + Ok(chunk) => { + if let Some(meter) = &self.meter { + match &chunk { + Some(Ok(bytes)) if !self.meter_failed => { + if self.pending_meter.len().saturating_add(bytes.len()) + > MAX_RESPONSE_BYTES + { + self.pending_meter.clear(); + self.meter_failed = true; + meter.incomplete(); + } else { + self.pending_meter.extend_from_slice(bytes); + while let Some((end, size)) = self + .pending_meter + .windows(2) + .position(|v| v == b"\n\n") + .map(|i| (i, 2)) + .or_else(|| { + self.pending_meter + .windows(4) + .position(|v| v == b"\r\n\r\n") + .map(|i| (i, 4)) + }) + { + let event = + self.pending_meter.drain(..end + size).collect::>(); + meter.raw(&event); + } + } + } + Some(Err(_)) => meter.incomplete(), + None if !self.pending_meter.is_empty() => meter.incomplete(), + _ => (), + } + } + chunk + } + Err(_) => { + if let Some(meter) = &self.meter { + meter.incomplete(); + } + Some(Err(UpstreamError::new("engine stream idle timeout"))) + } } } } @@ -234,6 +384,7 @@ impl Drop for RawResponseStream { /// route after another executor inspection. #[derive(Debug, Clone)] pub struct ObservedRoute { + meter: Option>, instance_id: String, generation: u64, process_identity: Option<(u32, u64)>, @@ -244,6 +395,10 @@ pub struct ObservedRoute { } impl ObservedRoute { + pub fn with_meter(mut self, meter: Arc) -> Self { + self.meter = Some(meter); + self + } pub fn new<'a>( instance_id: &str, generation: u64, @@ -252,14 +407,16 @@ impl ObservedRoute { allowed: impl IntoIterator, ) -> Result { if instance_id.is_empty() || instance_id.len() > 96 || generation == 0 || port == 0 { - return Err(UpstreamError("invalid observed engine identity")); + return Err(UpstreamError::new("invalid observed engine identity")); } let address_allowed = match address { IpAddr::V4(value) => value.is_private(), IpAddr::V6(value) => value.is_unique_local(), } || cfg!(test) && address.is_loopback(); if !address_allowed { - return Err(UpstreamError("engine address is outside a private bridge")); + return Err(UpstreamError::new( + "engine address is outside a private bridge", + )); } let allowed = allowed .into_iter() @@ -271,7 +428,7 @@ impl ObservedRoute { .iter() .any(|(method, path)| !valid_method(method) || !valid_path(path)) { - return Err(UpstreamError("recipe route allowlist is invalid")); + return Err(UpstreamError::new("recipe route allowlist is invalid")); } Ok(Self { instance_id: instance_id.into(), @@ -281,6 +438,7 @@ impl ObservedRoute { port, allowed, connections: Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT_CONNECTIONS)), + meter: None, }) } @@ -300,11 +458,11 @@ impl ObservedRoute { body_bytes: usize, ) -> Result { if body_bytes > MAX_BODY_BYTES { - return Err(UpstreamError("engine request body is too large")); + return Err(UpstreamError::rejected(RequestRejection::PayloadTooLarge)); } let key = (method.to_owned(), path.to_owned()); if !self.allowed.contains(&key) { - return Err(UpstreamError("engine method or route is not allowed")); + return Err(UpstreamError::new("engine method or route is not allowed")); } Ok(UpstreamRequest { method: key.0, @@ -327,9 +485,15 @@ impl ObservedRoute { timeout: Duration, ) -> Result { self.request(&request.method, &request.path, body.len())?; - tokio::time::timeout(timeout, self.send_inner(request, body)) + let response = tokio::time::timeout(timeout, self.send_inner(request, body)) .await - .map_err(|_| UpstreamError("engine request timed out"))? + .map_err(|_| UpstreamError::new("engine request timed out"))?; + if let Ok(response) = &response + && let Some(meter) = &self.meter + { + meter.json(&response.bytes, (200..300).contains(&response.status)); + } + response } pub async fn semantic_probe( @@ -343,14 +507,14 @@ impl ObservedRoute { let models = self.send(&models, &[]).await?; let models_status = models.status; let models: serde_json::Value = serde_json::from_slice(&models.bytes) - .map_err(|_| UpstreamError("engine model identity response is invalid"))?; + .map_err(|_| UpstreamError::new("engine model identity response is invalid"))?; let identity_matches = models["data"].as_array().is_some_and(|models| { models .iter() .any(|model| model["id"].as_str() == Some(served_model)) }); if !(200..300).contains(&models_status) || !identity_matches { - return Err(UpstreamError("engine served-model identity mismatch")); + return Err(UpstreamError::new("engine served-model identity mismatch")); } let body = serde_json::to_vec(&serde_json::json!({ "model": served_model, @@ -360,16 +524,16 @@ impl ObservedRoute { "ignore_eos": true, "stream": false })) - .map_err(|_| UpstreamError("semantic probe cannot be encoded"))?; + .map_err(|_| UpstreamError::new("semantic probe cannot be encoded"))?; let completion = self.request("POST", "/v1/completions", body.len())?; let completion = self.send_with_timeout(&completion, &body, timeout).await?; let completion_status = completion.status; let completion: serde_json::Value = serde_json::from_slice(&completion.bytes) - .map_err(|_| UpstreamError("semantic probe response is invalid"))?; + .map_err(|_| UpstreamError::new("semantic probe response is invalid"))?; if !(200..300).contains(&completion_status) || !valid_semantic_completion(&completion, served_model, max_tokens) { - return Err(UpstreamError("semantic completion contract rejected")); + return Err(UpstreamError::new("semantic completion contract rejected")); } Ok(()) } @@ -386,7 +550,7 @@ impl ObservedRoute { let models = self.request("GET", "/v1/models", 0)?; let models = self.send(&models, &[]).await?; let identity: serde_json::Value = serde_json::from_slice(&models.bytes) - .map_err(|_| UpstreamError("engine model identity response is invalid"))?; + .map_err(|_| UpstreamError::new("engine model identity response is invalid"))?; if !(200..300).contains(&models.status) || !identity["data"].as_array().is_some_and(|models| { models @@ -394,16 +558,16 @@ impl ObservedRoute { .any(|model| model["id"].as_str() == Some(served_model)) }) { - return Err(UpstreamError("engine served-model identity mismatch")); + return Err(UpstreamError::new("engine served-model identity mismatch")); } let body = serde_json::to_vec(&serde_json::json!({ "model":served_model,"input":input,"encoding_format":"float" })) - .map_err(|_| UpstreamError("embedding probe cannot be encoded"))?; + .map_err(|_| UpstreamError::new("embedding probe cannot be encoded"))?; let request = self.request("POST", "/v1/embeddings", body.len())?; let response = self.send_with_timeout(&request, &body, timeout).await?; if !(200..300).contains(&response.status) { - return Err(UpstreamError("embedding probe was rejected")); + return Err(UpstreamError::new("embedding probe was rejected")); } decode_embedding_response( &response.bytes, @@ -432,7 +596,7 @@ impl ObservedRoute { let models = self.request("GET", "/v1/models", 0)?; let models = self.send(&models, &[]).await?; let identity: serde_json::Value = serde_json::from_slice(&models.bytes) - .map_err(|_| UpstreamError("engine model identity response is invalid"))?; + .map_err(|_| UpstreamError::new("engine model identity response is invalid"))?; if !(200..300).contains(&models.status) || !identity["data"].as_array().is_some_and(|models| { models @@ -440,7 +604,7 @@ impl ObservedRoute { .any(|model| model["id"].as_str() == Some(served_model)) }) { - return Err(UpstreamError("engine served-model identity mismatch")); + return Err(UpstreamError::new("engine served-model identity mismatch")); } let body = serde_json::to_vec(&serde_json::json!({ "model":served_model, @@ -453,41 +617,43 @@ impl ObservedRoute { "max_tokens":max_tokens,"thinking_budget_tokens":if disable_thinking { 0 } else { -1 },"temperature":0,"stream":false, "chat_template_kwargs":{"enable_thinking":!disable_thinking,"reasoning_strength":if disable_thinking { "low" } else { "high" }} })) - .map_err(|_| UpstreamError("vision probe cannot be encoded"))?; + .map_err(|_| UpstreamError::new("vision probe cannot be encoded"))?; let request = self.request("POST", "/v1/chat/completions", body.len())?; let response = self.send_with_timeout(&request, &body, timeout).await?; if !(200..300).contains(&response.status) { - return Err(UpstreamError(match response.status { + return Err(UpstreamError::new(match response.status { 400..=499 => "vision probe rejected with HTTP 4xx", 500..=599 => "vision probe rejected with HTTP 5xx", _ => "vision probe returned a non-success HTTP status", })); } let value: serde_json::Value = serde_json::from_slice(&response.bytes) - .map_err(|_| UpstreamError("vision probe response is invalid"))?; + .map_err(|_| UpstreamError::new("vision probe response is invalid"))?; if value["model"] != served_model { - return Err(UpstreamError("vision probe model identity changed")); + return Err(UpstreamError::new("vision probe model identity changed")); } let choice = value["choices"] .as_array() .filter(|choices| choices.len() == 1) .map(|choices| &choices[0]) - .ok_or(UpstreamError("vision probe choice shape is invalid"))?; + .ok_or(UpstreamError::new("vision probe choice shape is invalid"))?; let content = choice["message"]["content"] .as_str() - .ok_or(UpstreamError("vision probe final content is missing"))?; + .ok_or(UpstreamError::new("vision probe final content is missing"))?; if !content .to_lowercase() .contains(&expected_text.to_lowercase()) { - return Err(UpstreamError("vision probe expected answer is missing")); + return Err(UpstreamError::new( + "vision probe expected answer is missing", + )); } let prompt_tokens = value["usage"]["prompt_tokens"].as_u64(); let completion_tokens = value["usage"]["completion_tokens"].as_u64(); if prompt_tokens.is_none_or(|tokens| tokens == 0) || completion_tokens.is_none_or(|tokens| tokens == 0 || tokens > u64::from(max_tokens)) { - return Err(UpstreamError("vision probe usage contract is invalid")); + return Err(UpstreamError::new("vision probe usage contract is invalid")); } Ok(()) } @@ -503,7 +669,7 @@ impl ObservedRoute { self.protocol_probe_inner(served_model, require_tools), ) .await - .map_err(|_| UpstreamError("engine protocol probe timed out"))? + .map_err(|_| UpstreamError::new("engine protocol probe timed out"))? } async fn protocol_probe_inner( @@ -516,7 +682,7 @@ impl ObservedRoute { "max_tokens":PROTOCOL_TOOL_MAX_TOKENS,"temperature":0,"stream":true, "stream_options":{"include_usage":true},"chat_template_kwargs":{"enable_thinking":false,"reasoning_strength":"low"} })) - .map_err(|_| UpstreamError("protocol probe cannot be encoded"))?; + .map_err(|_| UpstreamError::new("protocol probe cannot be encoded"))?; validate_protocol_stream(self.chat_stream(&body).await?, false).await?; if require_tools { let body = serde_json::to_vec(&serde_json::json!({ @@ -529,7 +695,7 @@ impl ObservedRoute { "parallel_tool_calls":true,"max_tokens":PROTOCOL_TOOL_MAX_TOKENS, "temperature":0,"stream":true,"stream_options":{"include_usage":true},"chat_template_kwargs":{"enable_thinking":false,"reasoning_strength":"low"} })) - .map_err(|_| UpstreamError("tool protocol probe cannot be encoded"))?; + .map_err(|_| UpstreamError::new("tool protocol probe cannot be encoded"))?; validate_protocol_stream(self.chat_stream(&body).await?, true).await?; } Ok(()) @@ -563,13 +729,13 @@ impl ObservedRoute { let permit = Arc::clone(&self.connections) .acquire_owned() .await - .map_err(|_| UpstreamError("engine connection pool is closed"))?; + .map_err(|_| UpstreamError::new("engine connection pool is closed"))?; let stream = tokio::net::TcpStream::connect((self.address, self.port)) .await - .map_err(|_| UpstreamError("engine endpoint is unreachable"))?; + .map_err(|_| UpstreamError::new("engine endpoint is unreachable"))?; let (mut sender, connection) = http1::handshake(TokioIo::new(stream)) .await - .map_err(|_| UpstreamError("engine HTTP handshake failed"))?; + .map_err(|_| UpstreamError::new("engine HTTP handshake failed"))?; let connection_task = tokio::spawn(async move { let _ = connection.await; }); @@ -579,14 +745,15 @@ impl ObservedRoute { .header(header::HOST, "engine.internal") .header(header::CONTENT_TYPE, "application/json") .body(Full::new(Bytes::copy_from_slice(body))) - .map_err(|_| UpstreamError("engine request is malformed"))?; + .map_err(|_| UpstreamError::new("engine request is malformed"))?; let response = sender .send_request(outbound) .await - .map_err(|_| UpstreamError("engine response failed"))?; + .map_err(|_| UpstreamError::new("engine response failed"))?; if !(200..300).contains(&response.status().as_u16()) { + let error = rejected_stream_response(response).await; connection_task.abort(); - return Err(UpstreamError("engine rejected response stream")); + return Err(error); } let (send, receiver) = tokio::sync::mpsc::channel(STREAM_CHANNEL_CAPACITY); let mut incoming = response.into_body(); @@ -594,7 +761,9 @@ impl ObservedRoute { let _permit = permit; while let Some(frame) = incoming.frame().await { let Ok(frame) = frame else { - let _ = send.send(Err(UpstreamError("engine stream failed"))).await; + let _ = send + .send(Err(UpstreamError::new("engine stream failed"))) + .await; break; }; let Ok(data) = frame.into_data() else { @@ -608,6 +777,9 @@ impl ObservedRoute { } }); Ok(RawResponseStream { + meter_failed: false, + pending_meter: Vec::new(), + meter: self.meter.clone(), receiver, task, connection_task, @@ -624,13 +796,13 @@ impl ObservedRoute { let permit = Arc::clone(&self.connections) .acquire_owned() .await - .map_err(|_| UpstreamError("engine connection pool is closed"))?; + .map_err(|_| UpstreamError::new("engine connection pool is closed"))?; let stream = tokio::net::TcpStream::connect((self.address, self.port)) .await - .map_err(|_| UpstreamError("engine endpoint is unreachable"))?; + .map_err(|_| UpstreamError::new("engine endpoint is unreachable"))?; let (mut sender, connection) = http1::handshake(TokioIo::new(stream)) .await - .map_err(|_| UpstreamError("engine HTTP handshake failed"))?; + .map_err(|_| UpstreamError::new("engine HTTP handshake failed"))?; let connection_task = tokio::spawn(async move { let _ = connection.await; }); @@ -640,13 +812,15 @@ impl ObservedRoute { .header(header::HOST, "engine.internal") .header(header::CONTENT_TYPE, "application/json") .body(Full::new(Bytes::copy_from_slice(body))) - .map_err(|_| UpstreamError("engine request is malformed"))?; + .map_err(|_| UpstreamError::new("engine request is malformed"))?; let response = sender .send_request(outbound) .await - .map_err(|_| UpstreamError("engine response failed"))?; + .map_err(|_| UpstreamError::new("engine response failed"))?; if !(200..300).contains(&response.status().as_u16()) { - return Err(UpstreamError("engine rejected completion stream")); + let error = rejected_stream_response(response).await; + connection_task.abort(); + return Err(error); } let (send, receiver) = tokio::sync::mpsc::channel(STREAM_CHANNEL_CAPACITY); let mut incoming = response.into_body(); @@ -661,7 +835,7 @@ impl ObservedRoute { pending.extend_from_slice(&data); if pending.len() > MAX_STREAM_EVENT_BYTES { let _ = send - .send(Err(UpstreamError("engine stream event is too large"))) + .send(Err(UpstreamError::new("engine stream event is too large"))) .await; break; } @@ -681,6 +855,7 @@ impl ObservedRoute { task, connection_task, idle_timeout, + meter: self.meter.clone(), }) } @@ -693,13 +868,13 @@ impl ObservedRoute { .connections .acquire() .await - .map_err(|_| UpstreamError("engine connection pool is closed"))?; + .map_err(|_| UpstreamError::new("engine connection pool is closed"))?; let stream = tokio::net::TcpStream::connect((self.address, self.port)) .await - .map_err(|_| UpstreamError("engine endpoint is unreachable"))?; + .map_err(|_| UpstreamError::new("engine endpoint is unreachable"))?; let (mut sender, connection) = http1::handshake(TokioIo::new(stream)) .await - .map_err(|_| UpstreamError("engine HTTP handshake failed"))?; + .map_err(|_| UpstreamError::new("engine HTTP handshake failed"))?; tokio::spawn(async move { let _ = connection.await; }); @@ -709,16 +884,16 @@ impl ObservedRoute { .header(header::HOST, "engine.internal") .header(header::CONTENT_TYPE, "application/json") .body(Full::new(Bytes::copy_from_slice(body))) - .map_err(|_| UpstreamError("engine request is malformed"))?; + .map_err(|_| UpstreamError::new("engine request is malformed"))?; let response = sender .send_request(outbound) .await - .map_err(|_| UpstreamError("engine response failed"))?; + .map_err(|_| UpstreamError::new("engine response failed"))?; let status = response.status().as_u16(); let bytes = Limited::new(response.into_body(), MAX_RESPONSE_BYTES) .collect() .await - .map_err(|_| UpstreamError("engine response is too large"))? + .map_err(|_| UpstreamError::new("engine response is too large"))? .to_bytes() .to_vec(); Ok(UpstreamResponse { status, bytes }) @@ -729,6 +904,28 @@ impl ObservedRoute { } } +async fn rejected_stream_response( + response: hyper::Response, +) -> UpstreamError { + let status = response.status().as_u16(); + let bytes = if matches!(status, 400 | 422) { + match tokio::time::timeout( + Duration::from_secs(2), + Limited::new(response.into_body(), 16 * 1024).collect(), + ) + .await + { + Ok(Ok(body)) => body.to_bytes(), + _ => Bytes::new(), + } + } else { + Bytes::new() + }; + RequestRejection::from_response(status, &bytes) + .map(UpstreamError::rejected) + .unwrap_or_else(|| UpstreamError::new("engine rejected response stream")) +} + async fn validate_protocol_stream( mut stream: CompletionStream, require_tool: bool, @@ -741,7 +938,7 @@ async fn validate_protocol_stream( match event? { GenerationEvent::ReasoningDelta { text } | GenerationEvent::TextDelta { text } => { if finish_reason.is_some() || text.is_empty() { - return Err(UpstreamError("engine protocol event order is invalid")); + return Err(UpstreamError::new("engine protocol event order is invalid")); } output = true; } @@ -752,7 +949,7 @@ async fn validate_protocol_stream( arguments, } => { if finish_reason.is_some() || !require_tool { - return Err(UpstreamError("engine protocol tool event is invalid")); + return Err(UpstreamError::new("engine protocol tool event is invalid")); } let tool = tools.entry(index).or_default(); if let Some(call_id) = call_id { @@ -767,7 +964,9 @@ async fn validate_protocol_stream( finish_reason: reason, } => { if finish_reason.is_some() || reason.is_none() { - return Err(UpstreamError("engine protocol finish event is invalid")); + return Err(UpstreamError::new( + "engine protocol finish event is invalid", + )); } finish_reason = reason; } @@ -777,7 +976,7 @@ async fn validate_protocol_stream( } => { if finish_reason.is_none() || usage || prompt_tokens == 0 || completion_tokens == 0 { - return Err(UpstreamError("engine protocol usage event is invalid")); + return Err(UpstreamError::new("engine protocol usage event is invalid")); } usage = true; } @@ -798,11 +997,13 @@ async fn validate_protocol_stream( { return Ok(()); } - return Err(UpstreamError("engine protocol stream is incomplete")); + return Err(UpstreamError::new("engine protocol stream is incomplete")); } } } - Err(UpstreamError("engine protocol stream ended before done")) + Err(UpstreamError::new( + "engine protocol stream ended before done", + )) } fn valid_semantic_completion( @@ -855,10 +1056,10 @@ fn decode_sse_events(bytes: &[u8]) -> Vec } let value: serde_json::Value = match serde_json::from_str(data) { Ok(value) => value, - Err(_) => return vec![Err(UpstreamError("engine stream event is invalid"))], + Err(_) => return vec![Err(UpstreamError::new("engine stream event is invalid"))], }; if value.get("error").is_some() { - return vec![Err(UpstreamError("engine stream returned an error"))]; + return vec![Err(UpstreamError::new("engine stream returned an error"))]; } let mut events = Vec::new(); let choice = value["choices"] @@ -886,7 +1087,7 @@ fn decode_sse_events(bytes: &[u8]) -> Vec .as_u64() .and_then(|value| value.try_into().ok()) else { - return vec![Err(UpstreamError("engine tool call index is invalid"))]; + return vec![Err(UpstreamError::new("engine tool call index is invalid"))]; }; events.push(Ok(GenerationEvent::ToolCallDelta { index, @@ -907,10 +1108,10 @@ fn decode_sse_events(bytes: &[u8]) -> Vec } if let Some(usage) = value.get("usage").filter(|usage| !usage.is_null()) { let Some(prompt_tokens) = usage["prompt_tokens"].as_u64() else { - return vec![Err(UpstreamError("engine stream usage is invalid"))]; + return vec![Err(UpstreamError::new("engine stream usage is invalid"))]; }; let Some(completion_tokens) = usage["completion_tokens"].as_u64() else { - return vec![Err(UpstreamError("engine stream usage is invalid"))]; + return vec![Err(UpstreamError::new("engine stream usage is invalid"))]; }; events.push(Ok(GenerationEvent::Usage { prompt_tokens, @@ -1264,6 +1465,7 @@ mod tests { }); let connection_task = tokio::spawn(std::future::pending()); let stream = CompletionStream { + meter: None, receiver, task, connection_task, @@ -1287,6 +1489,7 @@ mod tests { }); let connection_task = tokio::spawn(std::future::pending()); let mut stream = CompletionStream { + meter: None, receiver, task, connection_task, diff --git a/src/spark/wire.rs b/src/spark/wire.rs index 911aebc..2cf1b91 100644 --- a/src/spark/wire.rs +++ b/src/spark/wire.rs @@ -1356,3 +1356,50 @@ mod tests { ); } } +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] +pub struct LaunchSessionRequest { + pub id: String, + pub instance: String, + pub integration: String, + pub eco_mode: String, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] +pub struct SessionUsage { + pub requests: u64, + pub input_tokens: u64, + pub output_tokens: u64, + pub unknown_usage_requests: u64, + pub pending_requests: u64, + pub failed_requests: u64, + #[serde(default)] + pub long_context_input_tokens: u64, + #[serde(default)] + pub long_context_output_tokens: u64, + #[serde(default)] + pub accounting_errors: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] +pub struct LaunchSessionDocument { + pub schema: String, + pub id: String, + pub instance: String, + pub model: String, + pub integration: String, + pub eco_mode: String, + pub started_at: String, + pub finished_at: Option, + pub usage: SessionUsage, +} + +#[derive(Serialize, Deserialize)] +#[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] +pub struct LaunchSessionCreated { + pub session: LaunchSessionDocument, + pub bearer_token: Option, +} diff --git a/tests/eco_launch.rs b/tests/eco_launch.rs new file mode 100644 index 0000000..b65c32d --- /dev/null +++ b/tests/eco_launch.rs @@ -0,0 +1,205 @@ +use std::{ + io::Write, + os::unix::fs::PermissionsExt, + process::{Command, Stdio}, +}; + +#[test] +fn eco_mode_is_default_and_can_be_disabled() { + let help = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["fixture", "launch", "codex", "--help"]) + .output() + .unwrap(); + let help = String::from_utf8(help.stdout).unwrap(); + assert!(help.contains("--eco-mode")); + assert!(help.contains("[default: max]")); + let rejected = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["fixture", "launch", "codex", "--eco-mode=invalid"]) + .output() + .unwrap(); + assert_eq!(rejected.status.code(), Some(2)); +} + +#[test] +fn embedded_rewrite_preserves_shell_and_structured_output() { + for command in [ + "git status --porcelain", + "git status > result", + "git status | cat", + "echo $(git status)", + "cargo test --message-format=json", + "printf '%s' '$secret'", + ] { + let output = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["__eco", "rewrite", command]) + .output() + .unwrap(); + assert!(output.status.success()); + assert!(output.stdout.is_empty(), "{command}"); + } + let output = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["__eco", "rewrite", "git status"]) + .output() + .unwrap(); + assert!(output.status.success()); + assert!( + String::from_utf8(output.stdout) + .unwrap() + .contains("__eco exec") + ); +} + +#[test] +fn hooks_compress_only_supported_calls_and_collect_private_metrics() { + let root = tempfile::tempdir().unwrap(); + let bin = root.path().join("bin"); + std::fs::create_dir(&bin).unwrap(); + let cargo = bin.join("cargo"); + std::fs::write(&cargo,"#!/bin/sh\nprintf 'running 2 tests\\ntest first ... ok\\ntest second ... ok\\ntest result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s\\n'\nexit 0\n").unwrap(); + std::fs::set_permissions(&cargo, std::fs::Permissions::from_mode(0o700)).unwrap(); + let metrics = root.path().join("metrics"); + let binary = env!("CARGO_BIN_EXE_sparkplane"); + let mut hook = Command::new(binary) + .args(["__eco", "hook", "codex"]) + .env("SPARKPLANE_ECO_REWRITE_ENABLED", "1") + .env("SPARKPLANE_ECO_SESSION_DIR", &metrics) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + hook.stdin.take().unwrap().write_all(br#"{"hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{"command":"cargo test"}}"#).unwrap(); + let output = hook.wait_with_output().unwrap(); + let output: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let rewritten = output["hookSpecificOutput"]["updatedInput"]["command"] + .as_str() + .unwrap(); + let filtered = Command::new("/bin/sh") + .args(["-c", rewritten]) + .env("PATH", &bin) + .env("SPARKPLANE_ECO_SESSION_DIR", &metrics) + .output() + .unwrap(); + assert!(filtered.status.success()); + let raw = Command::new(binary) + .args(["__eco", "raw", "--", "cargo", "test"]) + .env("PATH", &bin) + .output() + .unwrap(); + assert!(raw.status.success()); + assert!(filtered.stdout.len() < raw.stdout.len()); + assert!(String::from_utf8_lossy(&filtered.stdout).contains("2 passed")); + assert_eq!( + std::fs::metadata(&metrics).unwrap().permissions().mode() & 0o777, + 0o700 + ); + for file in std::fs::read_dir(&metrics).unwrap() { + let path = file.unwrap().path(); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let record = std::fs::read_to_string(path).unwrap(); + assert!(!record.contains("first")); + assert!(!record.contains("cargo test")); + } + std::fs::write(&cargo,"#!/bin/sh\nprintf 'full stdout diagnostic\\n'\nprintf 'full stderr diagnostic\\n' >&2\nexit 7\n").unwrap(); + let failure = Command::new(binary) + .args(["__eco", "exec", "--", "cargo", "test"]) + .env("PATH", &bin) + .output() + .unwrap(); + assert_eq!(failure.status.code(), Some(7)); + assert_eq!(failure.stdout, b"full stdout diagnostic\n"); + assert_eq!(failure.stderr, b"full stderr diagnostic\n"); +} + +#[test] +fn disabled_hooks_emit_no_rewrite() { + let output = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["__eco", "rewrite", "git status"]) + .env("SPARKPLANE_ECO_REWRITE_ENABLED", "0") + .output() + .unwrap(); + assert!(output.status.success()); + assert!(output.stdout.is_empty()); +} + +#[test] +fn economics_json_is_a_retained_offline_report() { + use sparkplane::spark::economics::{self, Compression, Report}; + let root = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now().to_rfc3339(); + let mut report = Report { + schema: economics::SCHEMA.into(), + session_id: ulid::Ulid::new().to_string(), + host: "offline".into(), + model: "fixture".into(), + integration: "codex".into(), + eco_mode: "none".into(), + started_at: now.clone(), + finished_at: Some(now), + exit_code: Some(7), + inference: None, + compression: Compression { + status: "disabled".into(), + ..Default::default() + }, + catalog: economics::catalog().unwrap(), + comparisons: Vec::new(), + }; + report.calculate(); + economics::save(root.path(), &report).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args(["offline", "economics", "--json", "--config-dir"]) + .arg(root.path()) + .output() + .unwrap(); + assert!(output.status.success()); + assert!(output.stderr.is_empty()); + let value: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(value["session_id"], report.session_id); + assert_eq!(value["exit_code"], 7); + assert!(value["inference"].is_null()); + assert!(value["comparisons"][0]["inference_usd_nanos"].is_null()); +} + +#[test] +fn wrapper_forwards_termination_and_preserves_signal_exit_status() { + let root = tempfile::tempdir().unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_sparkplane")) + .args([ + "__eco", + "exec", + "--", + "sh", + "-c", + "printf ready > ready; exec sleep 30", + ]) + .current_dir(root.path()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !root.path().join("ready").exists() { + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + nix::sys::signal::kill( + nix::unistd::Pid::from_raw(child.id() as i32), + nix::sys::signal::Signal::SIGTERM, + ) + .unwrap(); + loop { + if let Some(status) = child.try_wait().unwrap() { + assert_eq!(status.code(), Some(143)); + break; + } + if std::time::Instant::now() >= deadline { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("eco child did not terminate"); + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } +} diff --git a/tests/gateway_image_upload.rs b/tests/gateway_image_upload.rs new file mode 100644 index 0000000..301260e --- /dev/null +++ b/tests/gateway_image_upload.rs @@ -0,0 +1,545 @@ +#![cfg(feature = "appliance")] + +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use image::{DynamicImage, ImageFormat, ImageReader, RgbImage}; +use sparkplane::spark::gateway::{self, GatewayProfile, ImageProtocol, IngressLimits}; + +fn profile() -> GatewayProfile { + let config: toml::Value = toml::from_str(include_str!( + "../configs/sparkplane/engines/vllm-qwen38-mmap.toml" + )) + .unwrap(); + let mut profile = GatewayProfile::text(); + profile.capabilities.insert("vision".into()); + profile.vision = Some(config["profiles"][0]["vision"].clone().try_into().unwrap()); + let vision = profile.vision.as_mut().unwrap(); + vision.max_width = 64; + vision.max_height = 32; + vision.max_bytes = 4096; + vision.max_total_bytes = 4096; + vision.max_count = 1; + profile +} + +fn png() -> Vec { + let mut seed = 0x12345678_u32; + let pixels = (0..1024 * 512 * 3) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed as u8 + }) + .collect(); + let image = DynamicImage::ImageRgb8(RgbImage::from_raw(1024, 512, pixels).unwrap()); + let mut out = std::io::Cursor::new(Vec::new()); + image.write_to(&mut out, ImageFormat::Png).unwrap(); + out.into_inner() +} + +fn responses(image: &[u8]) -> Vec { + serde_json::to_vec(&serde_json::json!({"input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"describe"},{"type":"input_image","image_url":format!("data:image/png;base64,{}",BASE64.encode(image)),"detail":"high"}]}],"stream":true,"store":false})).unwrap() +} + +fn check_image(url: &str, profile: &GatewayProfile) { + let (media, data) = url + .strip_prefix("data:") + .unwrap() + .split_once(";base64,") + .unwrap(); + let bytes = BASE64.decode(data).unwrap(); + let vision = profile.vision.as_ref().unwrap(); + assert!(vision.media_types.contains(media)); + assert!(bytes.len() <= vision.max_bytes); + let decoded = ImageReader::new(std::io::Cursor::new(&bytes)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + assert!(decoded.width() <= vision.max_width); + assert!(decoded.height() <= vision.max_height); + assert_eq!(decoded.width(), decoded.height() * 2); +} + +#[test] +fn large_uploads_fit_the_model_in_both_public_protocols() { + let image = png(); + let request = responses(&image); + assert!(request.len() > 1024 * 1024); + let profile = profile(); + let limits = IngressLimits::default(); + assert!(gateway::rewrite_responses_request_with_profile(&request, "served", &profile).is_err()); + let normalized = + gateway::normalize_image_request(&request, ImageProtocol::Responses, &profile, &limits) + .unwrap(); + let request = + gateway::rewrite_responses_request_with_profile(&normalized, "served", &profile).unwrap(); + let request: serde_json::Value = serde_json::from_slice(&request.body).unwrap(); + check_image( + request["messages"][0]["content"][1]["image_url"]["url"] + .as_str() + .unwrap(), + &profile, + ); + let request=serde_json::to_vec(&serde_json::json!({"model":"public","max_tokens":8,"messages":[{"role":"user","content":[{"type":"text","text":"describe"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":BASE64.encode(image)}}]}]})).unwrap(); + let normalized = + gateway::normalize_image_request(&request, ImageProtocol::Anthropic, &profile, &limits) + .unwrap(); + let request = + gateway::rewrite_anthropic_request_with_profile(&normalized, "served", &profile).unwrap(); + let request: serde_json::Value = serde_json::from_slice(&request.body).unwrap(); + check_image( + request["messages"][0]["content"][1]["image_url"]["url"] + .as_str() + .unwrap(), + &profile, + ); +} + +#[test] +fn native_responses_receives_normalized_images_and_preserves_other_fields() { + let mut profile = profile(); + profile.native_responses = true; + let normalized = gateway::normalize_image_request( + &responses(&png()), + ImageProtocol::Responses, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let request = + gateway::rewrite_native_responses_request_with_profile(&normalized, "public", &profile) + .unwrap(); + let request: serde_json::Value = serde_json::from_slice(&request.body).unwrap(); + assert_eq!(request["store"], false); + assert_eq!(request["input"][0]["content"][0]["text"], "describe"); + assert_eq!(request["input"][0]["content"][1]["detail"], "high"); + check_image( + request["input"][0]["content"][1]["image_url"] + .as_str() + .unwrap(), + &profile, + ); +} + +#[test] +fn upload_byte_pixel_and_model_image_count_limits_stay_finite() { + let request = responses(&png()); + let profile = profile(); + for limits in [ + IngressLimits { + request_body_bytes: 1024, + ..IngressLimits::default() + }, + IngressLimits { + image_bytes: 1024, + ..IngressLimits::default() + }, + IngressLimits { + image_pixels: 1024, + ..IngressLimits::default() + }, + IngressLimits { + image_dimension: 512, + ..IngressLimits::default() + }, + ] { + let error = + gateway::normalize_image_request(&request, ImageProtocol::Responses, &profile, &limits) + .unwrap_err(); + assert_eq!(error.code, "request_too_large"); + } + let mut request: serde_json::Value = serde_json::from_slice(&request).unwrap(); + let image = request["input"][0]["content"][1].clone(); + request["input"][0]["content"] + .as_array_mut() + .unwrap() + .push(image); + assert!( + gateway::normalize_image_request( + &serde_json::to_vec(&request).unwrap(), + ImageProtocol::Responses, + &profile, + &IngressLimits::default() + ) + .is_err() + ); +} + +#[test] +fn compatible_images_are_unchanged_and_resizing_preserves_transparency() { + let profile = profile(); + let image = DynamicImage::ImageRgb8(RgbImage::from_pixel(2, 1, image::Rgb([10, 20, 30]))); + let mut original = std::io::Cursor::new(Vec::new()); + image.write_to(&mut original, ImageFormat::Png).unwrap(); + let original = original.into_inner(); + let normalized = gateway::normalize_image_request( + &responses(&original), + ImageProtocol::Responses, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let normalized: serde_json::Value = serde_json::from_slice(&normalized).unwrap(); + assert_eq!( + normalized["input"][0]["content"][1]["image_url"], + format!("data:image/png;base64,{}", BASE64.encode(&original)) + ); + + let image = image::DynamicImage::ImageRgba8(image::RgbaImage::from_fn(128, 64, |_, _| { + image::Rgba([200, 30, 80, 0]) + })); + let mut png = std::io::Cursor::new(Vec::new()); + image.write_to(&mut png, ImageFormat::Png).unwrap(); + let normalized = gateway::normalize_image_request( + &responses(&png.into_inner()), + ImageProtocol::Responses, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let normalized: serde_json::Value = serde_json::from_slice(&normalized).unwrap(); + let url = normalized["input"][0]["content"][1]["image_url"] + .as_str() + .unwrap(); + check_image(url, &profile); + let bytes = BASE64 + .decode(url.split_once(";base64,").unwrap().1) + .unwrap(); + let decoded = ImageReader::new(std::io::Cursor::new(bytes)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + assert!(decoded.color().has_alpha()); + assert!(decoded.to_rgba8().pixels().all(|pixel| pixel[3] == 0)); +} + +#[test] +fn normalization_keeps_inline_only_and_verified_vision_requirements() { + let mut profile = profile(); + let mut request: serde_json::Value = serde_json::from_slice(&responses(&png())).unwrap(); + for invalid in [ + "https://example.com/picture.png", + "file:///tmp/picture.png", + "data:image/jpeg;base64,iVBORw0KGgo=", + ] { + request["input"][0]["content"][1]["image_url"] = invalid.into(); + assert!( + gateway::normalize_image_request( + &serde_json::to_vec(&request).unwrap(), + ImageProtocol::Responses, + &profile, + &IngressLimits::default() + ) + .is_err() + ); + } + profile.vision = None; + assert!( + gateway::normalize_image_request( + &responses(&png()), + ImageProtocol::Responses, + &profile, + &IngressLimits::default() + ) + .is_err() + ); +} + +#[test] +fn codex_view_image_tool_results_are_images_and_share_the_model_budget() { + let profile = profile(); + for (call, output) in [ + ("function_call", "function_call_output"), + ("custom_tool_call", "custom_tool_call_output"), + ] { + let image = serde_json::json!({"type":"input_image","image_url":format!("data:image/png;base64,{}",BASE64.encode(png())),"detail":"high"}); + let request = serde_json::json!({"input":[ + {"type":"message","role":"user","content":"inspect the screenshot"}, + {"type":call,"call_id":"call_view","name":"view_image","arguments":"{}","input":"image"}, + {"type":output,"call_id":"call_view","output":[{"type":"input_text","text":"Screenshot from the tool."},image]} + ],"stream":true}); + let normalized = gateway::normalize_image_request( + &serde_json::to_vec(&request).unwrap(), + ImageProtocol::Responses, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let rewritten = + gateway::rewrite_responses_request_with_profile(&normalized, "served", &profile) + .unwrap(); + let rewritten: serde_json::Value = serde_json::from_slice(&rewritten.body).unwrap(); + let tool = &rewritten["messages"][2]; + assert_eq!(tool["role"], "tool"); + assert_eq!(tool["tool_call_id"], "call_view"); + assert!( + tool["content"].is_array(), + "view_image output must not become base64 text" + ); + assert_eq!( + tool["content"][0], + serde_json::json!({"type":"text","text":"Screenshot from the tool."}) + ); + check_image( + tool["content"][1]["image_url"]["url"].as_str().unwrap(), + &profile, + ); + let mut native = profile.clone(); + native.native_responses = true; + let rewritten = + gateway::rewrite_native_responses_request_with_profile(&normalized, "public", &native) + .unwrap(); + let rewritten: serde_json::Value = serde_json::from_slice(&rewritten.body).unwrap(); + check_image( + rewritten["input"][2]["output"][1]["image_url"] + .as_str() + .unwrap(), + &native, + ); + let mut two_images = request.clone(); + two_images["input"][2]["output"] + .as_array_mut() + .unwrap() + .push(image); + assert!( + gateway::normalize_image_request( + &serde_json::to_vec(&two_images).unwrap(), + ImageProtocol::Responses, + &profile, + &IngressLimits::default() + ) + .is_err() + ); + assert!( + gateway::normalize_image_request( + &serde_json::to_vec(&request).unwrap(), + ImageProtocol::Responses, + &GatewayProfile::text(), + &IngressLimits::default() + ) + .is_err() + ); + } +} + +#[test] +fn successive_image_tool_results_reserve_bytes_for_every_frame() { + let config: toml::Value = toml::from_str(include_str!( + "../configs/sparkplane/engines/vllm-qwen38-mmap.toml" + )) + .unwrap(); + let mut profile = profile(); + profile.vision.as_mut().unwrap().max_count = config["profiles"][0]["vision"]["max_count"] + .as_integer() + .unwrap() as usize; + assert_eq!(profile.vision.as_ref().unwrap().max_count, 16); + let source = DynamicImage::ImageRgb8(RgbImage::from_fn(128, 64, |x, y| { + image::Rgb([ + (x * 31 + y * 17) as u8, + (x * 71 + y * 43) as u8, + (x * 19 + y * 83) as u8, + ]) + })); + let mut encoded = std::io::Cursor::new(Vec::new()); + source.write_to(&mut encoded, ImageFormat::Png).unwrap(); + let image = serde_json::json!({"type":"input_image","image_url":format!("data:image/png;base64,{}",BASE64.encode(encoded.into_inner())),"detail":"high"}); + for count in [2, 3, 16] { + let mut input = vec![ + serde_json::json!({"type":"message","role":"user","content":"compare the screenshots"}), + ]; + for index in 0..count { + let id = format!("view_{index}"); + input.push(serde_json::json!({"type":"function_call","name":"view_image","call_id":id,"arguments":"{}"})); + input.push(serde_json::json!({"type":"function_call_output","call_id":id,"output":[{"type":"input_text","text":format!("Frame {index}")},image]})); + if index + 1 < count { + input.push(serde_json::json!({"type":"message","role":"assistant","content":"Now inspect the next frame."})); + } + } + let request = + serde_json::to_vec(&serde_json::json!({"input":input,"stream":true})).unwrap(); + let normalized = gateway::normalize_image_request( + &request, + ImageProtocol::Responses, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let normalized: serde_json::Value = serde_json::from_slice(&normalized).unwrap(); + let mut total_bytes = 0; + let mut frames = 0; + let retained = if count == 16 { 10 } else { count }; + for (index, item) in normalized["input"].as_array().unwrap().iter().enumerate() { + if item["type"] != "function_call_output" { + continue; + } + assert_eq!( + item["output"][0]["text"], + format!("Frame {}", (index - 1) / 3) + ); + let Some(url) = item["output"][1]["image_url"].as_str() else { + continue; + }; + let bytes = BASE64 + .decode(url.split_once(";base64,").unwrap().1) + .unwrap(); + assert!(bytes.len() <= profile.vision.as_ref().unwrap().max_total_bytes / retained); + total_bytes += bytes.len(); + frames += 1; + } + assert_eq!(frames, retained); + assert!(total_bytes <= profile.vision.as_ref().unwrap().max_total_bytes); + let bytes = serde_json::to_vec(&normalized).unwrap(); + assert!( + gateway::rewrite_responses_request_with_profile(&bytes, "served", &profile).is_ok() + ); + let mut native = profile.clone(); + native.native_responses = true; + assert!( + gateway::rewrite_native_responses_request_with_profile(&bytes, "served", &native) + .is_ok() + ); + } +} + +#[test] +fn multiple_images_share_the_gateway_pixel_limit_and_keep_a_finite_count() { + let mut profile = profile(); + profile.vision.as_mut().unwrap().max_count = 16; + let source = DynamicImage::ImageRgb8(RgbImage::from_pixel(128, 64, image::Rgb([10, 20, 30]))); + let mut encoded = std::io::Cursor::new(Vec::new()); + source.write_to(&mut encoded, ImageFormat::Png).unwrap(); + let image = serde_json::json!({"type":"input_image","image_url":format!("data:image/png;base64,{}",BASE64.encode(encoded.into_inner()))}); + let make_request = |count| { + serde_json::to_vec(&serde_json::json!({"input":[{"role":"user","type":"message","content":vec![image.clone();count]}]})).unwrap() + }; + let limits = IngressLimits { + image_pixels: 128 * 64 + 1, + ..IngressLimits::default() + }; + assert!( + gateway::normalize_image_request( + &make_request(1), + ImageProtocol::Responses, + &profile, + &limits + ) + .is_ok() + ); + assert_eq!( + gateway::normalize_image_request( + &make_request(2), + ImageProtocol::Responses, + &profile, + &limits + ) + .unwrap_err() + .code, + "request_too_large" + ); + assert_eq!( + gateway::normalize_image_request( + &make_request(17), + ImageProtocol::Responses, + &profile, + &IngressLimits::default() + ) + .unwrap_err() + .message, + "image count exceeds the model profile limit" + ); +} + +#[test] +fn anthropic_nested_image_results_keep_calls_and_recent_frames() { + let mut profile = profile(); + profile.vision.as_mut().unwrap().max_count = 16; + let image = BASE64.encode(png()); + let mut messages = Vec::new(); + for index in 0..20 { + messages.push(serde_json::json!({"role":"assistant","content":[{"type":"text","text":"Inspect another frame."},{"type":"tool_use","id":format!("t{index}"),"name":"view_image","input":{"path":format!("/tmp/{index}.png")}}]})); + messages.push(serde_json::json!({"role":"user","content":[{"type":"tool_result","tool_use_id":format!("t{index}"),"content":[{"type":"text","text":format!("Frame {index}")},{"type":"image","source":{"type":"base64","media_type":"image/png","data":image}}]}]})); + } + let bytes = serde_json::to_vec( + &serde_json::json!({"model":"public","max_tokens":8,"messages":messages}), + ) + .unwrap(); + // Use small originals so the source body fits normal direct gateway ingress. + let source = gateway::normalize_image_request( + &bytes, + ImageProtocol::Anthropic, + &profile, + &IngressLimits::default(), + ) + .unwrap_err(); + assert_eq!(source.code, "request_too_large"); + let image = &profile.vision.as_ref().unwrap().health_image_base64; + let mut value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + for message in value["messages"].as_array_mut().unwrap() { + if message["role"] == "user" { + message["content"][0]["content"][1]["source"]["data"] = image.clone().into(); + } + } + let normalized = gateway::normalize_image_request( + &serde_json::to_vec(&value).unwrap(), + ImageProtocol::Anthropic, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + let rewritten = + gateway::rewrite_anthropic_request_with_profile(&normalized, "served", &profile).unwrap(); + let rewritten: serde_json::Value = serde_json::from_slice(&rewritten.body).unwrap(); + let tools = rewritten["messages"] + .as_array() + .unwrap() + .iter() + .filter(|message| message["role"] == "tool") + .collect::>(); + assert_eq!(tools.len(), 20); + assert_eq!(tools.last().unwrap()["tool_call_id"], "t19"); + assert!( + tools[0]["content"] + .as_str() + .unwrap() + .contains("Older image") + ); + assert_eq!(tools[19]["content"][0]["text"], "Frame 19"); + assert_eq!( + tools + .iter() + .filter(|tool| tool["content"].is_array()) + .count(), + 8 + ); +} + +#[test] +fn chat_vision_uses_the_same_retention_and_finite_profile_budget() { + let mut profile = profile(); + profile.vision.as_mut().unwrap().max_count = 16; + let url = format!( + "data:image/png;base64,{}", + profile.vision.as_ref().unwrap().health_image_base64 + ); + let mut messages = Vec::new(); + for index in 0..30 { + messages.push(serde_json::json!({"role":"user","content":[{"type":"text","text":format!("Frame {index}")},{"type":"image_url","image_url":{"url":url}}]})); + if index < 29 { + messages.push(serde_json::json!({"role":"assistant","content":"Observed frame."})); + } + } + let normalized = gateway::normalize_image_request( + &serde_json::to_vec(&serde_json::json!({"messages":messages})).unwrap(), + ImageProtocol::Chat, + &profile, + &IngressLimits::default(), + ) + .unwrap(); + assert!(gateway::rewrite_chat_request_with_profile(&normalized, "served", &profile).is_ok()); + assert!( + gateway::rewrite_chat_request_with_profile(&normalized, "served", &GatewayProfile::text()) + .is_err() + ); +} diff --git a/tests/image_history.rs b/tests/image_history.rs new file mode 100644 index 0000000..1275d6a --- /dev/null +++ b/tests/image_history.rs @@ -0,0 +1,104 @@ +use serde_json::{Value, json}; +use sparkplane::spark::image_history::{Protocol, project, project_reader}; + +fn frame(i: usize) -> Value { + json!({"type":"function_call_output","call_id":format!("call_{i}"),"output":[ + {"type":"input_text","text":format!("screenshot {i}")}, + {"type":"input_image","image_url":format!("data:image/png;base64,aW1hZ2U{i:04}")} + ]}) +} +fn images(value: &Value) -> usize { + match value { + Value::Array(a) => a.iter().map(images).sum(), + Value::Object(o) => { + usize::from(value["type"] == "input_image" || value["type"] == "image") + + o.values().map(images).sum::() + } + _ => 0, + } +} + +#[test] +fn five_hundred_views_keep_new_images_and_tool_pairing() { + let mut input = vec![]; + for i in 0..500 { + input.push(json!({"type":"function_call","call_id":format!("call_{i}"),"name":"view_image","arguments":format!("{{\"path\":\"/tmp/frame-{i}.png\"}}") })); + input.push(frame(i)); + if i != 499 { + input.push(json!({"type":"message","role":"assistant","content":format!("Observation {i}: button visible")})); + } + } + let original = json!({"model":"model","input":input,"store":false}); + let mut value = original.clone(); + let stats = project(&mut value, Protocol::Responses, 16, None).unwrap(); + assert!(images(&value) <= 12); + assert!(stats.evicted > 480); + assert_eq!(value["input"].as_array().unwrap().len(), 1499); + assert_eq!(value["input"][1]["call_id"], "call_0"); + assert_eq!(value["input"][1]["output"][0]["text"], "screenshot 0"); + assert!( + value["input"][1]["output"][1]["text"] + .as_str() + .unwrap() + .contains("/tmp/frame-0.png") + ); + assert_eq!(value["input"][1498], original["input"][1498]); + assert_eq!(value["input"][2], original["input"][2]); + let projected = project_reader( + std::io::Cursor::new(serde_json::to_vec(&original).unwrap()), + Protocol::Responses, + 16, + None, + ) + .unwrap(); + assert_eq!(projected.0, value); + let repeated = project(&mut value, Protocol::Responses, 16, None).unwrap(); + assert_eq!(repeated.evicted, 0); +} + +#[test] +fn unseen_batch_is_never_silently_discarded() { + let mut value = json!({"input":(0..17).map(frame).collect::>()}); + let before = value.clone(); + assert!(project(&mut value, Protocol::Responses, 16, None).is_err()); + assert_eq!(value, before); + let mut value = json!({"input":(0..16).map(frame).collect::>()}); + project(&mut value, Protocol::Responses, 16, None).unwrap(); + assert_eq!(images(&value), 16); +} + +#[test] +fn anthropic_nested_results_and_current_comparison_survive() { + let mut messages = vec![]; + for i in 0..20 { + messages.push(json!({"role":"assistant","content":[{"type":"tool_use","id":format!("t{i}"),"name":"screenshot","input":{}}]})); + messages.push(json!({"role":"user","content":[{"type":"tool_result","tool_use_id":format!("t{i}"),"content":[{"type":"text","text":"observed"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"aW1hZ2U="}}]}]})); + } + messages.push(json!({"role":"user","content":[{"type":"image","source":{"type":"base64","media_type":"image/png","data":"Y29tcGFyZTE="}},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"Y29tcGFyZTI="}}]})); + let mut value = json!({"messages":messages,"system":"preserve instructions"}); + let last = value["messages"][40].clone(); + project(&mut value, Protocol::Anthropic, 16, None).unwrap(); + assert!(images(&value) <= 12); + assert_eq!(value["messages"][40], last); + assert_eq!(value["messages"][1]["content"][0]["tool_use_id"], "t0"); + assert_eq!(value["system"], "preserve instructions"); +} + +#[test] +fn byte_pressure_evicts_processed_images_before_the_count_threshold() { + let image = format!("data:image/png;base64,{}", "a".repeat(6 * 1024 * 1024)); + let mut input = Vec::new(); + for index in 0..6 { + input.push(json!({"type":"message","role":"user","content":[{"type":"input_text","text":format!("Frame {index}")},{"type":"input_image","image_url":image}]})); + if index < 5 { + input.push(json!({"type":"message","role":"assistant","content":"Observed frame."})); + } + } + let mut request = json!({"input":input}); + let newest = request["input"][10].clone(); + let stats = project(&mut request, Protocol::Responses, 16, None).unwrap(); + assert_eq!(stats.evicted, 4); + assert_eq!(images(&request), 2); + assert_eq!(request["input"][10], newest); + assert!(serde_json::to_vec(&request).unwrap().len() < 16 * 1024 * 1024); +} diff --git a/tests/spark_anthropic_e2e.rs b/tests/spark_anthropic_e2e.rs index 8551b8d..e69fa1e 100644 --- a/tests/spark_anthropic_e2e.rs +++ b/tests/spark_anthropic_e2e.rs @@ -1,5 +1,7 @@ #![cfg(feature = "appliance")] +use sparkplane::spark::image_history; + #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] mod gateway; diff --git a/tests/spark_modalities_e2e.rs b/tests/spark_modalities_e2e.rs index 2bea587..518f5b3 100644 --- a/tests/spark_modalities_e2e.rs +++ b/tests/spark_modalities_e2e.rs @@ -1,5 +1,7 @@ #![cfg(feature = "appliance")] +use sparkplane::spark::image_history; + #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] mod gateway; diff --git a/tests/spark_openai_e2e.rs b/tests/spark_openai_e2e.rs index 8574e55..02892c4 100644 --- a/tests/spark_openai_e2e.rs +++ b/tests/spark_openai_e2e.rs @@ -1,5 +1,7 @@ #![cfg(feature = "appliance")] +use sparkplane::spark::image_history; + #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] mod gateway; diff --git a/tests/spark_qwen_vllm_optimization.rs b/tests/spark_qwen_vllm_optimization.rs index 3d28d01..3b148c5 100644 --- a/tests/spark_qwen_vllm_optimization.rs +++ b/tests/spark_qwen_vllm_optimization.rs @@ -32,6 +32,8 @@ fn qwen38_checkpoint_and_selected_engine_accept_inline_images() { vision.processor_sha256, "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516" ); + assert_eq!(vision.max_count, 16); + assert_eq!(vision.max_total_bytes, 524288); let image = serde_json::json!({"type":"input_image", "image_url":format!("data:image/png;base64,{}", vision.health_image_base64)}); let mut request = serde_json::json!({"input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"Name the color."},image]}]}); let rewrite = gateway::rewrite_responses_request_with_profile( @@ -59,6 +61,15 @@ fn qwen38_checkpoint_and_selected_engine_accept_inline_images() { ) .is_err() ); + request["input"][0]["content"] = serde_json::json!(vec![image.clone(); vision.max_count]); + assert!( + gateway::rewrite_responses_request_with_profile( + &serde_json::to_vec(&request).unwrap(), + "served-model", + &profile + ) + .is_ok() + ); request["input"][0]["content"] .as_array_mut() .unwrap() diff --git a/tests/spark_vllm_e2e.rs b/tests/spark_vllm_e2e.rs index 5ca3133..a6683d2 100644 --- a/tests/spark_vllm_e2e.rs +++ b/tests/spark_vllm_e2e.rs @@ -1,5 +1,7 @@ #![cfg(feature = "appliance")] +use sparkplane::spark::image_history; + #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] mod gateway; diff --git a/tests/standalone_contract.py b/tests/standalone_contract.py index d68548a..507deae 100644 --- a/tests/standalone_contract.py +++ b/tests/standalone_contract.py @@ -1,5 +1,7 @@ """Distribution boundary: this checkout must stand on its own.""" import pathlib +import hashlib +import json import re import tomllib import unittest @@ -8,6 +10,13 @@ class StandaloneContract(unittest.TestCase): + def test_vendored_rtk_sources_match_pinned_provenance(self): + root = ROOT / "crates/sparkplane-rtk" + provenance = json.loads((root / "PROVENANCE.json").read_text()) + for name, digest in provenance["files"].items(): + self.assertEqual(hashlib.sha256((root / "upstream" / name).read_bytes()).hexdigest(), digest, name) + self.assertEqual((root / "LICENSE").read_bytes(), (root / "upstream/LICENSE").read_bytes()) + def test_documentation_links_resolve_inside_this_checkout(self): for document in [ROOT / "README.md", *ROOT.glob("docs/**/*.md")]: for destination in re.findall(r"\]\(([^)]+)\)", document.read_text()):