From de5a0bae069e440dc70558988b85c2372735b579 Mon Sep 17 00:00:00 2001 From: softmaxe <94467584+softmaxe@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:00:24 +0800 Subject: [PATCH 1/3] feat: migrate validated usage kernels to Rust --- .github/workflows/ci.yml | 4 + .github/workflows/release.yml | 3 + .gitignore | 1 + Makefile | 12 +- Package.swift | 2 + README.md | 2 + README.zh-CN.md | 2 + Rust/QuotaBarCore/Cargo.lock | 107 +++++ Rust/QuotaBarCore/Cargo.toml | 20 + Rust/QuotaBarCore/src/aggregation.rs | 405 ++++++++++++++++++ Rust/QuotaBarCore/src/classifier.rs | 334 +++++++++++++++ Rust/QuotaBarCore/src/claude.rs | 346 +++++++++++++++ Rust/QuotaBarCore/src/lib.rs | 9 + Rust/QuotaBarCore/src/pricing.rs | 248 +++++++++++ Rust/QuotaBarCore/src/scanner.rs | 265 ++++++++++++ Scripts/build_rust.sh | 21 + Scripts/fixtures/rust_pipeline_hook.swift | 78 ++++ Scripts/setup_rust_ci.sh | 13 + Scripts/swift.sh | 25 ++ Scripts/validate_rust_pipeline.py | 395 +++++++++++++++++ Sources/CQuotaBarRust/include/QuotaBarRust.h | 85 ++++ Sources/CQuotaBarRust/module.modulemap | 4 + .../QuotaBarCore/Cost/ClaudeCodeAdapter.swift | 43 ++ .../QuotaBarCore/Cost/CostAggregator.swift | 134 ++++-- Sources/QuotaBarCore/Cost/CostPricing.swift | 67 ++- .../QuotaBarCore/Cost/JSONLogClassifier.swift | 207 +-------- .../QuotaBarCore/Cost/LogFileScanner.swift | 101 ++--- .../JSONLogClassifierDifferentialTests.swift | 137 ++++++ .../QuotaBarTests/RustAggregationTests.swift | 207 +++++++++ .../QuotaBarTests/RustClaudeParserTests.swift | 83 ++++ .../RustLogFileScannerTests.swift | 83 ++++ Sources/QuotaBarTests/RustPricingTests.swift | 193 +++++++++ .../QuotaBarTests/SwiftCostAggregator.swift | 106 +++++ .../SwiftJSONLogClassifier.swift | 216 ++++++++++ .../QuotaBarTests/SwiftLogLineScanner.swift | 90 ++++ Sources/QuotaBarTests/main.swift | 5 + docs/performance.md | 5 + docs/rust-migration.md | 222 ++++++++++ docs/rust-validation.md | 253 +++++++++++ rust-toolchain.toml | 4 + 40 files changed, 4203 insertions(+), 334 deletions(-) create mode 100644 Rust/QuotaBarCore/Cargo.lock create mode 100644 Rust/QuotaBarCore/Cargo.toml create mode 100644 Rust/QuotaBarCore/src/aggregation.rs create mode 100644 Rust/QuotaBarCore/src/classifier.rs create mode 100644 Rust/QuotaBarCore/src/claude.rs create mode 100644 Rust/QuotaBarCore/src/lib.rs create mode 100644 Rust/QuotaBarCore/src/pricing.rs create mode 100644 Rust/QuotaBarCore/src/scanner.rs create mode 100644 Scripts/build_rust.sh create mode 100644 Scripts/fixtures/rust_pipeline_hook.swift create mode 100644 Scripts/setup_rust_ci.sh create mode 100755 Scripts/validate_rust_pipeline.py create mode 100644 Sources/CQuotaBarRust/include/QuotaBarRust.h create mode 100644 Sources/CQuotaBarRust/module.modulemap create mode 100644 Sources/QuotaBarTests/JSONLogClassifierDifferentialTests.swift create mode 100644 Sources/QuotaBarTests/RustAggregationTests.swift create mode 100644 Sources/QuotaBarTests/RustClaudeParserTests.swift create mode 100644 Sources/QuotaBarTests/RustLogFileScannerTests.swift create mode 100644 Sources/QuotaBarTests/RustPricingTests.swift create mode 100644 Sources/QuotaBarTests/SwiftCostAggregator.swift create mode 100644 Sources/QuotaBarTests/SwiftJSONLogClassifier.swift create mode 100644 Sources/QuotaBarTests/SwiftLogLineScanner.swift create mode 100644 docs/rust-migration.md create mode 100644 docs/rust-validation.md create mode 100644 rust-toolchain.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cd0481..2f65ee9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,10 @@ jobs: with: persist-credentials: false + - name: Install pinned Rust toolchain + if: ${{ needs.changes.outputs.build == 'true' }} + run: bash Scripts/setup_rust_ci.sh + - name: Print tool versions if: ${{ needs.changes.outputs.build == 'true' }} run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4387cd2..805d3e5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -89,6 +89,9 @@ jobs: with: persist-credentials: false + - name: Install pinned Rust toolchain + run: bash Scripts/setup_rust_ci.sh + - name: Require the native menu bar SDK run: | set -euo pipefail diff --git a/.gitignore b/.gitignore index f2a8f8e..a1ccd3f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .build/ .swiftpm/ Package.resolved +Rust/**/target/ # Build output build/ diff --git a/Makefile b/Makefile index 6e0a5c5..2865676 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,13 @@ CONFIG ?= debug PROVIDER ?= codex PREVIEW_STATE ?= loaded SNAPSHOT_DIR ?= $(BUILD_DIR)/screenshots/card +TEST_RUNNER ?= SWIFT := Scripts/swift.sh BIN := $(BUILD_DIR)/$(CONFIG)/$(APP_NAME) DEBUG_BIN := $(BUILD_DIR)/debug/$(APP_NAME) LOG_SUBSYSTEM := com.quotabar.app -.PHONY: build run preview card-snapshot probe probe-cost benchmark-cost benchmark-startup logs kill test app readme-assets clean +.PHONY: build run preview card-snapshot probe probe-cost benchmark-cost benchmark-startup logs kill test rust-check app readme-assets clean build: $(SWIFT) build -c $(CONFIG) --product $(APP_NAME) @@ -67,9 +68,14 @@ app: ## Core and app regressions use a plain executable of assertions. Always a debug build: the ## suite imports the app with @testable, which needs a module compiled for testing. -test: +test: rust-check $(SWIFT) build -c debug --product $(APP_NAME)Tests - $(BUILD_DIR)/debug/$(APP_NAME)Tests + $(TEST_RUNNER) $(BUILD_DIR)/debug/$(APP_NAME)Tests + +rust-check: + cargo fmt --manifest-path Rust/QuotaBarCore/Cargo.toml -- --check + CARGO_TARGET_DIR="$(CURDIR)/.build/rust" MACOSX_DEPLOYMENT_TARGET=14.0 cargo test --locked --manifest-path Rust/QuotaBarCore/Cargo.toml + CARGO_TARGET_DIR="$(CURDIR)/.build/rust" MACOSX_DEPLOYMENT_TARGET=14.0 cargo clippy --locked --manifest-path Rust/QuotaBarCore/Cargo.toml --all-targets -- -D warnings clean: $(SWIFT) package clean diff --git a/Package.swift b/Package.swift index bcac18e..55f7585 100644 --- a/Package.swift +++ b/Package.swift @@ -11,9 +11,11 @@ let package = Package( .library(name: "QuotaBarCore", targets: ["QuotaBarCore"]), ], targets: [ + .systemLibrary(name: "CQuotaBarRust"), // Data layer: credentials, usage endpoints, snapshot models. No AppKit. .target( name: "QuotaBarCore", + dependencies: ["CQuotaBarRust"], resources: [.copy("Resources/UsageReport"), .copy("Resources/Pricing")], swiftSettings: [.swiftLanguageMode(.v5)] ), diff --git a/README.md b/README.md index 8841441..eac610a 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,8 @@ Codex quota requests use the `chatgpt_base_url` in `$CODEX_HOME/config.toml`, if Building requires full Xcode with a Swift 6 toolchain. The project uses Swift Package Manager. `Scripts/swift.sh` defaults to `/Applications/Xcode.app/Contents/Developer` without changing the global `xcode-select` setting. +Rust kernels also require `rustup` and the toolchain pinned in `rust-toolchain.toml`. Install rustup with `brew install rustup` and add its `bin` directory to PATH as Homebrew instructs. Use `make` or `Scripts/swift.sh` for builds: the wrapper builds the matching Cargo profile, links the static library, and invalidates SwiftPM's cache when Rust changes. Bare `swift build` and opening the package directly in Xcode do not run this integration. Prebuilt app users do not need Rust. See [Rust core boundaries and validation](docs/rust-migration.md). + The release workflow requires the macOS 27 SDK for native menu bar sessions on macOS 27. Local builds with older SDKs use the legacy menu bar handling. The minimum runtime remains macOS 14. ```bash diff --git a/README.zh-CN.md b/README.zh-CN.md index f1ef99a..5f9f25f 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -294,6 +294,8 @@ Codex 额度请求会使用 `$CODEX_HOME/config.toml` 中的 `chatgpt_base_url` 构建需要完整的 Xcode 及 Swift 6 工具链。项目使用 Swift Package Manager。`Scripts/swift.sh` 默认使用 `/Applications/Xcode.app/Contents/Developer`,不会修改全局 `xcode-select` 设置。 +Rust kernels also require `rustup` and the toolchain pinned in `rust-toolchain.toml`. Install rustup with `brew install rustup` and add its `bin` directory to PATH as Homebrew instructs. Use `make` or `Scripts/swift.sh` for builds: the wrapper builds the matching Cargo profile, links the static library, and invalidates SwiftPM's cache when Rust changes. Bare `swift build` and opening the package directly in Xcode do not run this integration. Prebuilt app users do not need Rust. See [Rust core boundaries and validation](docs/rust-migration.md). + 发布工作流要求 macOS 27 SDK,以支持 macOS 27 的原生菜单栏会话。使用较旧 SDK 的本地构建会采用旧版菜单栏处理方式。最低运行系统仍为 macOS 14。 ```bash diff --git a/Rust/QuotaBarCore/Cargo.lock b/Rust/QuotaBarCore/Cargo.lock new file mode 100644 index 0000000..673dc60 --- /dev/null +++ b/Rust/QuotaBarCore/Cargo.lock @@ -0,0 +1,107 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "itoa" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quotabar_core" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Rust/QuotaBarCore/Cargo.toml b/Rust/QuotaBarCore/Cargo.toml new file mode 100644 index 0000000..fc30231 --- /dev/null +++ b/Rust/QuotaBarCore/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "quotabar_core" +version = "0.1.0" +edition = "2024" +rust-version = "1.98" +publish = false +license = "AGPL-3.0-only" + +[lib] +crate-type = ["staticlib", "rlib"] + +[dependencies] +serde = { version = "=1.0.229", features = ["derive"] } +serde_json = "=1.0.154" + +[profile.release] +panic = "unwind" +lto = "thin" +codegen-units = 1 +overflow-checks = true diff --git a/Rust/QuotaBarCore/src/aggregation.rs b/Rust/QuotaBarCore/src/aggregation.rs new file mode 100644 index 0000000..57975e4 --- /dev/null +++ b/Rust/QuotaBarCore/src/aggregation.rs @@ -0,0 +1,405 @@ +//! Folds already priced, SQL-reduced rows. Swift supplies its actual day traversal order and +//! retains all strings, calendar operations, output models, and top-model tie policy. + +use std::{ + collections::HashMap, + panic::{AssertUnwindSafe, catch_unwind}, + slice, +}; + +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct QBAggregateTokens { + pub input: i64, + pub output: i64, + pub cache_write: i64, + pub cache_write_1h: i64, + pub cache_read: i64, +} + +#[repr(C)] +pub struct QBAggregateRow { + pub day: u64, + pub key: u64, + pub model: u64, + pub tokens: QBAggregateTokens, + pub cost: f64, + pub unpriced: i64, + pub priced: u8, + pub fallback: u8, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct QBAggregateGroup { + pub day: u64, + pub key: u64, + pub tokens: QBAggregateTokens, + pub cost: f64, + pub priced: u8, + pub fallback: u8, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct QBAggregateDay { + pub tokens: QBAggregateTokens, + pub cost: f64, + pub unpriced: i64, + pub priced: u8, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct QBAggregateModel { + pub cost: f64, + pub tokens: i64, + pub priced: u8, +} + +const INVALID_COUNTS: i32 = 1; +const TOKEN_OVERFLOW: i32 = 2; +const TOTAL_OVERFLOW: i32 = 3; +const WINDOW_OVERFLOW: i32 = 4; +const INVALID_COST: i32 = 5; +const INVALID_ARGUMENT: i32 = 6; +const INTERNAL_ERROR: i32 = 7; + +impl QBAggregateTokens { + fn total(self) -> Option { + if [ + self.input, + self.output, + self.cache_write, + self.cache_write_1h, + self.cache_read, + ] + .into_iter() + .any(|value| value < 0) + || self.cache_write_1h > self.cache_write + { + return None; + } + self.input + .checked_add(self.output)? + .checked_add(self.cache_write)? + .checked_add(self.cache_read) + } + + /// The order and distinct errors match TokenTotals.adding, including its final total check. + fn adding(self, other: Self) -> Result { + self.total().ok_or(INVALID_COUNTS)?; + other.total().ok_or(INVALID_COUNTS)?; + let add = |lhs: i64, rhs: i64| lhs.checked_add(rhs).ok_or(TOKEN_OVERFLOW); + let result = Self { + input: add(self.input, other.input)?, + output: add(self.output, other.output)?, + cache_write: add(self.cache_write, other.cache_write)?, + cache_write_1h: add(self.cache_write_1h, other.cache_write_1h)?, + cache_read: add(self.cache_read, other.cache_read)?, + }; + result.total().ok_or(TOTAL_OVERFLOW)?; + Ok(result) + } +} + +fn add_tokens(lhs: i64, rhs: i64) -> Result { + if lhs < 0 || rhs < 0 { + return Err(WINDOW_OVERFLOW); + } + lhs.checked_add(rhs).ok_or(WINDOW_OVERFLOW) +} + +fn add_cost(lhs: f64, rhs: f64) -> Result { + let result = lhs + rhs; + if lhs >= 0.0 && rhs >= 0.0 && result.is_finite() { + Ok(result) + } else { + Err(INVALID_COST) + } +} + +fn fold( + rows: &[QBAggregateRow], + groups: &mut [QBAggregateGroup], + days: &mut [QBAggregateDay], + models: &mut [QBAggregateModel], +) -> Result { + days.fill(QBAggregateDay::default()); + models.fill(QBAggregateModel::default()); + let mut indexes = HashMap::with_capacity(rows.len()); + let mut group_count = 0; + for row in rows { + let day = days + .get_mut(usize::try_from(row.day).map_err(|_| INVALID_ARGUMENT)?) + .ok_or(INVALID_ARGUMENT)?; + let model = models + .get_mut(usize::try_from(row.model).map_err(|_| INVALID_ARGUMENT)?) + .ok_or(INVALID_ARGUMENT)?; + let index = *indexes.entry((row.day, row.key)).or_insert_with(|| { + let index = group_count; + groups[index] = QBAggregateGroup { + day: row.day, + key: row.key, + ..Default::default() + }; + group_count += 1; + index + }); + let group = &mut groups[index]; + // Preserve the Swift fold's check and accumulation order exactly. + group.tokens = group.tokens.adding(row.tokens)?; + day.tokens = day.tokens.adding(row.tokens)?; + model.tokens = add_tokens(model.tokens, row.tokens.total().ok_or(INVALID_COUNTS)?)?; + if row.fallback != 0 { + group.fallback = 1; + } + if row.priced != 0 { + day.cost = add_cost(day.cost, row.cost)?; + day.priced = 1; + group.cost = add_cost(group.cost, row.cost)?; + group.priced = 1; + model.cost = add_cost(model.cost, row.cost)?; + model.priced = 1; + } + if row.unpriced > 0 { + day.unpriced = add_tokens(day.unpriced, row.unpriced)?; + } + } + Ok(group_count as u64) +} + +/// Folds rows synchronously into caller-owned buffers. A nonzero status invalidates all outputs. +/// Group capacity must be at least row_count. Null buffers are allowed only for zero counts. +/// +/// # Safety +/// Every nonempty buffer must reference aligned, initialized storage for its count. All buffers +/// must be disjoint and remain alive for the call; group_count must be writable and non-null. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn qb_aggregate_usage( + rows: *const QBAggregateRow, + row_count: u64, + day_count: u64, + model_count: u64, + groups: *mut QBAggregateGroup, + group_count: *mut u64, + days: *mut QBAggregateDay, + models: *mut QBAggregateModel, +) -> i32 { + if group_count.is_null() { + return INVALID_ARGUMENT; + } + // SAFETY: The caller guarantees writable storage for group_count. + unsafe { + *group_count = 0; + } + let Ok(row_count) = usize::try_from(row_count) else { + return INVALID_ARGUMENT; + }; + let Ok(day_count) = usize::try_from(day_count) else { + return INVALID_ARGUMENT; + }; + let Ok(model_count) = usize::try_from(model_count) else { + return INVALID_ARGUMENT; + }; + if row_count > isize::MAX as usize / std::mem::size_of::() + || row_count > isize::MAX as usize / std::mem::size_of::() + || day_count > isize::MAX as usize / std::mem::size_of::() + || model_count > isize::MAX as usize / std::mem::size_of::() + { + return INVALID_ARGUMENT; + } + if (row_count > 0 && (rows.is_null() || groups.is_null())) + || (day_count > 0 && days.is_null()) + || (model_count > 0 && models.is_null()) + { + return INVALID_ARGUMENT; + } + // SAFETY: Counts and pointers satisfy the caller's contract. Empty slices do not dereference + // nullable zero-length buffers. Rust borrows all storage only for this synchronous call. + let result = catch_unwind(AssertUnwindSafe(|| unsafe { + let rows = if row_count == 0 { + &[] + } else { + slice::from_raw_parts(rows, row_count) + }; + let groups = if row_count == 0 { + &mut [] + } else { + slice::from_raw_parts_mut(groups, row_count) + }; + let days = if day_count == 0 { + &mut [] + } else { + slice::from_raw_parts_mut(days, day_count) + }; + let models = if model_count == 0 { + &mut [] + } else { + slice::from_raw_parts_mut(models, model_count) + }; + fold(rows, groups, days, models) + })); + match result { + Ok(Ok(count)) => { + // SAFETY: The caller guarantees writable storage for group_count. + unsafe { + *group_count = count; + } + 0 + } + Ok(Err(status)) => status, + Err(_) => INTERNAL_ERROR, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ptr; + + fn row() -> QBAggregateRow { + QBAggregateRow { + day: 0, + key: 0, + model: 0, + tokens: QBAggregateTokens { + input: 1, + ..Default::default() + }, + cost: 1.0, + unpriced: 0, + priced: 1, + fallback: 0, + } + } + + fn run(rows: &[QBAggregateRow]) -> Result { + fold( + rows, + &mut vec![QBAggregateGroup::default(); rows.len()], + &mut [QBAggregateDay::default()], + &mut [QBAggregateModel::default()], + ) + } + + #[test] + fn fixed_width_layout() { + assert_eq!(std::mem::size_of::(), 40); + assert_eq!(std::mem::size_of::(), 88); + assert_eq!(std::mem::offset_of!(QBAggregateRow, tokens), 24); + assert_eq!(std::mem::offset_of!(QBAggregateRow, cost), 64); + assert_eq!(std::mem::size_of::(), 72); + assert_eq!(std::mem::size_of::(), 64); + assert_eq!(std::mem::size_of::(), 24); + } + + #[test] + fn token_errors_keep_their_distinct_statuses() { + let mut invalid = row(); + invalid.tokens.input = -1; + assert_eq!(run(&[invalid]), Err(INVALID_COUNTS)); + let mut maximum = row(); + maximum.tokens.input = i64::MAX; + assert_eq!(run(&[maximum, row()]), Err(TOKEN_OVERFLOW)); + let mut maximum = row(); + maximum.tokens.input = i64::MAX; + let mut output = row(); + output.tokens.input = 0; + output.tokens.output = 1; + assert_eq!(run(&[maximum, output]), Err(TOTAL_OVERFLOW)); + let mut maximum = row(); + maximum.unpriced = i64::MAX; + let mut next = row(); + next.unpriced = 1; + assert_eq!(run(&[maximum, next]), Err(WINDOW_OVERFLOW)); + } + + #[test] + fn cost_validation_and_optional_zero() { + for cost in [-1.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let mut invalid = row(); + invalid.cost = cost; + assert_eq!(run(&[invalid]), Err(INVALID_COST)); + } + let mut unpriced = row(); + unpriced.priced = 0; + unpriced.cost = f64::NAN; + assert_eq!(run(&[unpriced]), Ok(1)); + let mut zero = row(); + zero.cost = 0.0; + assert_eq!(run(&[zero]), Ok(1)); + } + + #[test] + fn abi_empty_and_invalid_arguments() { + let mut count = 99; + // SAFETY: Zero-length buffers may be null and count is live writable output storage. + unsafe { + assert_eq!( + qb_aggregate_usage( + ptr::null(), + 0, + 0, + 0, + ptr::null_mut(), + &mut count, + ptr::null_mut(), + ptr::null_mut() + ), + 0 + ); + assert_eq!(count, 0); + assert_eq!( + qb_aggregate_usage( + ptr::null(), + 1, + 0, + 0, + ptr::null_mut(), + &mut count, + ptr::null_mut(), + ptr::null_mut() + ), + INVALID_ARGUMENT + ); + assert_eq!( + qb_aggregate_usage( + ptr::null(), + 0, + 0, + 0, + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut() + ), + INVALID_ARGUMENT + ); + } + let mut invalid = row(); + invalid.day = 1; + assert_eq!(run(&[invalid]), Err(INVALID_ARGUMENT)); + } + + #[test] + fn abi_rejects_unrepresentable_slice_lengths_before_borrowing() { + let input = row(); + let mut group = QBAggregateGroup::default(); + let mut day = QBAggregateDay::default(); + let mut model = QBAggregateModel::default(); + let mut count = 99; + for (rows, days, models) in [(u64::MAX, 1, 1), (1, u64::MAX, 1), (1, 1, u64::MAX)] { + // SAFETY: Oversized counts are rejected before any buffer is borrowed or dereferenced. + assert_eq!( + unsafe { + qb_aggregate_usage( + &input, rows, days, models, &mut group, &mut count, &mut day, &mut model, + ) + }, + INVALID_ARGUMENT + ); + assert_eq!(count, 0); + } + } +} diff --git a/Rust/QuotaBarCore/src/classifier.rs b/Rust/QuotaBarCore/src/classifier.rs new file mode 100644 index 0000000..721ce8e --- /dev/null +++ b/Rust/QuotaBarCore/src/classifier.rs @@ -0,0 +1,334 @@ +//! Allocation-free structural classifier, deliberately matching the Swift prefix filter. +//! This is not a JSON validator: escaped keys/target values remain indeterminate, and +//! classification stops at the first matching key without validating trailing input. + +use std::ffi::{c_int, c_void}; +use std::ops::Range; + +unsafe extern "C" { + #[link_name = "memchr"] + fn c_memchr(data: *const c_void, byte: c_int, len: usize) -> *const c_void; +} + +/// Uses the same bounded libc search as the previous Swift implementation. +fn find_byte(bytes: &[u8], byte: u8) -> Option { + // SAFETY: The slice is readable for exactly its length. memchr returns either null + // or a pointer within this slice, so the offset stays within the same allocation. + unsafe { + let found = c_memchr(bytes.as_ptr().cast(), byte.into(), bytes.len()); + if found.is_null() { + None + } else { + Some(found.cast::().offset_from(bytes.as_ptr()) as usize) + } + } +} + +const MISSING: i32 = 0; +const INDETERMINATE: i32 = 1; +const OTHER: i32 = 2; + +struct ParsedString { + range: Range, + has_escape: bool, +} + +enum Lookup { + Found(T), + Indeterminate, + Missing, +} + +fn skip_whitespace(bytes: &[u8], index: &mut usize) { + while matches!(bytes.get(*index), Some(b' ' | b'\t' | b'\r' | b'\n')) { + *index += 1; + } +} + +fn string_range(bytes: &[u8], index: &mut usize) -> Option { + if bytes.get(*index) != Some(&b'"') { + return None; + } + *index += 1; + let start = *index; + // Most structural keys and type values fit in this prefix. Avoid two libc calls + // for those tiny strings while retaining bounded bulk searches for long content. + let prefix_end = (start + 16).min(bytes.len()); + while *index < prefix_end { + match bytes[*index] { + b'"' => { + let range = start..*index; + *index += 1; + return Some(ParsedString { + range, + has_escape: false, + }); + } + b'\\' => return escaped_string_range(bytes, index, start), + _ => *index += 1, + } + } + let end = *index + find_byte(&bytes[*index..], b'"')?; + let Some(escape) = find_byte(&bytes[*index..end], b'\\') else { + *index = end + 1; + return Some(ParsedString { + range: start..end, + has_escape: false, + }); + }; + *index += escape; + escaped_string_range(bytes, index, start) +} + +fn escaped_string_range(bytes: &[u8], index: &mut usize, start: usize) -> Option { + // Avoid a libc call per escape in dense strings. Each backslash consumes exactly + // one following byte, including quotes and backslashes, as in the Swift filter. + while let Some(&byte) = bytes.get(*index) { + if byte == b'\\' { + *index += 2; + continue; + } else if byte == b'"' { + let range = start..*index; + *index += 1; + return Some(ParsedString { + range, + has_escape: true, + }); + } + *index += 1; + } + None +} + +fn skip_value(bytes: &[u8], index: &mut usize) -> bool { + match bytes.get(*index) { + None => false, + Some(b'"') => string_range(bytes, index).is_some(), + Some(b'{' | b'[') => { + let mut depth = 0usize; + while let Some(&byte) = bytes.get(*index) { + match byte { + b'"' => { + if string_range(bytes, index).is_none() { + return false; + } + continue; + } + b'{' | b'[' => depth += 1, + b'}' | b']' => { + depth -= 1; + *index += 1; + if depth == 0 { + return true; + } + continue; + } + _ => {} + } + *index += 1; + } + false + } + Some(_) => { + while let Some(&byte) = bytes.get(*index) { + if matches!(byte, b',' | b'}' | b']' | b' ' | b'\t' | b'\r' | b'\n') { + return true; + } + *index += 1; + } + true + } + } +} + +#[inline] +fn value_start(bytes: &[u8], key: &[u8], object_start: usize) -> Lookup { + let mut index = object_start; + skip_whitespace(bytes, &mut index); + if bytes.get(index) != Some(&b'{') { + return Lookup::Indeterminate; + } + index += 1; + while index < bytes.len() { + skip_whitespace(bytes, &mut index); + if bytes.get(index) == Some(&b'}') { + return Lookup::Missing; + } + let Some(candidate) = string_range(bytes, &mut index) else { + return Lookup::Indeterminate; + }; + if candidate.has_escape { + return Lookup::Indeterminate; + } + skip_whitespace(bytes, &mut index); + if bytes.get(index) != Some(&b':') { + return Lookup::Indeterminate; + } + index += 1; + skip_whitespace(bytes, &mut index); + if bytes[candidate.range] == *key { + return Lookup::Found(index); + } + if !skip_value(bytes, &mut index) { + return Lookup::Indeterminate; + } + skip_whitespace(bytes, &mut index); + match bytes.get(index) { + Some(b',') => index += 1, + Some(b'}') => return Lookup::Missing, + _ => return Lookup::Indeterminate, + } + } + Lookup::Indeterminate +} + +#[inline] +fn string_value(bytes: &[u8], object_start: usize) -> Lookup { + let mut index = match value_start(bytes, b"type", object_start) { + Lookup::Found(index) => index, + Lookup::Indeterminate => return Lookup::Indeterminate, + Lookup::Missing => return Lookup::Missing, + }; + match string_range(bytes, &mut index) { + Some(value) if !value.has_escape => Lookup::Found(value), + _ => Lookup::Indeterminate, + } +} + +fn classify_top_level(bytes: &[u8]) -> i32 { + match string_value(bytes, 0) { + Lookup::Missing => MISSING, + Lookup::Indeterminate => INDETERMINATE, + Lookup::Found(value) => match &bytes[value.range] { + b"assistant" => 3, + b"event_msg" => 4, + b"turn_context" => 5, + _ => OTHER, + }, + } +} + +fn classify_payload(bytes: &[u8]) -> i32 { + let start = match value_start(bytes, b"payload", 0) { + Lookup::Found(start) => start, + Lookup::Indeterminate => return INDETERMINATE, + Lookup::Missing => return MISSING, + }; + match string_value(bytes, start) { + Lookup::Missing => MISSING, + Lookup::Indeterminate => INDETERMINATE, + Lookup::Found(value) => match &bytes[value.range] { + b"thread_settings_applied" => 3, + b"token_count" => 4, + _ => OTHER, + }, + } +} + +/// Classifies a borrowed JSONL prefix: 0 missing, 1 indeterminate, 2 other, +/// 3 assistant, 4 event_msg, 5 turn_context. Empty or null input is indeterminate. +/// +/// # Safety +/// For nonempty input, `data` must point to `len` readable bytes in one allocation +/// which remains immutable for this call. No pointer is retained. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn qb_classify_top_level(data: *const u8, len: usize) -> i32 { + if data.is_null() || len == 0 || len > isize::MAX as usize { + return INDETERMINATE; + } + // SAFETY: The caller supplies readable borrowed bytes; null and size are checked above. + classify_top_level(unsafe { std::slice::from_raw_parts(data, len) }) +} + +/// Classifies payload.type: 0 missing, 1 indeterminate, 2 other, +/// 3 thread_settings_applied, 4 token_count. Empty or null input is indeterminate. +/// +/// # Safety +/// For nonempty input, `data` must point to `len` readable bytes in one allocation +/// which remains immutable for this call. No pointer is retained. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn qb_classify_payload(data: *const u8, len: usize) -> i32 { + if data.is_null() || len == 0 || len > isize::MAX as usize { + return INDETERMINATE; + } + // SAFETY: The caller supplies readable borrowed bytes; null and size are checked above. + classify_payload(unsafe { std::slice::from_raw_parts(data, len) }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prefix_filter_contract() { + for (bytes, expected) in [ + (b"".as_slice(), INDETERMINATE), + (br#"{}"#, MISSING), + (br#"{"type":"assistant"}"#, 3), + (br#"{"type":"event_msg"}"#, 4), + (br#"{"type":"turn_context"}"#, 5), + (br#"{"type":"user"}"#, OTHER), + (br#"{"type":"assistant"garbage"#, 3), + (br#"{"type":"assistant","type":"user"}"#, 3), + (br#"{"ty\u0070e":"assistant"}"#, INDETERMINATE), + (br#"{"type":"assi\u0073tant"}"#, INDETERMINATE), + (br#"{"nested":{"type":"assistant"}}"#, MISSING), + (br#"{"skip":[{"type":"assistant"}],"type":"user"}"#, OTHER), + (br#"{"skip":],"type":"assistant"}"#, INDETERMINATE), + (br#"{"skip":[},"type":"assistant"}"#, 3), + (br#"{"skip":1,}"#, MISSING), + ] { + assert_eq!(classify_top_level(bytes), expected, "{bytes:?}"); + } + } + + #[test] + fn payload_contract() { + for (bytes, expected) in [ + (br#"{}"#.as_slice(), MISSING), + (br#"{"payload":{}}"#, MISSING), + (br#"{"payload":null}"#, INDETERMINATE), + (br#"{"payload":{"type":"token_count"}}"#, 4), + (br#"{"payload":{"type":"thread_settings_applied"}}"#, 3), + (br#"{"payload":{"type":"token_count""#, 4), + (br#"{"payload":{"type":"other"}}"#, OTHER), + (br#"{"payload":{"type":"token\u005fcount"}}"#, INDETERMINATE), + ] { + assert_eq!(classify_payload(bytes), expected, "{bytes:?}"); + } + } + + #[test] + fn empty_ffi_input_never_builds_a_null_slice() { + // SAFETY: Null/empty inputs are explicitly supported and never dereferenced. + unsafe { + for len in [0, 1, usize::MAX] { + assert_eq!(qb_classify_top_level(std::ptr::null(), len), INDETERMINATE); + assert_eq!(qb_classify_payload(std::ptr::null(), len), INDETERMINATE); + } + assert_eq!(qb_classify_top_level(b"x".as_ptr(), 0), INDETERMINATE); + assert_eq!( + qb_classify_payload(b"x".as_ptr(), usize::MAX), + INDETERMINATE + ); + } + } + + #[test] + fn escape_parity_and_large_truncated_strings() { + for count in 0..128 { + let mut bytes = br#"{"padding":""#.to_vec(); + bytes.extend(std::iter::repeat_n(b'\\', count)); + bytes.extend_from_slice(br#"","type":"assistant"}"#); + assert_eq!( + classify_top_level(&bytes), + if count % 2 == 0 { 3 } else { INDETERMINATE } + ); + } + let mut bytes = br#"{"padding":""#.to_vec(); + bytes.extend(std::iter::repeat_n(b'x', 1_048_576)); + assert_eq!(classify_top_level(&bytes), INDETERMINATE); + bytes.extend_from_slice(br#"","type":"assistant"}"#); + assert_eq!(classify_top_level(&bytes), 3); + } +} diff --git a/Rust/QuotaBarCore/src/claude.rs b/Rust/QuotaBarCore/src/claude.rs new file mode 100644 index 0000000..00e0e67 --- /dev/null +++ b/Rust/QuotaBarCore/src/claude.rs @@ -0,0 +1,346 @@ +// Adapted from QuotaBar's ClaudeCodeAdapter, itself adapted from CodexBar +// (MIT, Copyright 2026 Peter Steinberger). See THIRD_PARTY_NOTICES.md. + +use serde::de::{MapAccess, Visitor}; +use serde::{Deserialize, Deserializer}; +use serde_json::Value; +use std::ffi::c_void; +use std::marker::PhantomData; +use std::panic::catch_unwind; +use std::ptr; + +const PARSED: u32 = 0; +const IGNORED: u32 = 1; +const FALLBACK: u32 = 2; +const INTERNAL_ERROR: u32 = 3; +const INVALID_ARGUMENT: u32 = 4; + +/// Foundation keyed containers reject positional arrays, unlike Serde structs. +struct MapOnly(T); + +impl<'de, T: Deserialize<'de>> Deserialize<'de> for MapOnly { + fn deserialize>(deserializer: D) -> Result { + struct ObjectVisitor(PhantomData); + impl<'de, T: Deserialize<'de>> Visitor<'de> for ObjectVisitor { + type Value = MapOnly; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a JSON object") + } + + fn visit_map>(self, access: A) -> Result { + T::deserialize(serde::de::value::MapAccessDeserializer::new(access)).map(MapOnly) + } + } + deserializer.deserialize_map(ObjectVisitor(PhantomData)) + } +} + +// Unknown transcript content is skipped. Duplicate known keys and unexpected +// known-field shapes delegate to Foundation rather than changing loose decoding. +#[derive(Default, Deserialize)] +#[serde(default)] +struct Line { + #[serde(rename = "type")] + kind: Option, + timestamp: Option, + #[serde(rename = "requestId")] + request_id: Option, + uuid: Option, + message: Option>, +} + +#[derive(Default, Deserialize)] +#[serde(default)] +struct Message { + id: Option, + model: Option, + usage: Option>, +} + +#[derive(Default, Deserialize)] +#[serde(default)] +struct Usage { + input_tokens: Value, + output_tokens: Value, + cache_creation_input_tokens: Value, + cache_read_input_tokens: Value, + cache_creation: Option>, + speed: Option, +} + +#[derive(Default, Deserialize)] +#[serde(default)] +struct CacheCreation { + ephemeral_1h_input_tokens: Value, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ByteString { + ptr: *const u8, + len: usize, +} + +impl ByteString { + const EMPTY: Self = Self { + ptr: ptr::null(), + len: 0, + }; + + fn borrow(value: &str) -> Self { + Self { + ptr: value.as_ptr(), + len: value.len(), + } + } +} + +/// All views are valid until the unique owner is released by qb_claude_free. +#[repr(C)] +pub struct ClaudeResult { + status: u32, + counts: [i64; 5], + timestamp: ByteString, + model: ByteString, + message_id: ByteString, + request_id: ByteString, + uuid: ByteString, + speed: ByteString, + owner: *mut c_void, +} + +impl ClaudeResult { + fn empty(status: u32) -> Self { + Self { + status, + counts: [0; 5], + timestamp: ByteString::EMPTY, + model: ByteString::EMPTY, + message_id: ByteString::EMPTY, + request_id: ByteString::EMPTY, + uuid: ByteString::EMPTY, + speed: ByteString::EMPTY, + owner: ptr::null_mut(), + } + } +} + +struct OwnedRecord { + timestamp: String, + model: String, + message_id: String, + request_id: String, + uuid: String, + speed: String, +} + +fn scalar(value: &Value) -> Result { + if let Some(integer) = value.as_i64() { + return Ok(integer); + } + if value.is_f64() { + // Foundation tries exact Int decoding before Double, including decimal and + // exponent spellings. Delegate all floating encodings, not only values >2^53. + return Err(()); + } + if value.is_u64() { + // An unsigned value outside Int64 is also outside LooseScalar's Double bound. + return Ok(0); + } + if let Some(boolean) = value.as_bool() { + return Ok(i64::from(boolean)); + } + Ok(value + .as_str() + .and_then(|text| text.parse().ok()) + .unwrap_or(0)) +} + +fn decode(bytes: &[u8]) -> Result { + let line = serde_json::from_slice::>(bytes) + .map_err(|_| ())? + .0; + if line.kind.as_deref() != Some("assistant") { + return Ok(ClaudeResult::empty(IGNORED)); + } + let Some(message) = line.message.map(|value| value.0) else { + return Ok(ClaudeResult::empty(IGNORED)); + }; + let Some(usage) = message.usage.map(|value| value.0) else { + return Ok(ClaudeResult::empty(IGNORED)); + }; + let (Some(timestamp), Some(model)) = (line.timestamp, message.model) else { + return Ok(ClaudeResult::empty(IGNORED)); + }; + let write = scalar(&usage.cache_creation_input_tokens)?; + let write_1h = usage + .cache_creation + .as_ref() + .map(|value| scalar(&value.0.ephemeral_1h_input_tokens)) + .transpose()? + .unwrap_or(0); + let counts = [ + scalar(&usage.input_tokens)?, + scalar(&usage.output_tokens)?, + write, + write_1h.min(write), + scalar(&usage.cache_read_input_tokens)?, + ]; + let total = [counts[0], counts[1], counts[2], counts[4]] + .into_iter() + .try_fold(0_i64, i64::checked_add); + if counts.iter().any(|count| *count < 0) || !matches!(total, Some(value) if value > 0) { + return Ok(ClaudeResult::empty(IGNORED)); + } + let message_id = message.id.unwrap_or_default(); + let request_id = line.request_id.unwrap_or_default(); + let uuid = line.uuid.unwrap_or_default(); + if message_id.is_empty() && request_id.is_empty() && uuid.is_empty() { + // Foundation canonical JSON hashing is part of the persisted identity format. + return Err(()); + } + let owner = Box::new(OwnedRecord { + timestamp, + model, + message_id, + request_id, + uuid, + speed: usage.speed.unwrap_or_default(), + }); + let result = ClaudeResult { + status: PARSED, + counts, + timestamp: ByteString::borrow(&owner.timestamp), + model: ByteString::borrow(&owner.model), + message_id: ByteString::borrow(&owner.message_id), + request_id: ByteString::borrow(&owner.request_id), + uuid: ByteString::borrow(&owner.uuid), + speed: ByteString::borrow(&owner.speed), + owner: Box::into_raw(owner).cast(), + }; + Ok(result) +} + +fn contain_failure( + work: impl FnOnce() -> Result + std::panic::UnwindSafe, +) -> ClaudeResult { + match catch_unwind(work) { + Ok(Ok(result)) => result, + Ok(Err(())) => ClaudeResult::empty(FALLBACK), + Err(_) => ClaudeResult::empty(INTERNAL_ERROR), + } +} + +/// Decode one complete JSON record, including legal JSON whitespace newlines. +/// +/// # Safety +/// `bytes` must name `length` readable bytes for this call (or be null for zero). +/// `output` must name writable, aligned storage not overlapping the input. Its +/// previous owner, if any, must already have been released. No input is retained. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn qb_claude_decode( + bytes: *const u8, + length: usize, + output: *mut ClaudeResult, +) { + if output.is_null() { + return; + } + let result = if length > isize::MAX as usize || (bytes.is_null() && length != 0) { + ClaudeResult::empty(INVALID_ARGUMENT) + } else { + let input = if length == 0 { + &[] + } else { + // SAFETY: readable input and lifetime are supplied by the documented caller contract. + unsafe { std::slice::from_raw_parts(bytes, length) } + }; + contain_failure(|| decode(input)) + }; + // SAFETY: output storage belongs to the caller and is valid for this call. + unsafe { output.write(result) }; +} + +/// Release one result owner. Its record/string views must not be used afterward. +/// +/// # Safety +/// `owner` must be null or an unreleased pointer returned by qb_claude_decode. +/// The caller must release it exactly once, with no simultaneous readers. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn qb_claude_free(owner: *mut c_void) { + if !owner.is_null() { + // SAFETY: ownership is transferred back by the documented caller contract. + drop(unsafe { Box::from_raw(owner.cast::()) }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(usage: &str) -> String { + format!( + r#"{{"type":"assistant","timestamp":"2026-11-16T00:00:00Z","message":{{"id":"m","model":"claude-test","usage":{usage}}}}}"# + ) + } + + #[test] + fn preserves_token_domains_and_ownership() { + let text = record( + r#"{"input_tokens":"10","output_tokens":true,"cache_creation_input_tokens":5,"cache_creation":{"ephemeral_1h_input_tokens":9}}"#, + ); + let result = decode(text.as_bytes()).unwrap(); + assert_eq!(result.status, PARSED); + assert_eq!(result.counts, [10, 1, 5, 5, 0]); + // SAFETY: this result uniquely owns a live Rust allocation. + unsafe { qb_claude_free(result.owner) }; + assert_eq!( + decode(record(r#"{"input_tokens":-1}"#).as_bytes()) + .unwrap() + .status, + IGNORED + ); + assert_eq!( + decode(record(r#"{"input_tokens":9223372036854775807,"output_tokens":1}"#).as_bytes()) + .unwrap() + .status, + IGNORED + ); + } + + #[test] + fn defers_foundation_specific_representations() { + for usage in [ + r#"{"input_tokens":9007199254740993.0}"#, + r#"{"input_tokens":1.5}"#, + r#"{"input_tokens":1,"input_tokens":2}"#, + "[1]", + ] { + assert!(decode(record(usage).as_bytes()).is_err()); + } + } + + #[test] + fn abi_rejects_invalid_lengths_and_can_release_empty_results() { + let mut output = ClaudeResult::empty(IGNORED); + // SAFETY: output is writable; the deliberately invalid null input is rejected before dereference. + unsafe { qb_claude_decode(ptr::null(), 1, &mut output) }; + assert_eq!(output.status, INVALID_ARGUMENT); + assert!(output.owner.is_null()); + // SAFETY: null is an explicitly supported no-op release. + unsafe { qb_claude_free(output.owner) }; + } + + #[test] + fn abi_layout_and_unwinding_failures_are_explicit() { + assert_eq!(std::mem::size_of::(), 152); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::offset_of!(ClaudeResult, counts), 8); + assert_eq!(std::mem::offset_of!(ClaudeResult, timestamp), 48); + assert_eq!(std::mem::offset_of!(ClaudeResult, owner), 144); + let result = contain_failure(|| panic!("synthetic ABI failure")); + assert_eq!(result.status, INTERNAL_ERROR); + assert!(result.owner.is_null()); + } +} diff --git a/Rust/QuotaBarCore/src/lib.rs b/Rust/QuotaBarCore/src/lib.rs new file mode 100644 index 0000000..66ed6d6 --- /dev/null +++ b/Rust/QuotaBarCore/src/lib.rs @@ -0,0 +1,9 @@ +//! Synchronous parsing, pricing and streaming kernels behind QuotaBar's C ABI. +//! Swift retains application state, file transactions, dates and system integration. +#![deny(unsafe_op_in_unsafe_fn)] + +mod aggregation; +mod classifier; +mod claude; +mod pricing; +mod scanner; diff --git a/Rust/QuotaBarCore/src/pricing.rs b/Rust/QuotaBarCore/src/pricing.rs new file mode 100644 index 0000000..b812ec3 --- /dev/null +++ b/Rust/QuotaBarCore/src/pricing.rs @@ -0,0 +1,248 @@ +//! Numeric billing for one resolved model and token bucket. Catalog and dated-rate resolution +//! stay with the caller; all arithmetic deliberately retains Swift's original operation order. + +#[repr(C)] +pub struct QBModelPricing { + pub input: f64, + pub output: f64, + pub cache_write: f64, + pub cache_write_1h: f64, + pub cache_read: f64, + pub input_above: f64, + pub output_above: f64, + pub cache_write_above: f64, + pub cache_write_1h_above: f64, + pub cache_read_above: f64, + pub threshold_tokens: i64, + /// Bits 0..7 mark optional rates in field order; bit 8 marks the threshold. + pub present: u32, +} + +#[repr(C)] +pub struct QBPricingTokens { + pub input: i64, + pub output: i64, + pub cache_write: i64, + pub cache_write_1h: i64, + pub cache_read: i64, +} + +impl QBModelPricing { + fn optional(&self, bit: u32, value: f64) -> Option { + (self.present & (1 << bit) != 0).then_some(value) + } + + fn input_rate(&self, long_context: bool) -> f64 { + if long_context { + self.optional(3, self.input_above).unwrap_or(self.input) + } else { + self.input + } + } + + fn cache_write_1h_rate(&self, long_context: bool) -> f64 { + let above = if long_context { + self.optional(6, self.cache_write_1h_above) + } else { + None + }; + above + .or(self.optional(1, self.cache_write_1h)) + .unwrap_or_else(|| self.input_rate(long_context) * 2.0) + } + + /// Boolean counterpart of RateRules' diagnostics for the fixed public ModelPricing fields. + /// Validate every stated rate, even when this bucket does not use that rate. + fn valid(&self) -> bool { + let nonnegative = |rate: f64| rate.is_finite() && rate >= 0.0; + if !nonnegative(self.input) || !nonnegative(self.output) { + return false; + } + let optional = [ + self.cache_write, + self.cache_write_1h, + self.cache_read, + self.input_above, + self.output_above, + self.cache_write_above, + self.cache_write_1h_above, + self.cache_read_above, + ]; + for (bit, rate) in optional.into_iter().enumerate() { + if self.present & (1 << bit) != 0 && !nonnegative(rate) { + return false; + } + } + if self.present & (1 << 8) != 0 { + if self.threshold_tokens <= 0 || self.threshold_tokens >= 1_000_000_000_000 { + return false; + } + } else if self.present & 0b1111_1000 != 0 { + return false; + } + // Both implicit one-hour rates are validated, regardless of the selected context tier. + self.cache_write_1h_rate(false).is_finite() && self.cache_write_1h_rate(true).is_finite() + } + + fn cost(&self, tokens: &QBPricingTokens, long_context: bool) -> Option { + if !self.valid() + || [ + tokens.input, + tokens.output, + tokens.cache_write, + tokens.cache_write_1h, + tokens.cache_read, + ] + .into_iter() + .any(|count| count < 0) + { + return None; + } + let input = self.input_rate(long_context); + let above = |bit, value| { + if long_context { + self.optional(bit, value) + } else { + None + } + }; + let output = above(4, self.output_above).unwrap_or(self.output); + let write = above(5, self.cache_write_above) + .or(self.optional(0, self.cache_write)) + .unwrap_or(input); + let read = above(7, self.cache_read_above) + .or(self.optional(2, self.cache_read)) + .unwrap_or(input); + let write_1h_rate = self.cache_write_1h_rate(long_context); + let write_1h = tokens.cache_write_1h.min(tokens.cache_write) as f64; + // Subtract after integer-to-double conversion, matching the original rounding at 2^53. + let write_5m = tokens.cache_write as f64 - write_1h; + let millionths = tokens.input as f64 * input + + tokens.output as f64 * output + + write_5m * write + + write_1h * write_1h_rate + + tokens.cache_read as f64 * read; + if millionths.is_finite() { + return Some(millionths / 1_000_000.0); + } + let cost = tokens.input as f64 * (input / 1_000_000.0) + + tokens.output as f64 * (output / 1_000_000.0) + + write_5m * (write / 1_000_000.0) + + write_1h * (write_1h_rate / 1_000_000.0) + + tokens.cache_read as f64 * (read / 1_000_000.0); + (cost.is_finite() && cost >= 0.0).then_some(cost) + } +} + +/// Returns 1 and writes the cost when valid; returns 0 without writing otherwise. +/// +/// # Safety +/// Non-null pointers must point to aligned, initialized ABI structs or writable output storage. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn qb_pricing_cost( + pricing: *const QBModelPricing, + tokens: *const QBPricingTokens, + long_context: u8, + out_cost: *mut f64, +) -> u8 { + if pricing.is_null() || tokens.is_null() || out_cost.is_null() { + return 0; + } + // SAFETY: The caller provides valid storage according to the C ABI contract above. + match unsafe { (&*pricing).cost(&*tokens, long_context != 0) } { + Some(cost) => { + unsafe { *out_cost = cost }; + 1 + } + None => 0, + } +} + +/// Resolves the one-hour rate without validation, matching the public Swift accessor. +/// +/// # Safety +/// A non-null pointer must point to an aligned, initialized ABI struct. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn qb_pricing_cache_write_1h_rate( + pricing: *const QBModelPricing, + long_context: u8, +) -> f64 { + if pricing.is_null() { + return f64::NAN; + } + // SAFETY: The caller provides valid storage according to the C ABI contract above. + unsafe { (&*pricing).cache_write_1h_rate(long_context != 0) } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ptr; + + fn pricing() -> QBModelPricing { + QBModelPricing { + input: 3.0, + output: 15.0, + cache_write: 0.0, + cache_write_1h: 0.0, + cache_read: 0.0, + input_above: 0.0, + output_above: 0.0, + cache_write_above: 0.0, + cache_write_1h_above: 0.0, + cache_read_above: 0.0, + threshold_tokens: 0, + present: 0, + } + } + + fn tokens() -> QBPricingTokens { + QBPricingTokens { + input: 1_000_000, + output: 0, + cache_write: 0, + cache_write_1h: 0, + cache_read: 0, + } + } + + #[test] + fn abi_layout_is_fixed_width() { + assert_eq!(std::mem::size_of::(), 96); + assert_eq!(std::mem::offset_of!(QBModelPricing, threshold_tokens), 80); + assert_eq!(std::mem::offset_of!(QBModelPricing, present), 88); + assert_eq!(std::mem::size_of::(), 40); + } + + #[test] + fn invalid_calls_preserve_output_storage() { + let mut pricing = pricing(); + let tokens = tokens(); + let mut output = 42.0; + // SAFETY: All non-null pointers refer to live aligned values for each call. + unsafe { + assert_eq!(qb_pricing_cost(ptr::null(), &tokens, 0, &mut output), 0); + assert_eq!(qb_pricing_cost(&pricing, ptr::null(), 0, &mut output), 0); + assert_eq!(qb_pricing_cost(&pricing, &tokens, 0, ptr::null_mut()), 0); + pricing.output = f64::NAN; + assert_eq!(qb_pricing_cost(&pricing, &tokens, 0, &mut output), 0); + assert!(qb_pricing_cache_write_1h_rate(ptr::null(), 0).is_nan()); + } + assert_eq!(output, 42.0); + } + + #[test] + fn valid_call_writes_cost_and_keeps_inputs() { + let pricing = pricing(); + let tokens = tokens(); + let mut output = 0.0; + // SAFETY: Inputs and output are initialized local ABI values. + unsafe { + assert_eq!(qb_pricing_cost(&pricing, &tokens, 0, &mut output), 1); + assert_eq!(qb_pricing_cache_write_1h_rate(&pricing, 0), 6.0); + } + assert_eq!(output, 3.0); + assert_eq!(tokens.input, 1_000_000); + assert_eq!(pricing.input, 3.0); + } +} diff --git a/Rust/QuotaBarCore/src/scanner.rs b/Rust/QuotaBarCore/src/scanner.rs new file mode 100644 index 0000000..f576f46 --- /dev/null +++ b/Rust/QuotaBarCore/src/scanner.rs @@ -0,0 +1,265 @@ +//! Bounded JSONL streaming. Swift owns scan plans, transactions and callback lifetimes. +use std::ffi::{CStr, c_char, c_int, c_void}; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +const CHUNK_SIZE: usize = 1 << 20; +const LONG_LINE_PREFIX: usize = 64 * 1024; +const EIO: i32 = 5; +const EINVAL: i32 = 22; +const EINTR: i32 = 4; +// QuotaBar targets macOS; this is Darwin's O_CLOEXEC (O_RDONLY is zero). +const O_CLOEXEC: i32 = 0x0100_0000; + +unsafe extern "C" { + fn open(path: *const c_char, flags: c_int, ...) -> c_int; + fn read(fd: c_int, buffer: *mut c_void, count: usize) -> isize; + fn close(fd: c_int) -> c_int; + fn lseek(fd: c_int, offset: i64, whence: c_int) -> i64; + fn memchr(bytes: *const c_void, value: c_int, length: usize) -> *const c_void; +} + +struct Descriptor(c_int); +impl Drop for Descriptor { + fn drop(&mut self) { + // SAFETY: this descriptor is owned and closed exactly once. + unsafe { close(self.0) }; + } +} + +fn errno() -> i32 { + std::io::Error::last_os_error() + .raw_os_error() + .unwrap_or(EIO) +} + +fn read_lines( + path: &CStr, + offset: i64, + limit: Option, + mut wanted: impl FnMut(&[u8]) -> bool, + mut handle: impl FnMut(&[u8]), +) -> Result { + // SAFETY: path is terminated and borrowed for this call. + let descriptor = unsafe { open(path.as_ptr(), O_CLOEXEC) }; + if descriptor < 0 { + return Err(errno()); + } + let descriptor = Descriptor(descriptor); + let mut position = offset.max(0); + // SAFETY: descriptor is open; SEEK_SET is zero on Darwin. + if unsafe { lseek(descriptor.0, position, 0) } < 0 { + return Err(errno()); + } + let mut consumed = position; + let mut buffer = vec![0_u8; CHUNK_SIZE]; + let mut carry = Vec::new(); + let mut skipping = false; + + while limit.is_none_or(|limit| position < limit) { + let requested = limit.map_or(CHUNK_SIZE, |limit| { + CHUNK_SIZE.min((limit - position) as usize) + }); + let count = loop { + // SAFETY: requested never exceeds the writable buffer length. + let count = unsafe { read(descriptor.0, buffer.as_mut_ptr().cast(), requested) }; + if count >= 0 { + break count as usize; + } + let error = errno(); + if error != EINTR { + return Err(error); + } + }; + if count == 0 { + break; + } + + let mut start = 0; + while start < count { + // SAFETY: both the pointer and search length remain inside the initialized read. + let found = unsafe { memchr(buffer.as_ptr().add(start).cast(), 10, count - start) }; + if found.is_null() { + break; + } + // SAFETY: memchr returned a pointer into this buffer. + let newline = unsafe { found.cast::().offset_from(buffer.as_ptr()) as usize }; + if skipping { + skipping = false; + } else if carry.is_empty() { + let line = &buffer[start..newline]; + if !line.is_empty() && wanted(line) { + handle(line); + } + } else { + carry.extend_from_slice(&buffer[start..newline]); + if wanted(&carry) { + handle(&carry); + } + carry.clear(); + } + consumed = position + newline as i64 + 1; + start = newline + 1; + } + if start < count && !skipping { + let was_short = carry.len() < LONG_LINE_PREFIX; + carry.extend_from_slice(&buffer[start..count]); + if was_short && carry.len() >= LONG_LINE_PREFIX && !wanted(&carry) { + skipping = true; + carry.clear(); + } + } + position += count as i64; + } + Ok(consumed) +} + +type Wanted = extern "C" fn(*const u8, usize, *mut c_void) -> u8; +type Handle = extern "C" fn(*const u8, usize, *mut c_void); + +/// Returns zero or a POSIX errno. Callback buffers are borrowed only during each callback. +/// The context is opaque, and every callback runs synchronously on the calling thread. +/// +/// # Safety +/// `path` must point to a terminated C string, `consumed` must be writable, and callbacks must +/// accept borrowed buffers and the supplied context without unwinding or retaining pointers. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn qb_read_lines( + path: *const c_char, + offset: i64, + has_limit: u8, + limit: i64, + wanted: Option, + handle: Option, + context: *mut c_void, + consumed: *mut i64, +) -> i32 { + if path.is_null() || consumed.is_null() || wanted.is_none() || handle.is_none() { + return EINVAL; + } + let wanted = wanted.unwrap(); + let handle = handle.unwrap(); + let result = catch_unwind(AssertUnwindSafe(|| { + // SAFETY: the ABI caller supplies a valid terminated path. + let path = unsafe { CStr::from_ptr(path) }; + read_lines( + path, + offset, + (has_limit != 0).then_some(limit), + |line| wanted(line.as_ptr(), line.len(), context) != 0, + |line| handle(line.as_ptr(), line.len(), context), + ) + })); + match result { + Ok(Ok(offset)) => { + // SAFETY: the ABI caller supplies a writable output pointer. + unsafe { *consumed = offset }; + 0 + } + Ok(Err(error)) => error, + Err(_) => EIO, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + use std::sync::atomic::{AtomicU64, Ordering}; + + static NEXT: AtomicU64 = AtomicU64::new(0); + struct Fixture(std::path::PathBuf, CString); + impl Fixture { + fn new(bytes: &[u8]) -> Self { + let path = std::env::temp_dir().join(format!( + "quotabar-rust-scanner-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::write(&path, bytes).unwrap(); + let c_path = CString::new(path.as_os_str().as_encoded_bytes()).unwrap(); + Self(path, c_path) + } + } + impl Drop for Fixture { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } + } + + #[test] + fn respects_offsets_limits_and_partial_lines() { + let fixture = Fixture::new(b"\nfirst\nsecond\ntail"); + for (offset, limit, expected_offset, expected) in [ + (0, None, 14, vec![b"first".to_vec(), b"second".to_vec()]), + (0, Some(4), 1, vec![]), + (7, Some(13), 7, vec![]), + (7, Some(14), 14, vec![b"second".to_vec()]), + (-1, Some(7), 7, vec![b"first".to_vec()]), + (7, Some(1), 7, vec![]), + ] { + let mut lines = Vec::new(); + let consumed = read_lines( + &fixture.1, + offset, + limit, + |_| true, + |line| lines.push(line.to_vec()), + ) + .unwrap(); + assert_eq!((consumed, lines), (expected_offset, expected)); + } + } + + #[test] + fn skips_rejected_multichunk_lines_and_resumes() { + let mut bytes = vec![b'!'; CHUNK_SIZE * 3 + 37]; + bytes.extend_from_slice(b"\naccepted\npartial"); + let fixture = Fixture::new(&bytes); + let mut lengths = Vec::new(); + let mut lines = Vec::new(); + let consumed = read_lines( + &fixture.1, + 0, + None, + |line| { + lengths.push(line.len()); + line.first() != Some(&b'!') + }, + |line| lines.push(line.to_vec()), + ) + .unwrap(); + assert_eq!(consumed as usize, bytes.len() - 7); + assert_eq!(lines, vec![b"accepted".to_vec()]); + assert_eq!(lengths, vec![CHUNK_SIZE, 8]); + } + + #[test] + fn accepts_multichunk_lines_without_truncation() { + let mut bytes = vec![b'a'; CHUNK_SIZE * 2 + 73]; + bytes.push(b'\n'); + let fixture = Fixture::new(&bytes); + let mut lines = Vec::new(); + assert_eq!( + read_lines( + &fixture.1, + 0, + None, + |_| true, + |line| lines.push(line.to_vec()) + ) + .unwrap(), + bytes.len() as i64 + ); + assert_eq!(lines, vec![bytes[..bytes.len() - 1].to_vec()]); + } + + #[test] + fn reports_open_errors_without_callbacks() { + let fixture = Fixture::new(b"unused"); + std::fs::remove_file(&fixture.0).unwrap(); + assert_eq!( + read_lines(&fixture.1, 0, None, |_| panic!(), |_| panic!()), + Err(2) + ); + } +} diff --git a/Scripts/build_rust.sh b/Scripts/build_rust.sh new file mode 100644 index 0000000..cb04c5a --- /dev/null +++ b/Scripts/build_rust.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Build the native kernels with the same minimum OS and profile as SwiftPM. +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +profile="${1:-debug}" +case "$profile" in + debug) cargo_profile=dev ;; + release) cargo_profile=release ;; + *) echo "Unsupported Rust build profile: $profile" >&2; exit 1 ;; +esac + +if ! command -v cargo >/dev/null 2>&1; then + echo "Rust is required. Install rustup and the toolchain pinned in rust-toolchain.toml." >&2 + exit 1 +fi + +cd "$root" +export MACOSX_DEPLOYMENT_TARGET=14.0 +export CARGO_TARGET_DIR="$root/.build/rust" +cargo build --locked --manifest-path "$root/Rust/QuotaBarCore/Cargo.toml" --profile "$cargo_profile" diff --git a/Scripts/fixtures/rust_pipeline_hook.swift b/Scripts/fixtures/rust_pipeline_hook.swift new file mode 100644 index 0000000..10ad201 --- /dev/null +++ b/Scripts/fixtures/rust_pipeline_hook.swift @@ -0,0 +1,78 @@ +// Injected into a temporary probe source copy by validate_rust_pipeline.py. +// This hook always pins every log path, SQLite database and pricing input to synthetic data. +func validationTokens(_ value: TokenTotals) -> [String: Int] { + ["input": value.input, "output": value.output, "cacheWrite": value.cacheWrite, + "cacheWrite1h": value.cacheWrite1h, "cacheRead": value.cacheRead] +} + +func validationSnapshot(_ snapshot: CostSnapshot) -> [String: Any] { + let days: [[String: Any]] = snapshot.days.map { day in + let models: [[String: Any]] = day.byModel.map { key, value in + ["source": key.source.rawValue, "model": key.model, "fast": key.isFast, + "upstream": key.upstreamProvider as Any? ?? NSNull(), + "tokens": validationTokens(value.tokens), "cost": value.costUSD as Any? ?? NSNull(), + "loggedFallback": value.usesLoggedCostFallback] + }.sorted { lhs, rhs in + let left = try! JSONSerialization.data(withJSONObject: lhs, options: .sortedKeys) + let right = try! JSONSerialization.data(withJSONObject: rhs, options: .sortedKeys) + return left.lexicographicallyPrecedes(right) + } + return ["day": day.dayKey, "tokens": validationTokens(day.tokens), + "cost": day.costUSD as Any? ?? NSNull(), "unpricedTokens": day.unpricedTokens, + "models": models] + } + // scannedAt is deliberately omitted: it records the real wall clock of each independent run. + return ["tab": snapshot.tab.rawValue, "todayCost": snapshot.todayCostUSD, + "windowCost": snapshot.windowCostUSD, "latestTokens": snapshot.latestTokens, + "windowTokens": snapshot.windowTokens, "topModel": snapshot.topModel as Any? ?? NSNull(), + "unpriced": snapshot.hasUnpricedTokens, "days": days] +} + +func validationRun(database: String, fixture: String, tab: UsageTab, benchmark: Bool) async throws { + let root = URL(fileURLWithPath: fixture) + let env = ["CLAUDE_CONFIG_DIR": root.appendingPathComponent("claude").path, + "CODEX_HOME": root.appendingPathComponent("codex").path, + "OPENCODE_DATA_HOME": root.appendingPathComponent("opencode").path, + "PI_CODING_AGENT_DIR": root.appendingPathComponent("pi").path, + "PI_CODING_AGENT_SESSION_DIR": root.appendingPathComponent("pi/sessions").path] + let service = CostService(databaseURL: URL(fileURLWithPath: database), env: env, rateCard: RateCard()) + var snapshots: [String: Any] = [:] + var times: [String: Double] = [:] + let phases = benchmark ? ["initial", "unchanged", "append", "append-unchanged"] : ["scan"] + for phase in phases { + if phase == "append" { + let relative = tab == .claude ? "claude/projects/app/session-0.jsonl" : "codex/sessions/session-0.jsonl" + let data = try Data(contentsOf: root.appendingPathComponent("append.jsonl")) + let file = try FileHandle(forWritingTo: root.appendingPathComponent(relative)) + defer { try? file.close() } + _ = try file.seekToEnd() + try file.write(contentsOf: data) + } + let started = ContinuousClock.now + guard let snapshot = await service.refresh(tab) else { + throw NSError(domain: "RustPipelineValidation", code: 1, + userInfo: [NSLocalizedDescriptionKey: "Synthetic scan returned no snapshot"]) + } + let elapsed = started.duration(to: .now) + times[phase] = Double(elapsed.components.seconds) * 1_000 + + Double(elapsed.components.attoseconds) / 1_000_000_000_000_000 + snapshots[phase] = ["snapshot": validationSnapshot(snapshot), + "scanStatus": String(describing: await service.currentScanStatus(of: tab == .claude ? .claude : .codex))] + } + let data = try JSONSerialization.data(withJSONObject: ["phases": snapshots, "times_ms": times], + options: [.sortedKeys]) + print(String(decoding: data, as: UTF8.self)) +} + +if CommandLine.arguments.count == 6, CommandLine.arguments[1] == "--rust-validation", + let tab = UsageTab(rawValue: CommandLine.arguments[4]), + ["once", "benchmark"].contains(CommandLine.arguments[5]) { + do { + try await validationRun(database: CommandLine.arguments[2], fixture: CommandLine.arguments[3], + tab: tab, benchmark: CommandLine.arguments[5] == "benchmark") + exit(0) + } catch { + FileHandle.standardError.write(Data("Synthetic validation failed: \(error)\n".utf8)) + exit(3) + } +} diff --git a/Scripts/setup_rust_ci.sh b/Scripts/setup_rust_ci.sh new file mode 100644 index 0000000..5cd7f4e --- /dev/null +++ b/Scripts/setup_rust_ci.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Install the pinned toolchain without changing the runner's default toolchain. +set -euo pipefail +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if ! command -v rustup >/dev/null 2>&1; then + brew install rustup + rustup_bin="$(brew --prefix rustup)/bin" + export PATH="$rustup_bin:$PATH" + if [[ -n "${GITHUB_PATH:-}" ]]; then printf '%s\n' "$rustup_bin" >> "$GITHUB_PATH"; fi +fi +toolchain="$(awk -F '"' '/^channel[[:space:]]*=/ { print $2 }' "$root/rust-toolchain.toml")" +rustup toolchain install "$toolchain" --profile minimal --component rustfmt --component clippy +rustup run "$toolchain" rustc --version diff --git a/Scripts/swift.sh b/Scripts/swift.sh index 06b6b3c..a6d5b0b 100755 --- a/Scripts/swift.sh +++ b/Scripts/swift.sh @@ -12,4 +12,29 @@ if [[ ! -x "$xcode_swift" || ! -x "$DEVELOPER_DIR/usr/bin/xcodebuild" ]]; then fi export PATH="$DEVELOPER_DIR/Toolchains/XcodeDefault.xctoolchain/usr/bin:$PATH" + +# Keep every documented build/test/package entry point from linking a stale Rust archive. +case "${1:-}" in + build|run|test) + rust_profile=debug + previous_argument="" + for argument in "$@"; do + if [[ "$previous_argument" == -c || "$previous_argument" == --configuration ]]; then + rust_profile="$argument" + fi + case "$argument" in --configuration=*) rust_profile="${argument#*=}" ;; esac + previous_argument="$argument" + done + bash "$(dirname "${BASH_SOURCE[0]}")/build_rust.sh" "$rust_profile" + project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + rust_archive="$project_root/.build/rust/$rust_profile/libquotabar_core.a" + rust_digest="$(shasum -a 256 "$rust_archive" | cut -d ' ' -f 1)" + swift_command="$1" + shift + # A content-dependent compiler flag invalidates SwiftPM's link cache even + # when only Rust sources changed. Archives remain profile-specific. + exec "$xcode_swift" "$swift_command" \ + -Xlinker "$rust_archive" -Xswiftc "-DQUOTABAR_RUST_$rust_digest" "$@" + ;; +esac exec "$xcode_swift" "$@" diff --git a/Scripts/validate_rust_pipeline.py b/Scripts/validate_rust_pipeline.py new file mode 100755 index 0000000..d60c70b --- /dev/null +++ b/Scripts/validate_rust_pipeline.py @@ -0,0 +1,395 @@ +#!/usr/bin/env -S uv run --script +"""Compare isolated Swift/Rust builds through real scanning, SQLite and pricing.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import io +import json +from pathlib import Path +import shutil +import sqlite3 +import statistics +import subprocess +import tarfile +import time +from typing import Any + +BASELINE = "f8df98b129c9138ccb9e6290d6c801e805fed1be" +SOURCE_ROOTS = ["Package.swift", "rust-toolchain.toml", "Scripts/swift.sh", "Scripts/build_rust.sh", "Sources", "Rust/QuotaBarCore"] +SOURCE_SUFFIXES = {".swift", ".rs", ".h", ".c", ".modulemap", ".json", ".html", ".js", ".css", ".png", ".svg", ".toml", ".lock", ".sh"} + + +def save(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n") + + +def allowed_source(name: str) -> bool: + path = Path(name) + if path.is_absolute() or ".." in path.parts or any(part in {"target", ".build", ".git"} or part.startswith(".env") for part in path.parts): + return False + return path.suffix in SOURCE_SUFFIXES and any(name == root or name.startswith(root + "/") for root in SOURCE_ROOTS) + + +def copy_source(repository: Path, destination: Path, revision: str | None) -> dict[str, str]: + """Copy only build-source whitelist entries, never Git metadata or ignored local files.""" + destination.mkdir(parents=True) + hashes: dict[str, str] = {} + if revision is not None: + listed = subprocess.check_output(["git", "ls-tree", "-r", "-z", "--name-only", revision, "--", *SOURCE_ROOTS], cwd=repository) + paths = [name for name in listed.decode().split("\0") if name and allowed_source(name)] + archive = subprocess.check_output(["git", "archive", revision, "--", *paths], cwd=repository) + with tarfile.open(fileobj=io.BytesIO(archive)) as source: + for member in source: + if not member.isfile() or not allowed_source(member.name): + continue + stream = source.extractfile(member) + assert stream is not None + data = stream.read() + target = destination / member.name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + target.chmod(member.mode) + hashes[member.name] = hashlib.sha256(data).hexdigest() + else: + paths = subprocess.check_output(["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", *SOURCE_ROOTS], cwd=repository) + for name in sorted(set(paths.decode().split("\0"))): + if not name or not allowed_source(name): + continue + original = repository / name + if not original.is_file() or original.is_symlink(): + continue + target = destination / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(original, target) + hashes[name] = hashlib.sha256(target.read_bytes()).hexdigest() + return hashes + + +def build_probe(source: Path, hook: str, log: Path, source_hashes: dict[str, str] | None = None) -> Path: + main = source / "Sources/QuotaBarProbe/main.swift" + content = main.read_text() + marker = "let arguments = Array(CommandLine.arguments.dropFirst())" + if content.count(marker) != 1: + raise RuntimeError("Cannot find the unique probe dispatch insertion point") + main.write_text(content.replace(marker, hook + "\n" + marker)) + command = [str(source / "Scripts/swift.sh"), "build", "-c", "release", "--product", "QuotaBarProbe"] + save(log.with_suffix(".command.json"), {"command": command, "cwd": str(source)}) + with log.open("w") as output: + subprocess.run(command, cwd=source, stdout=output, stderr=subprocess.STDOUT, check=True) + binary = source / ".build/release/QuotaBarProbe" + save(binary.with_name(binary.name + ".validation.json"), { + "hook_sha256": hashlib.sha256(hook.encode()).hexdigest(), + "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "source_sha256": source_hashes, + }) + return binary + + +def database_state(database: Path) -> dict[str, Any]: + def cell(value: Any) -> Any: + return {"blob_hex": value.hex()} if isinstance(value, bytes) else value + with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection: + schema = connection.execute("SELECT type, name, tbl_name, sql FROM sqlite_master ORDER BY type, name").fetchall() + tables = connection.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").fetchall() + rows = {} + raw_rows = {} + for (name,) in tables: + quoted = name.replace('"', '""') + values = [[cell(value) for value in row] for row in connection.execute(f'SELECT * FROM "{quoted}"')] + raw_rows[name] = sorted(values, key=repr) + columns = [column[1] for column in connection.execute(f'PRAGMA table_info("{quoted}")')] + normalized = [list(row) for row in values] + if name == "file_cursor" and "resume_state" in columns: + index = columns.index("resume_state") + for row in normalized: + if row[index] is not None: + # Swift's JSONEncoder does not promise dictionary key order. Preserve + # every value while comparing this known JSON column semantically. + row[index] = json.dumps(json.loads(row[index]), sort_keys=True, separators=(",", ":"), ensure_ascii=False) + rows[name] = sorted(normalized, key=repr) + return {"schema": schema, "tables": rows, "raw_tables": raw_rows} + + +def equivalent_states(states: dict[str, Any]) -> bool: + def comparable(state: dict[str, Any]) -> dict[str, Any]: + return {"phases": state["phases"], "schema": state["database"]["schema"], "tables": state["database"]["tables"]} + return comparable(states["before"]) == comparable(states["after"]) + + +def probe(binary: Path, database: Path, fixture: Path, tab: str, mode: str, raw: Path) -> dict[str, Any]: + database.parent.mkdir(parents=True, exist_ok=True) + runtime = raw.parent / "runtime-tmp" + runtime.mkdir(exist_ok=True) + environment = {"PATH": "/usr/bin:/bin:/usr/sbin:/sbin", "TMPDIR": str(runtime) + "/", "LANG": "en_US.UTF-8", "TZ": "Asia/Shanghai"} + protected = [Path.home() / path for path in [".codex", ".claude", ".config/claude", ".local/share/opencode", ".pi", "Library/Keychains", "Library/Application Support/QuotaBar"]] + profile = "(version 1) (allow default) (deny network*) " + " ".join( + f"(deny file-read* file-write* (subpath {json.dumps(str(path))}))" for path in protected + ) + ' (deny process-exec (literal "/usr/bin/security"))' + sandbox = raw.parent / "probe.sb" + sandbox.write_text(profile + "\n") + command = ["/usr/bin/sandbox-exec", "-f", str(sandbox), str(binary), "--rust-validation", str(database), str(fixture), tab, mode] + started = time.perf_counter() + completed = subprocess.run(command, env=environment, capture_output=True, text=True) + wall_ms = (time.perf_counter() - started) * 1000 + save(raw, {"command": command, "environment": environment, "returncode": completed.returncode, + "stdout": completed.stdout, "stderr": completed.stderr, "process_wall_ms": wall_ms}) + completed.check_returncode() + result = json.loads(completed.stdout) + result["process_wall_ms"] = wall_ms + return result + + +def row(identifier: str, timestamp: str, output: int = 2) -> dict[str, Any]: + return {"type": "assistant", "timestamp": timestamp, "requestId": identifier, + "message": {"id": identifier, "model": "claude-opus-5", "usage": {"input_tokens": 10, "output_tokens": output}}} + + +def encoded(value: dict[str, Any]) -> str: + return json.dumps(value, ensure_ascii=False) + + +def differential(output: Path, binaries: dict[str, Path], timestamp: str) -> None: + root = output / "differential" + fixture = root / "fixture" + project = fixture / "claude/projects/app" + project.mkdir(parents=True) + transcript = project / "session.jsonl" + + def checkpoint(phase: str) -> None: + states = {} + for label, binary in binaries.items(): + database = root / label / "usage.sqlite" + result = probe(binary, database, fixture, "claude", "once", root / f"{phase}-{label}.raw.json") + states[label] = {"phases": result["phases"], "database": database_state(database)} + save(root / f"{phase}.json", states) + equal = equivalent_states(states) + print(json.dumps({"phase": phase, "equivalent": equal}), flush=True) + if not equal: + raise RuntimeError(f"Snapshot/schema/rows/cursor mismatch at {phase}; inspect {root / (phase + '.json')}") + + initial = [row("one", timestamp), row("one", timestamp, 7), row("other", timestamp)] + idless = row("", timestamp) + del idless["requestId"] + del idless["message"]["id"] + idless["message"]["content"] = [{"type": "text", "text": 'Synthetic escaped \\"text\\" and Unicode 世界'}] + initial.append(idless) + loose = row("loose", timestamp) + loose["message"]["usage"] = {"input_tokens": "10", "output_tokens": 2.9, "cache_read_input_tokens": True, + "cache_creation_input_tokens": 9, "cache_creation": {"ephemeral_1h_input_tokens": 3}, "speed": "FAST"} + initial.append(loose) + invalid = row("invalid", timestamp) + invalid["message"]["usage"] = {"input_tokens": -1, "output_tokens": 2} + initial.extend([invalid, {"type": "user", "message": {"usage": {"input_tokens": 999999}}}]) + transcript.write_text("\n".join(map(encoded, initial)) + '\n{"type":"assistant"') + checkpoint("initial-incomplete") + checkpoint("unchanged") + with transcript.open("a") as file: + file.write('}\n' + encoded(row("one", timestamp, 11)) + "\n" + encoded(row("append", timestamp)) + "\n") + checkpoint("append-and-replay") + copied = project / "copied-session.jsonl" + copied.write_bytes(transcript.read_bytes()) + checkpoint("copied-dedup") + transcript.write_text(encoded(row("replacement", timestamp)) + "\n") + checkpoint("rewrite-preserves-history") + unknown = row("unknown-price", timestamp) + unknown["message"]["model"] = "synthetic-unpriced-model" + with transcript.open("a") as file: + file.write(encoded(unknown) + "\n" + encoded(invalid) + "\n{malformed JSON\n") + checkpoint("unknown-and-invalid") + transcript.unlink() + copied.unlink() + checkpoint("deleted-retains-history") + + +def benchmark_fixture(root: Path, workload: str, timestamp: str) -> tuple[str, Path, int]: + """Match benchmark_classifier.py's 16 x 128 shape and add real append work.""" + tab = "codex" if workload == "codex" else "claude" + project = root / ("codex/sessions" if tab == "codex" else "claude/projects/app") + project.mkdir(parents=True) + padding = "" if workload in {"short", "codex"} else ('\\"' * 32768 if workload == "dense" else "x" * 65536) + + def event(index: int) -> dict[str, Any]: + return {"type": "event_msg", "timestamp": timestamp, "payload": {"type": "token_count", "info": { + "last_token_usage": {"input_tokens": 10, "cached_input_tokens": 2, "output_tokens": 2}, + "total_token_usage": {"input_tokens": index * 10, "output_tokens": index * 2}}}} + + for file_index in range(16): + with (project / f"session-{file_index}.jsonl").open("w") as file: + if tab == "codex": + file.write(encoded({"type": "turn_context", "timestamp": timestamp, "payload": {"model": "gpt-5.6-sol"}}) + "\n") + for line_index in range(128): + if tab == "claude": + value = {"padding": padding, **row(f"{file_index}-{line_index}", timestamp)} + value["type"] = "user" if line_index % 2 else "assistant" + else: + value = event(line_index + 1) if line_index % 2 == 0 else {"type": "event_msg", "payload": {"type": "agent_message", "message": "Synthetic text"}} + file.write(encoded(value) + "\n") + appended = [row(f"active-append-{index}", timestamp) if tab == "claude" else event(200 + index) for index in range(64)] + (root / "append.jsonl").write_text("\n".join(map(encoded, appended)) + "\n") + target = project / "session-0.jsonl" + return tab, target, sum(path.stat().st_size for path in project.glob("*.jsonl")) + + +def benchmarks(output: Path, binaries: dict[str, Path], timestamp: str, iterations: int, workloads: list[str]) -> None: + results: dict[str, Any] = {} + for workload in workloads: + root = output / "benchmarks" / workload + fixture = root / "fixture" + tab, target, byte_count = benchmark_fixture(fixture, workload, timestamp) + original_size = target.stat().st_size + samples: dict[str, list[dict[str, Any]]] = {label: [] for label in binaries} + for iteration in range(iterations): + states = {} + order = list(binaries.items()) + if iteration % 2: + order.reverse() + for label, binary in order: + with target.open("r+b") as file: + file.truncate(original_size) + database = root / f"{label}-{iteration}" / "usage.sqlite" + result = probe(binary, database, fixture, tab, "benchmark", root / f"{label}-{iteration}.raw.json") + phases = result["phases"] + if phases["initial"] != phases["unchanged"] or phases["append"] != phases["append-unchanged"]: + raise RuntimeError(f"Unchanged scan changed its snapshot: {workload}/{label}/{iteration}") + if phases["append"]["snapshot"]["windowTokens"] <= phases["initial"]["snapshot"]["windowTokens"]: + raise RuntimeError(f"Append fixture added no usage: {workload}/{label}/{iteration}") + states[label] = {"phases": phases, "database": database_state(database)} + samples[label].append({"times_ms": result["times_ms"], "process_wall_ms": result["process_wall_ms"]}) + save(root / f"comparison-{iteration}.json", states) + if not equivalent_states(states): + raise RuntimeError(f"Benchmark snapshot/database mismatch: {workload}/{iteration}") + summaries = {} + for label, runs in samples.items(): + metrics = {key: [sample["times_ms"][key] for sample in runs] for key in ["initial", "unchanged", "append", "append-unchanged"]} + metrics["process_wall"] = [sample["process_wall_ms"] for sample in runs] + summaries[label] = {key: {"median_ms": statistics.median(values), "min_ms": min(values), "max_ms": max(values), "runs_ms": values} for key, values in metrics.items()} + results[workload] = {"bytes": byte_count, "files": 16, "records_per_file": 128, "append_records": 64, "summary": summaries} + save(output / "benchmark-summary.json", results) + print(json.dumps({"workload": workload, "summary": summaries}), flush=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--revision", default=BASELINE) + parser.add_argument("--output", type=Path, default=Path("build/rust-validation/pipeline")) + parser.add_argument("--iterations", type=int, default=7) + parser.add_argument("--skip-benchmarks", action="store_true") + parser.add_argument("--workloads", choices=["short", "plain", "dense", "codex"], nargs="+", default=["short", "plain", "dense", "codex"]) + parser.add_argument("--ablate-aggregator-source", type=Path, help="Compare a validated candidate copy with only its old Swift aggregator restored") + parser.add_argument("--ablate-scanner-source", type=Path, help="Compare a validated candidate source copy with only its old Swift scanner restored") + parser.add_argument("--reuse-sources", type=Path, help="Reuse a previous build after matching every input source hash") + parser.add_argument("--before-probe", type=Path, help="Reuse a probe built with this exact hook; requires --after-probe") + parser.add_argument("--after-probe", type=Path, help="Reuse a probe built with this exact hook; requires --before-probe") + args = parser.parse_args() + if args.iterations < 1 or bool(args.before_probe) != bool(args.after_probe): + parser.error("iterations must be positive; reusable probes must be supplied together") + ablation_source = args.ablate_scanner_source or args.ablate_aggregator_source + if args.ablate_scanner_source and args.ablate_aggregator_source: + parser.error("choose one kernel for ablation") + if ablation_source and (args.before_probe or args.reuse_sources): + parser.error("kernel ablation cannot be combined with probe/source reuse") + if args.before_probe and args.reuse_sources: + parser.error("choose reusable probes or reusable source builds") + repository = args.repository.resolve() + output = args.output.resolve() + if output.exists(): + parser.error("output must be a new directory; existing validation evidence is never overwritten") + output.mkdir(parents=True) + shutil.copy2(Path(__file__), output / "script-used.py") + shutil.copy2(repository / "Scripts/fixtures/rust_pipeline_hook.swift", output / "hook-used.swift") + revision = subprocess.check_output(["git", "rev-parse", args.revision], cwd=repository, text=True).strip() + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT02:00:00Z") + metadata: dict[str, Any] = {"baseline_revision": revision, "fixture_timestamp": timestamp, "iterations": args.iterations, + "repository": str(repository), "workloads": args.workloads, "timezone": "Asia/Shanghai", "source_whitelist": SOURCE_ROOTS} + metadata["machine"] = { + "cpu": subprocess.check_output(["/usr/sbin/sysctl", "-n", "machdep.cpu.brand_string"], text=True).strip(), + "memory_bytes": int(subprocess.check_output(["/usr/sbin/sysctl", "-n", "hw.memsize"], text=True)), + "macos": subprocess.check_output(["/usr/bin/sw_vers", "-productVersion"], text=True).strip(), + "swift": subprocess.check_output([str(repository / "Scripts/swift.sh"), "--version"], text=True).strip(), + "rust": subprocess.check_output(["rustc", "--version"], text=True).strip() if shutil.which("rustc") else None, + } + hook = (repository / "Scripts/fixtures/rust_pipeline_hook.swift").read_text() + metadata["hook_sha256"] = hashlib.sha256(hook.encode()).hexdigest() + metadata["validation_script_sha256"] = hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + if ablation_source: + kernel = "scanner" if args.ablate_scanner_source else "aggregator" + filename = "LogFileScanner.swift" if kernel == "scanner" else "CostAggregator.swift" + symbol = b"qb_read_lines" if kernel == "scanner" else b"qb_aggregate_usage" + source = ablation_source.resolve() + original = source / ".build/release/QuotaBarProbe" + manifest = json.loads(original.with_name(original.name + ".validation.json").read_text()) + if manifest.get("hook_sha256") != metadata["hook_sha256"] or manifest.get("binary_sha256") != hashlib.sha256(original.read_bytes()).hexdigest(): + raise RuntimeError("Kernel ablation requires a validated probe with the current hook") + scanner = source / "Sources/QuotaBarCore/Cost" / filename + current = scanner.read_bytes() + if symbol not in current: + raise RuntimeError(f"Ablation source does not contain the Rust {kernel}") + preserved = original.with_name("QuotaBarProbeRust" + kernel.capitalize() + "-" + output.name) + if preserved.exists(): + raise RuntimeError("A preserved kernel probe already exists; do not overwrite validation evidence") + shutil.copy2(original, preserved) + save(preserved.with_name(preserved.name + ".validation.json"), manifest) + baseline = subprocess.check_output(["git", "show", f"{revision}:Sources/QuotaBarCore/Cost/{filename}"], cwd=repository) + scanner.write_bytes(baseline) + metadata["comparison"] = f"{kernel}-ablation: before=Rust {kernel}, after=Swift {kernel}; all other candidate code unchanged" + metadata["candidate_source"] = str(source) + metadata["kernel_sha256"] = {"before": hashlib.sha256(current).hexdigest(), "after": hashlib.sha256(baseline).hexdigest()} + save(output / "metadata.json", metadata) + command = [str(source / "Scripts/swift.sh"), "build", "-c", "release", "--product", "QuotaBarProbe"] + with (output / "ablation-build.log").open("w") as log: + subprocess.run(command, cwd=source, stdout=log, stderr=subprocess.STDOUT, check=True) + inputs = manifest.get("source_sha256") + if inputs is not None: + inputs = dict(inputs) + inputs[f"Sources/QuotaBarCore/Cost/{filename}"] = hashlib.sha256(baseline).hexdigest() + save(original.with_name(original.name + ".validation.json"), {"hook_sha256": metadata["hook_sha256"], "binary_sha256": hashlib.sha256(original.read_bytes()).hexdigest(), "source_sha256": inputs}) + metadata["before_source_sha256"] = manifest.get("source_sha256") + metadata["after_source_sha256"] = inputs + binaries = {"before": preserved, "after": original} + elif args.before_probe: + binaries = {"before": args.before_probe.resolve(), "after": args.after_probe.resolve()} + for label, binary in binaries.items(): + manifest = json.loads(binary.with_name(binary.name + ".validation.json").read_text()) + if manifest.get("hook_sha256") != metadata["hook_sha256"] or manifest.get("binary_sha256") != hashlib.sha256(binary.read_bytes()).hexdigest(): + raise RuntimeError(f"Reusable probe has mismatched validation hook or binary hash: {binary}") + metadata[f"{label}_source_sha256"] = manifest.get("source_sha256") + else: + previous = None + if args.reuse_sources: + previous = json.loads((args.reuse_sources / "metadata.json").read_text()) + if previous["baseline_revision"] != revision: + raise RuntimeError("Reusable source baseline revision does not match") + binaries = {} + for label, source_revision in [("before", revision), ("after", None)]: + source = output / "source" / label + metadata[f"{label}_source_sha256"] = copy_source(repository, source, source_revision) + if previous is not None: + if metadata[f"{label}_source_sha256"] != previous[f"{label}_source_sha256"]: + raise RuntimeError(f"Reusable {label} source hashes do not match the current source snapshot") + reused = args.reuse_sources.resolve() / "source" / label + # Replace only the injected entry point; every other source must still match. + for name, digest in metadata[f"{label}_source_sha256"].items(): + if name != "Sources/QuotaBarProbe/main.swift" and hashlib.sha256((reused / name).read_bytes()).hexdigest() != digest: + raise RuntimeError(f"Reusable source was modified: {reused / name}") + shutil.copy2(source / "Sources/QuotaBarProbe/main.swift", reused / "Sources/QuotaBarProbe/main.swift") + source = reused + metadata[f"{label}_reused_source"] = str(source) + save(output / "metadata.json", metadata) + print(f"Building {label} release probe in isolated source copy", flush=True) + binaries[label] = build_probe(source, hook, output / f"{label}-build.log", metadata[f"{label}_source_sha256"]) + metadata["binaries"] = {label: {"path": str(binary), "sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), "bytes": binary.stat().st_size} for label, binary in binaries.items()} + save(output / "metadata.json", metadata) + differential(output, binaries, timestamp) + if not args.skip_benchmarks: + benchmarks(output, binaries, timestamp, args.iterations, args.workloads) + metadata["completed"] = True + save(output / "metadata.json", metadata) + print(f"Validation completed: {output}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/Sources/CQuotaBarRust/include/QuotaBarRust.h b/Sources/CQuotaBarRust/include/QuotaBarRust.h new file mode 100644 index 0000000..439005c --- /dev/null +++ b/Sources/CQuotaBarRust/include/QuotaBarRust.h @@ -0,0 +1,85 @@ +#ifndef QUOTA_BAR_RUST_H +#define QUOTA_BAR_RUST_H + +#include +#include + +// Input buffers are borrowed synchronously. Output strings belong to owner; +// copy them before qb_claude_free and never free them with a Swift/C allocator. +typedef struct { const uint8_t *ptr; size_t len; } QBByteString; +typedef struct { + uint32_t status; + int64_t counts[5]; + QBByteString timestamp, model, message_id, request_id, uuid, speed; + void *owner; +} QBClaudeResult; + +enum { + QB_CLAUDE_PARSED = 0, + QB_CLAUDE_IGNORED = 1, + QB_CLAUDE_FALLBACK = 2, + QB_CLAUDE_INTERNAL_ERROR = 3, + QB_CLAUDE_INVALID_ARGUMENT = 4 +}; +void qb_claude_decode(const uint8_t *bytes, size_t length, QBClaudeResult *output); +void qb_claude_free(void *owner); + +// 0 missing, 1 indeterminate, 2 other. Top-level: 3 assistant, 4 event_msg, +// 5 turn_context. Payload: 3 thread_settings_applied, 4 token_count. +int32_t qb_classify_top_level(const uint8_t *bytes, size_t length); +int32_t qb_classify_payload(const uint8_t *bytes, size_t length); + +// Callbacks run synchronously on the caller's thread. Their byte views must not +// escape the callback. A successful scan returns the last complete-line offset; +// failures return POSIX errno and never publish an offset. +typedef uint8_t (*QBLineWanted)(const uint8_t *bytes, size_t length, void *context); +typedef void (*QBLineHandle)(const uint8_t *bytes, size_t length, void *context); +int32_t qb_read_lines(const char *path, int64_t offset, uint8_t has_limit, int64_t limit, + QBLineWanted wanted, QBLineHandle handle, void *context, int64_t *consumed); + +typedef struct { + double input, output, cache_write, cache_write_1h, cache_read; + double input_above, output_above, cache_write_above, cache_write_1h_above, cache_read_above; + int64_t threshold_tokens; + // Optional rates use bits 0...7 in the above order; threshold uses bit 8. + uint32_t present; +} QBModelPricing; +typedef struct { int64_t input, output, cache_write, cache_write_1h, cache_read; } QBPricingTokens; +uint8_t qb_pricing_cost(const QBModelPricing *rates, const QBPricingTokens *tokens, + uint8_t long_context, double *out_cost); +double qb_pricing_cache_write_1h_rate(const QBModelPricing *rates, uint8_t long_context); + +// Aggregation borrows disjoint caller-owned buffers synchronously. Group capacity is row_count. +// Null buffers are allowed for zero counts. A nonzero status invalidates every output buffer. +typedef struct { int64_t input, output, cache_write, cache_write_1h, cache_read; } QBAggregateTokens; +typedef struct { + uint64_t day, key, model; + QBAggregateTokens tokens; + double cost; + int64_t unpriced; + uint8_t priced, fallback; +} QBAggregateRow; +typedef struct { + uint64_t day, key; + QBAggregateTokens tokens; + double cost; + uint8_t priced, fallback; +} QBAggregateGroup; +typedef struct { QBAggregateTokens tokens; double cost; int64_t unpriced; uint8_t priced; } QBAggregateDay; +typedef struct { double cost; int64_t tokens; uint8_t priced; } QBAggregateModel; +enum { + QB_AGGREGATE_OK = 0, + QB_AGGREGATE_INVALID_COUNTS = 1, + QB_AGGREGATE_TOKEN_OVERFLOW = 2, + QB_AGGREGATE_TOTAL_OVERFLOW = 3, + QB_AGGREGATE_WINDOW_OVERFLOW = 4, + QB_AGGREGATE_INVALID_COST = 5, + QB_AGGREGATE_INVALID_ARGUMENT = 6, + QB_AGGREGATE_INTERNAL_ERROR = 7 +}; +int32_t qb_aggregate_usage(const QBAggregateRow *rows, uint64_t row_count, + uint64_t day_count, uint64_t model_count, + QBAggregateGroup *groups, uint64_t *group_count, + QBAggregateDay *days, QBAggregateModel *models); + +#endif diff --git a/Sources/CQuotaBarRust/module.modulemap b/Sources/CQuotaBarRust/module.modulemap new file mode 100644 index 0000000..65b45a9 --- /dev/null +++ b/Sources/CQuotaBarRust/module.modulemap @@ -0,0 +1,4 @@ +module CQuotaBarRust { + header "include/QuotaBarRust.h" + export * +} diff --git a/Sources/QuotaBarCore/Cost/ClaudeCodeAdapter.swift b/Sources/QuotaBarCore/Cost/ClaudeCodeAdapter.swift index e99a37a..a9b320e 100644 --- a/Sources/QuotaBarCore/Cost/ClaudeCodeAdapter.swift +++ b/Sources/QuotaBarCore/Cost/ClaudeCodeAdapter.swift @@ -8,6 +8,7 @@ import CryptoKit import Foundation +import CQuotaBarRust /// Turns Claude Code transcript lines into observed requests. package struct ClaudeCodeAdapter: AppendedLogAdapter { @@ -51,6 +52,48 @@ package struct ClaudeCodeAdapter: AppendedLogAdapter { } package struct Parser: AppendedLogParser { + private let fallback = FoundationParser() + + package init() {} + + package var savedState: String? { nil } + + package func request(in line: UnsafeRawBufferPointer) -> ObservedRequest? { + var result = QBClaudeResult() + qb_claude_decode(line.bindMemory(to: UInt8.self).baseAddress, line.count, &result) + defer { qb_claude_free(result.owner) } + if result.status == QB_CLAUDE_IGNORED { return nil } + guard result.status == QB_CLAUDE_PARSED else { + return self.fallback.request(in: line) + } + + // Keep Foundation's Unicode/date handling and persisted identity conventions. + let model = Self.string(result.model).trimmingCharacters(in: .whitespacesAndNewlines) + guard !model.isEmpty, model != "", + let date = ISO8601.parse(Self.string(result.timestamp)) else { return nil } + let messageID = Self.string(result.message_id) + let requestID = Self.string(result.request_id) + let key = messageID.isEmpty && requestID.isEmpty + ? "uuid:\(Self.string(result.uuid))" : "\(messageID)|\(requestID)" + let counts = result.counts + return ObservedRequest( + key: key, timestamp: date, model: model, + tokens: TokenTotals(input: Int(counts.0), output: Int(counts.1), cacheWrite: Int(counts.2), + cacheWrite1h: Int(counts.3), cacheRead: Int(counts.4)), + isFast: Self.string(result.speed).lowercased() == "fast" + ) + } + + private static func string(_ value: QBByteString) -> String { + guard let pointer = value.ptr, value.len > 0 else { return "" } + // Rust owns these validated UTF-8 bytes until the deferred release above. + return String(decoding: UnsafeBufferPointer(start: pointer, count: value.len), as: UTF8.self) + } + } + + /// Compatibility path for Foundation-specific decoding and ID-less canonical hashes. + /// It also remains the differential-test oracle for the native kernel. + package struct FoundationParser: AppendedLogParser { private let decoder = JSONDecoder() package init() {} diff --git a/Sources/QuotaBarCore/Cost/CostAggregator.swift b/Sources/QuotaBarCore/Cost/CostAggregator.swift index b6e9d10..99ade65 100644 --- a/Sources/QuotaBarCore/Cost/CostAggregator.swift +++ b/Sources/QuotaBarCore/Cost/CostAggregator.swift @@ -1,3 +1,4 @@ +import CQuotaBarRust import Foundation /// Folds priced recorded usage into the numbers the popover shows. @@ -19,52 +20,91 @@ enum CostAggregator { on: rateCard ) - var days: [CostDay] = [] - // Cost and token totals per model across the whole window, for picking the top model. - var modelCost: [String: Double] = [:] - var modelTokens: [String: Int] = [:] - var hasUnpriced = false - - for (dayKey, dayRows) in Dictionary(grouping: rows, by: \.day) { - var dayTokens: [ModelUsageKey: TokenTotals] = [:] - var checkedDayTokens = TokenTotals() - // Per-source/model cost stays nil until something prices it, so an unpriced row reads - // as "no price" in the breakdown rather than as zero dollars. - var dayCostByModel: [ModelUsageKey: Double] = [:] - var fallbacks: Set = [] - var dayCost = 0.0 - var dayPriced = false - var dayUnpricedTokens = 0 + return try Self.aggregate(groupedRows: Dictionary(grouping: rows, by: \.day), + tab: tab, todayKey: todayKey, now: now) + } + /// Keep Swift's actual day traversal order: model costs accumulate across days in this order. + /// The numeric kernel receives compact IDs; strings and public presentation models stay here. + static func aggregate(groupedRows: [String: [PricedUsage]], tab: UsageTab, + todayKey: String, now: Date) throws -> CostSnapshot { + var dayNames: [String] = [] + var keys: [ModelUsageKey] = [] + var modelIDs: [String: UInt64] = [:] + var modelNames: [String] = [] + var pricedModelNames: [String?] = [] + var input: [QBAggregateRow] = [] + input.reserveCapacity(groupedRows.values.reduce(0) { $0 + $1.count }) + for (day, dayRows) in groupedRows { + let dayID = UInt64(dayNames.count) + dayNames.append(day) + // Equal Swift strings can have different UTF-8 spellings. Each day's dictionary + // keeps that day's first key, independently of equivalent keys on other days. + var keyIDs: [ModelUsageKey: UInt64] = [:] for row in dayRows { - let usageKey = ModelUsageKey(source: row.source, model: row.model, isFast: row.key.isFast, upstreamProvider: row.key.upstreamProvider) - dayTokens[usageKey] = try dayTokens[usageKey, default: TokenTotals()].adding(row.tokens) - checkedDayTokens = try checkedDayTokens.adding(row.tokens) - modelTokens[row.model] = try self.add(modelTokens[row.model, default: 0], row.tokens.total) - if row.usesLoggedCostFallback { fallbacks.insert(usageKey) } - if let cost = row.cost { - dayCost = try self.add(dayCost, cost) - dayPriced = true - dayCostByModel[usageKey] = try self.add(dayCostByModel[usageKey, default: 0], cost) - modelCost[row.model] = try self.add(modelCost[row.model, default: 0], cost) + let key = ModelUsageKey(source: row.source, model: row.model, isFast: row.key.isFast, + upstreamProvider: row.key.upstreamProvider) + let keyID: UInt64 + if let value = keyIDs[key] { keyID = value } + else { keyID = UInt64(keys.count); keys.append(key); keyIDs[key] = keyID } + let modelID: UInt64 + if let value = modelIDs[row.model] { modelID = value } + else { + modelID = UInt64(modelNames.count) + modelNames.append(row.model) + pricedModelNames.append(nil) + modelIDs[row.model] = modelID } - if row.unpricedTokens > 0 { - hasUnpriced = true - dayUnpricedTokens = try self.add(dayUnpricedTokens, row.unpricedTokens) + // The original cost dictionary starts at the first priced occurrence, which + // can spell an equivalent model name differently from the token dictionary. + if row.cost != nil, pricedModelNames[Int(modelID)] == nil { + pricedModelNames[Int(modelID)] = row.model + } + input.append(QBAggregateRow(day: dayID, key: keyID, model: modelID, + tokens: QBAggregateTokens(input: Int64(row.tokens.input), output: Int64(row.tokens.output), + cache_write: Int64(row.tokens.cacheWrite), cache_write_1h: Int64(row.tokens.cacheWrite1h), + cache_read: Int64(row.tokens.cacheRead)), cost: row.cost ?? 0, + unpriced: Int64(row.unpricedTokens), priced: row.cost == nil ? 0 : 1, + fallback: row.usesLoggedCostFallback ? 1 : 0)) + } + } + var groups = [QBAggregateGroup](repeating: QBAggregateGroup(), count: input.count) + var dayValues = [QBAggregateDay](repeating: QBAggregateDay(), count: dayNames.count) + var modelValues = [QBAggregateModel](repeating: QBAggregateModel(), count: modelNames.count) + var groupCount: UInt64 = 0 + let status = input.withUnsafeBufferPointer { input in + groups.withUnsafeMutableBufferPointer { groups in + dayValues.withUnsafeMutableBufferPointer { days in + modelValues.withUnsafeMutableBufferPointer { models in + qb_aggregate_usage(input.baseAddress, UInt64(input.count), UInt64(days.count), UInt64(models.count), + groups.baseAddress, &groupCount, days.baseAddress, models.baseAddress) + } } } - - let byModel = Dictionary(uniqueKeysWithValues: dayTokens.lazy.map { key, tokens in - (key, ModelDayUsage(tokens: tokens, costUSD: dayCostByModel[key], usesLoggedCostFallback: fallbacks.contains(key))) - }) - - days.append(CostDay( - dayKey: dayKey, - byModel: byModel, - costUSD: dayPriced ? dayCost : nil, - unpricedTokens: dayUnpricedTokens - )) } + try Self.checkAggregationStatus(status) + var byDay = [[ModelUsageKey: ModelDayUsage]](repeating: [:], count: dayNames.count) + for group in groups.prefix(Int(groupCount)) { + byDay[Int(group.day)][keys[Int(group.key)]] = ModelDayUsage( + tokens: TokenTotals(input: Int(group.tokens.input), output: Int(group.tokens.output), + cacheWrite: Int(group.tokens.cache_write), cacheWrite1h: Int(group.tokens.cache_write_1h), + cacheRead: Int(group.tokens.cache_read)), + costUSD: group.priced == 0 ? nil : group.cost, usesLoggedCostFallback: group.fallback != 0) + } + var days = dayNames.indices.map { index in + CostDay(dayKey: dayNames[index], byModel: byDay[index], + costUSD: dayValues[index].priced == 0 ? nil : dayValues[index].cost, + unpricedTokens: Int(dayValues[index].unpriced)) + } + var modelCost: [String: Double] = [:] + var modelTokens: [String: Int] = [:] + for index in modelNames.indices { + if modelValues[index].priced != 0, let name = pricedModelNames[index] { + modelCost[name] = modelValues[index].cost + } + modelTokens[modelNames[index]] = Int(modelValues[index].tokens) + } + let hasUnpriced = dayValues.contains { $0.unpriced > 0 } days.sort { $0.dayKey < $1.dayKey } @@ -92,6 +132,20 @@ enum CostAggregator { ) } + private static func checkAggregationStatus(_ status: Int32) throws { + let message: String + switch status { + case 0: return + case 1: message = "token counts are negative or exceed the integer range" + case 2: message = "token aggregation exceeds the integer range" + case 3: message = "total tokens exceed the integer range" + case 4: message = "window token total exceeds the integer range" + case 5: message = "aggregate cost is not finite and nonnegative" + default: message = "Rust usage aggregation failed" + } + throw RecordedUsageReaderError.invalidData(message) + } + private static func add(_ lhs: Int, _ rhs: Int) throws -> Int { let result = lhs.addingReportingOverflow(rhs) guard lhs >= 0, rhs >= 0, !result.overflow else { diff --git a/Sources/QuotaBarCore/Cost/CostPricing.swift b/Sources/QuotaBarCore/Cost/CostPricing.swift index a5de3f8..5d28a30 100644 --- a/Sources/QuotaBarCore/Cost/CostPricing.swift +++ b/Sources/QuotaBarCore/Cost/CostPricing.swift @@ -5,6 +5,7 @@ // against published price lists. The rates themselves live in the price book // (`Resources/Pricing/price-book.json`), and `RateCard` decides which apply to a day. +import CQuotaBarRust import Foundation public struct ModelPricing: Sendable, Equatable { @@ -90,44 +91,42 @@ public struct ModelPricing: Sendable, Equatable { /// The one-hour cache-write rate actually billed, derived from the input rate when the /// rates do not state one. public func cacheWrite1hRate(longContext: Bool) -> Double { - let inputRate = (longContext ? self.inputAbove : nil) ?? self.input - if let stated = (longContext ? self.cacheWrite1hAbove : nil) ?? self.cacheWrite1h { return stated } - return inputRate * Self.oneHourCacheWriteMultiplier + var pricing = self.rustPricing + return qb_pricing_cache_write_1h_rate(&pricing, longContext ? 1 : 0) } /// Cost in USD for one bucket of tokens, or nil when rates or the amount are invalid. public func cost(for totals: TokenTotals, longContext: Bool) -> Double? { - guard Self.violations(in: self.rateValues).isEmpty, - [totals.input, totals.output, totals.cacheWrite, totals.cacheWrite1h, totals.cacheRead] - .allSatisfy({ $0 >= 0 }) else { return nil } - let million = 1_000_000.0 - let inputRate = (longContext ? self.inputAbove : nil) ?? self.input - let outputRate = (longContext ? self.outputAbove : nil) ?? self.output - // A model without its own cache rates bills cached tokens at the full input rate. - let cacheWriteRate = (longContext ? self.cacheWriteAbove : nil) ?? self.cacheWrite ?? inputRate - let cacheReadRate = (longContext ? self.cacheReadAbove : nil) ?? self.cacheRead ?? inputRate - - // Anthropic prices a one-hour cache write at twice the input rate, against 1.25x for the - // five-minute default. The table's cache-write column is the five-minute rate, so unless - // a one-hour rate is stated the longer TTL is derived the way Anthropic publishes it. - let write1h = Double(min(totals.cacheWrite1h, totals.cacheWrite)) - let write5m = Double(totals.cacheWrite) - write1h - - let millionths = Double(totals.input) * inputRate - + Double(totals.output) * outputRate - + write5m * cacheWriteRate - + write1h * self.cacheWrite1hRate(longContext: longContext) - + Double(totals.cacheRead) * cacheReadRate - if millionths.isFinite { return millionths / million } - - // Retry in USD when the intermediate amount in millionths overflowed. - // This preserves the existing rounding for ordinary rates. - let cost = Double(totals.input) * (inputRate / million) - + Double(totals.output) * (outputRate / million) - + write5m * (cacheWriteRate / million) - + write1h * (self.cacheWrite1hRate(longContext: longContext) / million) - + Double(totals.cacheRead) * (cacheReadRate / million) - return cost.isFinite && cost >= 0 ? cost : nil + var pricing = self.rustPricing + var tokens = QBPricingTokens( + input: Int64(totals.input), output: Int64(totals.output), + cache_write: Int64(totals.cacheWrite), cache_write_1h: Int64(totals.cacheWrite1h), + cache_read: Int64(totals.cacheRead) + ) + var cost = 0.0 + return qb_pricing_cost(&pricing, &tokens, longContext ? 1 : 0, &cost) != 0 ? cost : nil + } + + /// The fixed-width ABI keeps absent rates distinct from stated zero rates. + private var rustPricing: QBModelPricing { + var present: UInt32 = 0 + if self.cacheWrite != nil { present |= 1 << 0 } + if self.cacheWrite1h != nil { present |= 1 << 1 } + if self.cacheRead != nil { present |= 1 << 2 } + if self.inputAbove != nil { present |= 1 << 3 } + if self.outputAbove != nil { present |= 1 << 4 } + if self.cacheWriteAbove != nil { present |= 1 << 5 } + if self.cacheWrite1hAbove != nil { present |= 1 << 6 } + if self.cacheReadAbove != nil { present |= 1 << 7 } + if self.thresholdTokens != nil { present |= 1 << 8 } + return QBModelPricing( + input: self.input, output: self.output, + cache_write: self.cacheWrite ?? 0, cache_write_1h: self.cacheWrite1h ?? 0, + cache_read: self.cacheRead ?? 0, input_above: self.inputAbove ?? 0, + output_above: self.outputAbove ?? 0, cache_write_above: self.cacheWriteAbove ?? 0, + cache_write_1h_above: self.cacheWrite1hAbove ?? 0, cache_read_above: self.cacheReadAbove ?? 0, + threshold_tokens: Int64(self.thresholdTokens ?? 0), present: present + ) } } diff --git a/Sources/QuotaBarCore/Cost/JSONLogClassifier.swift b/Sources/QuotaBarCore/Cost/JSONLogClassifier.swift index b28524c..89653ca 100644 --- a/Sources/QuotaBarCore/Cost/JSONLogClassifier.swift +++ b/Sources/QuotaBarCore/Cost/JSONLogClassifier.swift @@ -1,8 +1,6 @@ -import Darwin -import Foundation +import CQuotaBarRust -/// Reads only the structural JSON fields needed to decide whether a JSONL record merits full -/// parsing. It never assumes key order or a maximum prefix length. +/// Reads structural JSON fields through the borrowed, allocation-free Rust prefix filter. package enum JSONLogClassifier { package enum TopLevelType: Equatable { case assistant @@ -19,199 +17,24 @@ package enum JSONLogClassifier { case other } - private struct ParsedString { - let range: Range - let hasEscape: Bool - } - - private enum Lookup { - case found(Value) - case indeterminate - case missing - } - - private static let typeKey = Array("type".utf8) - private static let payloadKey = Array("payload".utf8) - private static let assistant = Array("assistant".utf8) - private static let eventMessage = Array("event_msg".utf8) - private static let turnContext = Array("turn_context".utf8) - private static let threadSettingsApplied = Array("thread_settings_applied".utf8) - private static let tokenCount = Array("token_count".utf8) - package static func topLevelType(in buffer: UnsafeRawBufferPointer) -> TopLevelType? { - let value: ParsedString - switch self.stringValue(for: Self.typeKey, inObjectAt: 0, buffer: buffer) { - case let .found(found): value = found - case .indeterminate: return .indeterminate - case .missing: return nil + switch qb_classify_top_level(buffer.baseAddress?.assumingMemoryBound(to: UInt8.self), buffer.count) { + case 0: return nil + case 2: return .other + case 3: return .assistant + case 4: return .eventMessage + case 5: return .turnContext + default: return .indeterminate } - if self.matches(value.range, Self.assistant, buffer: buffer) { return .assistant } - if self.matches(value.range, Self.eventMessage, buffer: buffer) { return .eventMessage } - if self.matches(value.range, Self.turnContext, buffer: buffer) { return .turnContext } - return .other } package static func payloadType(in buffer: UnsafeRawBufferPointer) -> PayloadType? { - let payload: Int - switch self.valueStart(for: Self.payloadKey, inObjectAt: 0, buffer: buffer) { - case let .found(found): payload = found - case .indeterminate: return .indeterminate - case .missing: return nil - } - let value: ParsedString - switch self.stringValue(for: Self.typeKey, inObjectAt: payload, buffer: buffer) { - case let .found(found): value = found - case .indeterminate: return .indeterminate - case .missing: return nil - } - if self.matches(value.range, Self.threadSettingsApplied, buffer: buffer) { return .threadSettingsApplied } - if self.matches(value.range, Self.tokenCount, buffer: buffer) { return .tokenCount } - return .other - } - - private static func stringValue( - for key: [UInt8], - inObjectAt objectStart: Int, - buffer: UnsafeRawBufferPointer - ) -> Lookup { - let start: Int - switch self.valueStart(for: key, inObjectAt: objectStart, buffer: buffer) { - case let .found(found): start = found - case .indeterminate: return .indeterminate - case .missing: return .missing - } - var index = start - guard let value = self.stringRange(at: &index, buffer: buffer) else { return .indeterminate } - return value.hasEscape ? .indeterminate : .found(value) - } - - private static func valueStart( - for key: [UInt8], - inObjectAt objectStart: Int, - buffer: UnsafeRawBufferPointer - ) -> Lookup { - var index = objectStart - self.skipWhitespace(at: &index, buffer: buffer) - guard index < buffer.count, buffer[index] == UInt8(ascii: "{") else { return .indeterminate } - index += 1 - - while index < buffer.count { - self.skipWhitespace(at: &index, buffer: buffer) - if index < buffer.count, buffer[index] == UInt8(ascii: "}") { return .missing } - guard let candidate = self.stringRange(at: &index, buffer: buffer) else { return .indeterminate } - if candidate.hasEscape { return .indeterminate } - self.skipWhitespace(at: &index, buffer: buffer) - guard index < buffer.count, buffer[index] == UInt8(ascii: ":") else { return .indeterminate } - index += 1 - self.skipWhitespace(at: &index, buffer: buffer) - if self.matches(candidate.range, key, buffer: buffer) { return .found(index) } - guard self.skipValue(at: &index, buffer: buffer) else { return .indeterminate } - self.skipWhitespace(at: &index, buffer: buffer) - if index < buffer.count, buffer[index] == UInt8(ascii: ",") { - index += 1 - continue - } - if index < buffer.count, buffer[index] == UInt8(ascii: "}") { return .missing } - return .indeterminate - } - return .indeterminate - } - - private static func stringRange( - at index: inout Int, - buffer: UnsafeRawBufferPointer - ) -> ParsedString? { - guard index < buffer.count, buffer[index] == UInt8(ascii: "\"") else { return nil } - index += 1 - let start = index - guard let base = buffer.baseAddress, - let quote = memchr(base.advanced(by: index), Int32(UInt8(ascii: "\"")), buffer.count - index) else { - return nil - } - let end = base.distance(to: quote) - // Skip long plain strings in libc. Escaped strings retain the scalar traversal: searching - // for every escaped quote separately regresses on densely escaped transcript content. - guard let escape = memchr(base.advanced(by: index), Int32(UInt8(ascii: "\\")), end - index) else { - index = end + 1 - return ParsedString(range: start.. Bool { - guard index < buffer.count else { return false } - if buffer[index] == UInt8(ascii: "\"") { - return self.stringRange(at: &index, buffer: buffer) != nil - } - if buffer[index] == UInt8(ascii: "{") || buffer[index] == UInt8(ascii: "[") { - var depth = 0 - while index < buffer.count { - let byte = buffer[index] - if byte == UInt8(ascii: "\"") { - guard self.stringRange(at: &index, buffer: buffer) != nil else { return false } - continue - } - if byte == UInt8(ascii: "{") || byte == UInt8(ascii: "[") { depth += 1 } - if byte == UInt8(ascii: "}") || byte == UInt8(ascii: "]") { - depth -= 1 - index += 1 - if depth == 0 { return true } - continue - } - index += 1 - } - return false - } - while index < buffer.count { - let byte = buffer[index] - if byte == UInt8(ascii: ",") || byte == UInt8(ascii: "}") || byte == UInt8(ascii: "]") - || byte == UInt8(ascii: " ") || byte == UInt8(ascii: "\t") - || byte == UInt8(ascii: "\r") || byte == UInt8(ascii: "\n") { - return true - } - index += 1 - } - return true - } - - private static func skipWhitespace(at index: inout Int, buffer: UnsafeRawBufferPointer) { - while index < buffer.count { - switch buffer[index] { - case UInt8(ascii: " "), UInt8(ascii: "\t"), UInt8(ascii: "\r"), UInt8(ascii: "\n"): - index += 1 - default: - return - } - } - } - - private static func matches( - _ range: Range, - _ expected: [UInt8], - buffer: UnsafeRawBufferPointer - ) -> Bool { - guard range.count == expected.count else { return false } - for offset in expected.indices where buffer[range.lowerBound + offset] != expected[offset] { - return false + switch qb_classify_payload(buffer.baseAddress?.assumingMemoryBound(to: UInt8.self), buffer.count) { + case 0: return nil + case 2: return .other + case 3: return .threadSettingsApplied + case 4: return .tokenCount + default: return .indeterminate } - return true } } diff --git a/Sources/QuotaBarCore/Cost/LogFileScanner.swift b/Sources/QuotaBarCore/Cost/LogFileScanner.swift index f770d48..d525383 100644 --- a/Sources/QuotaBarCore/Cost/LogFileScanner.swift +++ b/Sources/QuotaBarCore/Cost/LogFileScanner.swift @@ -1,3 +1,4 @@ +import CQuotaBarRust import CryptoKit import Darwin import Foundation @@ -5,8 +6,6 @@ import Foundation /// Streams the newline-delimited records of a JSONL file, resuming from a byte offset. package enum LogFileScanner { private static let prefixDigestBytes = 64 * 1024 - private static let chunkSize = 1 << 20 - private static let longLinePrefixBytes = 64 * 1024 package struct ScanPlan { package let cursor: FileCursor @@ -94,75 +93,47 @@ package enum LogFileScanner { where isWanted: (UnsafeRawBufferPointer) -> Bool = { _ in true }, handle: (UnsafeRawBufferPointer) -> Void ) throws -> Int64 { - let start = max(0, offset) - let descriptor = Darwin.open(url.path, O_RDONLY | O_CLOEXEC) - guard descriptor >= 0 else { throw Self.posixError() } - defer { _ = Darwin.close(descriptor) } - guard Darwin.lseek(descriptor, off_t(start), SEEK_SET) >= 0 else { - throw Self.posixError() - } - - var buffer = [UInt8](repeating: 0, count: Self.chunkSize) - var carry: [UInt8] = [] - var skippingLine = false - var readPosition = start - var consumed = start - - while limit.map({ readPosition < $0 }) ?? true { - let requested = limit.map { min(Int64(Self.chunkSize), max(0, $0 - readPosition)) } - ?? Int64(Self.chunkSize) - guard requested > 0 else { break } - - let count: Int = try buffer.withUnsafeMutableBytes { storage in - var result: Int - repeat { - result = Darwin.read(descriptor, storage.baseAddress, Int(requested)) - } while result < 0 && errno == EINTR - guard result >= 0 else { throw Self.posixError() } - return result - } - guard count > 0 else { break } - - let chunkStart = readPosition - buffer.withUnsafeBytes { storage in - guard let base = storage.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return } - var lineStart = 0 - while lineStart < count, - let found = memchr(base.advanced(by: lineStart), Int32(UInt8(ascii: "\n")), count - lineStart) { - let newline = base.distance(to: found.assumingMemoryBound(to: UInt8.self)) - let length = newline - lineStart - if skippingLine { - skippingLine = false - } else if carry.isEmpty { - let line = UnsafeRawBufferPointer(start: base.advanced(by: lineStart), count: length) - if length > 0, isWanted(line) { - // Handlers parse with Foundation, whose autoreleased objects would - // otherwise pile up until the scan's task finishes. - autoreleasepool { handle(line) } - } - } else { - carry.append(contentsOf: UnsafeBufferPointer(start: base.advanced(by: lineStart), count: length)) - if carry.withUnsafeBytes(isWanted) { - autoreleasepool { carry.withUnsafeBytes(handle) } - } - carry.removeAll(keepingCapacity: true) + try withoutActuallyEscaping(isWanted) { wanted in + try withoutActuallyEscaping(handle) { handle in + let callbacks = LineCallbacks(wanted: wanted, handle: handle) + var consumed: Int64 = 0 + let error = withExtendedLifetime(callbacks) { + url.path.withCString { path in + qb_read_lines( + path, offset, limit == nil ? 0 : 1, limit ?? 0, + { bytes, count, context in + let callbacks = Unmanaged.fromOpaque(context!).takeUnretainedValue() + return callbacks.wanted(UnsafeRawBufferPointer(start: bytes, count: count)) ? 1 : 0 + }, + { bytes, count, context in + let callbacks = Unmanaged.fromOpaque(context!).takeUnretainedValue() + // Foundation handlers must release temporary objects after each record. + autoreleasepool { + callbacks.handle(UnsafeRawBufferPointer(start: bytes, count: count)) + } + }, + Unmanaged.passUnretained(callbacks).toOpaque(), &consumed + ) } - consumed = chunkStart + Int64(newline) + 1 - lineStart = newline + 1 } - if lineStart < count, !skippingLine { - let wasShort = carry.count < Self.longLinePrefixBytes - carry.append(contentsOf: UnsafeBufferPointer(start: base.advanced(by: lineStart), count: count - lineStart)) - if wasShort, carry.count >= Self.longLinePrefixBytes, !carry.withUnsafeBytes(isWanted) { - skippingLine = true - carry.removeAll(keepingCapacity: true) - } + guard error == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: error) ?? .EIO) } + return consumed } - readPosition += Int64(count) } + } - return consumed + /// Rust borrows this context synchronously; neither it nor a line buffer escapes the call. + private final class LineCallbacks { + let wanted: (UnsafeRawBufferPointer) -> Bool + let handle: (UnsafeRawBufferPointer) -> Void + + init(wanted: @escaping (UnsafeRawBufferPointer) -> Bool, + handle: @escaping (UnsafeRawBufferPointer) -> Void) { + self.wanted = wanted + self.handle = handle + } } private static func posixError() -> POSIXError { diff --git a/Sources/QuotaBarTests/JSONLogClassifierDifferentialTests.swift b/Sources/QuotaBarTests/JSONLogClassifierDifferentialTests.swift new file mode 100644 index 0000000..0ad51ec --- /dev/null +++ b/Sources/QuotaBarTests/JSONLogClassifierDifferentialTests.swift @@ -0,0 +1,137 @@ +import Foundation +import QuotaBarCore + +/// The pre-migration Swift classifier is the oracle, including permissive prefix semantics. +enum JSONLogClassifierDifferentialTests { + static func run() { + let fixtures = [ + "", "{}", "[]", "null", "{", "{\"type\":", "{\"type\":false}", + #"{"type":"assistant"}"#, #"{"type":"event_msg"}"#, #"{"type":"turn_context"}"#, + #"{"type":"assistant"trailing garbage"#, #"{"type":"assistant","type":"user"}"#, + #"{"type":"user","type":"assistant"}"#, #"{"ty\u0070e":"assistant"}"#, + #"{"type":"assi\u0073tant"}"#, #"{"x\u0079":0,"type":"assistant"}"#, + #"{"payload":{"type":"token_count"}}"#, #"{"payload":{"type":"thread_settings_applied"}}"#, + #"{"payload":{"type":"other"}}"#, #"{"payload":{}}"#, #"{"payload":null}"#, + #"{"payload":{"type":"token\u005fcount"}}"#, #"{"pay\u006coad":{"type":"token_count"}}"#, + #"{"skip":[{"type":"assistant"}],"type":"user"}"#, + #"{"skip":"\"type\":\"assistant\"","type":"user"}"#, + #"{"skip":[},"type":"assistant"}"#, #"{"skip":],"type":"assistant"}"#, + #"{"skip":,"type":"assistant"}"#, #"{"skip":1,}"#, + " \r\n\t{\"skip\":[1,{\"nested\":[true,null]}],\"payload\":{\"type\":\"token_count\"}}", + ].map { Array($0.utf8) } + + var prefixCount = 0 + for (fixtureIndex, fixture) in fixtures.enumerated() { + for count in 0...fixture.count { + guard Self.compare(Array(fixture.prefix(count)), label: "fixture \(fixtureIndex), prefix \(count)") else { + return + } + prefixCount += 1 + } + } + Harness.expect(prefixCount > 500, "Rust classifier matches Swift on every adversarial fixture prefix") + + var generator = Generator() + for iteration in 0..<10_000 { + let count = Int(generator.next() % 257) + let bytes = (0..> 24) } + guard Self.compare(bytes, label: "arbitrary bytes, seed 0x71626a736f6e, iteration \(iteration)") else { + return + } + } + Harness.expect(true, "Rust classifier matches Swift on 10,000 seeded arbitrary byte buffers") + + for iteration in 0..<10_000 { + var bytes = fixtures[Int(generator.next() % UInt64(fixtures.count))] + for _ in 0..<(1 + Int(generator.next() % 8)) { + let index = Int(generator.next() % UInt64(bytes.count + 1)) + let byte = UInt8(truncatingIfNeeded: generator.next() >> 24) + if index < bytes.count && generator.next() & 1 == 0 { + bytes[index] = byte + } else { + bytes.insert(byte, at: index) + } + } + guard Self.compare(bytes, label: "mutated fixture, iteration \(iteration)") else { return } + } + Harness.expect(true, "Rust classifier matches Swift on 10,000 seeded malformed structural fixtures") + + for plainCount in [0, 1, 15, 16, 17, 31, 32, 33] { + for suffix in ["", #"\""#, #"\\"#, #"\\\""#] { + let bytes = Array((#"{"padding":""# + String(repeating: "x", count: plainCount) + + suffix + #"","type":"assistant","payload":{"type":"token_count"}}"#).utf8) + for cut in 0...bytes.count { + guard Self.compare(Array(bytes.prefix(cut)), label: "scalar/libc boundary \(plainCount), prefix \(cut)") else { + return + } + } + } + } + for slashCount in 0..<128 { + let bytes = Array((#"{"padding":""# + String(repeating: "\\", count: slashCount) + + #"","type":"assistant","payload":{"type":"token_count"}}"#).utf8) + guard Self.compare(bytes, label: "escape parity \(slashCount)") else { return } + } + let nested = Array((#"{"padding":"# + String(repeating: "[", count: 16_384) + + #"{"type":"user"}"# + String(repeating: "]", count: 16_384) + + #", "type":"assistant","payload":{"type":"token_count"}}"#).utf8) + guard Self.compare(nested, label: "deep nested decoy uses bounded stack space") else { return } + for padding in [String(repeating: "x", count: 1_048_576), + String(repeating: #"a\"b\\c"#, count: 131_072)] { + let bytes = Array((#"{"padding":""# + padding + + #"","type":"assistant","payload":{"type":"token_count"}}"#).utf8) + let cuts = [0, 1, 12, 13, 31, 255, 65_535, 65_536, bytes.count / 2] + + Array((bytes.count - 64)...bytes.count) + for cut in cuts { + guard Self.compare(Array(bytes.prefix(cut)), label: "large string truncation \(cut)") else { return } + } + var invalidUTF8 = bytes + invalidUTF8[20] = 0xff + invalidUTF8[21] = 0 + guard Self.compare(invalidUTF8, label: "large string with non-UTF8 and NUL") else { return } + } + Harness.expect(true, "Rust classifier preserves large plain/escaped strings, truncation, NUL and escape parity") + let empty = UnsafeRawBufferPointer(start: nil, count: 0) + Harness.expectEqual(JSONLogClassifier.topLevelType(in: empty), .indeterminate, "Rust top-level accepts null empty buffer") + Harness.expectEqual(JSONLogClassifier.payloadType(in: empty), .indeterminate, "Rust payload accepts null empty buffer") + } + + private static func compare(_ bytes: [UInt8], label: String) -> Bool { + bytes.withUnsafeBytes { buffer in + let top = JSONLogClassifier.topLevelType(in: buffer) + let payload = JSONLogClassifier.payloadType(in: buffer) + let expectedTop = SwiftJSONLogClassifier.topLevelType(in: buffer) + let expectedPayload = SwiftJSONLogClassifier.payloadType(in: buffer) + let topMatches: Bool + switch (top, expectedTop) { + case (nil, nil), (.assistant?, .assistant?), (.eventMessage?, .eventMessage?), + (.turnContext?, .turnContext?), (.other?, .other?), (.indeterminate?, .indeterminate?): + topMatches = true + default: topMatches = false + } + let payloadMatches: Bool + switch (payload, expectedPayload) { + case (nil, nil), (.threadSettingsApplied?, .threadSettingsApplied?), + (.tokenCount?, .tokenCount?), (.other?, .other?), (.indeterminate?, .indeterminate?): + payloadMatches = true + default: payloadMatches = false + } + let matches = topMatches && payloadMatches + if !matches { + Harness.expect(false, "classifier differential mismatch at \(label): \(bytes.prefix(100)); " + + "top \(String(describing: top))/\(String(describing: expectedTop)), " + + "payload \(String(describing: payload))/\(String(describing: expectedPayload))") + } + return matches + } + } + + private struct Generator { + private var state: UInt64 = 0x71626a736f6e + + mutating func next() -> UInt64 { + state = state &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407 + return state + } + } +} diff --git a/Sources/QuotaBarTests/RustAggregationTests.swift b/Sources/QuotaBarTests/RustAggregationTests.swift new file mode 100644 index 0000000..484b9ae --- /dev/null +++ b/Sources/QuotaBarTests/RustAggregationTests.swift @@ -0,0 +1,207 @@ +import CQuotaBarRust +import Foundation +@testable import QuotaBarCore + +/// The same Swift day dictionary feeds both folds so cross-day floating-point order is identical. +enum RustAggregationTests { + private static let now = Date(timeIntervalSince1970: 1_795_000_000) + private static let today = "2026-11-30" + + static func run() { + Harness.expectEqual(MemoryLayout.size, 40, "aggregation token ABI size") + Harness.expectEqual(MemoryLayout.size, 88, "aggregation row ABI size") + Harness.expectEqual(MemoryLayout.size, 72, "aggregation group ABI size") + Harness.expectEqual(MemoryLayout.size, 64, "aggregation day ABI size") + Harness.expectEqual(MemoryLayout.size, 24, "aggregation model ABI size") + Harness.expectEqual(MemoryLayout.alignment, 8, "aggregation row ABI alignment") + Harness.expectEqual(MemoryLayout.alignment, 8, "aggregation group ABI alignment") + Harness.expectEqual(MemoryLayout.alignment, 8, "aggregation day ABI alignment") + Harness.expectEqual(MemoryLayout.alignment, 8, "aggregation model ABI alignment") + var failures: [String] = [] + var cases = 0 + func check(_ rows: [PricedUsage], _ name: String, tab: UsageTab = .pi, + allowedTies: Set? = nil) { + cases += 1 + let grouped = Dictionary(grouping: rows, by: \.day) + let swift = Result { try SwiftCostAggregator.aggregate(groupedRows: grouped, tab: tab, + todayKey: Self.today, now: Self.now) } + let rust = Result { try CostAggregator.aggregate(groupedRows: grouped, tab: tab, + todayKey: Self.today, now: Self.now) } + switch (swift, rust) { + case let (.success(expected), .success(actual)): + if !Self.equal(actual, expected, allowedTies: allowedTies) { failures.append(name) } + case let (.failure(expected), .failure(actual)): + if expected.localizedDescription != actual.localizedDescription { failures.append(name + " error text") } + default: failures.append(name + " success/error") + } + } + check([], "empty") + check([Self.row()], "single") + check([Self.row(model: "unknown"), Self.row(model: "")], "unknown names") + check([Self.row(cost: nil)], "unpriced") + check([Self.row(cost: 0), Self.row(model: "other", cost: nil)], "nil versus zero", allowedTies: ["model", "other"]) + check([Self.row(cost: -0.0)], "signed zero") + check([Self.row(cost: nil, unpriced: 1), Self.row(cost: 0, fallback: true)], "partial fallback") + check([Self.row(unpriced: -1)], "nonpositive unpriced is ignored") + let composed = "caf\u{e9}" + let decomposed = "cafe\u{301}" + for _ in 0..<16 { + check([Self.row(day: "2026-11-01", model: composed, cost: nil), + Self.row(day: "2026-11-02", model: decomposed)], "Unicode model spelling per day and top model") + check([Self.row(model: composed, cost: nil), Self.row(model: decomposed)], + "first token key differs from first cost key") + check([Self.row(model: decomposed, cost: nil), Self.row(model: composed)], + "reversed first token and cost spellings") + check([Self.row(day: "2026-11-01", upstream: composed), + Self.row(day: "2026-11-02", upstream: decomposed)], "Unicode upstream spelling per day") + } + for tab in UsageTab.allCases { + for count in [1, 7, 30] { + check(Self.fixture(days: count, models: 7), "mixed sources and tiers \(tab) \(count)", tab: tab) + } + } + for index in 0..<64 { + // Non-dyadic and extreme finite costs exercise exact accumulation bits and overflow. + let exponent = index % 4 == 0 ? 900 : (index % 4 == 1 ? -1_020 : 0) + let cost = Double(sign: .plus, exponent: exponent, significand: Double(index + 1) / 7) + check((0..<12).map { row in + Self.row(day: "2026-11-\(row % 3 + 10)", model: "model-\(row % 4)", + tokens: TokenTotals(input: (index + 1) * (row + 1)), + cost: cost * Double(row + 1) / 3) + }, "floating order \(index)") + } + for index in 0..<6 { + var tokens = TokenTotals(input: 1, output: 1, cacheWrite: 1, cacheWrite1h: 1, cacheRead: 1) + switch index { + case 0: tokens.input = -1 + case 1: tokens.output = -1 + case 2: tokens.cacheWrite = -1 + case 3: tokens.cacheWrite1h = -1 + case 4: tokens.cacheRead = -1 + default: tokens.cacheWrite1h = 2 + } + check([Self.row(tokens: tokens)], "invalid count \(index)") + } + check([Self.row(tokens: TokenTotals(input: Int.max, output: 1))], "invalid source total") + check([Self.row(tokens: TokenTotals(input: Int.max)), Self.row(tokens: TokenTotals(input: 1))], + "same-bucket token overflow") + check([Self.row(tokens: TokenTotals(input: Int.max)), Self.row(tokens: TokenTotals(output: 1))], + "cross-bucket total overflow") + check([Self.row(tokens: TokenTotals(input: Int.max)), Self.row(model: "other", tokens: TokenTotals(output: 1))], + "day cross-model total overflow") + check([Self.row(day: "2026-11-01", tokens: TokenTotals(input: Int.max)), + Self.row(day: "2026-11-02", tokens: TokenTotals(input: 1))], "model window overflow") + check([Self.row(day: "2026-11-01", tokens: TokenTotals(input: Int.max)), + Self.row(day: "2026-11-02", model: "other", tokens: TokenTotals(input: 1))], "window total overflow") + check([Self.row(unpriced: Int.max), Self.row(unpriced: 1)], "unpriced total overflow") + for cost in [-1.0, Double.nan, Double.infinity, -Double.infinity] { + check([Self.row(cost: cost)], "invalid cost \(cost)") + } + check([Self.row(cost: .greatestFiniteMagnitude), Self.row(cost: .greatestFiniteMagnitude)], "day cost overflow") + check([Self.row(day: "2026-11-01", cost: .greatestFiniteMagnitude), + Self.row(day: "2026-11-02", cost: .greatestFiniteMagnitude)], "model cost overflow") + check([Self.row(day: "2026-11-01", model: "a", cost: .greatestFiniteMagnitude), + Self.row(day: "2026-11-02", model: "b", cost: .greatestFiniteMagnitude)], "window cost overflow") + // The original Set.max comparator deliberately has no tertiary ordering for exact ties. + for _ in 0..<32 { + check([Self.row(model: "a"), Self.row(model: "b")], "top cost/token tie", allowedTies: ["a", "b"]) + check([Self.row(model: "a", cost: nil), Self.row(model: "b", cost: nil)], + "top unpriced tie", allowedTies: ["a", "b"]) + } + check([Self.row(model: "a", tokens: TokenTotals(input: 1)), + Self.row(model: "b", tokens: TokenTotals(input: 2))], "top cost tie breaks by tokens") + Harness.expectEqual(failures.count, 0, "Rust aggregation preserves snapshots, double bits and error text across \(cases) cases: \(failures.prefix(5))") + if ProcessInfo.processInfo.environment["QUOTABAR_BENCH_AGGREGATION"] == "1" { Self.benchmark() } + } + + private static func row(day: String = today, model: String = "model", source: CostUsageSource = .piAgent, + fast: Bool = false, long: Bool = false, upstream: String? = nil, + tokens: TokenTotals = TokenTotals(input: 1), cost: Double? = 1, + unpriced: Int = 0, fallback: Bool = false) -> PricedUsage { + PricedUsage(key: .init(day: day, source: source, model: model, longContext: long, isFast: fast, + upstreamProvider: upstream, pricingProvider: .codex), + tokens: tokens, cost: cost, unpricedTokens: unpriced, usesLoggedCostFallback: fallback) + } + + private static func fixture(days: Int, models: Int) -> [PricedUsage] { + var rows: [PricedUsage] = [] + for day in 1...days { + for model in 0..?) -> Bool { + let normalized = CostSnapshot(tab: actual.tab, days: actual.days, todayCostUSD: actual.todayCostUSD, + windowCostUSD: actual.windowCostUSD, latestTokens: actual.latestTokens, windowTokens: actual.windowTokens, + topModel: expected.topModel, hasUnpricedTokens: actual.hasUnpricedTokens, scannedAt: actual.scannedAt) + guard normalized == expected, + actual.todayCostUSD.bitPattern == expected.todayCostUSD.bitPattern, + actual.windowCostUSD.bitPattern == expected.windowCostUSD.bitPattern else { return false } + if let allowedTies { + guard let actualTop = actual.topModel, let expectedTop = expected.topModel, + allowedTies.contains(actualTop), allowedTies.contains(expectedTop) else { return false } + } else if actual.topModel.map({ Array($0.utf8) }) != expected.topModel.map({ Array($0.utf8) }) { return false } + for (a, b) in zip(actual.days, expected.days) { + guard a.costUSD?.bitPattern == b.costUSD?.bitPattern, + Array(a.dayKey.utf8) == Array(b.dayKey.utf8) else { return false } + for (key, usage) in a.byModel { + guard usage.costUSD?.bitPattern == b.byModel[key]?.costUSD?.bitPattern, + let expectedKey = b.byModel.keys.first(where: { $0 == key }), + Array(key.model.utf8) == Array(expectedKey.model.utf8), + key.upstreamProvider.map({ Array($0.utf8) }) == expectedKey.upstreamProvider.map({ Array($0.utf8) }) + else { return false } + } + } + return true + } + + private static func benchmark() { + @inline(never) + func measure(_ rows: [PricedUsage], rust: Bool, repeats: Int) throws -> (Double, Double) { + let start = DispatchTime.now().uptimeNanoseconds + var checksum = 0.0 + for _ in 0..= 2 { + rust.append(run % 2 == 0 ? first.0 : second.0) + swift.append(run % 2 == 0 ? second.0 : first.0) + } + } + print("aggregation_benchmark rows=\(rows.count) swift_ms=\(swift) rust_ffi_ms=\(rust)") + print("aggregation_benchmark rows=\(rows.count) swift_median_ms=\(swift.sorted()[3]) rust_ffi_median_ms=\(rust.sorted()[3])") + } + } catch { Harness.expect(false, "aggregation benchmark failed: \(error)") } + } +} diff --git a/Sources/QuotaBarTests/RustClaudeParserTests.swift b/Sources/QuotaBarTests/RustClaudeParserTests.swift new file mode 100644 index 0000000..9e4ae3a --- /dev/null +++ b/Sources/QuotaBarTests/RustClaudeParserTests.swift @@ -0,0 +1,83 @@ +import Foundation +import QuotaBarCore +import CQuotaBarRust + +/// The original decoder defines compatibility; adversarial input must not change recorded usage. +enum RustClaudeParserTests { + static func run() { + Harness.expect(MemoryLayout.size == 152 && MemoryLayout.alignment == 8 + && MemoryLayout.offset(of: \.counts) == 8 + && MemoryLayout.offset(of: \.timestamp) == 48 + && MemoryLayout.offset(of: \.owner) == 144, + "C-imported Claude result matches the Rust ABI layout") + let reference = ClaudeCodeAdapter.FoundationParser() + let candidate = ClaudeCodeAdapter.Parser() + var samples: [Data] = [] + func append(_ text: String) { samples.append(Data(text.utf8)) } + let prefix = #"{"type":"assistant","timestamp":"2026-11-16T12:34:56.123Z","requestId":"r","message":{"id":"m","model":"claude-test","usage":UREPLACE}}"# + let scalars = ["null", "true", "false", "0", "1", "-1", "1.9", "-0.9", "1e20", "1e308", "1e999", + "9007199254740993.0", "9007199254740993e0", "9.21e18", "-9.21e18", + "9223372036854775807", "9223372036854775808", #""+123""#, #"" 123""#, + #""123 ""#, #""00012""#, #""-1""#, #""1.5""#, #""9223372036854775808""#, + "{}", "[]", "[1]", #"{"unexpected":1}"#] + for field in ["input_tokens", "output_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"] { + for scalar in scalars { + let defaults = ["input_tokens", "output_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"] + .filter { $0 != field }.map { "\"\($0)\":10" }.joined(separator: ",") + append(prefix.replacingOccurrences(of: "UREPLACE", with: "{\(defaults),\"\(field)\":\(scalar)}")) + } + } + for usage in ["null", "[]", "[1]", "false", "1", "{}", + #"{"input_tokens":1,"input_tokens":2}"#, + #"{"input_tokens":1,"input_\u0074okens":2}"#, + #"{"input_tokens":9223372036854775807,"output_tokens":1}"#, + #"{"input_tokens":1,"cache_creation_input_tokens":3,"cache_creation":[2]}"#, + #"{"input_tokens":1,"cache_creation_input_tokens":3,"cache_creation":{"ephemeral_1h_input_tokens":-1}}"#] { + append(prefix.replacingOccurrences(of: "UREPLACE", with: usage)) + } + let valid = prefix.replacingOccurrences(of: "UREPLACE", with: #"{"input_tokens":10,"output_tokens":2,"speed":"FAST"}"#) + append(valid) + append(valid.replacingOccurrences(of: #""type":"assistant""#, with: #""type":"user","type":"assistant""#)) + append(valid.replacingOccurrences(of: "claude-test", with: #"\u0000model"#)) + append(valid.replacingOccurrences(of: "claude-test", with: #" \n\u00a0claude-test\t"#)) + append(valid.replacingOccurrences(of: "claude-test", with: "")) + append(valid.replacingOccurrences(of: #""requestId":"r","message":{"id":"m","model""#, + with: #""message":{"model""#)) + append(valid.replacingOccurrences(of: #""requestId":"r","message":{"id":"m","model""#, + with: #""uuid":"u\u0000id","message":{"model""#)) + append(valid.replacingOccurrences(of: "2026-11-16T12:34:56.123Z", with: "2026-11-16T20:34:56.123456789+08:00")) + if let object = try? JSONSerialization.jsonObject(with: Data(valid.utf8)), + let pretty = try? JSONSerialization.data(withJSONObject: object, options: [.prettyPrinted, .sortedKeys]) { + samples.append(pretty) + } + samples.append(Data([0xff, 0xfe, 0x7b, 0x7d])) + for end in valid.utf8.indices { + samples.append(Data(valid.utf8[..> 32) + samples.append(Data(bytes)) + } + var failures = 0 + for (index, sample) in samples.enumerated() { + let equal = sample.withUnsafeBytes { reference.request(in: $0) == candidate.request(in: $0) } + if !equal { + failures += 1 + Harness.expect(false, "Rust Claude parser differs from Foundation at fixture \(index)") + } + } + Harness.expect(failures == 0, "Rust Claude parser matches Foundation for \(samples.count) adversarial buffers") + + var output = QBClaudeResult() + qb_claude_decode(nil, 1, &output) + Harness.expect(output.status == QB_CLAUDE_INVALID_ARGUMENT && output.owner == nil, + "invalid FFI input reports failure without an owned allocation") + qb_claude_free(output.owner) + } +} diff --git a/Sources/QuotaBarTests/RustLogFileScannerTests.swift b/Sources/QuotaBarTests/RustLogFileScannerTests.swift new file mode 100644 index 0000000..964e783 --- /dev/null +++ b/Sources/QuotaBarTests/RustLogFileScannerTests.swift @@ -0,0 +1,83 @@ +import Foundation +import QuotaBarCore + +/// Differential checks include callback order, full borrowed bytes and consumed offsets. +enum RustLogFileScannerTests { + private struct Result: Equatable { + var inspected: [Data] = [] + var handled: [Data] = [] + var consumed: Int64? + var error: Int? + } + + static func run() { + let root = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("quotabar-rust-lines-\(UUID().uuidString)") + do { + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + let path = root.appendingPathComponent("synthetic 空格.jsonl") + let chunk = 1 << 20 + let fixtures: [Data] = [ + Data(), Data("\n\nshort\r\npartial".utf8), + Data(repeating: 97, count: chunk - 1) + Data("\nnext\n".utf8), + Data(repeating: 97, count: chunk) + Data("\nnext\n".utf8), + Data(repeating: 97, count: chunk * 2 + 31) + Data("\n\nlast\npartial".utf8), + Data(repeating: 33, count: chunk * 3 + 71) + Data("\naccepted\npartial".utf8), + Data([0, 255, 128, 10, 97, 0, 10]), + ] + for (index, bytes) in fixtures.enumerated() { + try bytes.write(to: path) + let bounds: [(Int64, Int64?)] = [ + (0, nil), (0, 0), (-1, 7), (0, -1), (0, 65_536), + (0, Int64(chunk)), (0, Int64(chunk + 1)), + (0, Int64(bytes.count)), (1, nil), + (Int64(chunk - 1), nil), (Int64(bytes.count + 10), nil), + ] + for (offset, limit) in bounds { + let swift = self.scan(path, offset: offset, limit: limit, rust: false) + let rust = self.scan(path, offset: offset, limit: limit, rust: true) + Harness.expectEqual(rust, swift, "Rust line scan matches Swift fixture \(index), offset \(offset), limit \(String(describing: limit))") + } + } + for url in [root, root.appendingPathComponent("missing.jsonl")] { + Harness.expectEqual( + self.scan(url, offset: 0, limit: nil, rust: true), + self.scan(url, offset: 0, limit: nil, rust: false), + "Rust line scan preserves open/read errors without callbacks" + ) + } + try Data("outer\ninner\n".utf8).write(to: path) + var outerCount = 0 + var nestedCount = 0 + let offset = try LogFileScanner.readLines(of: path, from: 0) { _ in + outerCount += 1 + _ = try? LogFileScanner.readLines(of: path, from: 0) { _ in nestedCount += 1 } + } + Harness.expectEqual(offset, 12, "Rust callback reentry preserves outer cursor") + Harness.expectEqual(outerCount, 2, "Rust callback reentry keeps outer records") + Harness.expectEqual(nestedCount, 4, "Rust callback contexts remain independent") + } catch { + Harness.expect(false, "Rust scanner fixture setup failed: \(error)") + } + } + + private static func scan(_ path: URL, offset: Int64, limit: Int64?, rust: Bool) -> Result { + var result = Result() + let wanted: (UnsafeRawBufferPointer) -> Bool = { bytes in + result.inspected.append(Data(bytes)) + return bytes.first != 33 + } + let handle: (UnsafeRawBufferPointer) -> Void = { bytes in result.handled.append(Data(bytes)) } + do { + if rust { + result.consumed = try LogFileScanner.readLines(of: path, from: offset, upTo: limit, + where: wanted, handle: handle) + } else { + result.consumed = try SwiftLogLineScanner.readLines(of: path, from: offset, upTo: limit, + where: wanted, handle: handle) + } + } catch { result.error = (error as NSError).code } + return result + } +} diff --git a/Sources/QuotaBarTests/RustPricingTests.swift b/Sources/QuotaBarTests/RustPricingTests.swift new file mode 100644 index 0000000..bc37c37 --- /dev/null +++ b/Sources/QuotaBarTests/RustPricingTests.swift @@ -0,0 +1,193 @@ +import Foundation +@testable import QuotaBarCore + +/// Differential checks retain the original Swift implementation as a test-only billing oracle. +enum RustPricingTests { + private struct Sample { + let pricing: ModelPricing + let tokens: TokenTotals + let longContext: Bool + } + + private struct Generator { + var state: UInt64 = 0x8f13_37a2_e9b0_c456 + mutating func next() -> UInt64 { + self.state = self.state &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407 + return self.state + } + } + + static func run() { + var samples: [Sample] = [] + let counts = [0, 1, 999_999, 1_000_000, 9_007_199_254_740_991, + 9_007_199_254_740_992, 9_007_199_254_740_993, Int.max] + // All optional masks include absent/present thresholds and both context tiers. + for mask in 0..<512 { + let rates = (0..<10).map { Double($0 + 1) / 8 } + for longContext in [false, true] { + var tokens = TokenTotals(input: counts[mask % counts.count], output: 17, + cacheWrite: counts[(mask + 1) % counts.count], cacheWrite1h: 1, cacheRead: 113) + // Public mutable fields can exceed their initializer's one-hour clamp. + if mask % 3 == 0 { tokens.cacheWrite1h = Int.max } + samples.append(Sample(pricing: Self.pricing(rates, mask: mask, threshold: 200_000), + tokens: tokens, longContext: longContext)) + } + } + let boundaries: [Double] = [0, -0.0, -1, .infinity, -.infinity, .nan, + .leastNonzeroMagnitude, .leastNormalMagnitude, .greatestFiniteMagnitude, + Double.greatestFiniteMagnitude / 2, (Double.greatestFiniteMagnitude / 2).nextUp] + for index in 0..<10 { + for rate in boundaries { + for mask in [0, 256, 511, 509, 447, 445] { + var rates = Array(repeating: 1.0, count: 10) + rates[index] = rate + for longContext in [false, true] { + samples.append(Sample(pricing: Self.pricing(rates, mask: mask, threshold: 200_000), + tokens: TokenTotals(input: Int.max, output: 3, cacheWrite: Int.max, + cacheWrite1h: 123, cacheRead: 17), longContext: longContext)) + } + } + } + } + for threshold in [Int.min, -1, 0, 1, 999_999_999_999, 1_000_000_000_000, Int.max] { + samples.append(Sample(pricing: Self.pricing(Array(repeating: 1, count: 10), + mask: 511, threshold: threshold), tokens: TokenTotals(input: 1), longContext: true)) + } + for longContext in [false, true] { + samples.append(Sample(pricing: Self.pricing( + Array(repeating: Double.greatestFiniteMagnitude / 4, count: 10), + mask: 511, threshold: 200_000), + tokens: TokenTotals(input: 8, output: 8, cacheWrite: 8, cacheWrite1h: 4, cacheRead: 8), + longContext: longContext)) + } + for index in 0..<5 { + var tokens = TokenTotals(input: 1, output: 1, cacheWrite: 1, cacheWrite1h: 1, cacheRead: 1) + switch index { + case 0: tokens.input = -1 + case 1: tokens.output = -1 + case 2: tokens.cacheWrite = -1 + case 3: tokens.cacheWrite1h = -1 + default: tokens.cacheRead = -1 + } + samples.append(Sample(pricing: ModelPricing(input: 1, output: 1), tokens: tokens, + longContext: false)) + } + var generator = Generator() + for index in 0..<4_096 { + // Alternate unrestricted IEEE values with positive finite values, including subnormals. + let rates = (0..<10).map { _ -> Double in + let bits = generator.next() + return Double(bitPattern: index % 2 == 0 ? bits : bits & 0x7fef_ffff_ffff_ffff) + } + let mask = Int(generator.next() % 512) + let tokens = TokenTotals(input: Int(generator.next() & 0x7fff_ffff_ffff_ffff), + output: Int(generator.next() % 1_000_000), cacheWrite: Int(generator.next() % 1_000_000), + cacheWrite1h: Int(generator.next() % 1_000_000), cacheRead: Int(generator.next() % 1_000_000)) + samples.append(Sample(pricing: Self.pricing(rates, mask: mask, threshold: 200_000), + tokens: tokens, longContext: index % 2 == 0)) + } + var costMismatches = 0 + var rateMismatches = 0 + for sample in samples { + let actual = sample.pricing.cost(for: sample.tokens, longContext: sample.longContext) + let expected = Self.swiftCost(sample.pricing, for: sample.tokens, longContext: sample.longContext) + if actual?.bitPattern != expected?.bitPattern { costMismatches += 1 } + let actualRate = sample.pricing.cacheWrite1hRate(longContext: sample.longContext) + let expectedRate = Self.swiftOneHour(sample.pricing, longContext: sample.longContext) + if actualRate.bitPattern != expectedRate.bitPattern { rateMismatches += 1 } + } + Harness.expectEqual(costMismatches, 0, "Rust pricing matches Swift optional results and exact double bits across \(samples.count) cases") + Harness.expectEqual(rateMismatches, 0, "Rust one-hour fallback matches Swift exact double bits across \(samples.count) cases") + if ProcessInfo.processInfo.environment["QUOTABAR_BENCH_PRICING"] == "1" { + Self.benchmark() + } + } + + private static func pricing(_ rates: [Double], mask: Int, threshold: Int) -> ModelPricing { + func optional(_ bit: Int) -> Double? { mask & (1 << bit) != 0 ? rates[bit + 2] : nil } + return ModelPricing(input: rates[0], output: rates[1], cacheWrite: optional(0), + cacheWrite1h: optional(1), cacheRead: optional(2), + thresholdTokens: mask & 256 != 0 ? threshold : nil, + inputAbove: optional(3), outputAbove: optional(4), cacheWriteAbove: optional(5), + cacheWrite1hAbove: optional(6), cacheReadAbove: optional(7)) + } + + private static func swiftOneHour(_ pricing: ModelPricing, longContext: Bool) -> Double { + let inputRate = (longContext ? pricing.inputAbove : nil) ?? pricing.input + if let stated = (longContext ? pricing.cacheWrite1hAbove : nil) ?? pricing.cacheWrite1h { return stated } + return inputRate * ModelPricing.oneHourCacheWriteMultiplier + } + + /// Original production validation and arithmetic, including the full RateRules dictionary path. + @inline(never) + private static func swiftCost(_ pricing: ModelPricing, for totals: TokenTotals, longContext: Bool) -> Double? { + guard ModelPricing.violations(in: pricing.rateValues).isEmpty, + [totals.input, totals.output, totals.cacheWrite, totals.cacheWrite1h, totals.cacheRead] + .allSatisfy({ $0 >= 0 }) else { return nil } + let million = 1_000_000.0 + let inputRate = (longContext ? pricing.inputAbove : nil) ?? pricing.input + let outputRate = (longContext ? pricing.outputAbove : nil) ?? pricing.output + // A model without its own cache rates bills cached tokens at the full input rate. + let cacheWriteRate = (longContext ? pricing.cacheWriteAbove : nil) ?? pricing.cacheWrite ?? inputRate + let cacheReadRate = (longContext ? pricing.cacheReadAbove : nil) ?? pricing.cacheRead ?? inputRate + + // Anthropic prices a one-hour cache write at twice the input rate, against 1.25x for the + // five-minute default. The table's cache-write column is the five-minute rate, so unless + // a one-hour rate is stated the longer TTL is derived the way Anthropic publishes it. + let write1h = Double(min(totals.cacheWrite1h, totals.cacheWrite)) + let write5m = Double(totals.cacheWrite) - write1h + + let millionths = Double(totals.input) * inputRate + + Double(totals.output) * outputRate + + write5m * cacheWriteRate + + write1h * Self.swiftOneHour(pricing, longContext: longContext) + + Double(totals.cacheRead) * cacheReadRate + if millionths.isFinite { return millionths / million } + + // Retry in USD when the intermediate amount in millionths overflowed. + // This preserves the existing rounding for ordinary rates. + let cost = Double(totals.input) * (inputRate / million) + + Double(totals.output) * (outputRate / million) + + write5m * (cacheWriteRate / million) + + write1h * (Self.swiftOneHour(pricing, longContext: longContext) / million) + + Double(totals.cacheRead) * (cacheReadRate / million) + return cost.isFinite && cost >= 0 ? cost : nil + } + + private static func benchmark() { + let samples = (0..<1_024).map { index in + Sample(pricing: Self.pricing((0..<10).map { Double($0 + 1) * 0.25 }, + mask: 256 | (index % 256), threshold: 200_000), + tokens: TokenTotals(input: index * 171, output: index * 31, + cacheWrite: index * 83, cacheWrite1h: index * 41, cacheRead: index * 193), + longContext: index % 2 == 0) + } + @inline(never) + func measure(rust: Bool) -> (Double, Double) { + let start = DispatchTime.now().uptimeNanoseconds + var total = 0.0 + for _ in 0..<8 { + for sample in samples { + let cost = rust + ? sample.pricing.cost(for: sample.tokens, longContext: sample.longContext) + : Self.swiftCost(sample.pricing, for: sample.tokens, longContext: sample.longContext) + total += cost ?? 0 + } + } + return (Double(DispatchTime.now().uptimeNanoseconds - start) / 1_000_000, total) + } + var swiftTimes: [Double] = [] + var rustTimes: [Double] = [] + for iteration in 0..<9 { + let first = measure(rust: iteration % 2 == 0) + let second = measure(rust: iteration % 2 != 0) + Harness.expectEqual(first.1.bitPattern, second.1.bitPattern, "pricing benchmark preserves checksum") + if iteration >= 2 { + rustTimes.append(iteration % 2 == 0 ? first.0 : second.0) + swiftTimes.append(iteration % 2 == 0 ? second.0 : first.0) + } + } + print("pricing_benchmark calls_per_run=8192 swift_ms=\(swiftTimes) rust_ffi_ms=\(rustTimes)") + print("pricing_benchmark swift_median_ms=\(swiftTimes.sorted()[3]) rust_ffi_median_ms=\(rustTimes.sorted()[3])") + } +} diff --git a/Sources/QuotaBarTests/SwiftCostAggregator.swift b/Sources/QuotaBarTests/SwiftCostAggregator.swift new file mode 100644 index 0000000..6f41e58 --- /dev/null +++ b/Sources/QuotaBarTests/SwiftCostAggregator.swift @@ -0,0 +1,106 @@ +import Foundation +@testable import QuotaBarCore + +/// Frozen pre-migration fold used only to verify the Rust implementation. +enum SwiftCostAggregator { + static func aggregate(groupedRows: [String: [PricedUsage]], tab: UsageTab, todayKey: String, now: Date) throws -> CostSnapshot { + var days: [CostDay] = [] + // Cost and token totals per model across the whole window, for picking the top model. + var modelCost: [String: Double] = [:] + var modelTokens: [String: Int] = [:] + var hasUnpriced = false + + for (dayKey, dayRows) in groupedRows { + var dayTokens: [ModelUsageKey: TokenTotals] = [:] + var checkedDayTokens = TokenTotals() + // Per-source/model cost stays nil until something prices it, so an unpriced row reads + // as "no price" in the breakdown rather than as zero dollars. + var dayCostByModel: [ModelUsageKey: Double] = [:] + var fallbacks: Set = [] + var dayCost = 0.0 + var dayPriced = false + var dayUnpricedTokens = 0 + + for row in dayRows { + let usageKey = ModelUsageKey(source: row.source, model: row.model, isFast: row.key.isFast, upstreamProvider: row.key.upstreamProvider) + dayTokens[usageKey] = try dayTokens[usageKey, default: TokenTotals()].adding(row.tokens) + checkedDayTokens = try checkedDayTokens.adding(row.tokens) + modelTokens[row.model] = try self.add(modelTokens[row.model, default: 0], row.tokens.total) + if row.usesLoggedCostFallback { fallbacks.insert(usageKey) } + if let cost = row.cost { + dayCost = try self.add(dayCost, cost) + dayPriced = true + dayCostByModel[usageKey] = try self.add(dayCostByModel[usageKey, default: 0], cost) + modelCost[row.model] = try self.add(modelCost[row.model, default: 0], cost) + } + if row.unpricedTokens > 0 { + hasUnpriced = true + dayUnpricedTokens = try self.add(dayUnpricedTokens, row.unpricedTokens) + } + } + + let byModel = Dictionary(uniqueKeysWithValues: dayTokens.lazy.map { key, tokens in + (key, ModelDayUsage(tokens: tokens, costUSD: dayCostByModel[key], usesLoggedCostFallback: fallbacks.contains(key))) + }) + + days.append(CostDay( + dayKey: dayKey, + byModel: byModel, + costUSD: dayPriced ? dayCost : nil, + unpricedTokens: dayUnpricedTokens + )) + } + + days.sort { $0.dayKey < $1.dayKey } + + var todayCostUSD = 0.0 + var windowCostUSD = 0.0 + var latestTokens = 0 + var windowTokens = 0 + for day in days { + if day.dayKey == todayKey { todayCostUSD = day.costUSD ?? 0 } + windowCostUSD = try self.add(windowCostUSD, day.costUSD ?? 0) + latestTokens = day.tokens.total + windowTokens = try self.add(windowTokens, latestTokens) + } + + return CostSnapshot( + tab: tab, + days: days, + todayCostUSD: todayCostUSD, + windowCostUSD: windowCostUSD, + latestTokens: latestTokens, + windowTokens: windowTokens, + topModel: Self.topModel(cost: modelCost, tokens: modelTokens), + hasUnpricedTokens: hasUnpriced, + scannedAt: now + ) + } + + private static func add(_ lhs: Int, _ rhs: Int) throws -> Int { + let result = lhs.addingReportingOverflow(rhs) + guard lhs >= 0, rhs >= 0, !result.overflow else { + throw RecordedUsageReaderError.invalidData("window token total exceeds the integer range") + } + return result.partialValue + } + + private static func add(_ lhs: Double, _ rhs: Double) throws -> Double { + let result = lhs + rhs + guard lhs >= 0, rhs >= 0, result.isFinite else { + throw RecordedUsageReaderError.invalidData("aggregate cost is not finite and nonnegative") + } + return result + } + + /// Highest accumulated cost wins; models with equal cost fall back to token count. + static func topModel(cost: [String: Double], tokens: [String: Int]) -> String? { + let candidates = Set(cost.keys).union(tokens.keys).filter(RateCard.isKnown) + return candidates.max { lhs, rhs in + let lhsCost = cost[lhs] ?? 0 + let rhsCost = cost[rhs] ?? 0 + if lhsCost != rhsCost { return lhsCost < rhsCost } + return (tokens[lhs] ?? 0) < (tokens[rhs] ?? 0) + } + } +} diff --git a/Sources/QuotaBarTests/SwiftJSONLogClassifier.swift b/Sources/QuotaBarTests/SwiftJSONLogClassifier.swift new file mode 100644 index 0000000..9f34cd7 --- /dev/null +++ b/Sources/QuotaBarTests/SwiftJSONLogClassifier.swift @@ -0,0 +1,216 @@ +import Darwin +import Foundation + +/// Pre-migration optimized Swift classifier, retained only as a differential-test oracle. +enum SwiftJSONLogClassifier { + enum TopLevelType: Equatable { + case assistant + case eventMessage + case indeterminate + case turnContext + case other + } + + enum PayloadType: Equatable { + case indeterminate + case threadSettingsApplied + case tokenCount + case other + } + + private struct ParsedString { + let range: Range + let hasEscape: Bool + } + + private enum Lookup { + case found(Value) + case indeterminate + case missing + } + + private static let typeKey = Array("type".utf8) + private static let payloadKey = Array("payload".utf8) + private static let assistant = Array("assistant".utf8) + private static let eventMessage = Array("event_msg".utf8) + private static let turnContext = Array("turn_context".utf8) + private static let threadSettingsApplied = Array("thread_settings_applied".utf8) + private static let tokenCount = Array("token_count".utf8) + + static func topLevelType(in buffer: UnsafeRawBufferPointer) -> TopLevelType? { + let value: ParsedString + switch self.stringValue(for: Self.typeKey, inObjectAt: 0, buffer: buffer) { + case let .found(found): value = found + case .indeterminate: return .indeterminate + case .missing: return nil + } + if self.matches(value.range, Self.assistant, buffer: buffer) { return .assistant } + if self.matches(value.range, Self.eventMessage, buffer: buffer) { return .eventMessage } + if self.matches(value.range, Self.turnContext, buffer: buffer) { return .turnContext } + return .other + } + + static func payloadType(in buffer: UnsafeRawBufferPointer) -> PayloadType? { + let payload: Int + switch self.valueStart(for: Self.payloadKey, inObjectAt: 0, buffer: buffer) { + case let .found(found): payload = found + case .indeterminate: return .indeterminate + case .missing: return nil + } + let value: ParsedString + switch self.stringValue(for: Self.typeKey, inObjectAt: payload, buffer: buffer) { + case let .found(found): value = found + case .indeterminate: return .indeterminate + case .missing: return nil + } + if self.matches(value.range, Self.threadSettingsApplied, buffer: buffer) { return .threadSettingsApplied } + if self.matches(value.range, Self.tokenCount, buffer: buffer) { return .tokenCount } + return .other + } + + private static func stringValue( + for key: [UInt8], + inObjectAt objectStart: Int, + buffer: UnsafeRawBufferPointer + ) -> Lookup { + let start: Int + switch self.valueStart(for: key, inObjectAt: objectStart, buffer: buffer) { + case let .found(found): start = found + case .indeterminate: return .indeterminate + case .missing: return .missing + } + var index = start + guard let value = self.stringRange(at: &index, buffer: buffer) else { return .indeterminate } + return value.hasEscape ? .indeterminate : .found(value) + } + + private static func valueStart( + for key: [UInt8], + inObjectAt objectStart: Int, + buffer: UnsafeRawBufferPointer + ) -> Lookup { + var index = objectStart + self.skipWhitespace(at: &index, buffer: buffer) + guard index < buffer.count, buffer[index] == UInt8(ascii: "{") else { return .indeterminate } + index += 1 + + while index < buffer.count { + self.skipWhitespace(at: &index, buffer: buffer) + if index < buffer.count, buffer[index] == UInt8(ascii: "}") { return .missing } + guard let candidate = self.stringRange(at: &index, buffer: buffer) else { return .indeterminate } + if candidate.hasEscape { return .indeterminate } + self.skipWhitespace(at: &index, buffer: buffer) + guard index < buffer.count, buffer[index] == UInt8(ascii: ":") else { return .indeterminate } + index += 1 + self.skipWhitespace(at: &index, buffer: buffer) + if self.matches(candidate.range, key, buffer: buffer) { return .found(index) } + guard self.skipValue(at: &index, buffer: buffer) else { return .indeterminate } + self.skipWhitespace(at: &index, buffer: buffer) + if index < buffer.count, buffer[index] == UInt8(ascii: ",") { + index += 1 + continue + } + if index < buffer.count, buffer[index] == UInt8(ascii: "}") { return .missing } + return .indeterminate + } + return .indeterminate + } + + private static func stringRange( + at index: inout Int, + buffer: UnsafeRawBufferPointer + ) -> ParsedString? { + guard index < buffer.count, buffer[index] == UInt8(ascii: "\"") else { return nil } + index += 1 + let start = index + guard let base = buffer.baseAddress, + let quote = memchr(base.advanced(by: index), Int32(UInt8(ascii: "\"")), buffer.count - index) else { + return nil + } + let end = base.distance(to: quote) + // Skip long plain strings in libc. Escaped strings retain the scalar traversal: searching + // for every escaped quote separately regresses on densely escaped transcript content. + guard let escape = memchr(base.advanced(by: index), Int32(UInt8(ascii: "\\")), end - index) else { + index = end + 1 + return ParsedString(range: start.. Bool { + guard index < buffer.count else { return false } + if buffer[index] == UInt8(ascii: "\"") { + return self.stringRange(at: &index, buffer: buffer) != nil + } + if buffer[index] == UInt8(ascii: "{") || buffer[index] == UInt8(ascii: "[") { + var depth = 0 + while index < buffer.count { + let byte = buffer[index] + if byte == UInt8(ascii: "\"") { + guard self.stringRange(at: &index, buffer: buffer) != nil else { return false } + continue + } + if byte == UInt8(ascii: "{") || byte == UInt8(ascii: "[") { depth += 1 } + if byte == UInt8(ascii: "}") || byte == UInt8(ascii: "]") { + depth -= 1 + index += 1 + if depth == 0 { return true } + continue + } + index += 1 + } + return false + } + while index < buffer.count { + let byte = buffer[index] + if byte == UInt8(ascii: ",") || byte == UInt8(ascii: "}") || byte == UInt8(ascii: "]") + || byte == UInt8(ascii: " ") || byte == UInt8(ascii: "\t") + || byte == UInt8(ascii: "\r") || byte == UInt8(ascii: "\n") { + return true + } + index += 1 + } + return true + } + + private static func skipWhitespace(at index: inout Int, buffer: UnsafeRawBufferPointer) { + while index < buffer.count { + switch buffer[index] { + case UInt8(ascii: " "), UInt8(ascii: "\t"), UInt8(ascii: "\r"), UInt8(ascii: "\n"): + index += 1 + default: + return + } + } + } + + private static func matches( + _ range: Range, + _ expected: [UInt8], + buffer: UnsafeRawBufferPointer + ) -> Bool { + guard range.count == expected.count else { return false } + for offset in expected.indices where buffer[range.lowerBound + offset] != expected[offset] { + return false + } + return true + } +} diff --git a/Sources/QuotaBarTests/SwiftLogLineScanner.swift b/Sources/QuotaBarTests/SwiftLogLineScanner.swift new file mode 100644 index 0000000..5b8a643 --- /dev/null +++ b/Sources/QuotaBarTests/SwiftLogLineScanner.swift @@ -0,0 +1,90 @@ +import Darwin +import Foundation + +/// Frozen f8df98b streaming implementation used only to check the Rust boundary. +enum SwiftLogLineScanner { + private static let chunkSize = 1 << 20 + private static let longLinePrefixBytes = 64 * 1024 + static func readLines( + of url: URL, + from offset: Int64, + upTo limit: Int64? = nil, + where isWanted: (UnsafeRawBufferPointer) -> Bool = { _ in true }, + handle: (UnsafeRawBufferPointer) -> Void + ) throws -> Int64 { + let start = max(0, offset) + let descriptor = Darwin.open(url.path, O_RDONLY | O_CLOEXEC) + guard descriptor >= 0 else { throw Self.posixError() } + defer { _ = Darwin.close(descriptor) } + guard Darwin.lseek(descriptor, off_t(start), SEEK_SET) >= 0 else { + throw Self.posixError() + } + + var buffer = [UInt8](repeating: 0, count: Self.chunkSize) + var carry: [UInt8] = [] + var skippingLine = false + var readPosition = start + var consumed = start + + while limit.map({ readPosition < $0 }) ?? true { + let requested = limit.map { min(Int64(Self.chunkSize), max(0, $0 - readPosition)) } + ?? Int64(Self.chunkSize) + guard requested > 0 else { break } + + let count: Int = try buffer.withUnsafeMutableBytes { storage in + var result: Int + repeat { + result = Darwin.read(descriptor, storage.baseAddress, Int(requested)) + } while result < 0 && errno == EINTR + guard result >= 0 else { throw Self.posixError() } + return result + } + guard count > 0 else { break } + + let chunkStart = readPosition + buffer.withUnsafeBytes { storage in + guard let base = storage.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return } + var lineStart = 0 + while lineStart < count, + let found = memchr(base.advanced(by: lineStart), Int32(UInt8(ascii: "\n")), count - lineStart) { + let newline = base.distance(to: found.assumingMemoryBound(to: UInt8.self)) + let length = newline - lineStart + if skippingLine { + skippingLine = false + } else if carry.isEmpty { + let line = UnsafeRawBufferPointer(start: base.advanced(by: lineStart), count: length) + if length > 0, isWanted(line) { + // Handlers parse with Foundation, whose autoreleased objects would + // otherwise pile up until the scan's task finishes. + autoreleasepool { handle(line) } + } + } else { + carry.append(contentsOf: UnsafeBufferPointer(start: base.advanced(by: lineStart), count: length)) + if carry.withUnsafeBytes(isWanted) { + autoreleasepool { carry.withUnsafeBytes(handle) } + } + carry.removeAll(keepingCapacity: true) + } + consumed = chunkStart + Int64(newline) + 1 + lineStart = newline + 1 + } + if lineStart < count, !skippingLine { + let wasShort = carry.count < Self.longLinePrefixBytes + carry.append(contentsOf: UnsafeBufferPointer(start: base.advanced(by: lineStart), count: count - lineStart)) + if wasShort, carry.count >= Self.longLinePrefixBytes, !carry.withUnsafeBytes(isWanted) { + skippingLine = true + carry.removeAll(keepingCapacity: true) + } + } + } + readPosition += Int64(count) + } + + return consumed + } + + private static func posixError() -> POSIXError { + POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + +} diff --git a/Sources/QuotaBarTests/main.swift b/Sources/QuotaBarTests/main.swift index 32399ec..a884623 100644 --- a/Sources/QuotaBarTests/main.swift +++ b/Sources/QuotaBarTests/main.swift @@ -197,6 +197,11 @@ await UsageRecorderTests.run() await UsageBackupTests.run() await ModelNameResolutionTests.run() ClaudeCodeAdapterTests.run() +RustClaudeParserTests.run() +JSONLogClassifierDifferentialTests.run() +RustPricingTests.run() +RustLogFileScannerTests.run() +RustAggregationTests.run() CodexRecordingTests.run() CodexAdapterTests.run() SnapshotRecordingTests.run() diff --git a/docs/performance.md b/docs/performance.md index f3150de..832d668 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -1,5 +1,10 @@ # Performance validation +The subsequent Rust migration and its final Swift/Rust comparison are documented in +[rust-migration.md](rust-migration.md) and [rust-validation.md](rust-validation.md). +The measurements below describe the earlier Swift optimization and remain historical +evidence; they are not the migrated application's benchmark results. + ## Scope The November 2026 optimization removes redundant report reads and display publications, diff --git a/docs/rust-migration.md b/docs/rust-migration.md new file mode 100644 index 0000000..1264076 --- /dev/null +++ b/docs/rust-migration.md @@ -0,0 +1,222 @@ +# Rust core migration + +The migration follows an explicit product preference: Rust is acceptable at performance +parity, provided the existing macOS interface, saved history, accounting and operational +behavior remain compatible. It does not attempt to replace the application with Rust. +The reference Swift implementation is commit `f8df98b129c9138ccb9e6290d6c801e805fed1be`. + +## Implemented boundaries + +P1 boundaries are record decoding, classification and numeric pricing. P2 boundaries +are reduced-row aggregation and streaming: both passed differential validation, with +streaming accepted for product-chain parity under the user's language preference. + +| Boundary | Implementation and code evidence | Rationale and retained ownership | +| --- | --- | --- | +| Claude record decoding | `ClaudeCodeAdapter.swift:54` calls `Rust/QuotaBarCore/src/claude.rs` through a fixed C result | Rust decodes ordinary records. Foundation still handles dates, Unicode trimming/lowercasing and ID-less canonical hashing. Unsupported or ambiguous encodings fall back to the original parser before recording. | +| JSON prefix classification | `JSONLogClassifier.swift:4`, `Rust/QuotaBarCore/src/classifier.rs` | Preserve the loose classifier contract, including truncated prefixes. Bounded libc searches retain the existing long-string optimization; a pure scalar Rust version was rejected experimentally. | +| Numeric model pricing | `CostPricing.swift:93`, `Rust/QuotaBarCore/src/pricing.rs` | One call validates and prices a complete token bucket, preserving operation order, optional-rate fallback, long-context pricing and checked overflow. RateCard model lookup, aliases, dated price books, unknown-price policy and public validation diagnostics remain Swift. | +| Bounded line streaming | `LogFileScanner.swift:89`, `Rust/QuotaBarCore/src/scanner.rs` | The synchronous reader owns a descriptor and byte buffers. File discovery, stat/digest resume planning, parser state, cursor commits and transactions remain Swift. See the separate validation report for the callback cost and final acceptance evidence. | +| Priced-row aggregation | `CostAggregator.swift`, `Rust/QuotaBarCore/src/aggregation.rs` | Rust folds already-priced rows into per-key/day/model totals. Swift supplies day/key identities and reconstructs existing presentation models; Calendar, SQL, pricing policy and top-model tie policy remain Swift. | + +The C header is [QuotaBarRust.h](../Sources/CQuotaBarRust/include/QuotaBarRust.h). +Swift wrappers use pointers and lengths, fixed-width numeric fields and explicit optional +flags. There is no JSON serialization of results, Rust executor, asynchronous callback, +new database connection, schema migration, settings change or UI implementation change. + +## Compatibility and FFI + +Each Claude call borrows one complete JSON record, not an entire JSONL file. The returned +strings belong to one Rust allocation; Swift copies them before a deferred +`qb_claude_free`. Embedded NUL is retained through length-delimited strings. The original +Foundation decoder remains a production compatibility path for floating token spellings, +duplicate known keys, unsupported shapes, ID-less canonical hashes and decoder failures. +The differential tests compare all request fields, including identity, token buckets, +dates and Fast mode. They also exercise malformed/truncated records and arbitrary bytes. + +Rust pricing validates counts and rates before arithmetic and preserves the Swift +floating-point operation sequence. Its tests compare optional results and exact `Double` +bit patterns across 6,454 cases, including all 512 optional-field masks, random finite and +non-finite values, negative counts and overflow fallback. This does not replace tests of +price-book policy: those continue to exercise the unchanged Swift layer. + +Aggregation crosses once per already-priced snapshot. Swift interns day, model and +usage keys into numeric IDs and owns the input/output buffers. Rust preserves the +original group-token, day-token, model-total, day/group/model-cost and unpriced-total +check order. Swift supplies its actual day-dictionary traversal order, preserving +cross-day floating-point accumulation, then sorts the resulting days as before. +Error codes map back to the original `RecordedUsageReaderError.invalidData` messages. +Top-model selection stays Swift, including its existing unspecified winner when both +cost and token totals are exactly tied. No new alphabetical tie-break is introduced. +Both sides assert ABI layout; impossible slice lengths are rejected before borrowing. +Usage-key interning is local to each day, and model token/cost dictionaries retain their +separate first key spellings. This preserves Swift's original UTF-8 representation even +when two model or upstream strings are canonically equivalent Unicode spellings. The +additional tests compare raw UTF-8 bytes, because Swift string equality alone would +hide that difference. + +Scanner callbacks run synchronously on the actor's current thread. Their byte views and +context cannot escape. The Swift wrapper keeps callback closures alive, retains the +per-record autorelease pool, and releases its context after the call. No second worker +or cancellation model is introduced. The existing scanner is synchronous too: task +cancellation does not interrupt an in-progress file read; cancellation and publication +continue at the existing Swift orchestration boundaries. + +The crate uses `panic = "unwind"`. Allocation/parsing and scanner entry points contain +unwinding panics and return fallback/error status; they do not unwind through Swift. +The classifier and fixed-size arithmetic paths avoid fallible allocations and use +checked/bounded operations. Unit tests assert C/Rust layouts, and a Claude unit test +injects an unwinding panic. This is not process isolation: invalid caller pointers, +aborting panics and allocation aborts are not made recoverable by `catch_unwind`. + +The choice of a small static library follows the official [Rust linkage reference](https://doc.rust-lang.org/reference/linkage.html). +Panic limitations follow [the standard library contract](https://doc.rust-lang.org/std/panic/fn.catch_unwind.html). +The deployment target uses the documented [Apple target environment variable](https://doc.rust-lang.org/rustc/platform-support/apple-darwin.html). +These references were checked during implementation. UniFFI is unnecessary for this +small internal synchronous ABI; no claim is made that it is intrinsically slower. + +## Modules retained in Swift + +Technical feasibility alone does not establish a compatible, useful migration. These +modules are intentionally retained; absence of a Rust port is not a claim that Rust +could never implement them. +Codex/Pi/OpenCode decoders and additional price-book transforms are P3 investigation +candidates. Database ownership, application concurrency, platform integration and UI +are excluded from this migration; they are not pending implementation work. + +| Module | Evidence and reason | Future experiment or alternative | +| --- | --- | --- | +| `RateCard` / `PriceBook` / `PricedUsage` | `RateCard.swift:38,72,120` and `PricedUsage.swift:34`: dated aliases, context thresholds, Fast availability, unknown prices and Pi logged-cost fallback | Numeric pricing is extracted. Cache repeated resolved lookups within a pinned refresh before adding strings and price-book ownership to FFI. | +| Codex adapter | `CodexAdapter.swift:86,162`: running totals, replay suppression and persisted parser state | Current full-chain Codex comparison exercises migrated shared kernels. A separate decoder port must compare saved state after every record, not just final totals. | +| Pi / OpenCode adapters | `PiAgentAdapter.swift:102`, `OpenCodeAdapter.swift:44`: fork identity, logged costs and coherent SQLite snapshots | Profile changed-file rescans; Swift per-file caching may eliminate more work than replacing parsing. No Rust decoder performance claim is made. | +| `UsageRecorder`, `CostCache`, backups | `UsageRecorder.swift:482`, `CostCache.swift:85,615,623`: one writer, WAL/FULL durability, per-file row/cursor/state/manifest commits and history-preserving merges | SQLite remains the same native engine. Do not split transaction ownership merely to move wrappers. Existing backup, rollback, rewrite and deduplication tests remain mandatory. | +| `CostService`, `UsageStore`, report updates | `CostService.swift:65,75,94`; `UsageStore.swift:227,257`: remote-host lease across suspension, synchronous record/read, atomic cost/status publication then completion event | Preserve actors and MainActor ownership. Existing concurrent refresh/report tests verify event ordering and coalescing. | +| HTTP, credentials, Keychain, PTY, remote sync | `ClaudeCredentials.swift:5`, `ClaudePTYRunner.swift:213`, `CodexCredentialCache.swift:42`, `RemoteLogMirror.swift:197`: Apple-signed requester identity, process cancellation, refresh generations and rsync-in-place guards | Keep platform adapters. Moving these without changing platform calls has no demonstrated CPU benefit and increases cancellation/authorization risk. Tests use fake transports and processes; real credentials were not exercised. | +| SwiftUI / AppKit / WebKit / settings | `UsageStore.swift`, `SettingsStore.swift`, `UsageReportWindowController.swift` | Preserve the required product surface and platform ownership. No Rust UI rewrite is proposed. | + +## Build and maintenance + +Install full Xcode and rustup (for example `brew install rustup`, with its bin directory +on `PATH`). `rust-toolchain.toml` pins Rust; `Cargo.lock` pins dependencies. Normal entry +points remain: + +```sh +make test +make app +Scripts/swift.sh build -c release --product QuotaBarProbe +``` + +`Scripts/swift.sh` runs a locked Cargo build before Swift build/run/test commands and +links the matching debug or release archive. A digest-derived Swift compiler flag makes +Rust-only changes invalidate SwiftPM's link cache. The archive is statically linked; +app users do not install Rust or receive a separate Rust dylib. Both toolchains target +macOS 14. The current package/release path is a native arm64 build. Cross-architecture +and universal builds are not implemented by this wrapper. + +The link-cache behavior was also tested in a minimal isolated package using byte-for-byte +copies of the actual wrappers: changing only a Rust function from returning `1` to `2` +changed both archive/executable hashes and the program's output, without cleaning or +changing Swift/C sources. Raw evidence is in `build/rust-validation/relink-test.md`. + +The local release app was assembled with `make app`; its existing ad-hoc signature +passes `codesign --verify --deep --strict`. The executable is arm64, reports minimum +macOS 14.0, and has no Rust dylib dependency. Its size is 7,551,312 bytes; the complete +app is approximately 8.5 MiB and the local ZIP approximately 3.1 MiB. The local bundle +retains version `0.0.34` from the repository's release tag; this is an uncommitted +development build, not a newly published release. No installed app was replaced or +launched against personal data. The ZIP is `build/QuotaBar-rust-local-arm64.zip`, with +a matching SHA-256 file. Hosted CI and release publication have not been run. +With the identical validation hook, the original and final release probes are +2,774,256 and 3,297,824 bytes respectively (+523,568 bytes). That is a like-for-like +probe binary comparison, not a measured app-bundle size delta. The delivery manifest +binds the packaged executable and ZIP to the 186 source-input hashes used for the +final benchmark; none of the UI or resource source files changed. + +Use the wrapper or Make targets. Bare `swift build`, opening the package directly in +Xcode, or consuming `QuotaBarCore` as an external Swift package does not automatically +build/link Rust. A future distributed SDK should package an XCFramework or supply a +separately validated build integration. The CI workflows now install the pinned toolchain +before their existing test/package steps; the older SDK runner and hosted release workflow +still require execution on GitHub before claiming those environments passed. + +Rust adds a pinned compiler and dependency graph, native boundary debugging, archive +cache invalidation and a second set of tests. An early pricing diagnostic placed the +Swift oracle outside the Core module and did not enable whole-module optimization. +Its apparent 79.363/0.109 ms result is therefore withdrawn as comparative performance +evidence. The implementation removes per-bucket dictionary construction/sorting, which +Swift could also optimize, but only the final production-chain measurements support the +performance conclusion. No general Rust-versus-Swift language speed ratio is claimed. + +The line reader was accepted only after an ablation held the other Rust kernels fixed. +With Rust versus Swift streaming, five-run initial-scan medians were 15.308/15.733 ms +(short), 43.179/42.941 ms (plain long strings), 275.545/275.911 ms (dense escapes), and +22.851/23.061 ms (Codex). Appended scans were likewise close. This supports parity in +the exercised product chain, not a streaming speedup. A minimal million-short-line +counter still regresses from 18.37 to 33.72 ms because of the two Swift callbacks; +that limitation is retained in the report rather than hidden behind other improvements. + +Aggregation is also a preference-based tradeoff at small row counts. The final isolated +same-Core/WMO benchmark (including input/output conversion) improves 120/840/4,800-row +folds from 0.175/1.293/7.269 ms to 0.087/0.598/3.398 ms. However, an independent +short-record full-chain confirmation with only aggregation switched measures +16.256 ms with Swift versus 16.647 ms with Rust (+0.391 ms, +2.4%), and appended +scans 2.186 versus 2.284 ms (+0.098 ms, +4.5%). Initial-scan ranges overlap. +This small overhead is accepted under the user's near-parity preference; the module +is not claimed to accelerate every refresh. The first five-run ablation and the +targeted seven-run confirmation are both retained, rather than selecting only the +faster-looking trial. Details are in `build/rust-validation/aggregation-experiment.md` +and the validation report. + +For the complete final migration against the original Swift revision, seven-run +initial-scan medians are 21.861/15.353 ms (short Claude), 83.578/44.452 ms (plain +long Claude), 401.844/277.387 ms (dense-escape Claude) and 23.317/23.089 ms (Codex). +Unchanged refreshes remain approximately equal. These include actual file scanning, +SQLite recording, pricing and snapshot construction through `CostService`, with +cross-language costs included. They do not measure network refreshes or imply an +equivalent percentage improvement in the whole application. + +## Validation and rollback + +The final integrated `make test` run passed 1,861 Swift harness assertions and 20 +Rust unit tests, plus `cargo fmt --check` and Clippy with warnings denied. The Swift +harness includes grouped differential corpora rather than counting every generated +input as a separate assertion: over 20,000 classifier cases, more than 2,000 parser +mutations/boundaries, 6,454 pricing cases, 230 aggregation cases and the scanner +boundary suite. The existing backup, authentication mocks, remote synchronization, +report publication and UI policy checks also ran. Execution used a sandbox denying +real provider credentials/data and network access; compilation ran outside that +sandbox because SwiftPM uses its own manifest sandbox. The pre-existing Swift 6-mode +warning in `AppCheck.swift` remains; the package's Swift 5 language mode is unchanged. + +See [validation methods and results](rust-validation.md) for the final repeated workload +measurements, differential database checks and UI snapshot comparison. Synthetic inputs +are intentional: the evaluation does not read personal logs, databases or credentials. +The unchanged UI source and fixture snapshots support interface compatibility, while +the complete test suite covers behavior. They cannot prove every possible real-world +log, OS release, Keychain prompt, live network race or assistive-technology interaction. + +All 32 paired offscreen PNGs (26 cards and six interaction states) are byte-identical. +The original earlier baseline is retained: 27 of its images already matched, while +five reset-clock views displayed the naturally different current minute. Rerendering +the unchanged baseline binary and final debug binary in the same minute produced the +32 identical pairs without editing any pixels or production code. No UI animation +logic changed, so no new motion implementation or animation recording is claimed. + +Seven fresh-process debug menu-construction measurements are 20.067 ms before and +19.975 ms after. Whole-process wall time, including the sandbox launcher, is +87.769/88.038 ms. All runs reported `menu_created=no`. These isolated startup checks +are effectively equal; they do not measure a cold disk, real account/network startup, +or the release app's complete interactive launch. + +The migration has no data-format or settings upgrade. Rollback is a code/build rollback: +restore the affected adapters/kernels and remove Rust build integration, then rebuild the +original Swift version. Existing histories and backups do not need conversion. The +original Claude Foundation path is already retained; frozen classifier, scanner and +pricing oracles remain in tests for future differential regression. Do not switch +implementations midway through a file transaction. + +Future changes must retain zero unexplained request/state/row/cursor differences; keep +full refresh and callback costs in measurements; rerun the backup, cancellation, +remote-sync and report-event suite; and follow `design-implementation.md` for affected +UI previews. A new speed claim needs representative end-to-end evidence, not only a +kernel microbenchmark. diff --git a/docs/rust-validation.md b/docs/rust-validation.md new file mode 100644 index 0000000..309d914 --- /dev/null +++ b/docs/rust-validation.md @@ -0,0 +1,253 @@ +# Rust migration validation + +## Scope and baseline + +The reference is commit `f8df98b129c9138ccb9e6290d6c801e805fed1be`. The final candidate +contains Rust kernels for Claude decoding, structural JSON classification, bounded +JSONL streaming, token-rate arithmetic and usage aggregation. See +[Migration boundaries](rust-migration.md) for their architecture. The migration keeps +Swift's file planning, inode/size checks, CryptoKit prefix digests and memoization, +SQLite transactions, scan cursors, recording rules, dates, authentication and UI. +Within the filesystem layer, only `LogFileScanner.readLines` moves its +read/line-walking loop to Rust. The Rust +function calls Swift's filter and handler synchronously with borrowed byte views; +Swift still drains each accepted record's autorelease pool. + +The implementation preserves the 1 MiB read buffer, strict absolute scan limit, +complete-line offset, incomplete trailing line, Darwin `O_CLOEXEC`, interrupted-read +retry, and 64 KiB threshold for early rejection of irrelevant long lines. An +accepted long line can still grow the carry buffer, as it did before. Rejected +lines are skipped without retaining the remainder. No file data is memory mapped. +`qb_read_lines` returns POSIX errno on failure and catches Rust panics as `EIO`. + +## Focused differential checks + +`RustLogFileScannerTests` compares the actual Swift-to-Rust boundary with the frozen +Swift implementation in `SwiftLogLineScanner.swift`. It checks all inspected +prefixes, accepted bytes, callback order, consumed offsets and errors, rather than +only counting lines. Cases include empty records, CRLF, incomplete tails, negative +and beyond-end offsets, limits inside records, boundaries at 1 MiB, multi-chunk +accepted/rejected records, arbitrary non-UTF-8 bytes and a Unicode filename. +Separate checks cover missing files, reading a directory and recursively scanning +from a callback. The production Swift wrapper and Rust static library passed all +82 focused assertions in an isolated optimized build; all four Rust scanner unit +tests and `cargo clippy --all-targets -- -D warnings` also passed. The existing recorder/scanner suite retains append, rewrite, +partial-line, persisted resume-state, deduplication and transaction coverage. + +Run these with the full suite: + +```sh +make test +``` + +The Rust unit tests independently cover offset/limit behavior, multi-chunk +acceptance/rejection and file-open errors: + +```sh +make rust-check +``` + +## Streaming measurement + +The final isolated comparison compiles the production Swift wrapper and Rust +reader beside the frozen Swift loop in the same optimized executable. Both the +filter and accepted-line handler cross FFI in the Rust case, including Swift's +per-record autorelease pool. Seven alternating rounds use three scans per +invocation and warm filesystem caches. The run had an exclusive timing window. +Every invocation returned identical consumed offsets, accepted counts and bytes. + +| Synthetic input | Bytes | Swift | Rust with both Swift callbacks | +| --- | ---: | ---: | ---: | +| 1,000,000 short lines | 16,000,000 | 18.37 ms | 33.72 ms | +| 2,048 lines with 64 KiB payloads | 134,219,776 | 11.92 ms | 11.84 ms | +| 16 rejected 8 MB lines | 128,000,032 | 11.16 ms | 11.14 ms | + +The minimal short-line workload exposes a regression of about 15 ns per line; +these callbacks only inspect a byte and count records. The long-line and rejected +large-line cases are effectively equal. No scanner speedup is claimed. The full +pipeline comparison includes the actual classifier, decoding, recording and +pricing work and determines the practical effect; a loop-only measurement cannot +substitute for it. This is also not a cold-disk or live-history measurement. + +An earlier exploratory version kept the first-byte rejection predicate in Rust +and crossed FFI only for the handler. It measured 18.45/10.06 ms on short lines, +but omitted the final filter boundary and therefore does not describe production +performance. The final table above supersedes those timings. + +Raw seven-run arrays, the standalone build harness and frozen baseline inputs are +retained in `/tmp/quota-rust-scanner-experiment/production` on the validation Mac. +The correctness checks are added to the source tree; the microbenchmark harness is a temporary +experiment and is not part of the product or standard test command. + +## Scanner ablation in the complete pipeline + +A separate five-round alternating comparison restored only the original Swift +`LogFileScanner.swift` in a temporary copy of the four-kernel candidate. It kept +the Rust Claude parser, classifier and pricing implementation identical. Both +binaries used the same fixture hook, offline prices and database path isolation. +The seven lifecycle phases and every benchmark's snapshots, schema, rows and +cursor contents matched. + +| Workload | Initial scan, Rust / Swift scanner | Append 64 records, Rust / Swift scanner | +| --- | ---: | ---: | +| Short Claude | 15.308 / 15.733 ms | 2.085 / 2.084 ms | +| Plain 64 KiB Claude | 43.179 / 42.941 ms | 2.108 / 2.059 ms | +| Dense escapes Claude | 275.545 / 275.911 ms | 2.134 / 2.109 ms | +| Codex | 22.851 / 23.061 ms | 1.996 / 2.042 ms | + +The complete-pipeline scanner comparison is effectively equal. It supports +retaining the Rust reader under the chosen preference for migration at comparable +performance, while preserving the short-line microbenchmark caveat above. It +does not attribute the other kernels' gains to the scanner. Raw results are in +`build/rust-validation/scanner-ablation`; `before` means the Rust scanner and +`after` means the restored Swift scanner in that run. + +## Final complete-pipeline results + +The final candidate includes the Unicode spelling preservation fix in aggregation. +All seven lifecycle checkpoints and all 28 alternating benchmark pairs matched +snapshots, schema, persisted rows and cursor contents. The full regression suite +passed 1,861 Swift assertions and 20 Rust tests. + +| Synthetic workload | Initial scan, Swift / final | Append 64 records, Swift / final | Unchanged scan, Swift / final | +| --- | ---: | ---: | ---: | +| Short Claude | 21.861 / 15.353 ms | 2.446 / 2.175 ms | 1.531 / 1.555 ms | +| Plain 64 KiB Claude | 83.578 / 44.452 ms | 2.404 / 2.080 ms | 1.638 / 1.662 ms | +| Dense escapes Claude | 401.844 / 277.387 ms | 2.563 / 2.143 ms | 1.745 / 1.713 ms | +| Codex | 23.317 / 23.089 ms | 2.016 / 1.981 ms | 0.904 / 0.906 ms | + +These are seven-run medians of the actual `CostService` pipeline, including file +planning, filtering, decoding, recording, SQLite and pricing. The first scan uses +a new database; unchanged scans and append scans reuse that process's service. +Codex and unchanged scans are effectively equal. The three Claude fixtures show +lower initial-import elapsed time, but are synthetic workloads rather than a +claim about every history or the app's startup time. + +`build/rust-validation/final-pipeline-unicode/benchmark-summary.json` retains all +runs and ranges. Its metadata binds every input source file and the fixture hook +to the executable hashes. The equally instrumented release probes are 2,774,256 +bytes before and 3,297,824 bytes after, an increase of 523,568 bytes (about 0.50 MiB). +Intermediate four-kernel and pre-Unicode-fix runs remain separately labeled and +are not the final artifact's results. In particular, the earlier `pipeline-final` +and `final-pipeline` directory names do not designate the corrected final build; +`final-pipeline-unicode` does. Its metadata also names the preserved final binary +used before the ablation copy was rebuilt. + +## Aggregation ablation and its tradeoff + +The corrected final candidate was compared with a temporary copy restoring only +`CostAggregator.swift` from the original revision. Every other Rust kernel stayed +the same. Five alternating rounds passed the same lifecycle and per-run database +and snapshot comparisons. + +| Workload | Initial scan, Rust / Swift aggregation | Append, Rust / Swift aggregation | +| --- | ---: | ---: | +| Short Claude | 17.140 / 16.270 ms | 2.333 / 2.160 ms | +| Plain 64 KiB Claude | 43.304 / 44.031 ms | 2.138 / 2.142 ms | +| Dense escapes Claude | 278.566 / 277.940 ms | 2.194 / 2.240 ms | +| Codex | 22.573 / 22.947 ms | 2.044 / 1.992 ms | + +The short fixture's difference warranted one targeted seven-round confirmation. +It measured 16.647 / 16.256 ms initially, an observed 0.391 ms (2.4%) overhead; +append measured 2.284 / 2.186 ms, an observed 0.098 ms (4.5%) overhead. Initial-scan +ranges overlap (15.71–17.56 / 15.12–17.27 ms). The migration retains this kernel +under the chosen preference for comparable performance: small-group conversion +work has a modest cost even though larger isolated aggregation workloads improve. +This is not evidence that aggregation accelerates every complete scan. No further +runs were added after that confirmation. + +Raw corrected ablation data is in `build/rust-validation/aggregator-ablation-unicode` +and `aggregator-short-confirmation`; in both, `before` means Rust aggregation and +`after` means the original Swift aggregation. The pre-correction +`aggregator-ablation` results remain intermediate evidence only. + +## Reproducing full pipeline validation + +```sh +uv run --no-project Scripts/validate_rust_pipeline.py \ + --revision f8df98b129c9138ccb9e6290d6c801e805fed1be \ + --output "$(mktemp -d /tmp/quota-rust-validation.XXXXXX)/run" --iterations 7 +``` + +The output must be a new directory. The script copies only whitelisted build +sources from the baseline revision and current working tree, and injects +`Scripts/fixtures/rust_pipeline_hook.swift` into those isolated probe copies. +The product probe gets no extra command or behavior. Source hashes, executable +hashes and sizes, commands, build logs, raw stdout/stderr, SQLite databases and +per-run timing arrays are retained. Reusable probes must have matching binary +and hook manifests; `--reuse-sources` additionally checks every source hash +before reusing build output. + +The hook supplies explicit synthetic source paths and fixed offline prices to +`CostService`. It never changes `HOME`. A runtime sandbox denies network access, +real provider directories, Keychains and QuotaBar's application data. No actual +provider or credential process is run. Every probe process has a new database; +its initial scan is followed by an unchanged scan, appending 64 records, and a +second unchanged scan. Wall elapsed includes the sandbox launcher and probe +serialization, so it is not application startup latency. + +The seven lifecycle checkpoints cover an incomplete final line, unchanged input, +append/replay, copied-file deduplication, a rewrite retaining Claude history, +unpriced/invalid input, and source deletion retaining recorded history. Every +SQLite column and schema entry is checked. Only `file_cursor.resume_state` is +parsed as JSON when comparing: Swift's encoder can reorder dictionary keys. +No values are discarded, and unnormalized rows remain under `raw_tables`. + +The development machine is an Apple M2 Pro with 32 GiB RAM, macOS 27.2.1, +Swift 6.4 and Rust 1.98. These are fresh-process, warm-filesystem synthetic +measurements, not cold-disk or personal-history measurements. Seven-run medians +should be read with their retained ranges; they do not imply a universal app +speedup or reduced memory usage. + +## Visual baseline + +An isolated source archive of the reference commit built the original debug app. +Only its documented offscreen commands ran: + +```sh +QuotaBar --dump-card build/rust-validation/before/card +QuotaBar --dump-interaction-states build/rust-validation/before/interactions +``` + +This captured 26 card variants and six interaction-state images using sample data. +The output is ignored build data. Neither command reads CLI credentials, scans +personal sessions or starts the normal application integrations. The same commands +on the final candidate initially produced 27 byte-identical PNGs. The five +`*-reset-clock.png` files differed because the sample factory uses the current +minute for its absolute reset labels. The original before and after images remain +under `build/rust-validation/before` and `after`. + +The untouched baseline binary and final binary were then rendered consecutively +within the same minute, without editing any pixels or source. All 32 resulting +PNGs are byte-identical: 26 card variants and six interaction states, including +light/dark, sign-in and refresh failure. They are in `paired-before` and +`paired-after`; `screenshot-comparison-paired.json` records every hash, and +`paired-render-metadata.json` records commands and wall times. UI source and motion +behavior are unchanged, so no animation comparison is claimed or needed. + +## Menu construction and process startup + +Seven alternating invocations of the original and final debug apps used the +existing isolated `--benchmark-menu-startup` flag. Every invocation reported +`menu_created=no`. Status-item construction medians were 20.067 / 19.975 ms; +external process-wall medians were 87.769 / 88.038 ms. The ranges overlap and no +startup improvement is claimed. + +The flag does not start provider integrations or scan logs. Both binaries ran +with the same runtime sandbox; process-wall time includes its launcher and normal +process loading/exit. These are fresh-process, warm-filesystem observations, not +cold-disk application-launch measurements. All 14 stdout/stderr records and timing +arrays are retained under `build/rust-validation/startup`. + +## Boundaries retained in Swift + +The SQLite recorder already uses SQLite's C engine and reuses prepared statements. +Moving its wrapper alone would not replace the query engine; changing transaction +batching would change failure boundaries. This migration retains one writable +connection, existing schema/backup compatibility, per-file transactions, cursor +updates and recorder merge rules. No SQLite performance gain is claimed. + +Network/authentication migration has no measured CPU benefit here. URLSession, +Keychain access through the Apple-signed `security` process, credential refresh +coordination and PTY cancellation remain in their existing platform layer. They +were reviewed from source, without invoking real providers or credential stores. diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..a1598cc --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.98.0" +profile = "minimal" +components = ["rustfmt", "clippy"] From 4b615abf49b5cbc4ee065e849738e230664856ba Mon Sep 17 00:00:00 2001 From: softmaxe <94467584+softmaxe@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:00:50 +0800 Subject: [PATCH 2/3] ci: cover Rust builds and dependency analysis --- .github/actionlint.yaml | 4 ++++ .github/dependabot.yml | 10 ++++++++++ .github/workflows/codeql.yml | 7 +++++++ docs/ci.md | 19 +++++++++++++++++-- 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..38f5e62 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +# The repository's macOS 27 SDK runner is a custom runner label. +self-hosted-runner: + labels: + - xcode-27 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 20a3bbd..f59c2b8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,13 @@ updates: - "*" commit-message: prefix: ci + - package-ecosystem: cargo + directory: /Rust/QuotaBarCore + schedule: + interval: monthly + groups: + rust-dependencies: + patterns: + - "*" + commit-message: + prefix: build diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d6e5a3d..8f75172 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -33,6 +33,9 @@ jobs: - language: javascript-typescript os: ubuntu-24.04 build-mode: none + - language: rust + os: ubuntu-24.04 + build-mode: none - language: actions os: ubuntu-24.04 build-mode: none @@ -42,6 +45,10 @@ jobs: with: persist-credentials: false + - name: Install pinned Rust toolchain for Swift linking + if: ${{ matrix.language == 'swift' }} + run: bash Scripts/setup_rust_ci.sh + - name: Initialize CodeQL uses: github/codeql-action/init@33d7b85da1a04171e211befe2c739303b9a4c066 # v4 with: diff --git a/docs/ci.md b/docs/ci.md index f56f85e..576bb05 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -6,10 +6,25 @@ in `Scripts/ci_requires_build.sh` lets both checks finish on Linux without a macOS build. Any other path runs the existing tests and release packaging. Classification errors fail the required checks. -CodeQL scans Swift, JavaScript/TypeScript, and Actions after pushes to `main`, +CodeQL scans Swift, Rust, JavaScript/TypeScript, and Actions after pushes to `main`, weekly, and on manual dispatch. It is independent of PR merging. Review scan failures and security alerts separately from the required PR checks. +Every Swift build job (PR checks, release packaging and CodeQL's manual Swift +build) installs the toolchain pinned in `rust-toolchain.toml` before invoking +`Scripts/swift.sh`. `make test` checks Rust formatting, unit tests and Clippy +before the Swift assertion suite. Packaging links the release static archive +and retains the existing signature and architecture checks. Rust source, Cargo +manifest/lockfile, C bridge and toolchain changes all require both macOS checks; +they are outside the documentation allowlist. + +The Rust CodeQL job uses `build-mode: none` on Linux, as supported by +[CodeQL's Rust analysis](https://github.blog/changelog/2025-10-14-codeql-scanning-rust-and-c-c-without-builds-is-now-generally-available/). +It does not execute the Darwin-specific native kernels. Dependabot checks the +Cargo manifest and lockfile in `Rust/QuotaBarCore` monthly, alongside the existing +GitHub Actions updates. Compiler updates remain explicit changes to the pinned +toolchain and must pass both macOS checks. + ## Activate the CodeQL workflow This is a one-time migration from GitHub default setup. Complete it after @@ -24,7 +39,7 @@ analysis uploads from custom workflows, so adding the YAML alone is not enough. `gh workflow enable codeql.yml`, then `gh workflow run codeql.yml --ref main`. 4. Inspect the resulting run using `gh run list --workflow codeql.yml` and - `gh run view `. Migration is complete when all three language jobs + `gh run view `. Migration is complete when all configured language jobs succeed, including analysis upload. Fix and rerun failures before considering the migration complete. From 50a830a9e40490ebe247db456ff07fb98f670133 Mon Sep 17 00:00:00 2001 From: softmaxe <94467584+softmaxe@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:03:34 +0800 Subject: [PATCH 3/3] ci: preserve Python coverage in advanced CodeQL analysis --- .github/workflows/codeql.yml | 3 +++ docs/ci.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8f75172..503376e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -33,6 +33,9 @@ jobs: - language: javascript-typescript os: ubuntu-24.04 build-mode: none + - language: python + os: ubuntu-24.04 + build-mode: none - language: rust os: ubuntu-24.04 build-mode: none diff --git a/docs/ci.md b/docs/ci.md index 576bb05..083a253 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -6,7 +6,7 @@ in `Scripts/ci_requires_build.sh` lets both checks finish on Linux without a macOS build. Any other path runs the existing tests and release packaging. Classification errors fail the required checks. -CodeQL scans Swift, Rust, JavaScript/TypeScript, and Actions after pushes to `main`, +CodeQL scans Swift, Rust, Python, JavaScript/TypeScript, and Actions after pushes to `main`, weekly, and on manual dispatch. It is independent of PR merging. Review scan failures and security alerts separately from the required PR checks.