From 2cb33c4d04362be8596083ef2040720ebd20df3b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 03:27:51 +0000 Subject: [PATCH 1/3] pricing: a models.dev row's context tier fails closed instead of base-pricing A live models.dev row can publish context tiers (`tiers` or the legacy `context_over_200k`). The catalog audit path priced every turn on such a row at its base rate, so a turn above the first tier was undercharged silently. `refused_catalog_pricing_audit` (was `invalid_catalog_pricing_audit`) now also returns `UnpricedReason::UnrepresentedTier` ("pricing tier unavailable") when the turn's total prompt input (`Usage::input_tokens`, cache read and write included) is at or above a context tier's size, and for any tier kind other than "context". It runs at both catalog call sites: the provider/catalog route and the OpenRouter bundled-only path. This is the first consumer of `ModelsDevCost::context_tiers()`, which already folds `tiers` and `context_over_200k` into one sorted list and had no caller until now. No new reason, string or locale key: the existing reason and its 15 translations are reused. Frozen quotes are untouched because they cannot carry tiers: provider-live costs are built from the four base fields (client.rs, two sites), cloud-facts patches replace the whole cost block (catalog_patch.rs), and custom_models accepts only the four keys (catalog/configured.rs). Tier rates themselves are still not applied; the 0.10.2 Known limits line now says so, and crates/tui/CHANGELOG.md is regenerated by scripts/sync-changelog.sh. Not model-visible: cost audits never reach a request, so no session event and no KV-cache effect. Pure computation, so #6149 does not apply. Evidence: - cargo test -p codewhale-tui --lib --locked -- pricing:: cost_status:: test result: ok. 130 passed; 0 failed (86 pricing::, 44 cost_status::), including malformed_catalog_row_has_an_explicit_runtime_reason and models_dev_live_rates_price_the_published_estimate (live refresh path, untiered row, still priced) - cargo fmt --all -- --check: clean - scripts/sync-changelog.sh --check: up to date - the tier edge itself is not exercised by a test (no new tests without founder request); the bundled catalog has no tiered rows Refs #6396 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BsXVvReYca2UxzimQtj7Mj --- CHANGELOG.md | 8 +++++--- crates/tui/CHANGELOG.md | 8 +++++--- crates/tui/src/pricing.rs | 28 ++++++++++++++++++---------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f28b3617d..81a752e0ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -421,9 +421,11 @@ launcher guidance. ([#6915](https://github.com/codewhale-hq/Codewhale/issues/6915)). - Idle CPU has not been measured on FreeBSD or Linux for this release ([#6728](https://github.com/codewhale-hq/Codewhale/issues/6728)). -- Models.dev prices with context tiers: a turn above a model's first tier can - be priced at its base rate. The bundled catalog has no tiered rows, so this - can only happen after a live models.dev refresh +- Models.dev prices with context tiers are not applied. A turn at or above a + model's first context tier reads as unpriced ("pricing tier unavailable") + and is counted as missing from the cost total, instead of being priced at + its base rate. The bundled catalog has no tiered rows, so this appears only + after a live models.dev refresh ([#6396](https://github.com/codewhale-hq/Codewhale/issues/6396)). - Shell commands run by delegated agents appear in `/jobs` and the work dock with their owner, not in the agent's own detail view; named terminal diff --git a/crates/tui/CHANGELOG.md b/crates/tui/CHANGELOG.md index 6284e161b6..42d6a9614a 100644 --- a/crates/tui/CHANGELOG.md +++ b/crates/tui/CHANGELOG.md @@ -421,9 +421,11 @@ launcher guidance. ([#6915](https://github.com/codewhale-hq/Codewhale/issues/6915)). - Idle CPU has not been measured on FreeBSD or Linux for this release ([#6728](https://github.com/codewhale-hq/Codewhale/issues/6728)). -- Models.dev prices with context tiers: a turn above a model's first tier can - be priced at its base rate. The bundled catalog has no tiered rows, so this - can only happen after a live models.dev refresh +- Models.dev prices with context tiers are not applied. A turn at or above a + model's first context tier reads as unpriced ("pricing tier unavailable") + and is counted as missing from the cost total, instead of being priced at + its base rate. The bundled catalog has no tiered rows, so this appears only + after a live models.dev refresh ([#6396](https://github.com/codewhale-hq/Codewhale/issues/6396)). - Shell commands run by delegated agents appear in `/jobs` and the work dock with their owner, not in the agent's own detail view; named terminal diff --git a/crates/tui/src/pricing.rs b/crates/tui/src/pricing.rs index c43572a23b..05b77a4600 100644 --- a/crates/tui/src/pricing.rs +++ b/crates/tui/src/pricing.rs @@ -1634,7 +1634,10 @@ fn audit_turn_cost_for_provider_on_endpoint_for_identity_at( VerifiedOffering::Absent => None, }; - if let Some(audit) = offering.as_ref().and_then(invalid_catalog_pricing_audit) { + if let Some(audit) = offering + .as_ref() + .and_then(|offering| refused_catalog_pricing_audit(offering, usage)) + { return audit.with_live_defect(live_defect); } @@ -1692,17 +1695,22 @@ fn audit_turn_cost_for_provider_on_endpoint_for_identity_at( } } -/// Convert malformed catalog numerics into an explicit runtime audit reason. +/// Convert malformed catalog numerics, or a context tier this turn reached, +/// into an explicit runtime audit reason. /// Keeping this distinct from the ordinary `None` projection prevents a bad /// published row from becoming indistinguishable from an absent price. -fn invalid_catalog_pricing_audit( +fn refused_catalog_pricing_audit( offering: &codewhale_config::catalog::CatalogOffering, + usage: &Usage, ) -> Option { - offering - .cost - .as_ref() - .is_some_and(|cost| !codewhale_config::pricing::catalog_cost_is_valid(cost)) - .then(|| TurnCostAudit::unpriced(UnpricedReason::InvalidPricingRow)) + let cost = offering.cost.as_ref()?; + if !codewhale_config::pricing::catalog_cost_is_valid(cost) { + return Some(TurnCostAudit::unpriced(UnpricedReason::InvalidPricingRow)); + } + cost.context_tiers() + .iter() + .any(|tier| tier.tier.kind != "context" || u64::from(usage.input_tokens) >= tier.tier.size) + .then(|| TurnCostAudit::unpriced(UnpricedReason::UnrepresentedTier)) } /// Outcome of checking a catalog row's pricing provenance against the route. @@ -2109,7 +2117,7 @@ fn audit_openrouter_immutable_pricing( ProviderKind::Openrouter, &canonical_model, ) { - if let Some(audit) = invalid_catalog_pricing_audit(&offering) { + if let Some(audit) = refused_catalog_pricing_audit(&offering, usage) { return audit; } if let Some(pricing) = effective_offering_pricing( @@ -2665,7 +2673,7 @@ mod tests { ..Default::default() }; - let audit = invalid_catalog_pricing_audit(&offering) + let audit = refused_catalog_pricing_audit(&offering, &Usage::default()) .expect("malformed row must become an explicit failed-closed audit"); assert!(!audit.is_priced()); assert_eq!( From e43d86e8be024f216d779befee615b33845ecbe3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 04:08:07 +0000 Subject: [PATCH 2/3] build-lock: honor CODEWHALE_BUILD_LOCK=0 and CODEWHALE_BUILD_LOCK_HELD itself The script's docstring says nested invocations see CODEWHALE_BUILD_LOCK_HELD and run unlocked, and its wait message tells the user to "set CODEWHALE_BUILD_LOCK=0 to skip". Neither was true: only scripts/dev-cache.sh checked the two variables before calling it. Every direct caller (CI's self-hosted lock holder, agents wrapping npm or cargo in build-lock.py) ignored both, and a nested direct call deadlocked on the lock its own parent held. The script now runs the command unlocked when CODEWHALE_BUILD_LOCK_HELD is non-empty or CODEWHALE_BUILD_LOCK is one of dev-cache.sh's falsey values (0, false, no, off, never, disabled), with the same exit-status mapping as the locked path. The locked path is unchanged. Evidence, scratch lock files only (never the real build.lock): - committed script, lock held by `-- sleep 8`: CODEWHALE_BUILD_LOCK=0 call -> timeout (exit 124), CODEWHALE_BUILD_LOCK_HELD=1 call -> timeout (124), `build-lock.py L -- python3 build-lock.py L -- echo` -> deadlock (124) - fixed script, same setups: all three exit 0 immediately; each of false/no/off/never/disabled exits 0 - still locks: CODEWHALE_BUILD_LOCK=1, yes, on, unset, and an empty CODEWHALE_BUILD_LOCK_HELD all wait while held (124 under a 2 s timeout) and run after release (exit 0); the locked child still sees CODEWHALE_BUILD_LOCK_HELD=1 - exit status: `sh -c 'exit 7'` -> 7 locked and bypassed; SIGTERM -> 143 locked and bypassed; bad usage -> 2 - python3 -m py_compile and ruff check: clean Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BsXVvReYca2UxzimQtj7Mj --- scripts/build-lock.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/build-lock.py b/scripts/build-lock.py index c4d3e2711e..44a4d9dffc 100755 --- a/scripts/build-lock.py +++ b/scripts/build-lock.py @@ -30,6 +30,11 @@ def main() -> int: print("usage: build-lock.py -- [args...]", file=sys.stderr) return 2 lock_path, command = sys.argv[1], sys.argv[3:] + if os.environ.get("CODEWHALE_BUILD_LOCK_HELD") or os.environ.get( + "CODEWHALE_BUILD_LOCK", "1" + ) in ("0", "false", "no", "off", "never", "disabled"): + status = subprocess.call(command) + return 128 - status if status < 0 else status env = dict(os.environ, CODEWHALE_BUILD_LOCK_HELD="1") try: import fcntl From ec242fe079751e2ffbc60bb2371c0755714fc32b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 04:17:55 +0000 Subject: [PATCH 3/3] WIP app-server: Windows recovers a dead owner through the Unix path UNVERIFIED WORK IN PROGRESS, pushed so it survives a move off the cloud session. Do not merge as is. Implements the Q4 plan for #6915 (founder: blocks the 0.10.2 tag; proof is the hosted Test (windows-latest) leg). Windows startup used connect_if_published and failed hard with "opening selected Windows owner; refusing fallback ... (os error 2)" once the owner died; the "stale-owner recovery is not qualified" guard in app-server lib.rs was unreachable. This joins Windows to the lease-first admission and the Unix recovery from a8cdccc1d6: - config/windows_identity.rs: windows_process_status(pid) -> Running/Exited/Absent from OpenProcess + creation time + WaitForSingleObject - config/private_directory.rs: POSIX-semantics handle deletion with the legacy fallback (same order as std::fs::remove_file) - app-server daemon_client.rs: require_dead_process / revalidate_* on Windows; require_same_selection shared with Unix; a NotFound pipe open maps to Unavailable - app-server daemon_windows.rs: bind takes `recovery`, retires the captured receipt by handle before creating the first pipe instance - app-server daemon_socket.rs: Unix bind uses require_same_selection - app-server lib.rs: the Windows-only ensure is gone; one bind call - tui runtime_api.rs / runtime_threads.rs: RuntimeHostAdmission, admit_runtime_host, try_acquire_for_host are cfg(any(unix, windows)); the Windows connect_if_published branch is deleted - cli/tests/owner_crash_reentry.rs: the founder-requested two-process test (spawn A, kill+reap, receipt still names dead A, spawn B, B owns it and serves A's thread) Verified so far (Linux only): - clippy -p codewhale-config -p codewhale-app-server -p codewhale-tui -p codewhale-cli --all-targets --all-features --locked -- -D warnings: exit 0, no warnings - cargo fmt --all -- --check: clean NOT yet run: owner_crash_reentry (Linux), app-server daemon_socket owner_bootstrap tests, tui runtime_api::tests::owner_bootstrap, any Windows compile (x86_64-pc-windows-gnu cross-check or hosted windows-latest), the three-lens review, the CHANGELOG change (drop the 0.10.2 Windows known-limit bullet, add the fix line, run scripts/sync-changelog.sh). Refs #6915 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BsXVvReYca2UxzimQtj7Mj --- crates/app-server/src/daemon_client.rs | 83 +++++++-- crates/app-server/src/daemon_socket.rs | 15 +- crates/app-server/src/daemon_windows.rs | 74 +++++--- crates/app-server/src/lib.rs | 9 - crates/cli/tests/owner_crash_reentry.rs | 216 ++++++++++++++++++++++++ crates/config/src/private_directory.rs | 30 +++- crates/config/src/windows_identity.rs | 36 ++++ crates/tui/src/runtime_api.rs | 20 +-- crates/tui/src/runtime_threads.rs | 2 +- 9 files changed, 414 insertions(+), 71 deletions(-) create mode 100644 crates/cli/tests/owner_crash_reentry.rs diff --git a/crates/app-server/src/daemon_client.rs b/crates/app-server/src/daemon_client.rs index e730884951..6f38db2c4b 100644 --- a/crates/app-server/src/daemon_client.rs +++ b/crates/app-server/src/daemon_client.rs @@ -120,27 +120,62 @@ mod platform { anyhow::Error::new(cause) .context("selected owner publication unavailable; canonical host recovery required") } - #[cfg(unix)] fn require_dead_process(&self) -> Result<()> { - use codewhale_config::private_directory::{UnixProcessStatus, unix_process_status}; let receipt = self.receipt(); + #[cfg(unix)] + let principal = PrivateDirectory::current_user_id().to_string(); + #[cfg(windows)] + let principal = + codewhale_config::windows_identity::CurrentWindowsUser::open()?.sid_string()?; anyhow::ensure!( - receipt.principal == PrivateDirectory::current_user_id().to_string() - && !receipt.process_start.is_empty(), + receipt.principal == principal && !receipt.process_start.is_empty(), "selected owner process identity is invalid" ); - match unix_process_status(receipt.pid)? { - UnixProcessStatus::Absent => Ok(()), - UnixProcessStatus::Present { start } if start == receipt.process_start => { - bail!("selected owner process is still present; refusing recovery") + #[cfg(unix)] + let (running, start) = { + use codewhale_config::private_directory::{UnixProcessStatus, unix_process_status}; + match unix_process_status(receipt.pid)? { + UnixProcessStatus::Absent => return Ok(()), + UnixProcessStatus::Present { start } => (true, start), } - UnixProcessStatus::Present { .. } => { - bail!("selected owner PID was reused; refusing recovery") + }; + #[cfg(windows)] + let (running, start) = { + use codewhale_config::windows_identity::{ + WindowsProcessStatus, windows_process_status, + }; + match windows_process_status(receipt.pid)? { + WindowsProcessStatus::Absent => return Ok(()), + WindowsProcessStatus::Running { start } => (true, start), + WindowsProcessStatus::Exited { start } => (false, start), } + }; + if start != receipt.process_start { + bail!("selected owner PID was reused; refusing recovery") + } + if running { + bail!("selected owner process is still present; refusing recovery") } + Ok(()) + } + pub(crate) fn require_same_selection( + &self, + owner: &RuntimeOwnerReceipt, + endpoint: &std::path::Path, + ) -> Result<()> { + let old = self.receipt(); + anyhow::ensure!( + old.version == owner.version + && old.data_dir == owner.data_dir + && old.execution_scope == owner.execution_scope + && old.socket_path == endpoint + && old.config_path == owner.config_path + && old.principal == owner.principal, + "captured stale owner belongs to another selected store/config/principal" + ); + Ok(()) } /// Caller retains the real Runtime lease; this checks evidence only. - #[cfg(unix)] pub(crate) fn revalidate_receipt(&self) -> Result { if !self.publication.receipt_is_current()? { return Ok(false); @@ -148,7 +183,6 @@ mod platform { self.require_dead_process()?; Ok(true) } - #[cfg(unix)] pub(crate) fn revalidate_all(&self) -> Result { if !self.publication.is_current()? { return Ok(false); @@ -156,7 +190,6 @@ mod platform { self.require_dead_process()?; Ok(true) } - #[cfg(unix)] pub async fn revalidate(self: &Arc) -> Result { let held = self.clone(); owner_work(move || held.revalidate_all()).await @@ -459,7 +492,29 @@ mod platform { }; #[cfg(windows)] let (stream, peer) = { - let (stream, process) = crate::daemon_windows::connect_owner(&receipt).await?; + let (stream, process) = match crate::daemon_windows::connect_owner(&receipt).await { + Ok(connected) => connected, + Err(error) => { + let held = publication.clone(); + anyhow::ensure!( + owner_work(move || held.is_current()).await?, + PublicationChangedBeforeAttach + ); + let missing = error + .downcast_ref::() + .filter(|cause| cause.kind() == std::io::ErrorKind::NotFound) + .map(|cause| match cause.raw_os_error() { + Some(code) => std::io::Error::from_raw_os_error(code), + None => std::io::Error::from(cause.kind()), + }); + if let Some(cause) = missing { + return Ok(HostOwnerProbe::Unavailable(Arc::new( + UnavailablePublication { publication, cause }, + ))); + } + return Err(error); + } + }; let peer = AuthorizedPeer { pid: process.pid(), start: process.start().to_string(), diff --git a/crates/app-server/src/daemon_socket.rs b/crates/app-server/src/daemon_socket.rs index 62c407f389..5554fc6b6d 100644 --- a/crates/app-server/src/daemon_socket.rs +++ b/crates/app-server/src/daemon_socket.rs @@ -999,18 +999,9 @@ mod platform { )) })? .1; - let old = previous.receipt(); - if old.version != owner.version - || old.data_dir != owner.data_dir - || old.execution_scope != owner.execution_scope - || old.socket_path != path - || old.config_path != owner.config_path - || old.principal != owner.principal - { - return Err(DaemonSocketError::State(anyhow::anyhow!( - "captured stale owner belongs to another selected store/config/principal" - ))); - } + previous + .require_same_selection(owner, &path) + .map_err(DaemonSocketError::State)?; if !previous .revalidate() .await diff --git a/crates/app-server/src/daemon_windows.rs b/crates/app-server/src/daemon_windows.rs index f8972f4bba..08c590384a 100644 --- a/crates/app-server/src/daemon_windows.rs +++ b/crates/app-server/src/daemon_windows.rs @@ -2,6 +2,7 @@ //! Pipe names route; private receipts plus held kernel process/token identity //! select the actual owner. No bearer or Engine authority crosses this wire. +use crate::daemon_client::UnavailablePublication; use crate::daemon_socket::{ AuthorizedPeer, ConnectionTasks, DaemonShutdownHandle, DaemonSocketError, DaemonSocketOptions, SocketPathInputs, connection_context, owner_work, run_authorized_connection, @@ -192,11 +193,12 @@ impl DaemonSocket { pub async fn bind_daemon_socket( options: DaemonSocketOptions, ) -> Result { - bind(options, None).await + bind(options, None, None).await } pub(crate) async fn bind_captured_owner( state: AppState, owner: RuntimeOwnerReceipt, + recovery: Option>, ) -> Result { bind( DaemonSocketOptions { @@ -204,17 +206,59 @@ pub(crate) async fn bind_captured_owner( config_path: None, }, Some((state, owner)), + recovery, ) .await } async fn bind( options: DaemonSocketOptions, captured: Option<(AppState, RuntimeOwnerReceipt)>, + recovery: Option>, ) -> Result { let explicit = options.socket_path; let path = owner_work(move || selected_pipe_path(explicit)) .await .map_err(DaemonSocketError::State)?; + let recovered = match recovery { + Some(previous) => { + let owner = &captured + .as_ref() + .ok_or_else(|| { + DaemonSocketError::State(anyhow::anyhow!( + "stale publication requires the actual captured Runtime owner" + )) + })? + .1; + previous + .require_same_selection(owner, &path) + .map_err(DaemonSocketError::State)?; + Some( + owner_work(move || { + let parent = previous.publication.parent.clone(); + anyhow::ensure!( + parent.is_at_selected_path()?, + "selected Windows owner parent changed" + ); + anyhow::ensure!( + previous.revalidate_receipt()?, + "captured stale owner receipt changed before retirement" + ); + anyhow::ensure!( + parent.retire_private_receipt( + &previous.publication.receipt_name, + &previous.publication.file + )?, + "captured stale owner receipt changed during retirement" + ); + drop(previous); + Ok(parent) + }) + .await + .map_err(DaemonSocketError::State)?, + ) + } + None => None, + }; let selected = path.clone(); let pipe = owner_work(move || create_pipe(&selected, true)) .await @@ -232,7 +276,10 @@ async fn bind( let owner = owner.clone(); Some( owner_work(move || { - let parent = Arc::new(PrivateDirectory::admit(&owner_directory()?)?); + let parent = match recovered { + Some(parent) => parent, + None => Arc::new(PrivateDirectory::admit(&owner_directory()?)?), + }; let mut guard = ReceiptGuard { parent: parent.clone(), file: None, @@ -241,23 +288,12 @@ async fn bind( parent.is_at_selected_path()?, "selected Windows owner parent changed" ); - if let Some((bytes, file)) = - parent.read_private_receipt("daemon.owner.json", 16384)? - { - let previous: RuntimeOwnerReceipt = serde_json::from_slice(&bytes)?; - anyhow::ensure!( - previous.version == owner.version - && previous.data_dir == owner.data_dir - && previous.execution_scope == owner.execution_scope - && previous.socket_path == owner.socket_path, - "old Windows owner receipt belongs to another selected store" - ); - anyhow::ensure!( - parent.retire_private_receipt("daemon.owner.json", &file)?, - "Windows owner receipt changed" - ); - drop(file); - } + anyhow::ensure!( + parent + .read_private_receipt("daemon.owner.json", 16384)? + .is_none(), + "owner publication appeared during bootstrap; refusing replacement" + ); let bytes = serde_json::to_vec(&owner)?; anyhow::ensure!(bytes.len() <= 16384, "Windows owner receipt exceeds bound"); parent.write_owned_file("daemon.owner.json", &bytes, false)?; diff --git a/crates/app-server/src/lib.rs b/crates/app-server/src/lib.rs index 23038d7a9c..5f548a4721 100644 --- a/crates/app-server/src/lib.rs +++ b/crates/app-server/src/lib.rs @@ -653,16 +653,7 @@ pub async fn bind_runtime_frontends( state.captured_routing = Some(routing); state.owner_frontend = owner_frontend; *state.runtime_bridge.lock().await = Some(Arc::new(Mutex::new(bridge))); - #[cfg(unix)] let daemon = daemon_socket::bind_captured_owner(state.clone(), owner, recovery).await?; - #[cfg(windows)] - let daemon = { - anyhow::ensure!( - recovery.is_none(), - "Windows stale-owner recovery is not qualified; refusing fallback" - ); - daemon_socket::bind_captured_owner(state.clone(), owner).await? - }; Ok((daemon, state)) } diff --git a/crates/cli/tests/owner_crash_reentry.rs b/crates/cli/tests/owner_crash_reentry.rs new file mode 100644 index 0000000000..d06fa860cb --- /dev/null +++ b/crates/cli/tests/owner_crash_reentry.rs @@ -0,0 +1,216 @@ +#![cfg(any(unix, windows))] + +use std::io::{BufRead, BufReader}; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc; +use std::time::Duration; + +use serde_json::{Value, json}; + +const TOKEN: &str = "owner-crash-reentry-fixture-token"; +const LISTENING: &str = "Runtime API listening on http://"; + +struct Fixture { + root: tempfile::TempDir, +} + +impl Fixture { + fn new() -> Self { + #[cfg(unix)] + let root = tempfile::Builder::new() + .prefix("cw-ocr-") + .tempdir_in( + std::path::Path::new("/tmp") + .canonicalize() + .expect("canonical /tmp"), + ) + .expect("fixture root"); + #[cfg(windows)] + let root = tempfile::tempdir().expect("fixture root"); + for name in ["home", "state", "tmp", "work"] { + std::fs::create_dir_all(root.path().join(name)).expect("fixture directory"); + } + std::fs::write(root.path().join("config.toml"), "").expect("fixture config"); + Self { root } + } + + fn path(&self, name: &str) -> PathBuf { + self.root.path().join(name) + } + + fn command(&self) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_codewhale")); + command.current_dir(self.path("work")).env_clear(); + for name in ["PATH", "SystemRoot", "WINDIR"] { + if let Some(value) = std::env::var_os(name) { + command.env(name, value); + } + } + let home = self.path("home"); + let tmp = self.path("tmp"); + command + .env("HOME", &home) + .env("USERPROFILE", &home) + .env("CODEWHALE_HOME", self.path("state")) + .env("CODEWHALE_SECRET_BACKEND", "file") + .env("CODEWHALE_TELEMETRY", "0") + .env("CODEWHALE_RUNTIME_TOKEN", TOKEN) + .env("TEMP", &tmp) + .env("TMP", &tmp) + .env("TMPDIR", &tmp) + .arg("--config") + .arg(self.path("config.toml")) + .arg("--no-project-config"); + command + } + + fn receipt(&self) -> (Vec, Value) { + #[cfg(unix)] + let name = "daemon.sock.owner.json"; + #[cfg(windows)] + let name = "daemon.owner.json"; + let bytes = std::fs::read(self.path("state").join("run").join(name)) + .expect("published owner receipt"); + let value = serde_json::from_slice(&bytes).expect("owner receipt JSON"); + (bytes, value) + } + + fn stderr(&self, label: &str) -> String { + std::fs::read_to_string(self.path(&format!("{label}.stderr"))).unwrap_or_default() + } +} + +struct Owner { + child: Child, + base: String, +} + +impl Drop for Owner { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn spawn_owner(fixture: &Fixture, label: &str) -> Owner { + let stderr = std::fs::File::create(fixture.path(&format!("{label}.stderr"))) + .expect("owner stderr capture"); + let mut child = fixture + .command() + .args(["app-server", "--http", "--host", "127.0.0.1", "--port", "0"]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::from(stderr)) + .spawn() + .expect("spawn codewhale app-server"); + let stdout = child.stdout.take().expect("owner stdout"); + let (ready, listening) = mpsc::channel(); + std::thread::spawn(move || { + let mut ready = Some(ready); + for line in BufReader::new(stdout).lines() { + let Ok(line) = line else { break }; + if let Some(address) = line.strip_prefix(LISTENING) + && let Some(ready) = ready.take() + { + let _ = ready.send(address.trim().to_string()); + } + } + }); + let mut owner = Owner { + child, + base: String::new(), + }; + match listening.recv_timeout(Duration::from_secs(120)) { + Ok(address) => owner.base = format!("http://{address}"), + Err(_) => { + let status = owner.child.try_wait().ok().flatten(); + drop(owner); + panic!( + "{label} never reported readiness (exit: {status:?})\nstderr:\n{}", + fixture.stderr(label) + ); + } + } + owner +} + +fn http() -> reqwest::blocking::Client { + reqwest::blocking::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(30)) + .build() + .expect("loopback HTTP client") +} + +#[test] +fn killed_owner_is_replaced_by_the_next_app_server() { + let fixture = Fixture::new(); + let client = http(); + + let mut first = spawn_owner(&fixture, "first"); + let (first_bytes, first_receipt) = fixture.receipt(); + assert_eq!(first_receipt["pid"], json!(first.child.id())); + let created = client + .post(format!("{}/v1/threads", first.base)) + .bearer_auth(TOKEN) + .json(&json!({})) + .send() + .expect("create thread on the first owner"); + let status = created.status(); + let created: Value = created.json().expect("created thread JSON"); + assert!(status.is_success(), "create thread: {status} {created}"); + let thread_id = created["id"] + .as_str() + .expect("created thread id") + .to_string(); + + first.child.kill().expect("kill the first owner"); + first.child.wait().expect("reap the first owner"); + let (dead_bytes, dead_receipt) = fixture.receipt(); + assert_eq!(dead_bytes, first_bytes); + assert_eq!(dead_receipt["pid"], json!(first.child.id())); + + let second = spawn_owner(&fixture, "second"); + let (_, second_receipt) = fixture.receipt(); + assert_eq!(second_receipt["pid"], json!(second.child.id())); + assert_ne!( + second_receipt["lease_generation"], + first_receipt["lease_generation"] + ); + assert_eq!(second_receipt["data_dir"], first_receipt["data_dir"]); + assert_eq!( + second_receipt["execution_scope"], + first_receipt["execution_scope"] + ); + + let listed: Value = client + .get(format!("{}/v1/threads", second.base)) + .bearer_auth(TOKEN) + .send() + .expect("list threads on the second owner") + .json() + .expect("thread list JSON"); + assert!( + listed + .as_array() + .expect("thread list") + .iter() + .any(|thread| thread["id"] == json!(thread_id)), + "{thread_id} missing from {listed}" + ); + + let attached = fixture + .command() + .args(["thread", "list"]) + .stdin(Stdio::null()) + .output() + .expect("run codewhale thread list"); + let stdout = String::from_utf8_lossy(&attached.stdout); + assert!( + attached.status.success() && stdout.contains(&thread_id), + "thread list through the second owner failed ({})\nstdout:\n{stdout}\nstderr:\n{}", + attached.status, + String::from_utf8_lossy(&attached.stderr) + ); +} diff --git a/crates/config/src/private_directory.rs b/crates/config/src/private_directory.rs index 173b5dd483..ed2f5ac8c2 100644 --- a/crates/config/src/private_directory.rs +++ b/crates/config/src/private_directory.rs @@ -776,10 +776,36 @@ fn reopen_windows_file_for_owner_security(file: &File, path: &Path) -> Result Result<()> { use std::os::windows::io::AsRawHandle as _; + use windows_sys::Win32::Foundation::{ + ERROR_INVALID_FUNCTION, ERROR_INVALID_PARAMETER, ERROR_NOT_SUPPORTED, + }; use windows_sys::Win32::Storage::FileSystem::{ - FILE_DISPOSITION_INFO, FileDispositionInfo, SetFileInformationByHandle, + FILE_DISPOSITION_FLAG_DELETE, FILE_DISPOSITION_FLAG_POSIX_SEMANTICS, FILE_DISPOSITION_INFO, + FILE_DISPOSITION_INFO_EX, FileDispositionInfo, FileDispositionInfoEx, + SetFileInformationByHandle, }; + let posix = FILE_DISPOSITION_INFO_EX { + Flags: FILE_DISPOSITION_FLAG_DELETE | FILE_DISPOSITION_FLAG_POSIX_SEMANTICS, + }; + if unsafe { + SetFileInformationByHandle( + file.as_raw_handle(), + FileDispositionInfoEx, + (&raw const posix).cast(), + std::mem::size_of::() as u32, + ) + } != 0 + { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + if !matches!( + error.raw_os_error().map(|code| code as u32), + Some(ERROR_INVALID_PARAMETER | ERROR_NOT_SUPPORTED | ERROR_INVALID_FUNCTION) + ) { + return Err(error).context("marking exact private file handle for deletion"); + } let disposition = FILE_DISPOSITION_INFO { DeleteFile: true }; // SAFETY: the disposition buffer has the documented structure and the // handle remains owned until after the call. Windows marks this exact file @@ -794,7 +820,7 @@ fn mark_windows_file_handle_for_deletion(file: &File) -> Result<()> { } == 0 { return Err(std::io::Error::last_os_error()) - .context("marking exact xAI OAuth file handle for deletion"); + .context("marking exact private file handle for deletion"); } Ok(()) } diff --git a/crates/config/src/windows_identity.rs b/crates/config/src/windows_identity.rs index 3b5158fc0e..0a88cb096e 100644 --- a/crates/config/src/windows_identity.rs +++ b/crates/config/src/windows_identity.rs @@ -181,6 +181,42 @@ impl WindowsPeerProcess { } } +pub enum WindowsProcessStatus { + Running { start: String }, + Exited { start: String }, + Absent, +} + +pub fn windows_process_status(pid: u32) -> Result { + use std::os::windows::io::AsRawHandle as _; + use windows_sys::Win32::Foundation::{ERROR_INVALID_PARAMETER, WAIT_OBJECT_0, WAIT_TIMEOUT}; + use windows_sys::Win32::Storage::FileSystem::SYNCHRONIZE; + use windows_sys::Win32::System::Threading::{ + GetProcessId, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, WaitForSingleObject, + }; + anyhow::ensure!(pid > 0, "invalid Windows process PID"); + let process = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION | SYNCHRONIZE, 0, pid) }; + if process.is_null() { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(ERROR_INVALID_PARAMETER as i32) { + return Ok(WindowsProcessStatus::Absent); + } + return Err(error).context("selected Windows process identity/liveness is uncertain"); + } + let process = unsafe { std::os::windows::io::OwnedHandle::from_raw_handle(process) }; + anyhow::ensure!( + unsafe { GetProcessId(process.as_raw_handle()) } == pid, + "Windows kernel process PID changed" + ); + let start = process_creation(process.as_raw_handle())?; + match unsafe { WaitForSingleObject(process.as_raw_handle(), 0) } { + WAIT_OBJECT_0 => Ok(WindowsProcessStatus::Exited { start }), + WAIT_TIMEOUT => Ok(WindowsProcessStatus::Running { start }), + _ => Err(std::io::Error::last_os_error()) + .context("selected Windows process identity/liveness is uncertain"), + } +} + fn process_creation(process: windows_sys::Win32::Foundation::HANDLE) -> Result { use windows_sys::Win32::Foundation::FILETIME; use windows_sys::Win32::System::Threading::GetProcessTimes; diff --git a/crates/tui/src/runtime_api.rs b/crates/tui/src/runtime_api.rs index 413a53088f..d20ce35b61 100644 --- a/crates/tui/src/runtime_api.rs +++ b/crates/tui/src/runtime_api.rs @@ -1639,7 +1639,7 @@ pub(crate) fn open_runtime_threads_for_host( Ok((manager, workshop_activation)) } -#[cfg(unix)] +#[cfg(any(unix, windows))] enum RuntimeHostAdmission { Attached(Box), Bootstrap { @@ -1651,7 +1651,7 @@ enum RuntimeHostAdmission { /// Observe first, then hold the existing store lease through all recovery. /// Only pre-write publication changes or lease contention can re-observe. -#[cfg(unix)] +#[cfg(any(unix, windows))] async fn admit_runtime_host( config_path: Option, selected_socket: Option, @@ -1735,14 +1735,14 @@ pub async fn run_http_server( Some(task_default_model.clone()), Some(options.workers), ); - #[cfg(unix)] + #[cfg(any(unix, windows))] let admission = admit_runtime_host( options.config_path.clone(), selected_control_socket(&options), RuntimeThreadManagerConfig::from_task_data_dir(task_cfg.data_dir.clone()).data_dir, ) .await?; - #[cfg(unix)] + #[cfg(any(unix, windows))] let (published, preheld_lock, existing_binding, recovery) = match admission { RuntimeHostAdmission::Attached(client) => (Some(*client), None, None, None), RuntimeHostAdmission::Bootstrap { @@ -1751,18 +1751,10 @@ pub async fn run_http_server( recovery, } => (None, Some(lock), binding, recovery), }; - #[cfg(windows)] - let published = codewhale_app_server::daemon_client::connect_if_published( - options.config_path.clone(), - selected_control_socket(&options), - ) - .await?; - #[cfg(not(unix))] + #[cfg(not(any(unix, windows)))] let preheld_lock = None; - #[cfg(not(unix))] + #[cfg(not(any(unix, windows)))] let existing_binding: Option = None; - #[cfg(windows)] - let recovery = None; #[cfg(any(unix, windows))] if let Some(control) = published { let selected_store = diff --git a/crates/tui/src/runtime_threads.rs b/crates/tui/src/runtime_threads.rs index 8a2f7dda92..3f0775e7e0 100644 --- a/crates/tui/src/runtime_threads.rs +++ b/crates/tui/src/runtime_threads.rs @@ -6728,7 +6728,7 @@ impl RuntimeProcessOwnerLock { } /// Host admission reuses this exact store lease, never a PID-file lock. - #[cfg(unix)] + #[cfg(any(unix, windows))] pub(crate) fn try_acquire_for_host(root: &Path, create: bool) -> Result> { if !create { checked_existing_runtime_store_dir(root)?;