From 7d8760f990d7ed332a9c609a1a013cc96b518f4f Mon Sep 17 00:00:00 2001 From: Nirupam Bhowmick <48842933+heydryft@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:20:59 +0100 Subject: [PATCH 1/6] perf(arckv): make candle's caching allocator reachable and on for decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `set_alloc_cache_enabled(true)` sat behind three stacked default-off gates: `probe && seq_len == 1 && env("ARC_CANDLE_ALLOC_CACHE")`. The first conjunct tied a general-purpose allocator to the V4 capture probe, so the only way to recycle a decode step's frees was to also be capturing. The ~11k allocations per token were a disabled feature, not a missing one. Replaced with a pure policy, `alloc_cache_action(seq_len, enabled, killed)`: * decode (`seq_len == 1`) -> Enable * prefill (`seq_len != 1`) -> DrainAndDisable * already in that state -> Leave Prefill draining is not incidental. The cache is keyed on exact byte count (`free: HashMap>`, no bucketing, no smallest-fit) and has no capacity bound and no eviction, so a prefill's large one-shot buffers would be parked for the process lifetime under a key nothing asks for again. `ARC_CANDLE_ALLOC_CACHE=0` is the kill switch. Any other value, and unset, leave the policy in force, so the `=1` the ops scripts pass still means what it always meant. The capture probe keeps its own gate for the graph-mode positions; only the allocator moved out from under it. Six host-runnable tests for the policy. The allocator has no tests at all in either repo — candle's `cuda_backend/{device,mod}.rs` carry no `#[cfg(test)]` and every arc-side exercise is `#[cfg(feature = "cuda")]` — so the decision of when it is on is now the part that CI can see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SpVNMpb13HkUXqSqbN1o9H --- mistralrs-core/src/pipeline/normal.rs | 200 ++++++++++++++++++++++++-- 1 file changed, 192 insertions(+), 8 deletions(-) diff --git a/mistralrs-core/src/pipeline/normal.rs b/mistralrs-core/src/pipeline/normal.rs index d412a7cd9..7f32c2563 100644 --- a/mistralrs-core/src/pipeline/normal.rs +++ b/mistralrs-core/src/pipeline/normal.rs @@ -98,6 +98,91 @@ pub struct NormalPipeline { autonomous_runner: Option, } +/// What a forward pass should do with candle's caching allocator before it +/// runs. See [`alloc_cache_action`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AllocCacheAction { + /// Turn the cache on. Frees start being recycled instead of returned to + /// the driver. + Enable, + /// Return every held buffer to the driver and turn the cache off. + DrainAndDisable, + /// Already in the right state; touch nothing. + Leave, +} + +/// Decide whether candle's caching allocator should be on for this forward. +/// +/// # Why this is not simply "on" +/// +/// The cache (fork `88d86a2`, `candle-core/src/cuda_backend/device.rs:42`) is +/// keyed on **exact byte count** — `free: HashMap>`, +/// looked up with `free.get_mut(&bytes)`. No bucketing, no smallest-fit, no +/// splitting. It also has **no capacity bound and no eviction**: the only ways +/// memory goes back to the driver are `set_alloc_cache_enabled(false)` and +/// `drain_alloc_cache_and_free()`. +/// +/// Those two facts together decide the policy: +/// +/// * **Decode wants it on.** A decode step allocates the same shapes 43 times +/// over (once per layer), and the shapes that do not depend on KV length — +/// hidden states, MLP and expert intermediates — repeat step after step. +/// Exact-size keying is a perfect fit for that traffic. +/// * **Prefill must drain it.** Prefill shapes scale with prompt length, are +/// large, and are hit once. Leaving the cache on across a prefill parks +/// those buffers for the process lifetime under a byte-size key nothing will +/// ever request again. That is the failure mode `ARC_NO_DEDICATED_DECODE` +/// already exists to work around, and it is why this returns +/// [`AllocCacheAction::DrainAndDisable`] rather than `Leave` on `seq_len != 1`. +/// +/// # The measurement this is waiting on +/// +/// Buffers whose size tracks KV length — the causal mask most obviously — +/// change size every decode step, so each step files one more never-reused +/// entry. That is bounded by `O(context^2)` bytes in the worst case and is +/// *not* bounded by this policy. It is small at short context (a `[1,1,1,kv]` +/// BF16 mask over 4k tokens sums to ~16 MB) and is not small at 128k. +/// +/// The fixed-capacity graph-mode path (`deepseek4.rs:4344-4353`, which swaps +/// the growing causal mask for `graph_mode_length_mask` at +/// `cfg_full.sliding_window`) removes that growth entirely, but it is reached +/// only under `ARC_V4_CAPTURE_PROBE`. Until the shape-invariance work lands, +/// `ARC_CANDLE_ALLOC_CACHE=0` is the kill switch, and the long-context +/// high-water mark is the number a GPU run should falsify this with. +/// +/// # Contract +/// +/// * `seq_len` — the forward's sequence length. `1` is decode. +/// * `enabled` — what candle reports *now* (`alloc_cache_enabled()`), so the +/// action is idempotent and we never drain a cache that is already off. +/// * `killed` — `ARC_CANDLE_ALLOC_CACHE=0` was set. +/// +/// Pure, so the policy is testable without a GPU — which matters, because the +/// allocator itself has no tests at all in either repo. +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn alloc_cache_action( + seq_len: usize, + enabled: bool, + killed: bool, +) -> AllocCacheAction { + let want = !killed && seq_len == 1; + match (want, enabled) { + (true, false) => AllocCacheAction::Enable, + (false, true) => AllocCacheAction::DrainAndDisable, + _ => AllocCacheAction::Leave, + } +} + +/// `ARC_CANDLE_ALLOC_CACHE=0` turns the caching allocator off entirely. +/// +/// Any other value — and *unset* — leaves the default policy in force. The +/// variable used to be the on-switch, so the value `1` the ops scripts pass +/// (`arc-tools/arcgraph_heap_probe.sh:191`) still means what it always meant. +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn alloc_cache_killed() -> bool { + std::env::var("ARC_CANDLE_ALLOC_CACHE").is_ok_and(|v| v == "0") +} + /// A loader for a "normal" (non-quantized) model. pub struct NormalLoader { inner: Box, @@ -1702,15 +1787,31 @@ impl Pipeline for NormalPipeline { "normal.rs:1554", ); } - // Enable the candle caching allocator for graph-capture - // safety (RUN-161). Idempotent; gated so it's off during - // model load and only active for decode. Warmup decode - // forwards populate the cache before capture. - if probe && seq_len == 1 && std::env::var_os("ARC_CANDLE_ALLOC_CACHE").is_some() - { - if let candle_core::Device::Cuda(cd) = self.device() { - cd.set_alloc_cache_enabled(true); + // Candle's caching allocator, driven by `alloc_cache_action`. + // + // This used to be `probe && seq_len == 1 && env(...)`, which + // made a general-purpose allocator reachable only when the + // V4 capture probe was also on. The cache recycles frees for + // ANY decode step — it is not capture machinery — and the + // three stacked default-off gates meant the ~11k allocations + // per token were a disabled feature rather than a missing + // one. It is now on for decode by default and drained on + // prefill; see `alloc_cache_action` for why prefill must + // drain rather than coast. + if let candle_core::Device::Cuda(cd) = self.device() { + match alloc_cache_action( + seq_len, + cd.alloc_cache_enabled(), + alloc_cache_killed(), + ) { + AllocCacheAction::Enable => cd.set_alloc_cache_enabled(true), + AllocCacheAction::DrainAndDisable => { + cd.set_alloc_cache_enabled(false) + } + AllocCacheAction::Leave => {} } + } + if probe && seq_len == 1 { // RUN-161 step 2b. Set the graph-mode device position: // drives RoPE + the fixed-capacity KV write slot, and // makes warmup forwards take the shape-constant path so @@ -2528,3 +2629,86 @@ impl AnyMoePipelineMixin for NormalPipeline { self.model.amoe_supported() } } + +/// The caching-allocator policy's contract. +/// +/// The allocator itself has **no tests at all** — not in this repo, and not in +/// the candle fork that implements it (`grep -rn "alloc_cache" --include="*.rs"` +/// over `candle-core` finds only `cuda_backend/device.rs` and +/// `cuda_backend/mod.rs`, with no `#[cfg(test)]` in either). Everything that +/// observes it needs a GPU, so nothing observes it in CI. +/// +/// The decision of *when* it is on does not need a GPU, so it is tested here. +#[cfg(test)] +mod alloc_cache_policy_tests { + use super::{alloc_cache_action, AllocCacheAction}; + + /// The change this policy exists to make: a decode step turns the cache on + /// without any capture probe being involved. + #[test] + fn decode_enables_the_cache() { + assert_eq!( + alloc_cache_action(1, false, false), + AllocCacheAction::Enable + ); + } + + /// Prefill must hand its buffers back. They are large, they are keyed by an + /// exact byte size no later request will ask for, and nothing evicts them. + #[test] + fn prefill_drains_the_cache() { + assert_eq!( + alloc_cache_action(512, true, false), + AllocCacheAction::DrainAndDisable + ); + } + + /// Called once per forward, so it must be a no-op in the steady state + /// rather than re-enabling (or re-draining) every step. + #[test] + fn steady_state_touches_nothing() { + assert_eq!(alloc_cache_action(1, true, false), AllocCacheAction::Leave); + assert_eq!( + alloc_cache_action(512, false, false), + AllocCacheAction::Leave + ); + } + + /// The kill switch has to work from either state, including turning off a + /// cache that a previous step already enabled — otherwise setting it + /// mid-run would leave the buffers stranded. + #[test] + fn kill_switch_disables_and_drains_from_either_state() { + assert_eq!( + alloc_cache_action(1, true, true), + AllocCacheAction::DrainAndDisable + ); + assert_eq!(alloc_cache_action(1, false, true), AllocCacheAction::Leave); + } + + /// A prompt that happens to be one token long is a prefill by every other + /// measure, but it allocates decode-shaped buffers, so the policy keys on + /// the shape it will actually see. Pinned so the equivalence is deliberate. + #[test] + fn one_token_prompt_is_treated_as_decode() { + assert_eq!( + alloc_cache_action(1, false, false), + AllocCacheAction::Enable + ); + } + + /// A zero-length forward is not decode. Guards against `seq_len == 0` + /// (which `dims2().unwrap_or((0, 0))` produces on a shape error) quietly + /// enabling the cache. + #[test] + fn degenerate_zero_length_forward_does_not_enable() { + assert_eq!( + alloc_cache_action(0, false, false), + AllocCacheAction::Leave + ); + assert_eq!( + alloc_cache_action(0, true, false), + AllocCacheAction::DrainAndDisable + ); + } +} From b0cf25a73411d6c01da65dea1e7a7bc2345a5248 Mon Sep 17 00:00:00 2001 From: Nirupam Bhowmick <48842933+heydryft@users.noreply.github.com> Date: Wed, 19 Aug 2026 23:48:23 +0100 Subject: [PATCH 2/6] perf(arckv): bound the caching allocator, and print the counters that prove it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Picks up candle b2a4dd80, which gives `AllocCache` a capacity and LRU eviction. The allocator had neither, and arc only drains it on a prefill, so a single long generation grew forever. Measured on an H200 over 2 600 tokens, `memory.used` at 2 Hz: **+6.04 MiB per decoded token with no plateau**, against +0.057 MiB/token for the same run with the cache off — so the growth is the cache's and nothing else's. `ARC_ALLOC_CACHE_MAX_MB` sets the cap; unset leaves candle's 1 GiB default; `0` restores the old unbounded behaviour for A/B. A typo deliberately does *not* fall back to unbounded. `ARC_ALLOC_CACHE_STATS=N` prints the allocator's counters every N decode steps: allocations per step, **frees per step**, hit rate, bytes held against the cap. Those are the numbers this has to be judged on. A green log is not evidence — an earlier arena here reported "accounting OK" and bit-identical output over 52 steps while silently bypassing itself for every buffer under 128 bytes (KERNEL_RULES.md:977-984). Allocations staying low *and* frees being non-zero is. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SpVNMpb13HkUXqSqbN1o9H --- Cargo.toml | 10 +- mistralrs-core/src/pipeline/normal.rs | 138 +++++++++++++++++++++++++- 2 files changed, 142 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b8dde1f07..225d8ecbe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,11 +49,11 @@ rust-version = "1.88" # pointer has broken CI on this repo before — a path/branch dependency that # advances under a merged PR turns a green build into a red one with no commit # here to blame. -candle-core = { git = "https://github.com/aeonmindai/candle.git", rev = "88d86a2019ee552923b217284e89847d2785dcdf" } -candle-nn = { git = "https://github.com/aeonmindai/candle.git", rev = "88d86a2019ee552923b217284e89847d2785dcdf" } -candle-flash-attn-v3 = { git = "https://github.com/aeonmindai/candle.git", rev = "88d86a2019ee552923b217284e89847d2785dcdf" } -candle-flash-attn = { git = "https://github.com/aeonmindai/candle.git", rev = "88d86a2019ee552923b217284e89847d2785dcdf" } -candle-metal-kernels = { git = "https://github.com/aeonmindai/candle.git", rev = "88d86a2019ee552923b217284e89847d2785dcdf" } +candle-core = { git = "https://github.com/aeonmindai/candle.git", rev = "b2a4dd80b82726730c3c4c90421fa3a0938be984" } +candle-nn = { git = "https://github.com/aeonmindai/candle.git", rev = "b2a4dd80b82726730c3c4c90421fa3a0938be984" } +candle-flash-attn-v3 = { git = "https://github.com/aeonmindai/candle.git", rev = "b2a4dd80b82726730c3c4c90421fa3a0938be984" } +candle-flash-attn = { git = "https://github.com/aeonmindai/candle.git", rev = "b2a4dd80b82726730c3c4c90421fa3a0938be984" } +candle-metal-kernels = { git = "https://github.com/aeonmindai/candle.git", rev = "b2a4dd80b82726730c3c4c90421fa3a0938be984" } axum = "0.8.8" anyhow = "1.0.100" diff --git a/mistralrs-core/src/pipeline/normal.rs b/mistralrs-core/src/pipeline/normal.rs index 7f32c2563..a69835923 100644 --- a/mistralrs-core/src/pipeline/normal.rs +++ b/mistralrs-core/src/pipeline/normal.rs @@ -183,6 +183,90 @@ pub(crate) fn alloc_cache_killed() -> bool { std::env::var("ARC_CANDLE_ALLOC_CACHE").is_ok_and(|v| v == "0") } +/// Retention cap for the caching allocator, in bytes, or `None` to leave +/// candle's own default (1 GiB) in force. +/// +/// `ARC_ALLOC_CACHE_MAX_MB=0` means unbounded — the pre-bounding behaviour, kept +/// reachable so the leak it causes can be A/B'd rather than argued about. +/// Measured unbounded on V4: **+6.04 MiB per decoded token with no plateau**, +/// against +0.057 MiB/token with the cache off entirely. +#[cfg_attr(not(feature = "cuda"), allow(dead_code))] +pub(crate) fn alloc_cache_capacity_bytes() -> Option { + parse_alloc_cache_capacity(std::env::var("ARC_ALLOC_CACHE_MAX_MB").ok().as_deref()) +} + +/// Pure half of [`alloc_cache_capacity_bytes`], so the parse is testable. +/// +/// `None` (unset, or unparseable) leaves candle's own default in force rather +/// than silently picking a different one — a typo in an ops script must not +/// quietly hand back the unbounded allocator this change exists to remove. +pub(crate) fn parse_alloc_cache_capacity(raw: Option<&str>) -> Option { + let mb: usize = raw?.trim().parse().ok()?; + Some(if mb == 0 { + usize::MAX + } else { + mb.saturating_mul(1024 * 1024) + }) +} + +/// Emit the allocator's counters every `ARC_ALLOC_CACHE_STATS` decode steps. +/// +/// The counters are what this cache has to be judged on. A green log — the +/// server did not crash, the output looked fine — is not evidence that the +/// cache is working: an earlier arena in this codebase reported "accounting OK" +/// and bit-identical output across 52 steps while silently bypassing itself for +/// every buffer under 128 bytes (`KERNEL_RULES.md:977-984`). What distinguishes +/// a working bounded cache from that is arithmetic, and it is printed here: +/// +/// * `alloc/step` — real `cuMemAllocAsync` calls per decode step. Low means the +/// cache is absorbing the ~11 k allocations a step makes. +/// * `free/step` — real `cuMemFreeAsync` calls per decode step. **Non-zero is +/// the point.** The unbounded cache's was exactly zero, forever, which is why +/// it grew without bound. +/// * `held` — bytes retained right now, against the cap. +#[cfg(feature = "cuda")] +fn report_alloc_cache_step(cd: &candle_core::CudaDevice, seq_len: usize) { + use std::sync::atomic::{AtomicU64, Ordering}; + static EVERY: std::sync::OnceLock = std::sync::OnceLock::new(); + let every = *EVERY.get_or_init(|| { + std::env::var("ARC_ALLOC_CACHE_STATS") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(0) + }); + if every == 0 || seq_len != 1 { + return; + } + static STEP: AtomicU64 = AtomicU64::new(0); + static LAST_ALLOC: AtomicU64 = AtomicU64::new(0); + static LAST_FREE: AtomicU64 = AtomicU64::new(0); + static LAST_STEP: AtomicU64 = AtomicU64::new(0); + let step = STEP.fetch_add(1, Ordering::Relaxed) + 1; + if step % every != 0 { + return; + } + let s = cd.alloc_cache_stats(); + let d_step = step - LAST_STEP.swap(step, Ordering::Relaxed); + let d_alloc = s.misses - LAST_ALLOC.swap(s.misses, Ordering::Relaxed); + let d_free = s.frees() - LAST_FREE.swap(s.frees(), Ordering::Relaxed); + let n = d_step.max(1) as f64; + tracing::info!( + "[alloc-cache] step {step} alloc/step {:.1} free/step {:.1} \ + hit-rate {:.4} held {:.1} MiB / cap {} high-water {:.1} MiB sizes {}", + d_alloc as f64 / n, + d_free as f64 / n, + s.hits as f64 / (s.hits + s.misses).max(1) as f64, + s.cached_bytes as f64 / (1024.0 * 1024.0), + if s.capacity_bytes == usize::MAX { + "unbounded".to_string() + } else { + format!("{} MiB", s.capacity_bytes / (1024 * 1024)) + }, + s.high_water_bytes as f64 / (1024.0 * 1024.0), + s.size_classes, + ); +} + /// A loader for a "normal" (non-quantized) model. pub struct NormalLoader { inner: Box, @@ -1804,12 +1888,18 @@ impl Pipeline for NormalPipeline { cd.alloc_cache_enabled(), alloc_cache_killed(), ) { - AllocCacheAction::Enable => cd.set_alloc_cache_enabled(true), + AllocCacheAction::Enable => { + cd.set_alloc_cache_enabled(true); + if let Some(cap) = alloc_cache_capacity_bytes() { + cd.set_alloc_cache_capacity(cap); + } + } AllocCacheAction::DrainAndDisable => { cd.set_alloc_cache_enabled(false) } AllocCacheAction::Leave => {} } + report_alloc_cache_step(cd, seq_len); } if probe && seq_len == 1 { // RUN-161 step 2b. Set the graph-mode device position: @@ -2712,3 +2802,49 @@ mod alloc_cache_policy_tests { ); } } + +/// The retention cap's parse. +/// +/// The cap is the whole of the fix — candle's caching allocator had none, and +/// retained 6.04 MiB per decoded token forever — so the way it is configured has +/// to be unambiguous. In particular a typo must not fall back to "unbounded". +#[cfg(test)] +mod alloc_cache_capacity_tests { + use super::parse_alloc_cache_capacity; + + #[test] + fn unset_leaves_candles_default_in_force() { + assert_eq!(parse_alloc_cache_capacity(None), None); + } + + /// A typo is not a licence to run unbounded. `None` means "don't touch it", + /// and candle's default is bounded, so a bad value degrades to bounded. + #[test] + fn an_unparseable_value_does_not_become_unbounded() { + for bad in ["", " ", "lots", "1GiB", "-1", "1.5"] { + assert_eq!(parse_alloc_cache_capacity(Some(bad)), None, "{bad:?}"); + } + } + + /// `0` is the documented escape hatch back to the old unbounded allocator, + /// kept reachable only so its leak can be re-measured rather than argued + /// about. + #[test] + fn zero_is_the_explicit_unbounded_opt_in() { + assert_eq!(parse_alloc_cache_capacity(Some("0")), Some(usize::MAX)); + } + + #[test] + fn megabytes_convert_and_do_not_overflow() { + assert_eq!(parse_alloc_cache_capacity(Some("1")), Some(1024 * 1024)); + assert_eq!( + parse_alloc_cache_capacity(Some(" 512 ")), + Some(512 * 1024 * 1024) + ); + assert_eq!( + parse_alloc_cache_capacity(Some(&usize::MAX.to_string())), + Some(usize::MAX), + "saturates instead of wrapping to a tiny cap" + ); + } +} From e999ef55b5c88aaa2dd37f90c7a26b3854795fde Mon Sep 17 00:00:00 2001 From: Nirupam Bhowmick <48842933+heydryft@users.noreply.github.com> Date: Wed, 19 Aug 2026 23:54:30 +0100 Subject: [PATCH 3/6] fix(build): report_alloc_cache_step takes a reference Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SpVNMpb13HkUXqSqbN1o9H --- mistralrs-core/src/pipeline/normal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mistralrs-core/src/pipeline/normal.rs b/mistralrs-core/src/pipeline/normal.rs index a69835923..4675ab926 100644 --- a/mistralrs-core/src/pipeline/normal.rs +++ b/mistralrs-core/src/pipeline/normal.rs @@ -1899,7 +1899,7 @@ impl Pipeline for NormalPipeline { } AllocCacheAction::Leave => {} } - report_alloc_cache_step(cd, seq_len); + report_alloc_cache_step(&cd, seq_len); } if probe && seq_len == 1 { // RUN-161 step 2b. Set the graph-mode device position: From 576e59995ef8ce5e9f0cd622e594b2f644f96296 Mon Sep 17 00:00:00 2001 From: Nirupam Bhowmick <48842933+heydryft@users.noreply.github.com> Date: Thu, 20 Aug 2026 00:22:50 +0100 Subject: [PATCH 4/6] =?UTF-8?q?chore(deps):=20candle=20859c49c8=20?= =?UTF-8?q?=E2=80=94=20allocator=20test=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test-only change on the candle side; `git diff b2a4dd80..859c49c8` touches nothing outside `#[cfg(test)] mod alloc_cache_tests`. The H200 numbers in the branch description were measured at b2a4dd80 and stand. Correction to that commit message: the allocator has **ten** tests, not eleven. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SpVNMpb13HkUXqSqbN1o9H --- Cargo.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 225d8ecbe..0f7f7935f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,11 +49,11 @@ rust-version = "1.88" # pointer has broken CI on this repo before — a path/branch dependency that # advances under a merged PR turns a green build into a red one with no commit # here to blame. -candle-core = { git = "https://github.com/aeonmindai/candle.git", rev = "b2a4dd80b82726730c3c4c90421fa3a0938be984" } -candle-nn = { git = "https://github.com/aeonmindai/candle.git", rev = "b2a4dd80b82726730c3c4c90421fa3a0938be984" } -candle-flash-attn-v3 = { git = "https://github.com/aeonmindai/candle.git", rev = "b2a4dd80b82726730c3c4c90421fa3a0938be984" } -candle-flash-attn = { git = "https://github.com/aeonmindai/candle.git", rev = "b2a4dd80b82726730c3c4c90421fa3a0938be984" } -candle-metal-kernels = { git = "https://github.com/aeonmindai/candle.git", rev = "b2a4dd80b82726730c3c4c90421fa3a0938be984" } +candle-core = { git = "https://github.com/aeonmindai/candle.git", rev = "859c49c8c4e24177378a816f7b3f3d284987c62f" } +candle-nn = { git = "https://github.com/aeonmindai/candle.git", rev = "859c49c8c4e24177378a816f7b3f3d284987c62f" } +candle-flash-attn-v3 = { git = "https://github.com/aeonmindai/candle.git", rev = "859c49c8c4e24177378a816f7b3f3d284987c62f" } +candle-flash-attn = { git = "https://github.com/aeonmindai/candle.git", rev = "859c49c8c4e24177378a816f7b3f3d284987c62f" } +candle-metal-kernels = { git = "https://github.com/aeonmindai/candle.git", rev = "859c49c8c4e24177378a816f7b3f3d284987c62f" } axum = "0.8.8" anyhow = "1.0.100" From 67ba08c034b35087617cd623d19cc9e09ccf16ed Mon Sep 17 00:00:00 2001 From: Nirupam Bhowmick <48842933+heydryft@users.noreply.github.com> Date: Thu, 20 Aug 2026 00:50:50 +0100 Subject: [PATCH 5/6] =?UTF-8?q?chore(deps):=20candle=2089ab14ef=20?= =?UTF-8?q?=E2=80=94=20no=20eviction=20inside=20the=20capture=20window?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes a free path this change introduced: leaving capture mode re-filed parked buffers through the evicting put, which could hand a private-pool pointer back to the driver at a moment arc-cuda-graph does not control. Decode is unaffected — `set_capture_mode` is only called during a capture. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SpVNMpb13HkUXqSqbN1o9H --- Cargo.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0f7f7935f..b8852835d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,11 +49,11 @@ rust-version = "1.88" # pointer has broken CI on this repo before — a path/branch dependency that # advances under a merged PR turns a green build into a red one with no commit # here to blame. -candle-core = { git = "https://github.com/aeonmindai/candle.git", rev = "859c49c8c4e24177378a816f7b3f3d284987c62f" } -candle-nn = { git = "https://github.com/aeonmindai/candle.git", rev = "859c49c8c4e24177378a816f7b3f3d284987c62f" } -candle-flash-attn-v3 = { git = "https://github.com/aeonmindai/candle.git", rev = "859c49c8c4e24177378a816f7b3f3d284987c62f" } -candle-flash-attn = { git = "https://github.com/aeonmindai/candle.git", rev = "859c49c8c4e24177378a816f7b3f3d284987c62f" } -candle-metal-kernels = { git = "https://github.com/aeonmindai/candle.git", rev = "859c49c8c4e24177378a816f7b3f3d284987c62f" } +candle-core = { git = "https://github.com/aeonmindai/candle.git", rev = "89ab14ef1216331a782539f533b99f6816708604" } +candle-nn = { git = "https://github.com/aeonmindai/candle.git", rev = "89ab14ef1216331a782539f533b99f6816708604" } +candle-flash-attn-v3 = { git = "https://github.com/aeonmindai/candle.git", rev = "89ab14ef1216331a782539f533b99f6816708604" } +candle-flash-attn = { git = "https://github.com/aeonmindai/candle.git", rev = "89ab14ef1216331a782539f533b99f6816708604" } +candle-metal-kernels = { git = "https://github.com/aeonmindai/candle.git", rev = "89ab14ef1216331a782539f533b99f6816708604" } axum = "0.8.8" anyhow = "1.0.100" From 3c4bb1d2dfab7a3d7fc4d2112d6181b10acc7e90 Mon Sep 17 00:00:00 2001 From: Nirupam Bhowmick <48842933+heydryft@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:56:53 +0100 Subject: [PATCH 6/6] fix(ci): spell 'unparsable' the way the typos gate expects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two sites in pipeline/normal.rs — a doc comment and a test name. No behaviour change; the Typos job was the only red check on this PR. Co-Authored-By: Claude Opus 5 (1M context) --- mistralrs-core/src/pipeline/normal.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mistralrs-core/src/pipeline/normal.rs b/mistralrs-core/src/pipeline/normal.rs index 4675ab926..0bcc24e8a 100644 --- a/mistralrs-core/src/pipeline/normal.rs +++ b/mistralrs-core/src/pipeline/normal.rs @@ -197,7 +197,7 @@ pub(crate) fn alloc_cache_capacity_bytes() -> Option { /// Pure half of [`alloc_cache_capacity_bytes`], so the parse is testable. /// -/// `None` (unset, or unparseable) leaves candle's own default in force rather +/// `None` (unset, or unparsable) leaves candle's own default in force rather /// than silently picking a different one — a typo in an ops script must not /// quietly hand back the unbounded allocator this change exists to remove. pub(crate) fn parse_alloc_cache_capacity(raw: Option<&str>) -> Option { @@ -2820,7 +2820,7 @@ mod alloc_cache_capacity_tests { /// A typo is not a licence to run unbounded. `None` means "don't touch it", /// and candle's default is bounded, so a bad value degrades to bounded. #[test] - fn an_unparseable_value_does_not_become_unbounded() { + fn an_unparsable_value_does_not_become_unbounded() { for bad in ["", " ", "lots", "1GiB", "-1", "1.5"] { assert_eq!(parse_alloc_cache_capacity(Some(bad)), None, "{bad:?}"); }