diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index eff9c4eb6..b1c231d70 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -23,6 +23,28 @@ bool g_use_simd = true; void kv_set_simd(bool on) { g_use_simd = on; } +size_t rerope_physical_end(const CacheHeader& header, bool sliding) { + + const size_t current = + static_cast(header.current_seq_len); + + if (!sliding) + return current; + + const size_t max_seq = + static_cast(header.max_seq_len); + + const size_t sink = + static_cast(header.sink_size); + + if (max_seq <= sink + 1) + return std::min(current, max_seq); + + return std::min( + current, + max_seq - sink - 1); +} + namespace { // Round half-to-even like numpy. recent_frac widens from float to a slightly-off double, so snap diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index c33a5e659..698688d18 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -27,7 +27,7 @@ struct CacheHeader { uint64_t reserved[3]; }; static_assert(sizeof(CacheHeader) == 64, "CacheHeader must be 64 bytes"); - +size_t rerope_physical_end(const CacheHeader& header, bool sliding); void kv_set_simd(bool on); // score s_i = -cos(k_i, mean(k)); keys pre-RoPE. diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 5383c89bf..86779e6e0 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -5222,10 +5222,10 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) auto* khdr = static_cast(kraw); size_t kv_heads = khdr->num_kv_heads, head_dim = khdr->head_dim; if (kv_heads == 0 || head_dim == 0) continue; - size_t hi = khdr->current_seq_len; + const bool sliding = cactus::kvcompress::is_sliding_layer(config_.layer_types, li); + size_t hi = cactus::kvcompress::rerope_physical_end(*khdr, sliding); size_t lo = std::min(khdr->sink_size, hi); - const double layer_theta = cactus::kvcompress::is_sliding_layer(config_.layer_types, li) - ? rope_local_theta : rope_theta; + const double layer_theta = sliding ? rope_local_theta : rope_theta; if (kdesc.precision == Precision::FP16) { auto* kbase = reinterpret_cast(static_cast(kraw) + kHeaderBytes); cactus::kvcompress::rerope_recent_fp16(kbase, kv_heads, head_dim, lo, hi, diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp index 76eaae518..2a11d719d 100644 --- a/cactus-engine/tests/test_kv_compress.cpp +++ b/cactus-engine/tests/test_kv_compress.cpp @@ -994,6 +994,283 @@ bool test_shrink_cache_buffer_preserves_rows() { return ok; } +bool test_global_shrink_then_sliding_int8_rerope_cycles() { + CactusGraph gb; + + const size_t kv_heads = 2; + const size_t head_dim = 128; + const size_t chunk = 300; + const size_t ceiling = 100000; + const size_t sliding_window = 512; + const size_t sink = 4; + const double theta = 1000000.0; + + const size_t groups = + (head_dim + kGroupSize - 1) / kGroupSize; + + const size_t int8_stride = + kv_heads * head_dim; + + const size_t scale_stride = + kv_heads * groups; + + size_t new_kv = + gb.input( + {chunk, kv_heads, head_dim}, + Precision::FP16); + + size_t global_state = + gb.kv_cache_state( + ceiling, + kv_heads, + head_dim, + /*window=*/0, + sink); + + size_t global_append = + gb.kv_cache_append( + new_kv, + global_state, + /*window=*/0, + sink); + + size_t sliding_state = + gb.kv_cache_state( + ceiling, + kv_heads, + head_dim, + sliding_window, + sink); + + size_t sliding_append = + gb.kv_cache_append( + new_kv, + sliding_state, + sliding_window, + sink); + + gb.retain_outputs({ + static_cast(global_state), + static_cast(global_append), + static_cast(sliding_state), + static_cast(sliding_append) + }); + + std::vector input( + chunk * int8_stride); + + for (size_t r = 0; r < chunk; ++r) { + for (size_t c = 0; c < int8_stride; ++c) { + float v = + std::sin( + 0.013f * static_cast(r) + + 0.007f * static_cast(c)); + + input[r * int8_stride + c] = + f32_to_f16(v); + } + } + + gb.set_input( + new_kv, + input.data(), + Precision::FP16); + + auto reset_len = [&](size_t state) { + auto* hdr = + reinterpret_cast( + gb.get_output(state)); + + hdr->current_seq_len = 0; + }; + + auto compact_global = [&]() -> size_t { + auto* raw = + static_cast( + gb.get_output(global_state)); + + auto* hdr = + reinterpret_cast(raw); + + const size_t n = + hdr->current_seq_len; + + int8_t* i8 = + reinterpret_cast( + raw + kHeaderBytes); + + float* scales = + reinterpret_cast( + raw + + kHeaderBytes + + hdr->max_seq_len * int8_stride); + + Params p; + p.sink = sink; + p.recent_frac = 0.30f; + p.abs_budget = 256; + + auto unrope = + unrope_table( + n, + head_dim, + theta); + + auto kept = + keepsets_from_int8( + i8, + scales, + n, + kv_heads, + head_dim, + kGroupSize, + unrope, + p); + + const size_t new_len = + kept.empty() + ? 0 + : kept[0].size(); + + compact_int8( + i8, + scales, + kv_heads, + head_dim, + kGroupSize, + kept, + unrope, + /*renumber=*/true); + + hdr->current_seq_len = new_len; + + return new_len; + }; + + for (int turn = 0; turn < 3; ++turn) { + if (turn > 0) { + reset_len(global_state); + reset_len(sliding_state); + } + + gb.execute(); + gb.execute(); + + auto* global_hdr = + reinterpret_cast( + gb.get_output(global_state)); + + auto* sliding_hdr = + reinterpret_cast( + gb.get_output(sliding_state)); + + if (global_hdr->current_seq_len != 600) + return false; + + // Metal sliding caches keep current_seq_len as a logical cursor. + if (sliding_hdr->current_seq_len != 600) + return false; + + if (sliding_hdr->max_seq_len != + sliding_window + sink + 1) + return false; + + const size_t hi = + cactus::kvcompress::rerope_physical_end( + *sliding_hdr, + true); + + // The logical cursor is 600, but only 512 physical ring rows + // may be accessed by re-RoPE. + if (hi != sliding_window) + return false; + + const size_t old_total = + global_hdr->current_seq_len; + + const size_t new_total = + compact_global(); + + if (new_total != 256) + return false; + + gb.shrink_cache_buffer( + global_state, + /*new_capacity=*/512); + + global_hdr = + reinterpret_cast( + gb.get_output(global_state)); + + if (global_hdr->current_seq_len != 256 || + global_hdr->max_seq_len != 512) + return false; + + const size_t delta = + old_total - new_total; + + auto* sliding_raw = + static_cast( + gb.get_output(sliding_state)); + + sliding_hdr = + reinterpret_cast( + sliding_raw); + + const size_t physical_hi = + cactus::kvcompress::rerope_physical_end( + *sliding_hdr, + true); + + const size_t lo = + std::min( + sliding_hdr->sink_size, + physical_hi); + + int8_t* sliding_i8 = + reinterpret_cast( + sliding_raw + kHeaderBytes); + + float* sliding_scales = + reinterpret_cast( + sliding_raw + + kHeaderBytes + + sliding_hdr->max_seq_len * + int8_stride); + + kv_set_simd(true); + + rerope_recent_int8( + sliding_i8, + sliding_scales, + kv_heads, + head_dim, + kGroupSize, + lo, + physical_hi, + theta, + -static_cast(delta)); + + for (size_t t = 0; t < physical_hi; ++t) { + for (size_t h = 0; h < kv_heads; ++h) { + const float* sc = + sliding_scales + + t * scale_stride + + h * groups; + + for (size_t g = 0; g < groups; ++g) { + if (!std::isfinite(sc[g]) || + sc[g] <= 0.0f) + return false; + } + } + } + } + + kv_set_simd(true); + return true; +} + int main() { TestUtils::TestRunner runner("KV Compress Free-Function Tests"); runner.run_test("cache_starts_small_and_grows", test_cache_starts_small_and_grows()); @@ -1025,6 +1302,7 @@ int main() { runner.run_test("empty_protect_per_head_uses_params_fallback", test_empty_protect_per_head_uses_params_fallback()); runner.run_test("all_heads_keep_special_across_cycles", test_all_heads_keep_special_across_cycles()); runner.run_test("shrink_cache_buffer_preserves_rows", test_shrink_cache_buffer_preserves_rows()); + runner.run_test("global_shrink_then_sliding_int8_rerope_cycles", test_global_shrink_then_sliding_int8_rerope_cycles()); runner.print_summary(); return runner.all_passed() ? 0 : 1; }