From 1e27909197a3ce59a15618dc328ca3ff538c6b07 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 14 May 2026 13:15:48 -0700 Subject: [PATCH 1/2] Add Gemma 4 MTP scaffolding --- cactus-engine/CMakeLists.txt | 2 + .../models/gemma4/gemma4_mtp_assistant.cpp | 571 ++++++++++++ .../models/gemma4/gemma4_mtp_assistant.h | 78 ++ cactus-engine/models/gemma4/model_gemma4.cpp | 838 +++++++++++++++++- cactus-engine/models/gemma4/model_gemma4.h | 51 ++ .../models/gemma4/model_gemma4_mm.cpp | 4 + cactus-engine/models/model.h | 1 + cactus-engine/models/model_lfm2vl.cpp | 4 + cactus-engine/src/complete.cpp | 698 ++++++++++++++- cactus-engine/src/decode_strategy.h | 134 +++ cactus-engine/src/engine.h | 3 +- cactus-engine/src/gemma4_mtp_decode.cpp | 411 +++++++++ cactus-engine/src/gemma4_mtp_decode.h | 111 +++ cactus-engine/src/mtp_completion.h | 125 +++ cactus-engine/src/mtp_sampler.h | 199 +++++ cactus-engine/src/utils.h | 56 +- cactus-engine/tests/chat.cpp | 45 +- cactus-engine/tests/test_decode_strategy.cpp | 165 ++++ cactus-engine/tests/test_gemma4_mtp_fast.cpp | 634 +++++++++++++ cactus-engine/tests/test_mtp_completion.cpp | 106 +++ cactus-engine/tests/test_mtp_sampler.cpp | 186 ++++ cactus-graph/CMakeLists.txt | 1 + cactus-graph/cactus_graph.h | 47 +- cactus-graph/src/builder.cpp | 47 + cactus-graph/src/cache_transaction.cpp | 165 ++++ cactus-graph/src/execute.cpp | 10 +- cactus-graph/src/ops_cache.cpp | 68 ++ cactus-graph/tests/test_cache_transaction.cpp | 148 ++++ cactus-kernels/cactus_kernels.h | 3 +- cactus-kernels/src/attention_hybrid.cpp | 84 +- cactus/build.sh | 2 +- python/cactus/cli/download.py | 50 ++ python/cactus/hf_gemma_assistant.py | 65 ++ python/cactus/hf_gemma_mtp_check.py | 456 ++++++++++ python/pyproject.toml | 5 +- python/requirements.txt | 3 +- tests/bench/README.md | 42 + tests/bench/analyze_mtp_cost_diagnostics.py | 190 ++++ tests/bench/e2e_cactus_decode.cpp | 379 ++++++++ tests/bench/e2e_models.json | 20 + tests/bench_e2e_decode.sh | 429 +++++++++ tests/mtp/gemma4_mtp_cases.json | 41 + tests/mtp/run_fast_gemma4_mtp.sh | 84 ++ 43 files changed, 6688 insertions(+), 73 deletions(-) create mode 100644 cactus-engine/models/gemma4/gemma4_mtp_assistant.cpp create mode 100644 cactus-engine/models/gemma4/gemma4_mtp_assistant.h create mode 100644 cactus-engine/src/decode_strategy.h create mode 100644 cactus-engine/src/gemma4_mtp_decode.cpp create mode 100644 cactus-engine/src/gemma4_mtp_decode.h create mode 100644 cactus-engine/src/mtp_completion.h create mode 100644 cactus-engine/src/mtp_sampler.h create mode 100644 cactus-engine/tests/test_decode_strategy.cpp create mode 100644 cactus-engine/tests/test_gemma4_mtp_fast.cpp create mode 100644 cactus-engine/tests/test_mtp_completion.cpp create mode 100644 cactus-engine/tests/test_mtp_sampler.cpp create mode 100644 cactus-graph/src/cache_transaction.cpp create mode 100644 cactus-graph/tests/test_cache_transaction.cpp create mode 100644 python/cactus/hf_gemma_assistant.py create mode 100644 python/cactus/hf_gemma_mtp_check.py create mode 100644 tests/bench/README.md create mode 100644 tests/bench/analyze_mtp_cost_diagnostics.py create mode 100644 tests/bench/e2e_cactus_decode.cpp create mode 100644 tests/bench/e2e_models.json create mode 100755 tests/bench_e2e_decode.sh create mode 100644 tests/mtp/gemma4_mtp_cases.json create mode 100755 tests/mtp/run_fast_gemma4_mtp.sh diff --git a/cactus-engine/CMakeLists.txt b/cactus-engine/CMakeLists.txt index a6ff0470f..fb078ce81 100644 --- a/cactus-engine/CMakeLists.txt +++ b/cactus-engine/CMakeLists.txt @@ -69,6 +69,7 @@ set(ENGINE_SOURCES src/init.cpp src/embed.cpp src/complete.cpp + src/gemma4_mtp_decode.cpp src/transcribe.cpp src/tokenizer.cpp src/npu.cpp @@ -80,6 +81,7 @@ set(MODEL_SOURCES models/model_qwen.cpp models/model_siglip2.cpp models/gemma4/model_gemma4.cpp + models/gemma4/gemma4_mtp_assistant.cpp models/gemma4/model_gemma4_mm.cpp models/gemma4/model_gemma4_audio.cpp models/gemma4/model_gemma4_vision.cpp diff --git a/cactus-engine/models/gemma4/gemma4_mtp_assistant.cpp b/cactus-engine/models/gemma4/gemma4_mtp_assistant.cpp new file mode 100644 index 000000000..dbb700e93 --- /dev/null +++ b/cactus-engine/models/gemma4/gemma4_mtp_assistant.cpp @@ -0,0 +1,571 @@ +#include "gemma4_mtp_assistant.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +namespace { + +constexpr size_t kTargetHiddenDim = 1536; +constexpr size_t kCentroidCount = 2048; +constexpr size_t kTopCentroidCount = 32; +constexpr size_t kTokensPerCentroid = 128; +constexpr size_t kTokenOrderingSize = kCentroidCount * kTokensPerCentroid; + +std::vector probs_from_logits(std::vector logits, float temperature, float top_p, size_t top_k, float min_p) { + if (logits.empty()) return {}; + if (temperature == 0.0f) { + std::vector probs(logits.size(), 0.0f); + auto it = std::max_element(logits.begin(), logits.end()); + probs[std::distance(logits.begin(), it)] = 1.0f; + return probs; + } + if (temperature > 0.0f) { + for (float& value : logits) value /= temperature; + } + if (top_k > 0 && top_k < logits.size()) { + std::vector sorted = logits; + std::nth_element(sorted.begin(), sorted.begin() + top_k - 1, sorted.end(), std::greater()); + float kth = sorted[top_k - 1]; + for (float& value : logits) { + if (value < kth) value = -std::numeric_limits::infinity(); + } + } + auto normalize = [](const std::vector& values) { + std::vector probs(values.size(), 0.0f); + float max_value = -std::numeric_limits::infinity(); + for (float value : values) max_value = std::max(max_value, value); + if (!std::isfinite(max_value)) return probs; + double sum = 0.0; + for (size_t i = 0; i < values.size(); ++i) { + if (std::isfinite(values[i])) { + probs[i] = std::exp(static_cast(values[i] - max_value)); + sum += probs[i]; + } + } + if (sum <= 0.0 || !std::isfinite(sum)) return probs; + for (float& prob : probs) prob = static_cast(prob / sum); + return probs; + }; + if (min_p > 0.0f) { + auto probs = normalize(logits); + float threshold = (*std::max_element(probs.begin(), probs.end())) * min_p; + for (size_t i = 0; i < logits.size(); ++i) { + if (probs[i] < threshold) logits[i] = -std::numeric_limits::infinity(); + } + } + if (top_p > 0.0f && top_p < 1.0f) { + std::vector> sorted; + sorted.reserve(logits.size()); + for (size_t i = 0; i < logits.size(); ++i) { + if (std::isfinite(logits[i])) sorted.emplace_back(logits[i], i); + } + std::sort(sorted.begin(), sorted.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); + std::vector sorted_logits; + sorted_logits.reserve(sorted.size()); + for (const auto& item : sorted) sorted_logits.push_back(item.first); + auto sorted_probs = normalize(sorted_logits); + float cumulative = 0.0f; + bool remove = false; + for (size_t i = 0; i < sorted.size(); ++i) { + cumulative += sorted_probs[i]; + if (remove) logits[sorted[i].second] = -std::numeric_limits::infinity(); + if (cumulative > top_p) remove = true; + } + } + auto probs = normalize(logits); + float sum = std::accumulate(probs.begin(), probs.end(), 0.0f); + if (sum <= 0.0f || !std::isfinite(sum)) { + std::fill(probs.begin(), probs.end(), 1.0f / static_cast(probs.size())); + } + return probs; +} + +std::vector> sparse_probs_from_logits_row( + const void* logits_ptr, Precision precision, size_t row_offset, size_t vocab_size, + float temperature, float top_p, size_t top_k, float min_p) { + using Entry = std::pair; + std::priority_queue, std::greater> top; + size_t k = std::min(top_k, vocab_size); + auto push_value = [&](float value, uint32_t token) { + if (top.size() < k) { + top.emplace(value, token); + } else if (value > top.top().first) { + top.pop(); + top.emplace(value, token); + } + }; + + if (precision == Precision::FP32) { + const float* src = static_cast(logits_ptr) + row_offset; + for (size_t i = 0; i < vocab_size; ++i) push_value(src[i], static_cast(i)); + } else if (precision == Precision::FP16) { + const __fp16* src = static_cast(logits_ptr) + row_offset; + for (size_t i = 0; i < vocab_size; ++i) push_value(static_cast(src[i]), static_cast(i)); + } else { + const int8_t* src = static_cast(logits_ptr) + row_offset; + for (size_t i = 0; i < vocab_size; ++i) push_value(static_cast(src[i]), static_cast(i)); + } + + std::vector> logits; + logits.reserve(top.size()); + while (!top.empty()) { + logits.emplace_back(top.top().second, top.top().first); + top.pop(); + } + if (temperature > 0.0f) { + for (auto& item : logits) item.second /= temperature; + } + std::sort(logits.begin(), logits.end(), [](const auto& a, const auto& b) { + return a.second > b.second; + }); + + auto normalize = [](const std::vector>& values) { + std::vector> probs; + probs.reserve(values.size()); + float max_value = -std::numeric_limits::infinity(); + for (const auto& item : values) max_value = std::max(max_value, item.second); + if (!std::isfinite(max_value)) return probs; + double sum = 0.0; + for (const auto& item : values) { + float prob = std::exp(static_cast(item.second - max_value)); + probs.emplace_back(item.first, prob); + sum += prob; + } + if (sum <= 0.0 || !std::isfinite(sum)) return std::vector>{}; + for (auto& item : probs) item.second = static_cast(item.second / sum); + return probs; + }; + + if (min_p > 0.0f) { + auto probs = normalize(logits); + if (!probs.empty()) { + float max_prob = 0.0f; + for (const auto& item : probs) max_prob = std::max(max_prob, item.second); + float threshold = max_prob * min_p; + std::vector> filtered; + filtered.reserve(logits.size()); + for (size_t i = 0; i < logits.size(); ++i) { + if (probs[i].second >= threshold) filtered.push_back(logits[i]); + } + logits = std::move(filtered); + } + } + if (top_p > 0.0f && top_p < 1.0f) { + auto probs = normalize(logits); + std::vector> filtered; + filtered.reserve(logits.size()); + float cumulative = 0.0f; + bool remove = false; + for (size_t i = 0; i < logits.size(); ++i) { + cumulative += i < probs.size() ? probs[i].second : 0.0f; + if (!remove) filtered.push_back(logits[i]); + if (cumulative > top_p) remove = true; + } + logits = std::move(filtered); + } + + auto probs = normalize(logits); + float sum = 0.0f; + for (const auto& item : probs) sum += item.second; + if (sum <= 0.0f || !std::isfinite(sum)) { + float uniform = probs.empty() ? 0.0f : 1.0f / static_cast(probs.size()); + for (auto& item : probs) item.second = uniform; + } + return probs; +} + +} + +bool Gemma4MtpAssistant::init(CactusGraph* graph, const std::string& assistant_path) { + if (!graph) { + throw std::invalid_argument("Gemma4MtpAssistant requires a graph"); + } + std::filesystem::path resolved_path; + const std::filesystem::path base_path(assistant_path); + const std::vector candidates = { + base_path, + base_path / "assistant", + }; + for (const auto& candidate : candidates) { + if (std::filesystem::exists(candidate / "assistant_cactus_manifest.json")) { + resolved_path = candidate; + break; + } + } + if (resolved_path.empty()) { + return false; + } + + graph_ = graph; + path_ = resolved_path.string(); + token_embeddings_ = graph_->mmap_embeddings(path_ + "/token_embeddings.weights"); + masked_centroids_ = graph_->mmap_weights(path_ + "/masked_embedding_centroids.weights"); + pre_projection_ = graph_->mmap_weights(path_ + "/pre_projection.weights"); + post_projection_ = graph_->mmap_weights(path_ + "/post_projection.weights"); + output_norm_ = graph_->mmap_weights(path_ + "/output_norm.weights"); + + layers_.resize(4); + for (uint32_t i = 0; i < layers_.size(); ++i) { + auto& layer = layers_[i]; + std::string prefix = path_ + "/layer_" + std::to_string(i) + "_"; + layer.input_norm = graph_->mmap_weights(prefix + "input_norm.weights"); + layer.attn_q = graph_->mmap_weights(prefix + "attn_q.weights"); + layer.attn_q_norm = graph_->mmap_weights(prefix + "attn_q_norm.weights"); + layer.attn_output = graph_->mmap_weights(prefix + "attn_output.weights"); + layer.post_attn_norm = graph_->mmap_weights(prefix + "post_attn_norm.weights"); + layer.pre_ffn_norm = graph_->mmap_weights(prefix + "pre_ffn_norm.weights"); + layer.ffn_gate = graph_->mmap_weights(prefix + "ffn_gate.weights"); + layer.ffn_up = graph_->mmap_weights(prefix + "ffn_up.weights"); + layer.ffn_down = graph_->mmap_weights(prefix + "ffn_down.weights"); + layer.post_ffn_norm = graph_->mmap_weights(prefix + "post_ffn_norm.weights"); + layer.layer_scalar = graph_->mmap_weights(prefix + "layer_scalar.weights"); + } + + std::ifstream ordering_file(path_ + "/masked_embedding_token_ordering.json"); + std::string ordering_json((std::istreambuf_iterator(ordering_file)), std::istreambuf_iterator()); + size_t values_pos = ordering_json.find("\"values\""); + size_t array_start = ordering_json.find('[', values_pos); + size_t array_end = ordering_json.find(']', array_start); + if (values_pos == std::string::npos || array_start == std::string::npos || array_end == std::string::npos) { + throw std::runtime_error("Invalid masked_embedding_token_ordering.json"); + } + token_ordering_.clear(); + token_ordering_.reserve(kTokenOrderingSize); + size_t pos = array_start + 1; + while (pos < array_end) { + while (pos < array_end + && (ordering_json[pos] == ',' || std::isspace(static_cast(ordering_json[pos])))) { + pos++; + } + if (pos >= array_end) break; + char* end_ptr = nullptr; + unsigned long value = std::strtoul(ordering_json.c_str() + pos, &end_ptr, 10); + if (end_ptr == ordering_json.c_str() + pos) { + throw std::runtime_error("Invalid Gemma 4 assistant token ordering value"); + } + token_ordering_.push_back(static_cast(value)); + pos = static_cast(end_ptr - ordering_json.c_str()); + } + if (token_ordering_.size() != kTokenOrderingSize) { + throw std::runtime_error("Unexpected Gemma 4 assistant token ordering size"); + } + + initialized_ = true; + return true; +} + +size_t Gemma4MtpAssistant::apply_rope(CactusGraph* gb, size_t tensor, size_t head_dim, size_t rot_dim, + float rope_freq, size_t position) { + if (rot_dim < head_dim) { + size_t half_dim = head_dim / 2; + size_t half_rot = rot_dim / 2; + size_t pass_len = half_dim - half_rot; + float adjusted_theta = std::pow(rope_freq, static_cast(rot_dim) / static_cast(head_dim)); + + auto left_rot = gb->slice(tensor, 3, 0, half_rot); + auto left_pass = gb->slice(tensor, 3, half_rot, pass_len); + auto right_rot = gb->slice(tensor, 3, half_dim, half_rot); + auto right_pass = gb->slice(tensor, 3, half_dim + half_rot, pass_len); + + auto rotated = gb->rope(gb->concat(left_rot, right_rot, 3), adjusted_theta, position); + auto rotated_left = gb->slice(rotated, 3, 0, half_rot); + auto rotated_right = gb->slice(rotated, 3, half_rot, half_rot); + + auto new_left = gb->concat(rotated_left, left_pass, 3); + auto new_right = gb->concat(rotated_right, right_pass, 3); + return gb->concat(new_left, new_right, 3); + } + return gb->rope(tensor, rope_freq, position); +} + +size_t Gemma4MtpAssistant::build_layer(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + const Gemma4Model::SharedCacheNodes& cache_nodes, size_t position) { + const auto& layer = layers_.at(layer_idx); + const bool is_full = layer_idx == 3; + const size_t head_dim = is_full ? 512 : 256; + const size_t num_heads = 4; + const size_t rot_dim = is_full ? 128 : head_dim; + const float rope_freq = is_full ? 1000000.0f : 10000.0f; + const size_t window = is_full ? 0 : 512; + const size_t cache_k = is_full ? cache_nodes.full_k : cache_nodes.sliding_k; + const size_t cache_v = is_full ? cache_nodes.full_v : cache_nodes.sliding_v; + if (cache_k == 0 || cache_v == 0) { + throw std::runtime_error("Gemma 4 MTP target shared KV cache is unavailable"); + } + + auto normed = gb->rms_norm(hidden, layer.input_norm, 1e-6f); + auto q = gb->matmul(normed, layer.attn_q, true, ComputeBackend::CPU); + q = gb->reshape(q, {1, 1, num_heads, head_dim}); + q = gb->reshape(gb->rms_norm(gb->reshape(q, {num_heads, head_dim}), layer.attn_q_norm, 1e-6f), + {1, 1, num_heads, head_dim}); + q = apply_rope(gb, q, head_dim, rot_dim, rope_freq, position); + + auto attn = gb->attention_cache_only(q, cache_k, cache_v, 1.0f, position, window); + attn = gb->matmul(gb->reshape(attn, {1, num_heads * head_dim}), layer.attn_output, true, ComputeBackend::CPU); + attn = gb->rms_norm(attn, layer.post_attn_norm, 1e-6f); + auto residual = gb->add(hidden, attn); + + auto mlp_in = gb->rms_norm(residual, layer.pre_ffn_norm, 1e-6f); + auto gate = gb->gelu(gb->matmul(mlp_in, layer.ffn_gate, true, ComputeBackend::CPU)); + auto up = gb->matmul(mlp_in, layer.ffn_up, true, ComputeBackend::CPU); + auto mlp = gb->matmul(gb->multiply(gate, up), layer.ffn_down, true, ComputeBackend::CPU); + mlp = gb->rms_norm(mlp, layer.post_ffn_norm, 1e-6f); + auto out = gb->add(residual, mlp); + return gb->multiply(out, layer.layer_scalar); +} + +std::vector Gemma4MtpAssistant::candidate_tokens_from_centroids(CactusGraph* gb, + size_t centroid_logits_node) const { + const auto& centroid_buf = gb->get_output_buffer(centroid_logits_node); + void* centroid_ptr = gb->get_output(centroid_logits_node); + const size_t centroid_count = centroid_buf.total_size; + if (centroid_count == 0 || centroid_count * kTokensPerCentroid > token_ordering_.size()) { + throw std::runtime_error("Gemma 4 MTP assistant centroid/token ordering shape mismatch"); + } + + std::vector> centroids; + centroids.reserve(centroid_count); + for (uint32_t i = 0; i < centroid_count; ++i) { + float value = 0.0f; + if (centroid_buf.precision == Precision::FP32) { + value = static_cast(centroid_ptr)[i]; + } else if (centroid_buf.precision == Precision::FP16) { + value = static_cast(static_cast(centroid_ptr)[i]); + } else { + value = static_cast(static_cast(centroid_ptr)[i]); + } + centroids.emplace_back(value, i); + } + const size_t top_count = std::min(kTopCentroidCount, centroids.size()); + std::partial_sort(centroids.begin(), centroids.begin() + top_count, centroids.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + + std::vector candidates; + candidates.reserve(top_count * kTokensPerCentroid); + for (size_t c = 0; c < top_count; ++c) { + size_t offset = static_cast(centroids[c].second) * kTokensPerCentroid; + for (size_t i = 0; i < kTokensPerCentroid; ++i) { + candidates.push_back(token_ordering_[offset + i]); + } + } + return candidates; +} + +uint32_t Gemma4MtpAssistant::argmax_masked_logits(CactusGraph* gb, size_t logits_node, size_t centroid_logits_node) const { + const auto& logits_buf = gb->get_output_buffer(logits_node); + const size_t vocab_size = logits_buf.shape.back(); + const size_t row_offset = (logits_buf.total_size / vocab_size - 1) * vocab_size; + void* logits_ptr = gb->get_output(logits_node); + auto candidates = candidate_tokens_from_centroids(gb, centroid_logits_node); + + auto logit_at = [&](uint32_t token) -> float { + if (logits_buf.precision == Precision::FP32) { + return static_cast(logits_ptr)[row_offset + token]; + } + if (logits_buf.precision == Precision::FP16) { + return static_cast(static_cast(logits_ptr)[row_offset + token]); + } + return static_cast(static_cast(logits_ptr)[row_offset + token]); + }; + + uint32_t best = candidates[0]; + float best_value = logit_at(best); + for (uint32_t token : candidates) { + float value = logit_at(token); + if (value > best_value) { + best_value = value; + best = token; + } + } + return best; +} + +uint32_t Gemma4MtpAssistant::argmax_masked_logits_from_hidden(CactusGraph* gb, size_t hidden_node, + size_t centroid_logits_node) const { + const auto& hidden_buf = gb->get_output_buffer(hidden_node); + if (hidden_buf.shape.size() != 2 || hidden_buf.shape[0] != 1 || hidden_buf.precision != Precision::FP16) { + throw std::runtime_error("Gemma 4 MTP assistant candidate argmax requires hidden [1, dim] FP16"); + } + size_t logits = gb->matmul(hidden_node, token_embeddings_, true, ComputeBackend::CPU); + gb->execute(); + return argmax_masked_logits(gb, logits, centroid_logits_node); +} + +std::pair Gemma4MtpAssistant::top2_masked_logits_from_hidden( + CactusGraph* gb, size_t hidden_node, size_t centroid_logits_node) const { + const auto& hidden_buf = gb->get_output_buffer(hidden_node); + if (hidden_buf.shape.size() != 2 || hidden_buf.shape[0] != 1 || hidden_buf.precision != Precision::FP16) { + throw std::runtime_error("Gemma 4 MTP assistant candidate top2 requires hidden [1, dim] FP16"); + } + + size_t logits = gb->matmul(hidden_node, token_embeddings_, true, ComputeBackend::CPU); + gb->execute(); + const auto& logits_buf = gb->get_output_buffer(logits); + const size_t vocab_size = logits_buf.shape.back(); + const size_t row_offset = (logits_buf.total_size / vocab_size - 1) * vocab_size; + void* logits_ptr = gb->get_output(logits); + auto candidates = candidate_tokens_from_centroids(gb, centroid_logits_node); + + auto logit_at = [&](uint32_t token) -> float { + if (logits_buf.precision == Precision::FP32) { + return static_cast(logits_ptr)[row_offset + token]; + } + if (logits_buf.precision == Precision::FP16) { + return static_cast(static_cast(logits_ptr)[row_offset + token]); + } + return static_cast(static_cast(logits_ptr)[row_offset + token]); + }; + + uint32_t first = candidates[0]; + uint32_t second = candidates[0]; + float first_value = -std::numeric_limits::infinity(); + float second_value = -std::numeric_limits::infinity(); + for (uint32_t token : candidates) { + float value = logit_at(token); + if (value > first_value) { + if (token != first) { + second = first; + second_value = first_value; + } + first = token; + first_value = value; + } else if (token != first && value > second_value) { + second = token; + second_value = value; + } + } + return {first, second}; +} + +Gemma4MtpAssistant::StepResult Gemma4MtpAssistant::draft_one( + uint32_t target_token, + size_t target_embedding_node, + const std::vector<__fp16>& target_hidden, + const Gemma4Model::SharedCacheNodes& cache_nodes, + size_t position, + float hidden_scale, + float temperature, + float top_p, + size_t top_k, + float min_p, + bool return_hidden, + bool return_second_token) { + if (!initialized_ || !graph_) { + throw std::runtime_error("Gemma4MtpAssistant is not initialized"); + } + if (target_hidden.size() != kTargetHiddenDim) { + throw std::runtime_error("Gemma4MtpAssistant expects 1536-wide target hidden vectors"); + } + + auto* gb = graph_; + gb->soft_reset_keep_pool(); + float embedding_scale = std::sqrt(static_cast(kTargetHiddenDim)) * 16.0f; + if (const char* value = std::getenv("CACTUS_GEMMA4_MTP_EMBED_SCALE")) { + embedding_scale *= std::strtof(value, nullptr); + } + if (const char* value = std::getenv("CACTUS_GEMMA4_MTP_HIDDEN_SCALE")) { + hidden_scale = std::strtof(value, nullptr); + } + auto token_input = gb->input({1}, Precision::FP32); + float token_value = static_cast(target_token); + gb->set_input(token_input, &token_value, Precision::FP32); + auto target_embedding = gb->scalar_multiply(gb->embedding(target_embedding_node, token_input), embedding_scale); + + auto hidden_input = gb->input({1, kTargetHiddenDim}, Precision::FP16); + gb->set_input(hidden_input, target_hidden.data(), Precision::FP16); + if (hidden_scale != 1.0f) { + hidden_input = gb->scalar_multiply(hidden_input, hidden_scale); + } + auto input = gb->concat(target_embedding, hidden_input, 1); + + auto hidden = gb->matmul(input, pre_projection_, true, ComputeBackend::CPU); + for (uint32_t i = 0; i < layers_.size(); ++i) { + hidden = build_layer(gb, hidden, i, cache_nodes, position); + } + hidden = gb->rms_norm(hidden, output_norm_, 1e-6f); + + const bool sparse_sampled = temperature > 0.0f && top_k > 0; + const bool greedy_candidate_logits = temperature == 0.0f && return_second_token; + size_t logits = 0; + if (!greedy_candidate_logits) { + logits = gb->matmul(hidden, token_embeddings_, true, ComputeBackend::CPU); + } + size_t centroid_logits = 0; + if (!sparse_sampled) { + centroid_logits = gb->matmul(hidden, masked_centroids_, true, ComputeBackend::CPU); + } + size_t projected = 0; + if (return_hidden) { + projected = gb->matmul(hidden, post_projection_, true, ComputeBackend::CPU); + } + gb->execute(); + + StepResult result; + if (!sparse_sampled) { + if (greedy_candidate_logits && return_second_token) { + auto top2 = top2_masked_logits_from_hidden(gb, hidden, centroid_logits); + result.token = top2.first; + result.second_token = top2.second; + result.has_second_token = top2.second != top2.first; + } else { + result.token = argmax_masked_logits(gb, logits, centroid_logits); + } + } + if (temperature > 0.0f) { + const auto& logits_buf = gb->get_output_buffer(logits); + const size_t vocab_size = logits_buf.shape.back(); + const size_t row_offset = (logits_buf.total_size / vocab_size - 1) * vocab_size; + void* logits_ptr = gb->get_output(logits); + if (top_k > 0) { + result.sparse_probabilities = sparse_probs_from_logits_row( + logits_ptr, logits_buf.precision, row_offset, vocab_size, + temperature, top_p, top_k, min_p); + if (!result.sparse_probabilities.empty()) { + result.token = result.sparse_probabilities.front().first; + } + } else { + std::vector logits_values(vocab_size); + if (logits_buf.precision == Precision::FP32) { + const float* src = static_cast(logits_ptr) + row_offset; + std::copy(src, src + vocab_size, logits_values.begin()); + } else if (logits_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(logits_ptr) + row_offset; + for (size_t i = 0; i < vocab_size; ++i) logits_values[i] = static_cast(src[i]); + } else { + const int8_t* src = static_cast(logits_ptr) + row_offset; + for (size_t i = 0; i < vocab_size; ++i) logits_values[i] = static_cast(src[i]); + } + result.probabilities = probs_from_logits(std::move(logits_values), temperature, top_p, top_k, min_p); + } + } + if (return_hidden) { + result.hidden.resize(kTargetHiddenDim); + const auto& projected_buf = gb->get_output_buffer(projected); + void* projected_ptr = gb->get_output(projected); + if (projected_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(projected_ptr); + std::copy(src, src + kTargetHiddenDim, result.hidden.begin()); + } else if (projected_buf.precision == Precision::FP32) { + const float* src = static_cast(projected_ptr); + for (size_t i = 0; i < kTargetHiddenDim; ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } else { + const int8_t* src = static_cast(projected_ptr); + for (size_t i = 0; i < kTargetHiddenDim; ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } + } + return result; +} + +} +} diff --git a/cactus-engine/models/gemma4/gemma4_mtp_assistant.h b/cactus-engine/models/gemma4/gemma4_mtp_assistant.h new file mode 100644 index 000000000..47ae3c933 --- /dev/null +++ b/cactus-engine/models/gemma4/gemma4_mtp_assistant.h @@ -0,0 +1,78 @@ +#pragma once + +#include "model_gemma4.h" +#include "cactus_graph.h" + +#include +#include +#include + +namespace cactus { +namespace engine { + +class Gemma4MtpAssistant { +public: + struct StepResult { + uint32_t token = 0; + uint32_t second_token = 0; + bool has_second_token = false; + std::vector<__fp16> hidden; + std::vector probabilities; + std::vector> sparse_probabilities; + }; + + bool init(CactusGraph* graph, const std::string& assistant_path); + bool initialized() const { return initialized_; } + + StepResult draft_one(uint32_t target_token, + size_t target_embedding_node, + const std::vector<__fp16>& target_hidden, + const Gemma4Model::SharedCacheNodes& cache_nodes, + size_t position, + float hidden_scale = 1.0f / 16.0f, + float temperature = 0.0f, + float top_p = 0.0f, + size_t top_k = 0, + float min_p = 0.0f, + bool return_hidden = true, + bool return_second_token = false); + +private: + struct LayerWeights { + size_t input_norm = 0; + size_t attn_q = 0; + size_t attn_q_norm = 0; + size_t attn_output = 0; + size_t post_attn_norm = 0; + size_t pre_ffn_norm = 0; + size_t ffn_gate = 0; + size_t ffn_up = 0; + size_t ffn_down = 0; + size_t post_ffn_norm = 0; + size_t layer_scalar = 0; + }; + + size_t apply_rope(CactusGraph* gb, size_t tensor, size_t head_dim, size_t rot_dim, + float rope_freq, size_t position); + size_t build_layer(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + const Gemma4Model::SharedCacheNodes& cache_nodes, size_t position); + std::vector candidate_tokens_from_centroids(CactusGraph* gb, size_t centroid_logits_node) const; + uint32_t argmax_masked_logits(CactusGraph* gb, size_t logits_node, size_t centroid_logits_node) const; + uint32_t argmax_masked_logits_from_hidden(CactusGraph* gb, size_t hidden_node, size_t centroid_logits_node) const; + std::pair top2_masked_logits_from_hidden(CactusGraph* gb, size_t hidden_node, size_t centroid_logits_node) const; + + CactusGraph* graph_ = nullptr; + std::string path_; + bool initialized_ = false; + + size_t token_embeddings_ = 0; + size_t masked_centroids_ = 0; + size_t pre_projection_ = 0; + size_t post_projection_ = 0; + size_t output_norm_ = 0; + std::vector layers_; + std::vector token_ordering_; +}; + +} +} diff --git a/cactus-engine/models/gemma4/model_gemma4.cpp b/cactus-engine/models/gemma4/model_gemma4.cpp index 6a67d77ce..e4d374e29 100644 --- a/cactus-engine/models/gemma4/model_gemma4.cpp +++ b/cactus-engine/models/gemma4/model_gemma4.cpp @@ -1,8 +1,12 @@ #include "model_gemma4.h" #include "cactus_graph.h" +#include #include #include #include +#include +#include +#include #include namespace cactus { @@ -14,6 +18,728 @@ Gemma4Model::Gemma4Model(const Config& config) : Model(config) { weight_nodes_.layers.resize(config.num_layers); } +static uint32_t gemma4_argmax_logits(CactusGraph* gb, size_t logits_node) { + const auto& logits_buf = gb->get_output_buffer(logits_node); + if (logits_buf.shape.empty()) { + throw std::runtime_error("Expected logits tensor"); + } + + const size_t vocab_size = logits_buf.shape.back(); + const size_t row_offset = (logits_buf.total_size / vocab_size - 1) * vocab_size; + void* logits_ptr = gb->get_output(logits_node); + + uint32_t best = 0; + float best_value = -std::numeric_limits::infinity(); + if (logits_buf.precision == Precision::FP32) { + const float* src = static_cast(logits_ptr) + row_offset; + for (size_t i = 0; i < vocab_size; ++i) { + if (src[i] > best_value) { + best_value = src[i]; + best = static_cast(i); + } + } + } else if (logits_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(logits_ptr) + row_offset; + for (size_t i = 0; i < vocab_size; ++i) { + float value = static_cast(src[i]); + if (value > best_value) { + best_value = value; + best = static_cast(i); + } + } + } else { + const int8_t* src = static_cast(logits_ptr) + row_offset; + for (size_t i = 0; i < vocab_size; ++i) { + float value = static_cast(src[i]); + if (value > best_value) { + best_value = value; + best = static_cast(i); + } + } + } + return best; +} + +static std::vector gemma4_argmax_logits_rows(CactusGraph* gb, size_t logits_node) { + const auto& logits_buf = gb->get_output_buffer(logits_node); + if (logits_buf.shape.size() != 2) { + throw std::runtime_error("Expected logits tensor [T, vocab]"); + } + + const size_t rows = logits_buf.shape[0]; + const size_t vocab_size = logits_buf.shape[1]; + void* logits_ptr = gb->get_output(logits_node); + std::vector result(rows, 0); + + for (size_t row = 0; row < rows; ++row) { + uint32_t best = 0; + float best_value = -std::numeric_limits::infinity(); + if (logits_buf.precision == Precision::FP32) { + const float* src = static_cast(logits_ptr) + row * vocab_size; + for (size_t i = 0; i < vocab_size; ++i) { + if (src[i] > best_value) { + best_value = src[i]; + best = static_cast(i); + } + } + } else if (logits_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(logits_ptr) + row * vocab_size; + for (size_t i = 0; i < vocab_size; ++i) { + float value = static_cast(src[i]); + if (value > best_value) { + best_value = value; + best = static_cast(i); + } + } + } else { + const int8_t* src = static_cast(logits_ptr) + row * vocab_size; + for (size_t i = 0; i < vocab_size; ++i) { + float value = static_cast(src[i]); + if (value > best_value) { + best_value = value; + best = static_cast(i); + } + } + } + result[row] = best; + } + return result; +} + +static bool gemma4_can_argmax_output_projection_rows(CactusGraph* gb, size_t hidden_node, size_t output_weight_node) { + (void)gb; + (void)hidden_node; + (void)output_weight_node; + return false; +} + +static bool gemma4_argmax_output_projection_rows(CactusGraph* gb, size_t hidden_node, + size_t output_weight_node, + std::vector& result) { + (void)gb; + (void)hidden_node; + (void)output_weight_node; + result.clear(); + return false; +} + +static std::vector gemma4_probs_from_logits(std::vector logits, float temperature, + float top_p, size_t top_k, float min_p) { + if (logits.empty()) return {}; + if (temperature == 0.0f) { + std::vector probs(logits.size(), 0.0f); + auto it = std::max_element(logits.begin(), logits.end()); + probs[std::distance(logits.begin(), it)] = 1.0f; + return probs; + } + if (temperature > 0.0f) { + for (float& value : logits) value /= temperature; + } + if (top_k > 0 && top_k < logits.size()) { + std::vector sorted = logits; + std::nth_element(sorted.begin(), sorted.begin() + top_k - 1, sorted.end(), std::greater()); + float kth = sorted[top_k - 1]; + for (float& value : logits) { + if (value < kth) value = -std::numeric_limits::infinity(); + } + } + auto normalize = [](const std::vector& values) { + std::vector probs(values.size(), 0.0f); + float max_value = -std::numeric_limits::infinity(); + for (float value : values) max_value = std::max(max_value, value); + if (!std::isfinite(max_value)) return probs; + double sum = 0.0; + for (size_t i = 0; i < values.size(); ++i) { + if (std::isfinite(values[i])) { + probs[i] = std::exp(static_cast(values[i] - max_value)); + sum += probs[i]; + } + } + if (sum <= 0.0 || !std::isfinite(sum)) return probs; + for (float& prob : probs) prob = static_cast(prob / sum); + return probs; + }; + if (min_p > 0.0f) { + auto probs = normalize(logits); + float max_prob = *std::max_element(probs.begin(), probs.end()); + float threshold = max_prob * min_p; + for (size_t i = 0; i < logits.size(); ++i) { + if (probs[i] < threshold) logits[i] = -std::numeric_limits::infinity(); + } + } + if (top_p > 0.0f && top_p < 1.0f) { + std::vector> sorted; + sorted.reserve(logits.size()); + for (size_t i = 0; i < logits.size(); ++i) { + if (std::isfinite(logits[i])) sorted.emplace_back(logits[i], i); + } + std::sort(sorted.begin(), sorted.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); + std::vector sorted_logits; + sorted_logits.reserve(sorted.size()); + for (const auto& item : sorted) sorted_logits.push_back(item.first); + auto sorted_probs = normalize(sorted_logits); + float cumulative = 0.0f; + bool remove = false; + for (size_t i = 0; i < sorted.size(); ++i) { + cumulative += sorted_probs[i]; + if (remove) { + logits[sorted[i].second] = -std::numeric_limits::infinity(); + } + if (cumulative > top_p) remove = true; + } + } + auto probs = normalize(logits); + float sum = std::accumulate(probs.begin(), probs.end(), 0.0f); + if (sum <= 0.0f || !std::isfinite(sum)) { + std::fill(probs.begin(), probs.end(), 1.0f / static_cast(probs.size())); + } + return probs; +} + +static std::vector> gemma4_sparse_probs_from_top_logits( + std::vector> logits, float temperature, float top_p, float min_p) { + if (logits.empty()) return {}; + if (temperature > 0.0f) { + for (auto& item : logits) item.second /= temperature; + } + std::sort(logits.begin(), logits.end(), [](const auto& a, const auto& b) { + return a.second > b.second; + }); + + auto normalize = [](const std::vector>& values) { + std::vector> probs; + probs.reserve(values.size()); + float max_value = -std::numeric_limits::infinity(); + for (const auto& item : values) max_value = std::max(max_value, item.second); + if (!std::isfinite(max_value)) return probs; + double sum = 0.0; + for (const auto& item : values) { + float prob = std::exp(static_cast(item.second - max_value)); + probs.emplace_back(item.first, prob); + sum += prob; + } + if (sum <= 0.0 || !std::isfinite(sum)) return std::vector>{}; + for (auto& item : probs) item.second = static_cast(item.second / sum); + return probs; + }; + + if (min_p > 0.0f) { + auto probs = normalize(logits); + if (!probs.empty()) { + float max_prob = 0.0f; + for (const auto& item : probs) max_prob = std::max(max_prob, item.second); + float threshold = max_prob * min_p; + std::vector> filtered; + filtered.reserve(logits.size()); + for (size_t i = 0; i < logits.size(); ++i) { + if (probs[i].second >= threshold) filtered.push_back(logits[i]); + } + logits = std::move(filtered); + } + } + + if (top_p > 0.0f && top_p < 1.0f) { + auto probs = normalize(logits); + std::vector> filtered; + filtered.reserve(logits.size()); + float cumulative = 0.0f; + bool remove = false; + for (size_t i = 0; i < logits.size(); ++i) { + cumulative += i < probs.size() ? probs[i].second : 0.0f; + if (!remove) filtered.push_back(logits[i]); + if (cumulative > top_p) remove = true; + } + logits = std::move(filtered); + } + + auto probs = normalize(logits); + float sum = 0.0f; + for (const auto& item : probs) sum += item.second; + if (sum <= 0.0f || !std::isfinite(sum)) { + float uniform = probs.empty() ? 0.0f : 1.0f / static_cast(probs.size()); + for (auto& item : probs) item.second = uniform; + } + return probs; +} + +static std::vector> gemma4_sparse_probs_from_logits_row( + const void* logits_ptr, Precision precision, size_t row_offset, size_t vocab_size, + float temperature, float top_p, size_t top_k, float min_p) { + using Entry = std::pair; + std::priority_queue, std::greater> top; + size_t k = std::min(top_k, vocab_size); + auto push_value = [&](float value, uint32_t token) { + if (top.size() < k) { + top.emplace(value, token); + } else if (value > top.top().first) { + top.pop(); + top.emplace(value, token); + } + }; + + if (precision == Precision::FP32) { + const float* src = static_cast(logits_ptr) + row_offset; + for (size_t i = 0; i < vocab_size; ++i) push_value(src[i], static_cast(i)); + } else if (precision == Precision::FP16) { + const __fp16* src = static_cast(logits_ptr) + row_offset; + for (size_t i = 0; i < vocab_size; ++i) push_value(static_cast(src[i]), static_cast(i)); + } else { + const int8_t* src = static_cast(logits_ptr) + row_offset; + for (size_t i = 0; i < vocab_size; ++i) push_value(static_cast(src[i]), static_cast(i)); + } + + std::vector> logits; + logits.reserve(top.size()); + while (!top.empty()) { + logits.emplace_back(top.top().second, top.top().first); + top.pop(); + } + return gemma4_sparse_probs_from_top_logits(std::move(logits), temperature, top_p, min_p); +} + +static std::vector> gemma4_prob_rows(CactusGraph* gb, size_t logits_node, + float temperature, float top_p, + size_t top_k, float min_p) { + const auto& logits_buf = gb->get_output_buffer(logits_node); + if (logits_buf.shape.size() != 2) { + throw std::runtime_error("Expected logits tensor [T, vocab]"); + } + const size_t rows = logits_buf.shape[0]; + const size_t vocab_size = logits_buf.shape[1]; + void* logits_ptr = gb->get_output(logits_node); + std::vector> result; + result.reserve(rows); + for (size_t row = 0; row < rows; ++row) { + std::vector logits(vocab_size); + if (logits_buf.precision == Precision::FP32) { + const float* src = static_cast(logits_ptr) + row * vocab_size; + std::copy(src, src + vocab_size, logits.begin()); + } else if (logits_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(logits_ptr) + row * vocab_size; + for (size_t i = 0; i < vocab_size; ++i) logits[i] = static_cast(src[i]); + } else { + const int8_t* src = static_cast(logits_ptr) + row * vocab_size; + for (size_t i = 0; i < vocab_size; ++i) logits[i] = static_cast(src[i]); + } + result.push_back(gemma4_probs_from_logits(std::move(logits), temperature, top_p, top_k, min_p)); + } + return result; +} + +static std::vector>> gemma4_sparse_prob_rows( + CactusGraph* gb, size_t logits_node, float temperature, float top_p, size_t top_k, float min_p) { + const auto& logits_buf = gb->get_output_buffer(logits_node); + if (logits_buf.shape.size() != 2) { + throw std::runtime_error("Expected logits tensor [T, vocab]"); + } + const size_t rows = logits_buf.shape[0]; + const size_t vocab_size = logits_buf.shape[1]; + void* logits_ptr = gb->get_output(logits_node); + std::vector>> result; + result.reserve(rows); + for (size_t row = 0; row < rows; ++row) { + result.push_back(gemma4_sparse_probs_from_logits_row( + logits_ptr, logits_buf.precision, row * vocab_size, vocab_size, + temperature, top_p, top_k, min_p)); + } + return result; +} + +Gemma4Model::GreedyStepWithHidden Gemma4Model::decode_greedy_with_hidden(const std::vector& tokens) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + if (tokens.empty()) + throw std::runtime_error("Token sequence cannot be empty"); + + auto* gb = static_cast(graph_handle_); + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + + auto final_hidden = forward(tokens, true); + auto last_hidden = gb->index(final_hidden, tokens.size() - 1, 0); + const auto& last_hidden_buf = gb->get_output_buffer(last_hidden); + size_t hidden_dim = last_hidden_buf.shape[0]; + auto hidden_row = gb->reshape(last_hidden, {1, hidden_dim}); + + auto logits = gb->matmul(hidden_row, output_weight_node_id_, true, backend); + if (config_.final_logit_softcapping > 0.0f) { + float inv_cap = 1.0f / config_.final_logit_softcapping; + logits = gb->scalar_multiply(logits, inv_cap); + logits = gb->tanh(logits); + logits = gb->scalar_multiply(logits, config_.final_logit_softcapping); + } + + gb->execute(); + + GreedyStepWithHidden result; + result.token = gemma4_argmax_logits(gb, logits); + result.hidden.resize(hidden_dim); + const auto& hidden_buf = gb->get_output_buffer(hidden_row); + void* hidden_ptr = gb->get_output(hidden_row); + if (hidden_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(hidden_ptr); + std::copy(src, src + hidden_dim, result.hidden.begin()); + } else if (hidden_buf.precision == Precision::FP32) { + const float* src = static_cast(hidden_ptr); + for (size_t i = 0; i < hidden_dim; ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } else { + const int8_t* src = static_cast(hidden_ptr); + for (size_t i = 0; i < hidden_dim; ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } + + post_execute_updates(gb, tokens.size()); + cache_total_seq_len_ += tokens.size(); + record_sampled_token(result.token); + return result; +} + +std::vector Gemma4Model::decode_greedy_tokens(const std::vector& tokens) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + if (tokens.empty()) + throw std::runtime_error("Token sequence cannot be empty"); + + auto* gb = static_cast(graph_handle_); + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + + auto final_hidden = forward(tokens, true); + auto logits = gb->matmul(final_hidden, output_weight_node_id_, true, backend); + if (config_.final_logit_softcapping > 0.0f) { + float inv_cap = 1.0f / config_.final_logit_softcapping; + logits = gb->scalar_multiply(logits, inv_cap); + logits = gb->tanh(logits); + logits = gb->scalar_multiply(logits, config_.final_logit_softcapping); + } + + gb->execute(); + std::vector result = gemma4_argmax_logits_rows(gb, logits); + post_execute_updates(gb, tokens.size()); + cache_total_seq_len_ += tokens.size(); + for (uint32_t token : result) record_sampled_token(token); + return result; +} + +Gemma4Model::GreedyBatchWithHidden Gemma4Model::decode_greedy_tokens_with_hidden(const std::vector& tokens) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + if (tokens.empty()) + throw std::runtime_error("Token sequence cannot be empty"); + + auto* gb = static_cast(graph_handle_); + + auto final_hidden = forward(tokens, true); + const auto& final_hidden_buf = gb->get_output_buffer(final_hidden); + if (final_hidden_buf.shape.size() != 2) { + throw std::runtime_error("Expected Gemma 4 hidden tensor [T, hidden]"); + } + size_t hidden_dim = final_hidden_buf.shape[1]; + + GreedyBatchWithHidden result; + result.hidden_dim = hidden_dim; + result.hidden.resize(tokens.size() * hidden_dim); + + if (gemma4_can_argmax_output_projection_rows(gb, final_hidden, output_weight_node_id_)) { + gb->execute(); + if (!gemma4_argmax_output_projection_rows(gb, final_hidden, output_weight_node_id_, result.tokens)) { + throw std::runtime_error("Gemma 4 output projection argmax failed"); + } + } else { + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + auto logits = gb->matmul(final_hidden, output_weight_node_id_, true, backend); + if (config_.final_logit_softcapping > 0.0f) { + float inv_cap = 1.0f / config_.final_logit_softcapping; + logits = gb->scalar_multiply(logits, inv_cap); + logits = gb->tanh(logits); + logits = gb->scalar_multiply(logits, config_.final_logit_softcapping); + } + + gb->execute(); + result.tokens = gemma4_argmax_logits_rows(gb, logits); + } + + const auto& hidden_buf = gb->get_output_buffer(final_hidden); + void* hidden_ptr = gb->get_output(final_hidden); + if (hidden_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(hidden_ptr); + std::copy(src, src + result.hidden.size(), result.hidden.begin()); + } else if (hidden_buf.precision == Precision::FP32) { + const float* src = static_cast(hidden_ptr); + for (size_t i = 0; i < result.hidden.size(); ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } else { + const int8_t* src = static_cast(hidden_ptr); + for (size_t i = 0; i < result.hidden.size(); ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } + + post_execute_updates(gb, tokens.size()); + cache_total_seq_len_ += tokens.size(); + for (uint32_t token : result.tokens) record_sampled_token(token); + return result; +} + +Gemma4Model::GreedyBatchWithHidden Gemma4Model::decode_greedy_tokens_with_hidden_early_stop( + const std::vector& tokens, + const std::vector& expected_drafts) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + if (tokens.empty()) + throw std::runtime_error("Token sequence cannot be empty"); + + auto* gb = static_cast(graph_handle_); + + auto final_hidden = forward(tokens, true); + const auto& final_hidden_buf = gb->get_output_buffer(final_hidden); + if (final_hidden_buf.shape.size() != 2) { + throw std::runtime_error("Expected Gemma 4 hidden tensor [T, hidden]"); + } + size_t hidden_dim = final_hidden_buf.shape[1]; + + GreedyBatchWithHidden result; + result.hidden_dim = hidden_dim; + result.hidden.resize(tokens.size() * hidden_dim); + + if (!gemma4_can_argmax_output_projection_rows(gb, final_hidden, output_weight_node_id_)) { + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + auto logits = gb->matmul(final_hidden, output_weight_node_id_, true, backend); + if (config_.final_logit_softcapping > 0.0f) { + float inv_cap = 1.0f / config_.final_logit_softcapping; + logits = gb->scalar_multiply(logits, inv_cap); + logits = gb->tanh(logits); + logits = gb->scalar_multiply(logits, config_.final_logit_softcapping); + } + gb->execute(); + result.tokens = gemma4_argmax_logits_rows(gb, logits); + } else { + gb->execute(); + size_t max_rows = tokens.size(); + std::vector row_tokens; + if (!gemma4_argmax_output_projection_rows(gb, final_hidden, output_weight_node_id_, row_tokens)) { + throw std::runtime_error("Gemma 4 output projection argmax failed"); + } + result.tokens.reserve(row_tokens.size()); + for (size_t row = 0; row < max_rows; ++row) { + uint32_t token = row_tokens[row]; + result.tokens.push_back(token); + if (row < expected_drafts.size() && token != expected_drafts[row]) { + break; + } + } + } + + const auto& hidden_buf = gb->get_output_buffer(final_hidden); + void* hidden_ptr = gb->get_output(final_hidden); + if (hidden_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(hidden_ptr); + std::copy(src, src + result.hidden.size(), result.hidden.begin()); + } else if (hidden_buf.precision == Precision::FP32) { + const float* src = static_cast(hidden_ptr); + for (size_t i = 0; i < result.hidden.size(); ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } else { + const int8_t* src = static_cast(hidden_ptr); + for (size_t i = 0; i < result.hidden.size(); ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } + + post_execute_updates(gb, tokens.size()); + cache_total_seq_len_ += tokens.size(); + for (uint32_t token : result.tokens) record_sampled_token(token); + return result; +} + +Gemma4Model::GreedyBatchWithHidden Gemma4Model::decode_greedy_tokens_with_hidden_tree( + const std::vector& tokens, + const std::vector<__fp16>& new_token_attention_mask) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + if (tokens.empty()) + throw std::runtime_error("Token sequence cannot be empty"); + + auto* gb = static_cast(graph_handle_); + auto final_hidden = forward_with_attention_mask(tokens, true, &new_token_attention_mask); + const auto& final_hidden_buf = gb->get_output_buffer(final_hidden); + if (final_hidden_buf.shape.size() != 2) { + throw std::runtime_error("Expected Gemma 4 hidden tensor [T, hidden]"); + } + size_t hidden_dim = final_hidden_buf.shape[1]; + + GreedyBatchWithHidden result; + result.hidden_dim = hidden_dim; + result.hidden.resize(tokens.size() * hidden_dim); + + if (gemma4_can_argmax_output_projection_rows(gb, final_hidden, output_weight_node_id_)) { + gb->execute(); + if (!gemma4_argmax_output_projection_rows(gb, final_hidden, output_weight_node_id_, result.tokens)) { + throw std::runtime_error("Gemma 4 output projection argmax failed"); + } + } else { + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + auto logits = gb->matmul(final_hidden, output_weight_node_id_, true, backend); + if (config_.final_logit_softcapping > 0.0f) { + float inv_cap = 1.0f / config_.final_logit_softcapping; + logits = gb->scalar_multiply(logits, inv_cap); + logits = gb->tanh(logits); + logits = gb->scalar_multiply(logits, config_.final_logit_softcapping); + } + gb->execute(); + result.tokens = gemma4_argmax_logits_rows(gb, logits); + } + + const auto& hidden_buf = gb->get_output_buffer(final_hidden); + void* hidden_ptr = gb->get_output(final_hidden); + if (hidden_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(hidden_ptr); + std::copy(src, src + result.hidden.size(), result.hidden.begin()); + } else if (hidden_buf.precision == Precision::FP32) { + const float* src = static_cast(hidden_ptr); + for (size_t i = 0; i < result.hidden.size(); ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } else { + const int8_t* src = static_cast(hidden_ptr); + for (size_t i = 0; i < result.hidden.size(); ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } + + post_execute_updates(gb, tokens.size()); + cache_total_seq_len_ += tokens.size(); + return result; +} + +Gemma4Model::GreedyBatchWithHidden Gemma4Model::decode_tokens_with_hidden_and_probs( + const std::vector& tokens, float temperature, float top_p, size_t top_k, float min_p) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + if (tokens.empty()) + throw std::runtime_error("Token sequence cannot be empty"); + + auto* gb = static_cast(graph_handle_); + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + + auto final_hidden = forward(tokens, true); + const auto& final_hidden_buf = gb->get_output_buffer(final_hidden); + if (final_hidden_buf.shape.size() != 2) { + throw std::runtime_error("Expected Gemma 4 hidden tensor [T, hidden]"); + } + size_t hidden_dim = final_hidden_buf.shape[1]; + + auto logits = gb->matmul(final_hidden, output_weight_node_id_, true, backend); + if (config_.final_logit_softcapping > 0.0f) { + float inv_cap = 1.0f / config_.final_logit_softcapping; + logits = gb->scalar_multiply(logits, inv_cap); + logits = gb->tanh(logits); + logits = gb->scalar_multiply(logits, config_.final_logit_softcapping); + } + + gb->execute(); + + GreedyBatchWithHidden result; + result.tokens = gemma4_argmax_logits_rows(gb, logits); + result.probabilities = gemma4_prob_rows(gb, logits, temperature, top_p, top_k, min_p); + result.hidden_dim = hidden_dim; + result.hidden.resize(tokens.size() * hidden_dim); + + const auto& hidden_buf = gb->get_output_buffer(final_hidden); + void* hidden_ptr = gb->get_output(final_hidden); + if (hidden_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(hidden_ptr); + std::copy(src, src + result.hidden.size(), result.hidden.begin()); + } else if (hidden_buf.precision == Precision::FP32) { + const float* src = static_cast(hidden_ptr); + for (size_t i = 0; i < result.hidden.size(); ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } else { + const int8_t* src = static_cast(hidden_ptr); + for (size_t i = 0; i < result.hidden.size(); ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } + + post_execute_updates(gb, tokens.size()); + cache_total_seq_len_ += tokens.size(); + return result; +} + +Gemma4Model::GreedyBatchWithHidden Gemma4Model::decode_tokens_with_hidden_and_sparse_probs( + const std::vector& tokens, float temperature, float top_p, size_t top_k, float min_p) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + if (tokens.empty()) + throw std::runtime_error("Token sequence cannot be empty"); + if (top_k == 0) + throw std::runtime_error("Sparse sampled Gemma 4 decode requires top_k > 0"); + + auto* gb = static_cast(graph_handle_); + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + + auto final_hidden = forward(tokens, true); + const auto& final_hidden_buf = gb->get_output_buffer(final_hidden); + if (final_hidden_buf.shape.size() != 2) { + throw std::runtime_error("Expected Gemma 4 hidden tensor [T, hidden]"); + } + size_t hidden_dim = final_hidden_buf.shape[1]; + + auto logits = gb->matmul(final_hidden, output_weight_node_id_, true, backend); + if (config_.final_logit_softcapping > 0.0f) { + float inv_cap = 1.0f / config_.final_logit_softcapping; + logits = gb->scalar_multiply(logits, inv_cap); + logits = gb->tanh(logits); + logits = gb->scalar_multiply(logits, config_.final_logit_softcapping); + } + + gb->execute(); + + GreedyBatchWithHidden result; + result.tokens = gemma4_argmax_logits_rows(gb, logits); + result.sparse_probabilities = gemma4_sparse_prob_rows(gb, logits, temperature, top_p, top_k, min_p); + result.hidden_dim = hidden_dim; + result.hidden.resize(tokens.size() * hidden_dim); + + const auto& hidden_buf = gb->get_output_buffer(final_hidden); + void* hidden_ptr = gb->get_output(final_hidden); + if (hidden_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(hidden_ptr); + std::copy(src, src + result.hidden.size(), result.hidden.begin()); + } else if (hidden_buf.precision == Precision::FP32) { + const float* src = static_cast(hidden_ptr); + for (size_t i = 0; i < result.hidden.size(); ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } else { + const int8_t* src = static_cast(hidden_ptr); + for (size_t i = 0; i < result.hidden.size(); ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + } + + post_execute_updates(gb, tokens.size()); + cache_total_seq_len_ += tokens.size(); + return result; +} + +std::vector<__fp16> Gemma4Model::token_embedding(uint32_t token) { + auto embedding = get_token_embeddings({token}); + const float scale = std::sqrt(static_cast(config_.hidden_dim)); + for (__fp16& value : embedding) { + value = static_cast<__fp16>(static_cast(value) * scale); + } + return embedding; +} + +Gemma4Model::SharedCacheNodes Gemma4Model::shared_cache_nodes_for_mtp() const { + SharedCacheNodes nodes; + for (uint32_t i = 0; i < first_shared_layer_ && i < config_.layer_types.size(); ++i) { + if (config_.layer_types[i] == "global") { + nodes.full_k = graph_cache_k_nodes_[i]; + nodes.full_v = graph_cache_v_nodes_[i]; + } else { + nodes.sliding_k = graph_cache_k_nodes_[i]; + nodes.sliding_v = graph_cache_v_nodes_[i]; + } + } + return nodes; +} + +std::vector Gemma4Model::cache_state_nodes_for_mtp() const { + std::vector nodes; + for (uint32_t i = 0; i < first_shared_layer_ && i < graph_cache_k_nodes_.size(); ++i) { + if (graph_cache_k_nodes_[i] != 0) nodes.push_back(graph_cache_k_nodes_[i]); + if (i < graph_cache_v_nodes_.size() && graph_cache_v_nodes_[i] != 0) nodes.push_back(graph_cache_v_nodes_[i]); + } + return nodes; +} + +bool Gemma4Model::can_use_mtp_tree_attention(size_t token_count) const { + if (config_.sliding_window == 0 || token_count == 0) return true; + return cache_total_seq_len_ + token_count <= config_.sliding_window; +} + bool Gemma4Model::is_global_layer(uint32_t idx) const { return idx < config_.layer_types.size() && (config_.layer_types[idx] == "global" || config_.layer_types[idx] == "full_attention"); @@ -217,26 +943,44 @@ size_t Gemma4Model::build_per_layer_input(CactusGraph* gb, size_t hidden, size_t size_t Gemma4Model::apply_partial_rope(CactusGraph* gb, size_t tensor, size_t head_dim, size_t rot_dim, float rope_freq, size_t position_offset) { - if (rot_dim < head_dim) { - size_t half_dim = head_dim / 2; - size_t half_rot = rot_dim / 2; - size_t pass_len = half_dim - half_rot; - float adjusted_theta = std::pow(rope_freq, static_cast(rot_dim) / static_cast(head_dim)); - - auto left_rot = gb->slice(tensor, 3, 0, half_rot); - auto left_pass = gb->slice(tensor, 3, half_rot, pass_len); - auto right_rot = gb->slice(tensor, 3, half_dim, half_rot); - auto right_pass = gb->slice(tensor, 3, half_dim + half_rot, pass_len); - - auto rotated = gb->rope(gb->concat(left_rot, right_rot, 3), adjusted_theta, position_offset); - auto rotated_left = gb->slice(rotated, 3, 0, half_rot); - auto rotated_right = gb->slice(rotated, 3, half_rot, half_rot); + auto apply_uniform = [&](size_t input, size_t offset) { + if (rot_dim < head_dim) { + size_t half_dim = head_dim / 2; + size_t half_rot = rot_dim / 2; + size_t pass_len = half_dim - half_rot; + float adjusted_theta = std::pow(rope_freq, static_cast(rot_dim) / static_cast(head_dim)); + + auto left_rot = gb->slice(input, 3, 0, half_rot); + auto left_pass = gb->slice(input, 3, half_rot, pass_len); + auto right_rot = gb->slice(input, 3, half_dim, half_rot); + auto right_pass = gb->slice(input, 3, half_dim + half_rot, pass_len); + + auto rotated = gb->rope(gb->concat(left_rot, right_rot, 3), adjusted_theta, offset); + auto rotated_left = gb->slice(rotated, 3, 0, half_rot); + auto rotated_right = gb->slice(rotated, 3, half_rot, half_rot); + + auto new_left = gb->concat(rotated_left, left_pass, 3); + auto new_right = gb->concat(rotated_right, right_pass, 3); + return gb->concat(new_left, new_right, 3); + } + return gb->rope(input, rope_freq, offset); + }; - auto new_left = gb->concat(rotated_left, left_pass, 3); - auto new_right = gb->concat(rotated_right, right_pass, 3); - return gb->concat(new_left, new_right, 3); + const auto& tensor_shape = gb->get_output_buffer(tensor).shape; + if (tensor_shape.size() == 4 + && active_position_deltas_.size() == tensor_shape[1] + && tensor_shape[1] > 1) { + std::vector rows; + rows.reserve(tensor_shape[1]); + for (size_t i = 0; i < tensor_shape[1]; ++i) { + rows.push_back(apply_uniform( + gb->slice(tensor, 1, i, 1), + position_offset + active_position_deltas_[i])); + } + return gb->cat(rows, 1); } - return gb->rope(tensor, rope_freq, position_offset); + + return apply_uniform(tensor, position_offset); } size_t Gemma4Model::build_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, @@ -289,11 +1033,22 @@ size_t Gemma4Model::build_attention(CactusGraph* gb, size_t input, uint32_t laye gb->kv_cache_append(k4, graph_cache_k_nodes_[cache_src], window, cache_sink_size_); gb->kv_cache_append(v4, graph_cache_v_nodes_[cache_src], window, cache_sink_size_); } - attn = gb->attention_cached(q4, k4, v4, - graph_cache_k_nodes_[cache_src], graph_cache_v_nodes_[cache_src], - attention_scale_, position_offset, window); + if (active_attention_mask_node_ != 0) { + attn = gb->attention_cached_masked(q4, k4, v4, + graph_cache_k_nodes_[cache_src], graph_cache_v_nodes_[cache_src], + active_attention_mask_node_, attention_scale_, position_offset, window); + } else { + attn = gb->attention_cached(q4, k4, v4, + graph_cache_k_nodes_[cache_src], graph_cache_v_nodes_[cache_src], + attention_scale_, position_offset, window); + } } else { - attn = gb->attention(q4, k4, v4, attention_scale_, position_offset, window); + if (active_attention_mask_node_ != 0) { + attn = gb->attention_masked(q4, k4, v4, active_attention_mask_node_, + attention_scale_, true, backend, false, position_offset, window); + } else { + attn = gb->attention(q4, k4, v4, attention_scale_, position_offset, window); + } } return gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), layer.attn_output_weight, true, backend); @@ -464,6 +1219,13 @@ size_t Gemma4Model::forward_from_embeddings(CactusGraph* gb, size_t hidden, size } size_t Gemma4Model::forward(const std::vector& tokens, bool use_cache) { + return forward_with_attention_mask(tokens, use_cache, nullptr); +} + +size_t Gemma4Model::forward_with_attention_mask( + const std::vector& tokens, + bool use_cache, + const std::vector<__fp16>* new_token_attention_mask) { if (!initialized_ || !graph_handle_) throw std::runtime_error("Model not initialized - call init() first"); if (tokens.empty()) @@ -471,6 +1233,28 @@ size_t Gemma4Model::forward(const std::vector& tokens, bool use_cache) auto* gb = static_cast(graph_handle_); gb->soft_reset(); + size_t previous_attention_mask_node = active_attention_mask_node_; + std::vector previous_position_deltas = std::move(active_position_deltas_); + if (new_token_attention_mask) { + if (new_token_attention_mask->size() != tokens.size() * tokens.size()) { + throw std::runtime_error("Gemma 4 attention mask must be [tokens, tokens]"); + } + active_attention_mask_node_ = gb->input({1, tokens.size(), tokens.size()}, Precision::FP16); + gb->set_input(active_attention_mask_node_, new_token_attention_mask->data(), Precision::FP16); + if (tokens.size() == 3) { + active_position_deltas_ = {0, 1, 1}; + } else if (tokens.size() == 4) { + active_position_deltas_ = {0, 1, 1, 2}; + } else if (tokens.size() == 5) { + active_position_deltas_ = {0, 1, 1, 2, 3}; + } else { + active_position_deltas_.resize(tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) active_position_deltas_[i] = i; + } + } else { + active_attention_mask_node_ = 0; + active_position_deltas_.clear(); + } std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); @@ -490,7 +1274,10 @@ size_t Gemma4Model::forward(const std::vector& tokens, bool use_cache) for (size_t i = 0; i < tokens.size(); i++) input_data[i] = static_cast(tokens[i]); gb->set_input(token_input, input_data.data(), Precision::FP32); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + auto result = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + active_attention_mask_node_ = previous_attention_mask_node; + active_position_deltas_ = std::move(previous_position_deltas); + return result; } size_t token_input, pli_input; @@ -501,7 +1288,10 @@ size_t Gemma4Model::forward(const std::vector& tokens, bool use_cache) hidden = apply_transformer_layer(gb, hidden, pli, i, backend, use_cache, pos_offset); set_token_inputs(gb, token_input, pli_input, tokens); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + auto result = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + active_attention_mask_node_ = previous_attention_mask_node; + active_position_deltas_ = std::move(previous_position_deltas); + return result; } size_t Gemma4Model::forward_split(const std::vector& tokens, bool use_cache) { diff --git a/cactus-engine/models/gemma4/model_gemma4.h b/cactus-engine/models/gemma4/model_gemma4.h index de4c6c60f..bb8a04ab6 100644 --- a/cactus-engine/models/gemma4/model_gemma4.h +++ b/cactus-engine/models/gemma4/model_gemma4.h @@ -2,6 +2,8 @@ #include "../model.h" +#include + bool test_gemma4_vision(bool expect_npu); bool test_gemma4_audio(bool expect_npu); @@ -15,6 +17,50 @@ class Gemma4Model : public Model { explicit Gemma4Model(const Config& config); ~Gemma4Model() override = default; + struct GreedyStepWithHidden { + uint32_t token = 0; + std::vector<__fp16> hidden; + }; + + struct GreedyBatchWithHidden { + std::vector tokens; + std::vector<__fp16> hidden; + std::vector> probabilities; + std::vector>> sparse_probabilities; + size_t hidden_dim = 0; + }; + + struct SharedCacheNodes { + size_t sliding_k = 0; + size_t sliding_v = 0; + size_t full_k = 0; + size_t full_v = 0; + }; + + GreedyStepWithHidden decode_greedy_with_hidden(const std::vector& tokens); + std::vector decode_greedy_tokens(const std::vector& tokens); + GreedyBatchWithHidden decode_greedy_tokens_with_hidden(const std::vector& tokens); + GreedyBatchWithHidden decode_greedy_tokens_with_hidden_early_stop( + const std::vector& tokens, + const std::vector& expected_drafts); + GreedyBatchWithHidden decode_greedy_tokens_with_hidden_tree( + const std::vector& tokens, + const std::vector<__fp16>& new_token_attention_mask); + GreedyBatchWithHidden decode_tokens_with_hidden_and_probs(const std::vector& tokens, + float temperature, float top_p, + size_t top_k, float min_p); + GreedyBatchWithHidden decode_tokens_with_hidden_and_sparse_probs(const std::vector& tokens, + float temperature, float top_p, + size_t top_k, float min_p); + std::vector<__fp16> token_embedding(uint32_t token); + void prefill_for_mtp(const std::vector& tokens, size_t chunk_size) { prefill(tokens, chunk_size); } + size_t token_embedding_node_for_mtp() const { return embedding_node_id_; } + SharedCacheNodes shared_cache_nodes_for_mtp() const; + size_t cache_position() const { return cache_total_seq_len_; } + void set_cache_position_for_mtp(size_t position) { cache_total_seq_len_ = position; } + std::vector cache_state_nodes_for_mtp() const; + bool can_use_mtp_tree_attention(size_t token_count) const; + protected: size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; @@ -48,6 +94,8 @@ class Gemma4Model : public Model { private: size_t forward_split(const std::vector& tokens, bool use_cache); + size_t forward_with_attention_mask(const std::vector& tokens, bool use_cache, + const std::vector<__fp16>* new_token_attention_mask); std::pair build_preamble_and_embed(CactusGraph* gb, size_t seq_len, ComputeBackend backend, size_t& token_input, size_t& pli_input); @@ -112,6 +160,8 @@ class Gemma4Model : public Model { std::vector kv_share_map_; std::vector shared_k_nodes_; std::vector shared_v_nodes_; + size_t active_attention_mask_node_ = 0; + std::vector active_position_deltas_; std::vector<__fp16> v_norm_ones_weight_; size_t v_norm_ones_node_ = 0; @@ -349,6 +399,7 @@ class Gemma4MmModel : public Model { float min_p = 0.15f, float repetition_penalty = 1.1f); void reset_cache() override; + size_t get_cache_size() const override; std::vector get_image_embeddings(const std::string& image_path) override; std::vector get_audio_embeddings(const std::vector& audio_features) override; void compact_kv_cache() override; diff --git a/cactus-engine/models/gemma4/model_gemma4_mm.cpp b/cactus-engine/models/gemma4/model_gemma4_mm.cpp index d8632ae3a..1c242093c 100644 --- a/cactus-engine/models/gemma4/model_gemma4_mm.cpp +++ b/cactus-engine/models/gemma4/model_gemma4_mm.cpp @@ -57,6 +57,10 @@ void Gemma4MmModel::reset_cache() { language_model_.reset_cache(); } +size_t Gemma4MmModel::get_cache_size() const { + return language_model_.get_cache_size(); +} + void Gemma4MmModel::compact_kv_cache() { language_model_.compact_kv_cache(); } diff --git a/cactus-engine/models/model.h b/cactus-engine/models/model.h index 5e60c43e8..a9574b30c 100644 --- a/cactus-engine/models/model.h +++ b/cactus-engine/models/model.h @@ -217,6 +217,7 @@ class Lfm2VlModel : public Model { void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, const std::string& profile_file = "") override; + size_t get_cache_size() const override; uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, float temperature = -1.0f, diff --git a/cactus-engine/models/model_lfm2vl.cpp b/cactus-engine/models/model_lfm2vl.cpp index 898f6c2ba..5174c574c 100644 --- a/cactus-engine/models/model_lfm2vl.cpp +++ b/cactus-engine/models/model_lfm2vl.cpp @@ -459,6 +459,10 @@ void Lfm2VlModel::prefill_with_images(const std::vector& tokens, const last_token_count_ = tokens.size(); } +size_t Lfm2VlModel::get_cache_size() const { + return language_model_.get_cache_size(); +} + Lfm2VlModel::ForwardImageResult Lfm2VlModel::forward_images( CactusGraph* gb, const std::vector& tokens, diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index c79bd794a..06df887f3 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -1,14 +1,25 @@ #include "../cactus_engine.h" #include "cloud.h" +#include "gemma4_mtp_decode.h" #include "utils.h" #include "telemetry.h" #include "cactus_kernels.h" #include "wav.h" +#include "../models/gemma4/gemma4_mtp_assistant.h" +#include "../models/gemma4/model_gemma4.h" #include +#include #include +#include #include +#include +#include #include +#include #include +#include +#include +#include #include using namespace cactus::engine; @@ -21,6 +32,197 @@ namespace { std::vector> extract_schema_property_types(const std::string& schema); std::vector extract_schema_required(const std::string& schema); +Gemma4Model* gemma4_language_model(Model* model) { + if (auto* gemma4 = dynamic_cast(model)) { + return gemma4; + } + if (auto* mm = dynamic_cast(model)) { + return &mm->language_model(); + } + return nullptr; +} + +std::string default_gemma4_mtp_assistant_path(Model* model) { + if (!model) { + return {}; + } + std::filesystem::path assistant_path = std::filesystem::path(model->get_model_folder_path()) / "assistant"; + return assistant_path.string(); +} + +std::string token_list_for_trace(const std::vector& tokens) { + std::ostringstream out; + out << "["; + for (size_t i = 0; i < tokens.size(); ++i) { + if (i > 0) out << ","; + out << tokens[i]; + } + out << "]"; + return out.str(); +} + +std::string env_string(const char* key, const std::string& fallback = "") { + const char* value = std::getenv(key); + return value && value[0] ? std::string(value) : fallback; +} + +std::string csv_escape_field(const std::string& value) { + if (value.find_first_of(",\"\n\r") == std::string::npos) return value; + std::string out = "\""; + for (char c : value) { + if (c == '"') out += "\"\""; + else out += c; + } + out += "\""; + return out; +} + +std::string bool_field(bool value) { + return value ? "true" : "false"; +} + +std::string double_field(double value) { + std::ostringstream out; + out << std::fixed << std::setprecision(3) << value; + return out.str(); +} + +std::string join_ms(const std::vector& values) { + std::ostringstream out; + out << std::fixed << std::setprecision(3); + for (size_t i = 0; i < values.size(); ++i) { + if (i > 0) out << ";"; + out << values[i]; + } + return out.str(); +} + +void append_csv_line(const std::filesystem::path& path, + const std::string& header, + const std::vector& fields) { + bool write_header = !std::filesystem::exists(path) || std::filesystem::file_size(path) == 0; + std::ofstream out(path, std::ios::app); + if (!out) { + throw std::runtime_error("Unable to open diagnostic trace CSV: " + path.string()); + } + if (write_header) out << header << "\n"; + for (size_t i = 0; i < fields.size(); ++i) { + if (i > 0) out << ","; + out << csv_escape_field(fields[i]); + } + out << "\n"; +} + +struct DiagnosticTraceContext { + bool enabled = false; + std::filesystem::path dir; + std::string run_id; + std::string model; + std::string prompt_name; + std::string shape; + std::string rep; + + static DiagnosticTraceContext from_env() { + DiagnosticTraceContext ctx; + std::string dir = env_string("CACTUS_GEMMA4_MTP_TRACE_CSV_DIR"); + if (dir.empty()) return ctx; + ctx.run_id = env_string("CACTUS_MTP_DIAG_RUN_ID"); + if (ctx.run_id.empty()) return ctx; + ctx.enabled = true; + ctx.dir = dir; + std::filesystem::create_directories(ctx.dir); + ctx.model = env_string("CACTUS_MTP_DIAG_MODEL", "model"); + ctx.prompt_name = env_string("CACTUS_MTP_DIAG_PROMPT", "prompt"); + ctx.shape = env_string("CACTUS_MTP_DIAG_SHAPE", "unknown"); + ctx.rep = env_string("CACTUS_MTP_DIAG_REP", "0"); + return ctx; + } + + void write_round(size_t round_index, + size_t generated_start, + size_t generated_end, + size_t target_batch_m, + size_t assistant_pass_count, + size_t drafted_tokens, + size_t accepted_drafts, + bool rejected, + bool alt_branch_accepted, + bool emitted_extra_target_token, + double target_forward_ms, + double assistant_total_ms, + const std::vector& assistant_step_ms, + double sampling_or_argmax_ms, + double kv_transaction_ms, + double callback_stream_ms, + double loop_overhead_ms, + double round_total_ms) const { + if (!enabled || generated_end <= generated_start) return; + static const std::string header = + "run_id,model,prompt_name,shape,rep,round_index,generated_token_start,generated_token_end," + "generated_tokens_emitted,target_batch_m,assistant_pass_count,drafted_tokens,accepted_drafts," + "rejected,alt_branch_accepted,emitted_extra_target_token,target_forward_ms,assistant_total_ms," + "assistant_step_ms,sampling_or_argmax_ms,kv_transaction_ms,callback_stream_ms,loop_overhead_ms,round_total_ms"; + append_csv_line(dir / "round_trace.csv", header, { + run_id, model, prompt_name, shape, rep, + std::to_string(round_index), + std::to_string(generated_start), + std::to_string(generated_end), + std::to_string(generated_end - generated_start), + std::to_string(target_batch_m), + std::to_string(assistant_pass_count), + std::to_string(drafted_tokens), + std::to_string(accepted_drafts), + bool_field(rejected), + bool_field(alt_branch_accepted), + bool_field(emitted_extra_target_token), + double_field(target_forward_ms), + double_field(assistant_total_ms), + join_ms(assistant_step_ms), + double_field(sampling_or_argmax_ms), + double_field(kv_transaction_ms), + double_field(callback_stream_ms), + double_field(loop_overhead_ms), + double_field(round_total_ms), + }); + } + + void write_token(size_t token_position, + uint32_t token_id, + size_t round_index, + size_t token_index_in_round, + const std::string& source, + size_t round_emitted, + double target_forward_ms, + double assistant_total_ms, + double other_ms, + double round_total_ms) const { + if (!enabled || round_emitted == 0) return; + static const std::string header = + "run_id,model,prompt_name,shape,rep,token_position,token_id,round_index,token_index_in_round," + "source,round_generated_tokens_emitted,round_target_forward_ms,round_assistant_total_ms," + "round_other_ms,round_total_ms,allocated_target_forward_ms,allocated_assistant_ms," + "allocated_other_ms,allocated_total_ms"; + double denom = static_cast(round_emitted); + append_csv_line(dir / "token_trace.csv", header, { + run_id, model, prompt_name, shape, rep, + std::to_string(token_position), + std::to_string(token_id), + std::to_string(round_index), + std::to_string(token_index_in_round), + source, + std::to_string(round_emitted), + double_field(target_forward_ms), + double_field(assistant_total_ms), + double_field(other_ms), + double_field(round_total_ms), + double_field(target_forward_ms / denom), + double_field(assistant_total_ms / denom), + double_field(other_ms / denom), + double_field(round_total_ms / denom), + }); + } +}; + std::string extract_last_user_query(const std::vector& messages) { for (auto it = messages.rbegin(); it != messages.rend(); ++it) { if (it->role == "user") { @@ -397,6 +599,11 @@ bool prompt_context_matches( if (prompt.context_token_count < handle->processed_tokens.size()) { return false; } + size_t cache_size = handle->model->get_cache_size(); + size_t desired_cache_size = prompt.context_token_count > 0 ? prompt.context_token_count - 1 : 0; + if (cache_size > handle->processed_tokens.size() || cache_size > desired_cache_size) { + return false; + } if (!std::equal(handle->processed_tokens.begin(), handle->processed_tokens.end(), prompt.tokens.begin())) { return false; } @@ -518,10 +725,13 @@ PrefillResult do_prefill( ) { PrefillResult result = {}; bool has_images = prompt.has_images(); + size_t cache_size = handle->model->get_cache_size(); + size_t desired_cache_size = target_tokens.empty() ? 0 : target_tokens.size() - 1; result.was_prefix = prompt_context_matches(handle, prompt); result.was_exact_match = result.was_prefix && - target_tokens.size() == handle->processed_tokens.size(); + target_tokens.size() == handle->processed_tokens.size() && + cache_size == desired_cache_size; if (result.was_exact_match) { return result; @@ -532,8 +742,9 @@ PrefillResult do_prefill( reset_cache(handle); tokens_to_process = target_tokens; } else { + size_t cached_tokens = std::min(cache_size, handle->processed_tokens.size()); tokens_to_process.assign( - target_tokens.begin() + handle->processed_tokens.size(), + target_tokens.begin() + cached_tokens, target_tokens.end() ); } @@ -652,12 +863,62 @@ int cactus_complete( handle->should_stop = false; auto* tokenizer = handle->model->get_tokenizer(); auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, true, true, pcm_buffer, pcm_buffer_size); + if (prompt.options.mtp && prompt.options.mtp_assistant_path.empty()) { + prompt.options.mtp_assistant_path = default_gemma4_mtp_assistant_path(handle->model.get()); + } + + bool mtp_assistant_available = false; + if (prompt.options.mtp) { + mtp_assistant_available = !prompt.options.mtp_assistant_path.empty() + && std::filesystem::exists(prompt.options.mtp_assistant_path); + if (prompt.options.mtp_required && !mtp_assistant_available) { + throw std::runtime_error("Gemma 4 MTP assistant is unavailable: " + prompt.options.mtp_assistant_path); + } + } CACTUS_LOG_DEBUG("complete", "Prompt tokens: " << prompt.tokens.size() << ", max_tokens: " << prompt.options.max_tokens); bool has_images = prompt.has_images(); bool has_audio = prompt.has_audio(); + Gemma4Model* mtp_target = gemma4_language_model(handle->model.get()); + Gemma4MtpAssistant mtp_assistant; + bool mtp_enabled = false; + size_t mtp_drafted_tokens = 0; + size_t mtp_accepted_tokens = 0; + size_t mtp_rejected_tokens = 0; + size_t mtp_rounds = 0; + double mtp_assistant_draft_ms = 0.0; + double mtp_target_verify_ms = 0.0; + double mtp_sampling_or_argmax_ms = 0.0; + double mtp_kv_transaction_ms = 0.0; + double mtp_callback_stream_ms = 0.0; + std::string mtp_fallback_reason; + if (prompt.options.mtp) { + const bool text_only = !has_images && !has_audio && prompt.tools.empty(); + const bool short_completion_gate = !prompt.options.mtp_required + && prompt.options.temperature == 0.0f + && prompt.options.mtp_max_draft_tokens >= 2 + && prompt.options.max_tokens <= 32; + if (short_completion_gate) { + mtp_fallback_reason = "short_completion"; + } else if (!mtp_target) { + mtp_fallback_reason = "target_not_gemma4_text"; + } else if (!text_only) { + mtp_fallback_reason = "only_text_supported"; + } else if (!mtp_assistant_available) { + mtp_fallback_reason = "assistant_unavailable"; + } else { + auto* graph = static_cast(mtp_target->graph_handle_); + mtp_enabled = mtp_assistant.init(graph, prompt.options.mtp_assistant_path); + if (!mtp_enabled) { + mtp_fallback_reason = "assistant_load_failed"; + } + } + if (prompt.options.mtp_required && !mtp_enabled) { + throw std::runtime_error("Gemma 4 native MTP unavailable: " + mtp_fallback_reason); + } + } const bool has_gemma4_mixed_media = prompt.model_type == Config::ModelType::GEMMA4 && has_images && has_audio; auto decode_gemma4_mixed_media = [&](const std::vector& tokens, float* out_entropy) -> uint32_t { auto* gemma4_mm = dynamic_cast(handle->model.get()); @@ -680,6 +941,12 @@ int cactus_complete( double time_to_first_token = 0.0; float first_token_entropy = 0.0f; uint32_t next_token; + std::vector<__fp16> mtp_prev_hidden; + std::mt19937 mtp_rng(prompt.options.seed == 0 ? std::random_device{}() : static_cast(prompt.options.seed)); + const bool mtp_sparse_sampled = prompt.options.temperature > 0.0f && prompt.options.top_k > 0; + const bool mtp_trace = env_flag_enabled("CACTUS_GEMMA4_MTP_TRACE"); + const DiagnosticTraceContext diag_trace = DiagnosticTraceContext::from_env(); + size_t diagnostic_round_index = 0; size_t prompt_tokens; if ((has_gemma4_mixed_media || has_audio) && !handle->processed_tokens.empty()) { @@ -711,7 +978,47 @@ int cactus_complete( } else { auto prefill_result = do_prefill(handle, prompt, prompt.tokens); prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size(); - next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); + if (mtp_enabled && prompt.options.temperature == 0.0f) { + std::vector first_input; + if (prefill_result.was_exact_match || prefill_result.remaining_tokens.empty()) { + if (handle->processed_tokens.empty()) { + throw std::runtime_error("Cannot generate from empty prompt"); + } + first_input = {handle->processed_tokens.back()}; + } else { + first_input = prefill_result.remaining_tokens; + } + auto first = mtp_target->decode_greedy_with_hidden(first_input); + next_token = first.token; + mtp_prev_hidden = std::move(first.hidden); + } else if (mtp_enabled) { + std::vector first_input; + if (prefill_result.was_exact_match || prefill_result.remaining_tokens.empty()) { + if (handle->processed_tokens.empty()) { + throw std::runtime_error("Cannot generate from empty prompt"); + } + first_input = {handle->processed_tokens.back()}; + } else { + first_input = prefill_result.remaining_tokens; + } + auto first = mtp_sparse_sampled + ? mtp_target->decode_tokens_with_hidden_and_sparse_probs( + first_input, prompt.options.temperature, prompt.options.top_p, + prompt.options.top_k, prompt.options.min_p) + : mtp_target->decode_tokens_with_hidden_and_probs( + first_input, prompt.options.temperature, prompt.options.top_p, + prompt.options.top_k, prompt.options.min_p); + next_token = mtp_sparse_sampled + ? gemma4_mtp_sample_sparse_distribution(first.sparse_probabilities.back(), mtp_rng) + : gemma4_mtp_sample_distribution(first.probabilities.back(), mtp_rng); + size_t hidden_row = first.hidden_dim == 0 ? 0 : first.hidden.size() / first.hidden_dim - 1; + mtp_prev_hidden.assign( + first.hidden.begin() + hidden_row * first.hidden_dim, + first.hidden.begin() + (hidden_row + 1) * first.hidden_dim); + mtp_target->record_sampled_token(next_token); + } else { + next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); + } } handle->processed_tokens = prompt.tokens; @@ -776,44 +1083,349 @@ int cactus_complete( callback(new_text.c_str(), next_token, user_data); } - for (size_t i = 1; i < prompt.options.max_tokens; i++) { - if (handle->should_stop) break; - - float token_entropy = 0.0f; - if (has_gemma4_mixed_media) { - next_token = decode_gemma4_mixed_media(handle->processed_tokens, &token_entropy); - } else if (has_audio) { - next_token = handle->model->decode_with_audio( - handle->processed_tokens, prompt.audio_features, - prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, - "", &token_entropy, - prompt.options.min_p, prompt.options.repetition_penalty); - } else { - next_token = decode(handle->model, {next_token}, prompt.options, &token_entropy); - } - handle->processed_tokens.push_back(next_token); - generated_tokens.push_back(next_token); - - entropy.add(token_entropy); - - if (entropy.rolling_confidence() < prompt.options.confidence_threshold) { - entropy.spike_handoff = true; - maybe_start_cloud_handoff("", {}); - } + auto consume_token = [&](uint32_t token) { + handle->processed_tokens.push_back(token); + generated_tokens.push_back(token); + entropy.add(0.0f); if (prompt.options.force_tools && !prompt.tools.empty()) { - handle->model->update_tool_constraints(next_token); + handle->model->update_tool_constraints(token); } if (matches_stop_sequence(generated_tokens, stop_token_sequences)) { trim_stop_suffix(generated_tokens, stop_token_sequences, prompt.options.include_stop_sequences); - break; + return true; } if (callback) { - std::string new_text = tokenizer->decode({next_token}); - callback(new_text.c_str(), next_token, user_data); + std::string new_text = tokenizer->decode({token}); + callback(new_text.c_str(), token, user_data); + } + return false; + }; + + auto run_standard_decode = [&](size_t produced) { + for (size_t i = produced; i < prompt.options.max_tokens; i++) { + if (handle->should_stop) break; + + float token_entropy = 0.0f; + size_t generated_start = generated_tokens.size(); + auto round_start = std::chrono::steady_clock::now(); + auto target_start = round_start; + if (has_gemma4_mixed_media) { + next_token = decode_gemma4_mixed_media(handle->processed_tokens, &token_entropy); + } else if (has_audio) { + next_token = handle->model->decode_with_audio( + handle->processed_tokens, prompt.audio_features, + prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, + "", &token_entropy, + prompt.options.min_p, prompt.options.repetition_penalty); + } else { + next_token = decode(handle->model, {next_token}, prompt.options, &token_entropy); + } + auto target_end = std::chrono::steady_clock::now(); + handle->processed_tokens.push_back(next_token); + generated_tokens.push_back(next_token); + + entropy.add(token_entropy); + + if (entropy.rolling_confidence() < prompt.options.confidence_threshold) { + entropy.spike_handoff = true; + maybe_start_cloud_handoff("", {}); + } + + if (prompt.options.force_tools && !prompt.tools.empty()) { + handle->model->update_tool_constraints(next_token); + } + + if (matches_stop_sequence(generated_tokens, stop_token_sequences)) { + trim_stop_suffix(generated_tokens, stop_token_sequences, prompt.options.include_stop_sequences); + break; + } + + double callback_ms = 0.0; + if (callback) { + auto callback_start = std::chrono::steady_clock::now(); + std::string new_text = tokenizer->decode({next_token}); + callback(new_text.c_str(), next_token, user_data); + auto callback_end = std::chrono::steady_clock::now(); + callback_ms = std::chrono::duration(callback_end - callback_start).count(); + } + auto round_end = std::chrono::steady_clock::now(); + double target_ms = std::chrono::duration(target_end - target_start).count(); + double round_total_ms = std::chrono::duration(round_end - round_start).count(); + double loop_overhead_ms = round_total_ms - target_ms - callback_ms; + if (loop_overhead_ms < 0.0) loop_overhead_ms = 0.0; + size_t generated_end = generated_tokens.size(); + size_t round_index = diagnostic_round_index++; + diag_trace.write_round( + round_index, + generated_start, + generated_end, + 1, + 0, + 0, + 0, + false, + false, + true, + target_ms, + 0.0, + {}, + 0.0, + 0.0, + callback_ms, + loop_overhead_ms, + round_total_ms); + if (generated_end > generated_start) { + double other_ms = callback_ms + loop_overhead_ms; + diag_trace.write_token( + generated_end - 1, + next_token, + round_index, + 0, + "baseline_target", + generated_end - generated_start, + target_ms, + 0.0, + other_ms, + round_total_ms); + } + } + }; + + auto rebuild_mtp_cache_for_invariant_check = [&]() { + if (!mtp_target || prompt.tokens.empty() || generated_tokens.empty()) return; + mtp_target->reset_cache(); + if (prompt.tokens.size() > 1) { + std::vector prompt_prefix(prompt.tokens.begin(), prompt.tokens.end() - 1); + mtp_target->prefill_for_mtp(prompt_prefix, mtp_target->get_prefill_chunk_size()); + } + + std::vector committed; + committed.reserve(prompt.tokens.size() + generated_tokens.size()); + committed.insert(committed.end(), prompt.tokens.begin(), prompt.tokens.end()); + committed.insert(committed.end(), generated_tokens.begin(), generated_tokens.end()); + for (size_t i = prompt.tokens.size() - 1; i + 1 < committed.size(); ++i) { + mtp_target->prefill_for_mtp({committed[i]}, 1); + mtp_target->record_sampled_token(committed[i + 1]); } + }; + + auto check_mtp_cache_position_invariant = [&]() { + if (!mtp_target || prompt.tokens.empty() || generated_tokens.empty()) return; + size_t expected_position = prompt.tokens.size() + generated_tokens.size() - 1; + size_t actual_position = mtp_target->cache_position(); + if (actual_position != expected_position) { + throw std::runtime_error( + "Gemma 4 MTP cache position invariant failed: position " + + std::to_string(actual_position) + + " != committed prefix " + + std::to_string(expected_position)); + } + }; + + auto check_mtp_cache_invariant = [&](uint32_t last_token) { + if (!prompt.options.mtp_cache_invariant_check + || !mtp_target + || prompt.options.temperature != 0.0f + || prompt.tokens.empty() + || generated_tokens.empty()) { + return; + } + CactusGraph* mtp_graph = static_cast(mtp_target->graph_handle_); + auto cache_state_nodes = mtp_target->cache_state_nodes_for_mtp(); + if (cache_state_nodes.empty()) return; + + size_t current_position = mtp_target->cache_position(); + auto current_txn = mtp_graph->begin_kv_cache_transaction(cache_state_nodes); + uint32_t current_next = mtp_target->decode_greedy_with_hidden({last_token}).token; + current_txn.rollback(); + mtp_graph->apply_pending_kv_cache_sequence_lengths(); + mtp_target->set_cache_position_for_mtp(current_position); + + rebuild_mtp_cache_for_invariant_check(); + size_t replay_position = mtp_target->cache_position(); + auto replay_cache_state_nodes = mtp_target->cache_state_nodes_for_mtp(); + auto replay_txn = mtp_graph->begin_kv_cache_transaction(replay_cache_state_nodes); + uint32_t replay_next = mtp_target->decode_greedy_with_hidden({last_token}).token; + replay_txn.rollback(); + mtp_graph->apply_pending_kv_cache_sequence_lengths(); + mtp_target->set_cache_position_for_mtp(replay_position); + + if (current_next != replay_next) { + throw std::runtime_error( + "Gemma 4 MTP cache invariant failed: live next token " + + std::to_string(current_next) + + " != replayed next token " + + std::to_string(replay_next)); + } + }; + + if (mtp_enabled) { + size_t produced = 1; + bool stopped = false; + size_t mtp_previous_n_matches = 0; + Gemma4MtpSamplingOptions mtp_sampling_options{ + .temperature = prompt.options.temperature, + .top_p = prompt.options.top_p, + .top_k = prompt.options.top_k, + .min_p = prompt.options.min_p, + }; + Gemma4MtpDecodeStrategy mtp_strategy(*mtp_target, mtp_assistant); + while (produced < prompt.options.max_tokens && !handle->should_stop) { + size_t remaining = prompt.options.max_tokens - produced; + if (remaining == 0) break; + size_t generated_start = generated_tokens.size(); + auto round_start = std::chrono::steady_clock::now(); + + size_t draft_limit = std::min( + prompt.options.mtp_max_draft_tokens, + remaining > 1 ? remaining - 1 : 1); + auto cache_nodes = mtp_target->shared_cache_nodes_for_mtp(); + const size_t assistant_position = handle->processed_tokens.empty() ? 0 : handle->processed_tokens.size() - 1; + + auto round = mtp_strategy.decode_round( + next_token, + mtp_prev_hidden, + cache_nodes, + assistant_position, + draft_limit, + remaining, + mtp_sampling_options, + mtp_sparse_sampled, + mtp_rng); + mtp_rounds++; + auto& draft = round.draft; + auto& verified = round.verification; + mtp_drafted_tokens += draft.tokens.size(); + mtp_accepted_tokens += verified.accepted; + mtp_assistant_draft_ms += draft.assistant_draft_ms; + mtp_target_verify_ms += verified.target_verify_ms; + mtp_sampling_or_argmax_ms += draft.sampling_or_argmax_ms + verified.sampling_or_argmax_ms; + mtp_kv_transaction_ms += verified.kv_transaction_ms; + + if (mtp_trace) { + std::cerr + << "[mtp_trace] round=" << mtp_rounds + << " input_len=" << handle->processed_tokens.size() + << " previous_n_matches=" << mtp_previous_n_matches + << " assistant_position_id=" << assistant_position + << " assistant_input_token=" << next_token + << " assistant_draft_tokens=" << token_list_for_trace(draft.tokens) + << " target_verify_input_tokens=" << token_list_for_trace(verified.verify_tokens) + << " target_argmax_tokens=" << token_list_for_trace(verified.target_tokens) + << " accepted_count=" << verified.accepted + << " valid_committed_tokens=" << token_list_for_trace(verified.output_tokens) + << " target_cache_crop_length=" << verified.target_cache_crop_length + << " assistant_draft_ms=" << draft.assistant_draft_ms + << " target_verify_ms=" << verified.target_verify_ms + << " sampling_or_argmax_ms=" << (draft.sampling_or_argmax_ms + verified.sampling_or_argmax_ms) + << " kv_transaction_ms=" << verified.kv_transaction_ms + << "\n"; + } + mtp_previous_n_matches = verified.accepted; + + if (!verified.next_hidden.empty()) { + mtp_prev_hidden = std::move(verified.next_hidden); + } + + if (verified.rejected) { + mtp_rejected_tokens++; + } + + double round_callback_ms = 0.0; + for (uint32_t token : verified.output_tokens) { + next_token = token; + produced++; + if (prompt.options.temperature > 0.0f) { + mtp_target->record_sampled_token(token); + } + auto callback_start = std::chrono::steady_clock::now(); + stopped = consume_token(token); + auto callback_end = std::chrono::steady_clock::now(); + double callback_ms = std::chrono::duration(callback_end - callback_start).count(); + round_callback_ms += callback_ms; + mtp_callback_stream_ms += callback_ms; + if (stopped || produced >= prompt.options.max_tokens) break; + } + auto round_end = std::chrono::steady_clock::now(); + size_t generated_end = generated_tokens.size(); + size_t emitted_count = generated_end > generated_start ? generated_end - generated_start : 0; + double round_total_ms = std::chrono::duration(round_end - round_start).count(); + double round_sampling_ms = draft.sampling_or_argmax_ms + verified.sampling_or_argmax_ms; + double named_ms = draft.assistant_draft_ms + verified.target_verify_ms + + round_sampling_ms + verified.kv_transaction_ms + round_callback_ms; + double loop_overhead_ms = round_total_ms - named_ms; + if (loop_overhead_ms < 0.0) loop_overhead_ms = 0.0; + bool alt_branch_accepted = draft.has_alt_first_token + && !verified.output_tokens.empty() + && verified.output_tokens[0] == draft.alt_first_token + && (draft.tokens.empty() || draft.tokens[0] != draft.alt_first_token); + size_t round_index = diagnostic_round_index++; + diag_trace.write_round( + round_index, + generated_start, + generated_end, + verified.verify_tokens.size(), + draft.assistant_step_ms.size(), + draft.tokens.size() + (draft.has_alt_first_token ? 1 : 0), + verified.accepted, + verified.rejected, + alt_branch_accepted, + verified.emitted_extra_target_token, + verified.target_verify_ms, + draft.assistant_draft_ms, + draft.assistant_step_ms, + round_sampling_ms, + verified.kv_transaction_ms, + round_callback_ms, + loop_overhead_ms, + round_total_ms); + if (emitted_count > 0) { + std::vector sources; + sources.reserve(emitted_count); + if (alt_branch_accepted) { + sources.push_back("accepted_alt_draft"); + while (sources.size() < emitted_count) { + sources.push_back(prompt.options.temperature > 0.0f ? "sampled_target" : "free_target"); + } + } else { + for (size_t i = 0; i < emitted_count; ++i) { + if (i < verified.accepted) { + sources.push_back("accepted_main_draft"); + } else if (verified.rejected) { + sources.push_back("rejection_target"); + } else { + sources.push_back(prompt.options.temperature > 0.0f ? "sampled_target" : "free_target"); + } + } + } + double other_ms = round_sampling_ms + verified.kv_transaction_ms + round_callback_ms + loop_overhead_ms; + for (size_t i = 0; i < emitted_count; ++i) { + diag_trace.write_token( + generated_start + i, + verified.output_tokens[i], + round_index, + i, + sources[i], + emitted_count, + verified.target_verify_ms, + draft.assistant_draft_ms, + other_ms, + round_total_ms); + } + } + + if (!stopped && produced < prompt.options.max_tokens && !generated_tokens.empty()) { + check_mtp_cache_position_invariant(); + check_mtp_cache_invariant(next_token); + } + + if (stopped) break; + } + } else { + run_standard_decode(1); } } else { trim_stop_suffix(generated_tokens, stop_token_sequences, prompt.options.include_stop_sequences); @@ -891,10 +1503,32 @@ int cactus_complete( } const bool handoff_succeeded = cloud_used; + std::string mtp_json; + if (prompt.options.mtp) { + std::ostringstream mtp; + mtp << "{"; + mtp << "\"requested\":true,"; + mtp << "\"enabled\":" << (mtp_enabled ? "true" : "false") << ","; + mtp << "\"assistant_available\":" << (mtp_assistant_available ? "true" : "false") << ","; + mtp << "\"max_draft_tokens\":" << prompt.options.mtp_max_draft_tokens << ","; + mtp << "\"drafted_tokens\":" << mtp_drafted_tokens << ","; + mtp << "\"accepted_tokens\":" << mtp_accepted_tokens << ","; + mtp << "\"rejected_tokens\":" << mtp_rejected_tokens << ","; + mtp << "\"rounds\":" << mtp_rounds << ","; + mtp << std::fixed << std::setprecision(3); + mtp << "\"assistant_draft_ms\":" << mtp_assistant_draft_ms << ","; + mtp << "\"target_verify_ms\":" << mtp_target_verify_ms << ","; + mtp << "\"sampling_or_argmax_ms\":" << mtp_sampling_or_argmax_ms << ","; + mtp << "\"kv_transaction_ms\":" << mtp_kv_transaction_ms << ","; + mtp << "\"callback_stream_ms\":" << mtp_callback_stream_ms << ","; + mtp << "\"fallback_reason\":\"" << escape_json_string(mtp_enabled ? "" : mtp_fallback_reason) << "\""; + mtp << "}"; + mtp_json = mtp.str(); + } std::string result = construct_response_json(primary_response, primary_function_calls, time_to_first_token, total_time_ms, prefill_tps, decode_tps, prompt_tokens, completion_tokens, confidence, handoff_succeeded, - thinking_text); + thinking_text, {}, mtp_json); if (result.length() >= buffer_size) { handle_error_response("Response buffer too small", response_buffer, buffer_size); diff --git a/cactus-engine/src/decode_strategy.h b/cactus-engine/src/decode_strategy.h new file mode 100644 index 000000000..79e661725 --- /dev/null +++ b/cactus-engine/src/decode_strategy.h @@ -0,0 +1,134 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +struct DecodeOptions { + bool mtp_enabled = false; + bool mtp_required = false; + float temperature = -1.0f; + float top_p = -1.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + size_t top_k = 0; + size_t max_draft_tokens = 6; +}; + +struct DecodeStepResult { + uint32_t token = 0; + float entropy = 0.0f; +}; + +struct DraftBatch { + std::vector tokens; +}; + +struct TargetDecodeState { + std::vector input_tokens; +}; + +struct AcceptedTokenBatch { + std::vector tokens; + std::vector texts; + std::vector entropies; + size_t drafted_tokens = 0; + size_t accepted_draft_tokens = 0; + size_t rejected_tokens = 0; + std::vector rejected_draft_token_ids; +}; + +class MissingAssistantError : public std::runtime_error { +public: + explicit MissingAssistantError(const std::string& message) : std::runtime_error(message) {} +}; + +class TargetDecoder { +public: + virtual ~TargetDecoder() = default; + virtual DecodeStepResult decode_one(const std::vector& input_tokens, + const DecodeOptions& options) = 0; +}; + +class AssistantDraftProvider { +public: + virtual ~AssistantDraftProvider() = default; + virtual DraftBatch draft(const TargetDecodeState& state, size_t max_draft_tokens) = 0; +}; + +class SpeculativeVerifier { +public: + virtual ~SpeculativeVerifier() = default; + virtual AcceptedTokenBatch verify(const DraftBatch& draft, + const TargetDecodeState& state, + const DecodeOptions& options) = 0; +}; + +class DecodeStrategy { +public: + virtual ~DecodeStrategy() = default; + virtual AcceptedTokenBatch decode_next(const std::vector& input_tokens, + const DecodeOptions& options) = 0; + virtual std::string name() const = 0; +}; + +class StandardDecodeStrategy final : public DecodeStrategy { +public: + explicit StandardDecodeStrategy(TargetDecoder& target) : target_(target) {} + + AcceptedTokenBatch decode_next(const std::vector& input_tokens, + const DecodeOptions& options) override { + DecodeStepResult step = target_.decode_one(input_tokens, options); + return AcceptedTokenBatch{ + .tokens = {step.token}, + .entropies = {step.entropy}, + }; + } + + std::string name() const override { return "standard"; } + +private: + TargetDecoder& target_; +}; + +class Gemma4MtpDecodeStrategy final : public DecodeStrategy { +public: + Gemma4MtpDecodeStrategy(TargetDecoder& target, + AssistantDraftProvider& assistant, + SpeculativeVerifier& verifier) + : target_(target), assistant_(assistant), verifier_(verifier) {} + + AcceptedTokenBatch decode_next(const std::vector& input_tokens, + const DecodeOptions& options) override { + (void)target_; + TargetDecodeState state{.input_tokens = input_tokens}; + DraftBatch draft = assistant_.draft(state, options.max_draft_tokens); + return verifier_.verify(draft, state, options); + } + + std::string name() const override { return "gemma4_mtp"; } + +private: + TargetDecoder& target_; + AssistantDraftProvider& assistant_; + SpeculativeVerifier& verifier_; +}; + +inline std::unique_ptr make_decode_strategy(TargetDecoder& target, + AssistantDraftProvider* assistant, + SpeculativeVerifier* verifier, + const DecodeOptions& options) { + if (!options.mtp_enabled) { + return std::make_unique(target); + } + if (!assistant || !verifier) { + if (options.mtp_required) { + throw MissingAssistantError("Gemma 4 MTP assistant is unavailable"); + } + return std::make_unique(target); + } + return std::make_unique(target, *assistant, *verifier); +} diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 463046464..782baa16f 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -566,6 +566,7 @@ class Model { const Config& get_config() const { return config_; } Tokenizer* get_tokenizer() const { return tokenizer_.get(); } + const std::string& get_model_folder_path() const { return model_folder_path_; } const std::vector& get_debug_nodes() const; virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); @@ -611,7 +612,7 @@ class Model { void set_cache_window(size_t window_size, size_t sink_size = 4); - size_t get_cache_size() const; + virtual size_t get_cache_size() const; bool load_npu_prefill(const std::string& model_path); bool has_npu_prefill() const; diff --git a/cactus-engine/src/gemma4_mtp_decode.cpp b/cactus-engine/src/gemma4_mtp_decode.cpp new file mode 100644 index 000000000..0c1dd5430 --- /dev/null +++ b/cactus-engine/src/gemma4_mtp_decode.cpp @@ -0,0 +1,411 @@ +#include "gemma4_mtp_decode.h" + +#include "mtp_sampler.h" +#include "cactus_graph.h" + +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +namespace { + +std::vector rejection_adjusted_distribution(const std::vector& target, + const std::vector& assistant) { + std::vector adjusted(target.size(), 0.0f); + float sum = 0.0f; + for (size_t i = 0; i < target.size() && i < assistant.size(); ++i) { + float value = std::max(target[i] - assistant[i], 0.0f); + adjusted[i] = value; + sum += value; + } + if (sum <= 0.0f || !std::isfinite(sum)) { + adjusted = target; + sum = std::accumulate(adjusted.begin(), adjusted.end(), 0.0f); + } + if (sum <= 0.0f || !std::isfinite(sum)) { + std::fill(adjusted.begin(), adjusted.end(), adjusted.empty() ? 0.0f : 1.0f / static_cast(adjusted.size())); + return adjusted; + } + for (float& value : adjusted) value /= sum; + return adjusted; +} + +std::vector<__fp16> hidden_row(const Gemma4Model::GreedyBatchWithHidden& batch, size_t row) { + if (batch.hidden_dim == 0 || row >= batch.hidden.size() / batch.hidden_dim) { + return {}; + } + return std::vector<__fp16>( + batch.hidden.begin() + row * batch.hidden_dim, + batch.hidden.begin() + (row + 1) * batch.hidden_dim); +} + +double elapsed_ms(std::chrono::steady_clock::time_point start, + std::chrono::steady_clock::time_point end) { + return std::chrono::duration(end - start).count(); +} + +std::vector<__fp16> tree_mask(size_t token_count) { + std::vector<__fp16> mask(token_count * token_count, static_cast<__fp16>(0)); + for (size_t row = 0; row < token_count; ++row) { + mask[row * token_count] = static_cast<__fp16>(1); + if (row == 1 || row >= 3) mask[row * token_count + 1] = static_cast<__fp16>(1); + if (row == 2) mask[row * token_count + 2] = static_cast<__fp16>(1); + for (size_t col = 3; col <= row; ++col) { + mask[row * token_count + col] = static_cast<__fp16>(1); + } + } + return mask; +} + +size_t tree_main_tokens_for_draft_limit(size_t draft_limit) { + const char* env = std::getenv("CACTUS_GEMMA4_MTP_TREE_MAIN_TOKENS"); + if (env && env[0] != '\0') { + char* end = nullptr; + unsigned long value = std::strtoul(env, &end, 10); + if (end != env && *end == '\0' && value >= 1 && value <= 3 && draft_limit == value + 1) { + return static_cast(value); + } + return 0; + } + return 0; +} + +} + +uint32_t gemma4_mtp_sample_distribution(const std::vector& probabilities, std::mt19937& rng) { + if (probabilities.empty()) return 0; + std::uniform_real_distribution dist(0.0f, 1.0f); + float sample = dist(rng); + float cumulative = 0.0f; + for (size_t i = 0; i < probabilities.size(); ++i) { + cumulative += std::max(0.0f, probabilities[i]); + if (sample <= cumulative) return static_cast(i); + } + return static_cast(probabilities.size() - 1); +} + +uint32_t gemma4_mtp_sample_sparse_distribution( + const std::vector>& probabilities, + std::mt19937& rng) { + if (probabilities.empty()) return 0; + std::uniform_real_distribution dist(0.0f, 1.0f); + return mtp_sample_sparse(probabilities, dist(rng)); +} + +Gemma4AssistantDraftProvider::Gemma4AssistantDraftProvider(Gemma4Model& target, + Gemma4MtpAssistant& assistant) + : target_(target), assistant_(assistant) {} + +Gemma4MtpDraftBatch Gemma4AssistantDraftProvider::draft( + uint32_t input_token, + const std::vector<__fp16>& previous_hidden, + const Gemma4Model::SharedCacheNodes& cache_nodes, + size_t assistant_position, + size_t draft_limit, + const Gemma4MtpSamplingOptions& options, + bool sparse_sampled, + std::mt19937& rng) { + Gemma4MtpDraftBatch batch; + batch.tokens.reserve(draft_limit); + if (options.temperature > 0.0f) { + if (sparse_sampled) { + batch.sparse_probabilities.reserve(draft_limit); + } else { + batch.probabilities.reserve(draft_limit); + } + } + + std::vector<__fp16> assistant_hidden = previous_hidden; + uint32_t assistant_input_token = input_token; + const size_t tree_main_tokens = tree_main_tokens_for_draft_limit(draft_limit); + const bool use_tree = options.temperature == 0.0f + && tree_main_tokens > 0 + && target_.can_use_mtp_tree_attention(tree_main_tokens + 2); + const size_t main_draft_limit = use_tree ? tree_main_tokens : draft_limit; + batch.assistant_step_ms.reserve(main_draft_limit); + for (size_t i = 0; i < main_draft_limit; ++i) { + auto draft_start = std::chrono::steady_clock::now(); + auto step = assistant_.draft_one( + assistant_input_token, + target_.token_embedding_node_for_mtp(), + assistant_hidden, + cache_nodes, + assistant_position, + i == 0 ? (1.0f / 16.0f) : 1.0f, + options.temperature, + options.top_p, + options.top_k, + options.min_p, + i + 1 < main_draft_limit, + use_tree && i == 0); + auto draft_end = std::chrono::steady_clock::now(); + double step_ms = elapsed_ms(draft_start, draft_end); + batch.assistant_draft_ms += step_ms; + batch.assistant_step_ms.push_back(step_ms); + + uint32_t draft_token = step.token; + if (use_tree && i == 0 && step.has_second_token) { + batch.alt_first_token = step.second_token; + batch.has_alt_first_token = true; + } + if (options.temperature > 0.0f) { + auto sample_start = std::chrono::steady_clock::now(); + if (sparse_sampled) { + draft_token = gemma4_mtp_sample_sparse_distribution(step.sparse_probabilities, rng); + batch.sparse_probabilities.push_back(std::move(step.sparse_probabilities)); + } else { + draft_token = gemma4_mtp_sample_distribution(step.probabilities, rng); + batch.probabilities.push_back(std::move(step.probabilities)); + } + auto sample_end = std::chrono::steady_clock::now(); + batch.sampling_or_argmax_ms += elapsed_ms(sample_start, sample_end); + } + + batch.tokens.push_back(draft_token); + if (i + 1 < main_draft_limit) { + assistant_hidden = std::move(step.hidden); + } + assistant_input_token = draft_token; + } + return batch; +} + +Gemma4SpeculativeVerifier::Gemma4SpeculativeVerifier(Gemma4Model& target) : target_(target) {} + +Gemma4MtpVerifyResult Gemma4SpeculativeVerifier::verify_and_commit( + uint32_t input_token, + const Gemma4MtpDraftBatch& draft, + size_t remaining_tokens, + const Gemma4MtpSamplingOptions& options, + bool sparse_sampled, + std::mt19937& rng) { + Gemma4MtpVerifyResult result; + result.verify_tokens.reserve(draft.tokens.size() + 1); + result.verify_tokens.push_back(input_token); + const bool use_tree = options.temperature == 0.0f + && draft.has_alt_first_token + && !draft.tokens.empty() + && draft.tokens.size() <= 3 + && target_.can_use_mtp_tree_attention(draft.tokens.size() + 2); + if (use_tree) { + result.verify_tokens.push_back(draft.tokens[0]); + result.verify_tokens.push_back(draft.alt_first_token); + result.verify_tokens.insert(result.verify_tokens.end(), draft.tokens.begin() + 1, draft.tokens.end()); + } else { + result.verify_tokens.insert(result.verify_tokens.end(), draft.tokens.begin(), draft.tokens.end()); + } + + auto* graph = static_cast(target_.graph_handle_); + size_t verifier_cache_start = target_.cache_position(); + auto kv_start = std::chrono::steady_clock::now(); + auto cache_txn = graph->begin_kv_cache_transaction(target_.cache_state_nodes_for_mtp()); + auto kv_end = std::chrono::steady_clock::now(); + result.kv_transaction_ms += elapsed_ms(kv_start, kv_end); + + Gemma4Model::GreedyBatchWithHidden target_batch; + auto verify_start = std::chrono::steady_clock::now(); + if (use_tree) { + target_batch = target_.decode_greedy_tokens_with_hidden_tree( + result.verify_tokens, tree_mask(result.verify_tokens.size())); + } else if (options.temperature == 0.0f) { + target_batch = target_.decode_greedy_tokens_with_hidden_early_stop( + result.verify_tokens, draft.tokens); + } else if (sparse_sampled) { + target_batch = target_.decode_tokens_with_hidden_and_sparse_probs( + result.verify_tokens, options.temperature, options.top_p, options.top_k, options.min_p); + } else { + target_batch = target_.decode_tokens_with_hidden_and_probs( + result.verify_tokens, options.temperature, options.top_p, options.top_k, options.min_p); + } + auto verify_end = std::chrono::steady_clock::now(); + result.target_verify_ms += elapsed_ms(verify_start, verify_end); + result.target_tokens = target_batch.tokens; + + auto sample_start = std::chrono::steady_clock::now(); + if (options.temperature == 0.0f) { + if (use_tree) { + if (target_batch.tokens.size() != result.verify_tokens.size()) { + throw std::runtime_error("Gemma 4 tree verifier target row count mismatch"); + } + auto main_row = [](size_t draft_index) { + return draft_index == 0 ? size_t{1} : draft_index + 2; + }; + auto prediction_row = [&](size_t draft_index) { + return draft_index == 0 ? size_t{0} : main_row(draft_index - 1); + }; + result.committed_verify_indices.push_back(0); + result.next_hidden_row = 0; + + for (size_t i = 0; i < draft.tokens.size(); ++i) { + const size_t pred_row = prediction_row(i); + if (target_batch.tokens[pred_row] != draft.tokens[i]) { + if (i == 0 && target_batch.tokens[0] == draft.alt_first_token) { + result.output_tokens.push_back(draft.alt_first_token); + result.accepted = 1; + result.next_hidden_row = 2; + if (result.output_tokens.size() < remaining_tokens) { + result.output_tokens.push_back(target_batch.tokens[2]); + result.emitted_extra_target_token = true; + } + result.committed_verify_indices = {0, 2}; + } else { + result.output_tokens.push_back(target_batch.tokens[pred_row]); + result.rejected = true; + } + break; + } + result.output_tokens.push_back(draft.tokens[i]); + result.accepted = i + 1; + result.next_hidden_row = main_row(i); + result.committed_verify_indices.push_back(result.next_hidden_row); + } + + if (!result.rejected + && !result.emitted_extra_target_token + && result.accepted == draft.tokens.size() + && result.output_tokens.size() < remaining_tokens) { + result.output_tokens.push_back(target_batch.tokens[result.next_hidden_row]); + result.emitted_extra_target_token = true; + } + } else { + const size_t produced = target_batch.tokens.size(); + const size_t draft_size = draft.tokens.size(); + for (; result.accepted < draft_size && result.accepted < produced; ++result.accepted) { + if (draft.tokens[result.accepted] != target_batch.tokens[result.accepted]) { + result.output_tokens.push_back(target_batch.tokens[result.accepted]); + result.rejected = true; + break; + } + result.output_tokens.push_back(draft.tokens[result.accepted]); + } + if (!result.rejected && result.output_tokens.size() < remaining_tokens + && produced > draft_size) { + result.output_tokens.push_back(target_batch.tokens[draft_size]); + result.emitted_extra_target_token = true; + } + } + } else { + for (; result.accepted < draft.tokens.size(); ++result.accepted) { + uint32_t y = draft.tokens[result.accepted]; + float p_y = 0.0f; + float q_y = 0.0f; + if (sparse_sampled) { + p_y = mtp_sparse_probability_at(target_batch.sparse_probabilities[result.accepted], y); + q_y = mtp_sparse_probability_at(draft.sparse_probabilities[result.accepted], y); + } else { + const auto& p = target_batch.probabilities[result.accepted]; + const auto& q = draft.probabilities[result.accepted]; + p_y = y < p.size() ? p[y] : 0.0f; + q_y = y < q.size() ? q[y] : 0.0f; + } + + float accept_probability = q_y <= 0.0f ? 1.0f : std::min(1.0f, p_y / q_y); + std::uniform_real_distribution dist(0.0f, 1.0f); + if (dist(rng) <= accept_probability) { + result.output_tokens.push_back(y); + continue; + } + + if (sparse_sampled) { + auto adjusted = mtp_rejection_adjusted_sparse_distribution( + target_batch.sparse_probabilities[result.accepted], + draft.sparse_probabilities[result.accepted]); + result.output_tokens.push_back(gemma4_mtp_sample_sparse_distribution(adjusted, rng)); + } else { + auto adjusted = rejection_adjusted_distribution( + target_batch.probabilities[result.accepted], + draft.probabilities[result.accepted]); + result.output_tokens.push_back(gemma4_mtp_sample_distribution(adjusted, rng)); + } + result.rejected = true; + break; + } + if (!result.rejected && result.output_tokens.size() < remaining_tokens) { + if (sparse_sampled) { + result.output_tokens.push_back(gemma4_mtp_sample_sparse_distribution( + target_batch.sparse_probabilities[draft.tokens.size()], rng)); + } else { + result.output_tokens.push_back(gemma4_mtp_sample_distribution( + target_batch.probabilities[draft.tokens.size()], rng)); + } + result.emitted_extra_target_token = true; + } + } + auto sample_end = std::chrono::steady_clock::now(); + result.sampling_or_argmax_ms += elapsed_ms(sample_start, sample_end); + + if (use_tree) { + for (uint32_t token : result.output_tokens) { + target_.record_sampled_token(token); + } + } + + if (!result.output_tokens.empty()) { + size_t row = use_tree + ? result.next_hidden_row + : (options.temperature == 0.0f ? result.accepted : result.output_tokens.size() - 1); + result.next_hidden = hidden_row(target_batch, row); + } + + size_t committed_cache_tokens = result.output_tokens.size(); + result.target_cache_crop_length = verifier_cache_start + committed_cache_tokens; + kv_start = std::chrono::steady_clock::now(); + if (use_tree) { + cache_txn.commit_selected(result.committed_verify_indices); + graph->apply_pending_kv_cache_sequence_lengths(); + target_.set_cache_position_for_mtp(result.target_cache_crop_length); + } else if (committed_cache_tokens < result.verify_tokens.size()) { + cache_txn.commit_prefix(committed_cache_tokens); + graph->apply_pending_kv_cache_sequence_lengths(); + target_.set_cache_position_for_mtp(result.target_cache_crop_length); + } else { + cache_txn.commit_all(); + graph->apply_pending_kv_cache_sequence_lengths(); + } + kv_end = std::chrono::steady_clock::now(); + result.kv_transaction_ms += elapsed_ms(kv_start, kv_end); + + return result; +} + +Gemma4MtpDecodeStrategy::Gemma4MtpDecodeStrategy(Gemma4Model& target, + Gemma4MtpAssistant& assistant) + : draft_provider_(target, assistant), verifier_(target) {} + +Gemma4MtpRoundResult Gemma4MtpDecodeStrategy::decode_round( + uint32_t input_token, + const std::vector<__fp16>& previous_hidden, + const Gemma4Model::SharedCacheNodes& cache_nodes, + size_t assistant_position, + size_t draft_limit, + size_t remaining_tokens, + const Gemma4MtpSamplingOptions& options, + bool sparse_sampled, + std::mt19937& rng) { + Gemma4MtpRoundResult round; + round.draft = draft_provider_.draft( + input_token, + previous_hidden, + cache_nodes, + assistant_position, + draft_limit, + options, + sparse_sampled, + rng); + round.verification = verifier_.verify_and_commit( + input_token, + round.draft, + remaining_tokens, + options, + sparse_sampled, + rng); + return round; +} + +} +} diff --git a/cactus-engine/src/gemma4_mtp_decode.h b/cactus-engine/src/gemma4_mtp_decode.h new file mode 100644 index 000000000..1723b29c3 --- /dev/null +++ b/cactus-engine/src/gemma4_mtp_decode.h @@ -0,0 +1,111 @@ +#pragma once + +#include "../models/gemma4/gemma4_mtp_assistant.h" +#include "../models/gemma4/model_gemma4.h" + +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +struct Gemma4MtpSamplingOptions { + float temperature = 0.0f; + float top_p = 0.0f; + size_t top_k = 0; + float min_p = 0.0f; +}; + +struct Gemma4MtpDraftBatch { + std::vector tokens; + uint32_t alt_first_token = 0; + bool has_alt_first_token = false; + std::vector> probabilities; + std::vector>> sparse_probabilities; + std::vector assistant_step_ms; + double assistant_draft_ms = 0.0; + double sampling_or_argmax_ms = 0.0; +}; + +struct Gemma4MtpVerifyResult { + std::vector output_tokens; + std::vector verify_tokens; + std::vector target_tokens; + std::vector committed_verify_indices; + std::vector<__fp16> next_hidden; + size_t next_hidden_row = 0; + size_t accepted = 0; + bool rejected = false; + bool emitted_extra_target_token = false; + size_t target_cache_crop_length = 0; + double target_verify_ms = 0.0; + double sampling_or_argmax_ms = 0.0; + double kv_transaction_ms = 0.0; +}; + +struct Gemma4MtpRoundResult { + Gemma4MtpDraftBatch draft; + Gemma4MtpVerifyResult verification; +}; + +uint32_t gemma4_mtp_sample_distribution(const std::vector& probabilities, std::mt19937& rng); +uint32_t gemma4_mtp_sample_sparse_distribution( + const std::vector>& probabilities, + std::mt19937& rng); + +class Gemma4AssistantDraftProvider { +public: + Gemma4AssistantDraftProvider(Gemma4Model& target, Gemma4MtpAssistant& assistant); + + Gemma4MtpDraftBatch draft(uint32_t input_token, + const std::vector<__fp16>& previous_hidden, + const Gemma4Model::SharedCacheNodes& cache_nodes, + size_t assistant_position, + size_t draft_limit, + const Gemma4MtpSamplingOptions& options, + bool sparse_sampled, + std::mt19937& rng); + +private: + Gemma4Model& target_; + Gemma4MtpAssistant& assistant_; +}; + +class Gemma4SpeculativeVerifier { +public: + explicit Gemma4SpeculativeVerifier(Gemma4Model& target); + + Gemma4MtpVerifyResult verify_and_commit(uint32_t input_token, + const Gemma4MtpDraftBatch& draft, + size_t remaining_tokens, + const Gemma4MtpSamplingOptions& options, + bool sparse_sampled, + std::mt19937& rng); + +private: + Gemma4Model& target_; +}; + +class Gemma4MtpDecodeStrategy { +public: + Gemma4MtpDecodeStrategy(Gemma4Model& target, Gemma4MtpAssistant& assistant); + + Gemma4MtpRoundResult decode_round(uint32_t input_token, + const std::vector<__fp16>& previous_hidden, + const Gemma4Model::SharedCacheNodes& cache_nodes, + size_t assistant_position, + size_t draft_limit, + size_t remaining_tokens, + const Gemma4MtpSamplingOptions& options, + bool sparse_sampled, + std::mt19937& rng); + +private: + Gemma4AssistantDraftProvider draft_provider_; + Gemma4SpeculativeVerifier verifier_; +}; + +} +} diff --git a/cactus-engine/src/mtp_completion.h b/cactus-engine/src/mtp_completion.h new file mode 100644 index 000000000..443403d4e --- /dev/null +++ b/cactus-engine/src/mtp_completion.h @@ -0,0 +1,125 @@ +#pragma once + +#include "decode_strategy.h" + +#include +#include +#include +#include + +struct AcceptedToken { + uint32_t id = 0; + std::string text; +}; + +struct MtpCompletionResult { + bool stopped = false; + size_t tokens_consumed = 0; +}; + +struct MtpCompletionMetrics { + std::string fallback_reason; + size_t drafted_tokens = 0; + size_t accepted_tokens = 0; + size_t rejected_tokens = 0; + size_t mtp_rounds = 0; +}; + +struct MtpAvailability { + bool requested = false; + bool required = false; + bool assistant_loaded = false; + bool supports_prompt = true; +}; + +struct MtpFallbackDecision { + bool use_standard_decode = false; + bool error = false; + std::string reason; +}; + +inline MtpFallbackDecision decide_mtp_fallback(const MtpAvailability& availability) { + if (!availability.requested) { + return MtpFallbackDecision{.use_standard_decode = true, .reason = "mtp_disabled"}; + } + if (!availability.supports_prompt) { + return MtpFallbackDecision{ + .use_standard_decode = !availability.required, + .error = availability.required, + .reason = "unsupported_prompt", + }; + } + if (!availability.assistant_loaded) { + return MtpFallbackDecision{ + .use_standard_decode = !availability.required, + .error = availability.required, + .reason = "assistant_unavailable", + }; + } + return MtpFallbackDecision{.reason = ""}; +} + +class MtpCompletionAccumulator { +public: + void set_stream_callback(std::function callback) { + callback_ = std::move(callback); + } + + void set_stop_sequences(std::vector stop_sequences) { + stop_sequences_ = std::move(stop_sequences); + } + + MtpCompletionResult consume_accepted_batch(const AcceptedTokenBatch& batch) { + metrics_.drafted_tokens += batch.drafted_tokens; + metrics_.rejected_tokens += batch.rejected_tokens; + metrics_.mtp_rounds++; + + MtpCompletionResult result; + for (size_t i = 0; i < batch.tokens.size(); ++i) { + AcceptedToken token{ + .id = batch.tokens[i], + .text = i < batch.texts.size() ? batch.texts[i] : "", + }; + + token_history_.push_back(token.id); + output_text_ += token.text; + metrics_.accepted_tokens++; + result.tokens_consumed++; + + if (callback_) { + callback_(token); + } + + if (has_stop_sequence()) { + result.stopped = true; + break; + } + } + + return result; + } + + void record_fallback(const MtpFallbackDecision& decision) { + metrics_.fallback_reason = decision.reason; + } + + const std::string& output_text() const { return output_text_; } + const std::vector& token_history() const { return token_history_; } + const MtpCompletionMetrics& metrics() const { return metrics_; } + +private: + bool has_stop_sequence() const { + for (const auto& stop : stop_sequences_) { + if (!stop.empty() && output_text_.find(stop) != std::string::npos) { + return true; + } + } + return false; + } + + std::function callback_; + std::vector stop_sequences_; + std::string output_text_; + std::vector token_history_; + MtpCompletionMetrics metrics_; +}; diff --git a/cactus-engine/src/mtp_sampler.h b/cactus-engine/src/mtp_sampler.h new file mode 100644 index 000000000..78bcce2d8 --- /dev/null +++ b/cactus-engine/src/mtp_sampler.h @@ -0,0 +1,199 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +struct MtpDistribution { + std::vector probabilities; +}; + +struct MtpDraftBatch { + std::vector tokens; + std::vector probabilities; +}; + +struct MtpVerificationResult { + std::vector output_tokens; + size_t accepted_draft_tokens = 0; + bool rejected = false; +}; + +class MtpDeterministicRng { +public: + explicit MtpDeterministicRng(std::initializer_list values) : values_(values) {} + + float uniform() { + if (values_.empty()) return 0.0f; + float value = values_[index_ % values_.size()]; + index_++; + return std::clamp(value, 0.0f, 1.0f); + } + +private: + std::vector values_; + size_t index_ = 0; +}; + +inline uint32_t mtp_argmax(const MtpDistribution& dist) { + if (dist.probabilities.empty()) { + throw std::invalid_argument("MTP distribution must not be empty"); + } + return static_cast(std::distance( + dist.probabilities.begin(), + std::max_element(dist.probabilities.begin(), dist.probabilities.end()))); +} + +inline uint32_t mtp_sample(const MtpDistribution& dist, float u) { + if (dist.probabilities.empty()) { + throw std::invalid_argument("MTP distribution must not be empty"); + } + float cumulative = 0.0f; + for (size_t i = 0; i < dist.probabilities.size(); ++i) { + cumulative += std::max(0.0f, dist.probabilities[i]); + if (u <= cumulative) { + return static_cast(i); + } + } + return static_cast(dist.probabilities.size() - 1); +} + +inline uint32_t mtp_sample_sparse(const std::vector>& probabilities, float u) { + if (probabilities.empty()) { + throw std::invalid_argument("MTP sparse distribution must not be empty"); + } + float cumulative = 0.0f; + for (const auto& item : probabilities) { + cumulative += std::max(0.0f, item.second); + if (u <= cumulative) { + return item.first; + } + } + return probabilities.back().first; +} + +inline float mtp_sparse_probability_at(const std::vector>& probabilities, + uint32_t token) { + for (const auto& item : probabilities) { + if (item.first == token) return item.second; + } + return 0.0f; +} + +inline MtpDistribution rejection_adjusted_distribution(const MtpDistribution& target, + const MtpDistribution& assistant) { + if (target.probabilities.size() != assistant.probabilities.size()) { + throw std::invalid_argument("MTP target and assistant distributions must have same vocabulary size"); + } + + MtpDistribution adjusted; + adjusted.probabilities.resize(target.probabilities.size()); + float sum = 0.0f; + for (size_t i = 0; i < target.probabilities.size(); ++i) { + float value = std::max(target.probabilities[i] - assistant.probabilities[i], 0.0f); + adjusted.probabilities[i] = value; + sum += value; + } + + if (sum <= 0.0f || !std::isfinite(sum)) { + adjusted = target; + sum = std::accumulate(adjusted.probabilities.begin(), adjusted.probabilities.end(), 0.0f); + } + + if (sum <= 0.0f || !std::isfinite(sum)) { + float uniform = adjusted.probabilities.empty() ? 0.0f : 1.0f / static_cast(adjusted.probabilities.size()); + std::fill(adjusted.probabilities.begin(), adjusted.probabilities.end(), uniform); + return adjusted; + } + + for (float& value : adjusted.probabilities) { + value /= sum; + } + return adjusted; +} + +inline std::vector> mtp_rejection_adjusted_sparse_distribution( + const std::vector>& target, + const std::vector>& assistant) { + std::vector> adjusted; + adjusted.reserve(target.size()); + float sum = 0.0f; + for (const auto& item : target) { + float value = std::max(item.second - mtp_sparse_probability_at(assistant, item.first), 0.0f); + if (value > 0.0f) { + adjusted.emplace_back(item.first, value); + sum += value; + } + } + if (sum <= 0.0f || !std::isfinite(sum)) { + return target; + } + for (auto& item : adjusted) item.second /= sum; + return adjusted; +} + +inline MtpVerificationResult verify_greedy_mtp_draft(const MtpDraftBatch& draft, + const std::vector& target) { + if (target.size() < draft.tokens.size() + 1) { + throw std::invalid_argument("MTP target distributions must include one extra next-token distribution"); + } + + MtpVerificationResult result; + for (size_t i = 0; i < draft.tokens.size(); ++i) { + uint32_t target_token = mtp_argmax(target[i]); + if (draft.tokens[i] != target_token) { + result.output_tokens.push_back(target_token); + result.rejected = true; + return result; + } + result.output_tokens.push_back(draft.tokens[i]); + result.accepted_draft_tokens++; + } + + result.output_tokens.push_back(mtp_argmax(target[draft.tokens.size()])); + return result; +} + +inline MtpVerificationResult verify_sampled_mtp_draft(const MtpDraftBatch& draft, + const std::vector& target, + MtpDeterministicRng& rng) { + if (target.size() < draft.tokens.size() + 1) { + throw std::invalid_argument("MTP target distributions must include one extra next-token distribution"); + } + if (draft.probabilities.size() < draft.tokens.size()) { + throw std::invalid_argument("MTP draft must include assistant probabilities for each token"); + } + + MtpVerificationResult result; + for (size_t i = 0; i < draft.tokens.size(); ++i) { + uint32_t y = draft.tokens[i]; + const auto& q = draft.probabilities[i]; + const auto& p = target[i]; + if (y >= q.probabilities.size() || y >= p.probabilities.size()) { + throw std::out_of_range("MTP draft token outside distribution vocabulary"); + } + + float q_y = q.probabilities[y]; + float p_y = p.probabilities[y]; + float accept_probability = q_y <= 0.0f ? 1.0f : std::min(1.0f, p_y / q_y); + + if (rng.uniform() <= accept_probability) { + result.output_tokens.push_back(y); + result.accepted_draft_tokens++; + continue; + } + + MtpDistribution adjusted = rejection_adjusted_distribution(p, q); + result.output_tokens.push_back(mtp_sample(adjusted, rng.uniform())); + result.rejected = true; + return result; + } + + result.output_tokens.push_back(mtp_sample(target[draft.tokens.size()], rng.uniform())); + return result; +} diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index ab0b60823..0eb6bf736 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -443,7 +443,7 @@ struct InferenceOptions { float temperature = 0.0f; float top_p = 0.0f; float min_p = 0.15f; - float repetition_penalty = 1.1f; + float repetition_penalty = 1.0f; float confidence_threshold = -1.0f; size_t top_k = 0; size_t max_tokens = 100; @@ -457,6 +457,12 @@ struct InferenceOptions { bool auto_handoff = true; bool handoff_with_images = true; bool enable_thinking_if_supported = false; + bool mtp = false; + bool mtp_required = false; + bool mtp_cache_invariant_check = false; + size_t mtp_max_draft_tokens = 0; + size_t seed = 0; + std::string mtp_assistant_path; }; } // namespace ffi @@ -1421,6 +1427,48 @@ inline InferenceOptions parse_inference_options_json(const std::string& json) { options.enable_thinking_if_supported = (json.substr(pos, 4) == "true"); } + pos = json.find("\"mtp\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.mtp = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"mtp_required\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.mtp_required = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"mtp_cache_invariant_check\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.mtp_cache_invariant_check = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"mtp_max_draft_tokens\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.mtp_max_draft_tokens = std::stoul(json.substr(pos)); + } + + pos = json.find("\"seed\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.seed = std::stoul(json.substr(pos)); + } + + options.mtp_assistant_path = json_string_field(json, "mtp_assistant_path"); + if (!options.mtp_assistant_path.empty() || options.mtp_required) { + options.mtp = true; + } + if (options.mtp_max_draft_tokens == 0) { + options.mtp = false; + options.mtp_required = false; + } + pos = json.find("\"stop_sequences\""); if (pos != std::string::npos) { pos = json.find('[', pos); @@ -1581,7 +1629,8 @@ inline std::string construct_response_json(const std::string& regular_response, float confidence = 0.0f, bool cloud_handoff = false, const std::string& thinking = "", - const std::vector& segments = {}) { + const std::vector& segments = {}, + const std::string& mtp_json = "") { std::ostringstream json; json << "{"; json << "\"success\":true,"; @@ -1613,6 +1662,9 @@ inline std::string construct_response_json(const std::string& regular_response, json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; json << "\"prefill_tokens\":" << prompt_tokens << ","; json << "\"decode_tokens\":" << completion_tokens << ","; + if (!mtp_json.empty()) { + json << "\"mtp\":" << mtp_json << ","; + } json << "\"total_tokens\":" << (prompt_tokens + completion_tokens); json << "}"; return json.str(); diff --git a/cactus-engine/tests/chat.cpp b/cactus-engine/tests/chat.cpp index 9a5eeb1b1..a5e0f4192 100644 --- a/cactus-engine/tests/chat.cpp +++ b/cactus-engine/tests/chat.cpp @@ -338,7 +338,8 @@ std::string build_messages(const std::string& system_prompt, void print_usage(const char* argv0) { std::cerr << "Usage: " << argv0 << " [--system ] [--image ] [--audio ]" - << " [--prompt ] [--thinking]\n"; + << " [--prompt ] [--thinking]" + << " [--temperature ] [--top-k ] [--mtp-max-draft ]\n"; } } // namespace @@ -355,6 +356,9 @@ int main(int argc, char** argv) { std::string current_audio; std::string initial_prompt; bool thinking = false; + float temperature = 0.7f; + int top_k = 40; + int mtp_max_draft_tokens = 0; for (int i = 2; i < argc; ++i) { std::string arg = argv[i]; @@ -368,6 +372,15 @@ int main(int argc, char** argv) { initial_prompt = argv[++i]; } else if (arg == "--thinking") { thinking = true; + } else if (arg == "--temperature" && i + 1 < argc) { + temperature = std::stof(argv[++i]); + } else if (arg == "--top-k" && i + 1 < argc) { + top_k = std::stoi(argv[++i]); + } else if (arg == "--mtp-max-draft" && i + 1 < argc) { + mtp_max_draft_tokens = std::stoi(argv[++i]); + } else { + print_usage(argv[0]); + return 1; } } @@ -379,6 +392,18 @@ int main(int argc, char** argv) { std::cerr << "Audio file not found: " << current_audio << "\n"; return 1; } + if (temperature < 0.0f) { + std::cerr << "--temperature must be non-negative\n"; + return 1; + } + if (top_k < 0) { + std::cerr << "--top-k must be non-negative\n"; + return 1; + } + if (mtp_max_draft_tokens < 0) { + std::cerr << "--mtp-max-draft must be non-negative\n"; + return 1; + } std::cout << "Loading model from " << model_path << "...\n"; cactus_model_t model = cactus_init(model_path.c_str(), nullptr, false); @@ -484,11 +509,19 @@ int main(int argc, char** argv) { } history.push_back({"user", input}); std::string messages = build_messages(system_prompt, history, current_image, current_audio, attach_media); - std::string options = "{\"temperature\":0.7,\"top_p\":0.95,\"top_k\":40,\"max_tokens\":" - + std::to_string(kMaxTokens) - + ",\"enable_thinking_if_supported\":" + (thinking ? "true" : "false") - + ",\"auto_handoff\":false,\"confidence_threshold\":0.0" - + ",\"stop_sequences\":[\"<|im_end|>\",\"\"]}"; + bool mtp_requested = mtp_max_draft_tokens > 0; + std::ostringstream options_stream; + options_stream << "{\"temperature\":" << temperature + << ",\"top_p\":0.95,\"top_k\":" << top_k + << ",\"max_tokens\":" << kMaxTokens + << ",\"enable_thinking_if_supported\":" << (thinking ? "true" : "false") + << ",\"auto_handoff\":false,\"confidence_threshold\":0.0"; + std::string options = options_stream.str(); + if (mtp_requested) { + options += ",\"mtp\":true"; + options += ",\"mtp_max_draft_tokens\":" + std::to_string(mtp_max_draft_tokens); + } + options += ",\"stop_sequences\":[\"<|im_end|>\",\"\"]}"; if (!current_image.empty()) std::cout << "[image: " << current_image << "]\n"; if (!current_audio.empty()) std::cout << "[audio: " << current_audio << "]\n"; diff --git a/cactus-engine/tests/test_decode_strategy.cpp b/cactus-engine/tests/test_decode_strategy.cpp new file mode 100644 index 000000000..bc3ba39b0 --- /dev/null +++ b/cactus-engine/tests/test_decode_strategy.cpp @@ -0,0 +1,165 @@ +#include "test_utils.h" +#include "src/decode_strategy.h" + +#include +#include +#include +#include + +using namespace TestUtils; + +class FakeTargetDecoder final : public TargetDecoder { +public: + explicit FakeTargetDecoder(std::vector tokens) : tokens_(std::move(tokens)) {} + + DecodeStepResult decode_one(const std::vector& input_tokens, + const DecodeOptions& options) override { + seen_inputs.push_back(input_tokens); + seen_options.push_back(options); + uint32_t token = tokens_.at(index_++); + return DecodeStepResult{.token = token, .entropy = 0.0f}; + } + + std::vector> seen_inputs; + std::vector seen_options; + +private: + std::vector tokens_; + size_t index_ = 0; +}; + +class FakeAssistantDraftProvider final : public AssistantDraftProvider { +public: + explicit FakeAssistantDraftProvider(std::vector drafts) : drafts_(std::move(drafts)) {} + + DraftBatch draft(const TargetDecodeState& state, size_t max_draft_tokens) override { + seen_states.push_back(state); + seen_limits.push_back(max_draft_tokens); + return drafts_.at(index_++); + } + + std::vector seen_states; + std::vector seen_limits; + +private: + std::vector drafts_; + size_t index_ = 0; +}; + +class FakeVerifier final : public SpeculativeVerifier { +public: + explicit FakeVerifier(std::vector batches) : batches_(std::move(batches)) {} + + AcceptedTokenBatch verify(const DraftBatch& draft, + const TargetDecodeState& state, + const DecodeOptions& options) override { + seen_drafts.push_back(draft); + seen_states.push_back(state); + seen_options.push_back(options); + return batches_.at(index_++); + } + + std::vector seen_drafts; + std::vector seen_states; + std::vector seen_options; + +private: + std::vector batches_; + size_t index_ = 0; +}; + +bool test_standard_strategy_is_null_object_fallback() { + FakeTargetDecoder target({42}); + StandardDecodeStrategy strategy(target); + + DecodeOptions options; + options.temperature = 0.0f; + AcceptedTokenBatch batch = strategy.decode_next({7, 8}, options); + + return batch.tokens == std::vector{42} + && batch.entropies.size() == 1 + && target.seen_inputs == std::vector>{{7, 8}}; +} + +bool test_mtp_strategy_returns_accepted_token_batch() { + FakeTargetDecoder target({99}); + FakeAssistantDraftProvider assistant({ + DraftBatch{.tokens = {10, 11, 12}}, + }); + FakeVerifier verifier({ + AcceptedTokenBatch{ + .tokens = {10, 11, 7}, + .entropies = {0.1f, 0.2f, 0.3f}, + .drafted_tokens = 3, + .accepted_draft_tokens = 2, + .rejected_tokens = 1, + }, + }); + Gemma4MtpDecodeStrategy strategy(target, assistant, verifier); + + DecodeOptions options; + options.temperature = 0.0f; + options.max_draft_tokens = 3; + AcceptedTokenBatch batch = strategy.decode_next({1, 2, 3}, options); + + return batch.tokens == std::vector({10, 11, 7}) + && batch.drafted_tokens == 3 + && batch.accepted_draft_tokens == 2 + && batch.rejected_tokens == 1 + && assistant.seen_limits == std::vector{3} + && verifier.seen_drafts.size() == 1; +} + +bool test_strategy_factory_selects_standard_when_mtp_disabled() { + FakeTargetDecoder target({5}); + FakeAssistantDraftProvider assistant({}); + FakeVerifier verifier({}); + + DecodeOptions options; + options.mtp_enabled = false; + std::unique_ptr strategy = make_decode_strategy(target, &assistant, &verifier, options); + + return strategy->name() == std::string("standard"); +} + +bool test_strategy_factory_selects_mtp_when_available() { + FakeTargetDecoder target({5}); + FakeAssistantDraftProvider assistant({}); + FakeVerifier verifier({}); + + DecodeOptions options; + options.mtp_enabled = true; + options.max_draft_tokens = 6; + std::unique_ptr strategy = make_decode_strategy(target, &assistant, &verifier, options); + + return strategy->name() == std::string("gemma4_mtp"); +} + +bool test_strategy_factory_errors_when_mtp_required_but_unavailable() { + FakeTargetDecoder target({5}); + + DecodeOptions options; + options.mtp_enabled = true; + options.mtp_required = true; + + try { + (void)make_decode_strategy(target, nullptr, nullptr, options); + } catch (const MissingAssistantError& err) { + return std::string(err.what()).find("assistant") != std::string::npos; + } + + return false; +} + +int main() { + TestRunner runner("Decode Strategy Tests"); + + runner.run_test("standard_null_object", test_standard_strategy_is_null_object_fallback()); + runner.run_test("mtp_returns_batch", test_mtp_strategy_returns_accepted_token_batch()); + runner.run_test("factory_standard", test_strategy_factory_selects_standard_when_mtp_disabled()); + runner.run_test("factory_mtp", test_strategy_factory_selects_mtp_when_available()); + runner.run_test("factory_missing_required", test_strategy_factory_errors_when_mtp_required_but_unavailable()); + + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus-engine/tests/test_gemma4_mtp_fast.cpp b/cactus-engine/tests/test_gemma4_mtp_fast.cpp new file mode 100644 index 000000000..feb05d8ba --- /dev/null +++ b/cactus-engine/tests/test_gemma4_mtp_fast.cpp @@ -0,0 +1,634 @@ +#include "test_utils.h" +#include "models/gemma4/gemma4_mtp_assistant.h" +#include "models/gemma4/model_gemma4.h" +#include "src/mtp_sampler.h" +#include "src/utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace TestUtils; +using namespace EngineTestUtils; +using cactus::engine::Gemma4Model; +using cactus::engine::Gemma4MtpAssistant; + +static const char* target_path() { + return std::getenv("CACTUS_TEST_GEMMA4_TARGET"); +} + +static const char* assistant_path() { + return std::getenv("CACTUS_TEST_GEMMA4_ASSISTANT"); +} + +static bool has_real_model_env() { + return target_path() && std::strlen(target_path()) > 0 + && assistant_path() && std::strlen(assistant_path()) > 0; +} + +static int max_tokens() { + const char* value = std::getenv("CACTUS_TEST_GEMMA4_MAX_TOKENS"); + return value && std::strlen(value) > 0 ? std::atoi(value) : 24; +} + +static bool run_complete_collect_ids(cactus_model_t model, + const char* messages, + const char* options, + std::vector& token_ids, + std::string& json_response) { + StreamingData data; + data.model = model; + char response[8192] = {0}; + + int result = cactus_complete(model, messages, response, sizeof(response), + options, nullptr, stream_callback, &data, nullptr, 0); + json_response = response; + token_ids = data.token_ids; + + return result > 0 && !token_ids.empty(); +} + +static uint32_t sample_sparse_for_test(const std::vector>& probabilities, + std::mt19937& rng) { + if (probabilities.empty()) return 0; + std::uniform_real_distribution dist(0.0f, 1.0f); + return mtp_sample_sparse(probabilities, dist(rng)); +} + +static std::vector prompt_tokens_for_messages(cactus_model_t model, const char* messages) { + auto* handle = static_cast(model); + std::vector image_paths; + std::vector audio_paths; + auto parsed = cactus::ffi::parse_messages_json(messages, image_paths, &audio_paths); + auto* tokenizer = handle->model->get_tokenizer(); + std::string prompt = tokenizer->format_chat_prompt(parsed, true, "", false); + if (prompt.find("ERROR:") == 0) { + throw std::runtime_error(prompt.substr(6)); + } + return tokenizer->encode(prompt); +} + +static std::vector<__fp16> hidden_row_for_test(const Gemma4Model::GreedyBatchWithHidden& batch, + size_t row) { + if (batch.hidden_dim == 0 || row >= batch.hidden.size() / batch.hidden_dim) { + throw std::runtime_error("Gemma 4 MTP test hidden row is unavailable"); + } + return std::vector<__fp16>( + batch.hidden.begin() + row * batch.hidden_dim, + batch.hidden.begin() + (row + 1) * batch.hidden_dim); +} + +static std::vector run_manual_sampled_mtp_reference(cactus_model_t model, + const char* messages, + const std::string& assistant_dir, + size_t max_tokens, + size_t max_draft, + uint32_t seed) { + auto* handle = static_cast(model); + auto* target = dynamic_cast(handle->model.get()); + if (!target) { + if (auto* mm = dynamic_cast(handle->model.get())) { + target = &mm->language_model(); + } + } + if (!target) { + throw std::runtime_error("Gemma 4 MTP sampled reference requires a Gemma4Model target"); + } + + auto* graph = static_cast(target->graph_handle_); + Gemma4MtpAssistant assistant; + if (!assistant.init(graph, assistant_dir)) { + throw std::runtime_error("Gemma 4 MTP sampled reference assistant load failed"); + } + + const float temperature = 0.7f; + const float top_p = 0.95f; + const size_t top_k = 64; + const float min_p = 0.0f; + + target->reset_cache(); + std::vector prompt_tokens = prompt_tokens_for_messages(model, messages); + if (prompt_tokens.empty()) { + throw std::runtime_error("Gemma 4 MTP sampled reference prompt is empty"); + } + if (prompt_tokens.size() > 1) { + std::vector prefix(prompt_tokens.begin(), prompt_tokens.end() - 1); + target->prefill_for_mtp(prefix, target->get_prefill_chunk_size()); + } + + std::mt19937 rng(seed); + auto first = target->decode_tokens_with_hidden_and_sparse_probs( + {prompt_tokens.back()}, temperature, top_p, top_k, min_p); + uint32_t next_token = sample_sparse_for_test(first.sparse_probabilities.back(), rng); + target->record_sampled_token(next_token); + + std::vector generated = {next_token}; + std::vector<__fp16> mtp_prev_hidden = hidden_row_for_test( + first, first.hidden.size() / first.hidden_dim - 1); + size_t produced = 1; + + while (produced < max_tokens) { + size_t remaining = max_tokens - produced; + size_t draft_limit = std::min(max_draft, remaining > 1 ? remaining - 1 : 1); + auto cache_nodes = target->shared_cache_nodes_for_mtp(); + size_t assistant_position = prompt_tokens.size() + generated.size() - 1; + + std::vector draft_tokens; + std::vector>> draft_probabilities; + draft_tokens.reserve(draft_limit); + draft_probabilities.reserve(draft_limit); + + std::vector<__fp16> assistant_hidden = mtp_prev_hidden; + uint32_t assistant_input_token = next_token; + for (size_t i = 0; i < draft_limit; ++i) { + auto draft = assistant.draft_one( + assistant_input_token, + target->token_embedding_node_for_mtp(), + assistant_hidden, + cache_nodes, + assistant_position, + i == 0 ? (1.0f / 16.0f) : 1.0f, + temperature, + top_p, + top_k, + min_p); + uint32_t draft_token = sample_sparse_for_test(draft.sparse_probabilities, rng); + draft_tokens.push_back(draft_token); + draft_probabilities.push_back(std::move(draft.sparse_probabilities)); + assistant_hidden = std::move(draft.hidden); + assistant_input_token = draft_token; + } + + std::vector verify_tokens; + verify_tokens.reserve(draft_tokens.size() + 1); + verify_tokens.push_back(next_token); + verify_tokens.insert(verify_tokens.end(), draft_tokens.begin(), draft_tokens.end()); + + size_t verifier_cache_start = target->cache_position(); + auto cache_txn = graph->begin_kv_cache_transaction(target->cache_state_nodes_for_mtp()); + auto verifier = target->decode_tokens_with_hidden_and_sparse_probs( + verify_tokens, temperature, top_p, top_k, min_p); + + bool rejected = false; + size_t accepted = 0; + std::vector output_batch; + for (; accepted < draft_tokens.size(); ++accepted) { + uint32_t y = draft_tokens[accepted]; + float p_y = mtp_sparse_probability_at(verifier.sparse_probabilities[accepted], y); + float q_y = mtp_sparse_probability_at(draft_probabilities[accepted], y); + float accept_probability = q_y <= 0.0f ? 1.0f : std::min(1.0f, p_y / q_y); + std::uniform_real_distribution dist(0.0f, 1.0f); + if (dist(rng) <= accept_probability) { + output_batch.push_back(y); + continue; + } + auto adjusted = mtp_rejection_adjusted_sparse_distribution( + verifier.sparse_probabilities[accepted], + draft_probabilities[accepted]); + output_batch.push_back(sample_sparse_for_test(adjusted, rng)); + rejected = true; + break; + } + if (!rejected && output_batch.size() < remaining) { + output_batch.push_back(sample_sparse_for_test( + verifier.sparse_probabilities[draft_tokens.size()], rng)); + } + + if (!output_batch.empty()) { + mtp_prev_hidden = hidden_row_for_test(verifier, output_batch.size() - 1); + } + + size_t committed_cache_tokens = output_batch.size(); + if (committed_cache_tokens < verify_tokens.size()) { + cache_txn.commit_prefix(committed_cache_tokens); + graph->apply_pending_kv_cache_sequence_lengths(); + target->set_cache_position_for_mtp(verifier_cache_start + committed_cache_tokens); + } else { + cache_txn.commit_all(); + graph->apply_pending_kv_cache_sequence_lengths(); + } + + for (uint32_t token : output_batch) { + next_token = token; + produced++; + target->record_sampled_token(token); + generated.push_back(token); + if (produced >= max_tokens) break; + } + } + + return generated; +} + +bool test_gemma4_mtp_greedy_matches_standard_decode() { + if (!has_real_model_env()) { + std::cout << "Skipping: CACTUS_TEST_GEMMA4_TARGET and CACTUS_TEST_GEMMA4_ASSISTANT are not set\n"; + return true; + } + + cactus_model_t model = cactus_init(target_path(), nullptr, false); + if (!model) return false; + + std::string normal_options = "{" + "\"temperature\":0," + "\"top_p\":0," + "\"max_tokens\":" + std::to_string(max_tokens()) + "," + "\"telemetry_enabled\":false" + "}"; + + const std::vector prompts = { + "who are you", + "Write one short sentence about desert rain.", + "Count from 1 to 100, separated by commas.", + "Write a short Python function that adds two numbers.", + "Return a JSON object with keys name, role, and status." + }; + + bool ok = true; + for (const auto& prompt : prompts) { + std::string messages = "[{\"role\":\"user\",\"content\":\"" + prompt + "\"}]"; + std::vector normal_ids; + std::string normal_json; + bool normal_ok = run_complete_collect_ids(model, messages.c_str(), normal_options.c_str(), normal_ids, normal_json); + ok = ok && normal_ok; + for (int draft : {1, 2, 3, 4}) { + cactus_reset(model); + std::string mtp_options = "{" + "\"temperature\":0," + "\"top_p\":0," + "\"max_tokens\":" + std::to_string(max_tokens()) + "," + "\"telemetry_enabled\":false," + "\"mtp\":true," + "\"mtp_required\":true," + "\"mtp_max_draft_tokens\":" + std::to_string(draft) + "," + "\"mtp_assistant_path\":\"" + std::string(assistant_path()) + "\"" + "}"; + std::vector mtp_ids; + std::string mtp_json; + bool mtp_ok = run_complete_collect_ids(model, messages.c_str(), mtp_options.c_str(), mtp_ids, mtp_json); + ok = ok + && mtp_ok + && normal_ids == mtp_ids + && json_number(mtp_json, "drafted_tokens") > 0.0 + && mtp_json.find("\"enabled\":true") != std::string::npos + && mtp_json.find("low_acceptance") == std::string::npos + && mtp_json.find("mtp") != std::string::npos; + } + cactus_reset(model); + } + cactus_destroy(model); + + return ok; +} + +bool test_gemma4_mtp_short_chat_accepts_drafts() { + if (!has_real_model_env()) { + std::cout << "Skipping: CACTUS_TEST_GEMMA4_TARGET and CACTUS_TEST_GEMMA4_ASSISTANT are not set\n"; + return true; + } + + cactus_model_t model = cactus_init(target_path(), nullptr, false); + if (!model) return false; + + const char* messages = R"([ + {"role": "user", "content": "who are you"} + ])"; + + std::string normal_options = "{" + "\"temperature\":0," + "\"top_k\":1," + "\"max_tokens\":48," + "\"telemetry_enabled\":false" + "}"; + + std::string mtp_options = "{" + "\"temperature\":0," + "\"top_k\":1," + "\"max_tokens\":48," + "\"telemetry_enabled\":false," + "\"mtp\":true," + "\"mtp_required\":true," + "\"mtp_max_draft_tokens\":2," + "\"mtp_assistant_path\":\"" + std::string(assistant_path()) + "\"" + "}"; + + std::vector normal_ids; + std::vector mtp_ids; + std::string normal_json; + std::string mtp_json; + + bool normal_ok = run_complete_collect_ids(model, messages, normal_options.c_str(), normal_ids, normal_json); + cactus_reset(model); + bool mtp_ok = run_complete_collect_ids(model, messages, mtp_options.c_str(), mtp_ids, mtp_json); + + cactus_destroy(model); + + double drafted = json_number(mtp_json, "drafted_tokens"); + double accepted = json_number(mtp_json, "accepted_tokens"); + double rejected = json_number(mtp_json, "rejected_tokens"); + + return normal_ok + && mtp_ok + && normal_ids == mtp_ids + && drafted >= 12.0 + && accepted >= 0.6 * drafted + && rejected <= std::max(3.0, 0.25 * drafted) + && mtp_json.find("\"enabled\":true") != std::string::npos + && mtp_json.find("low_acceptance") == std::string::npos; +} + +bool test_gemma4_mtp_cache_invariant() { + if (!has_real_model_env()) { + std::cout << "Skipping: CACTUS_TEST_GEMMA4_TARGET and CACTUS_TEST_GEMMA4_ASSISTANT are not set\n"; + return true; + } + + cactus_model_t model = cactus_init(target_path(), nullptr, false); + if (!model) return false; + + const char* messages = R"([ + {"role": "user", "content": "Write one short sentence about desert rain."} + ])"; + + std::string normal_options = "{" + "\"temperature\":0," + "\"top_p\":0," + "\"max_tokens\":" + std::to_string(max_tokens()) + "," + "\"telemetry_enabled\":false" + "}"; + + std::string mtp_options = "{" + "\"temperature\":0," + "\"top_p\":0," + "\"max_tokens\":" + std::to_string(max_tokens()) + "," + "\"telemetry_enabled\":false," + "\"mtp\":true," + "\"mtp_required\":true," + "\"mtp_cache_invariant_check\":true," + "\"mtp_max_draft_tokens\":2," + "\"mtp_assistant_path\":\"" + std::string(assistant_path()) + "\"" + "}"; + + std::vector normal_ids; + std::vector mtp_ids; + std::string normal_json; + std::string mtp_json; + + bool normal_ok = run_complete_collect_ids(model, messages, normal_options.c_str(), normal_ids, normal_json); + cactus_reset(model); + bool mtp_ok = run_complete_collect_ids(model, messages, mtp_options.c_str(), mtp_ids, mtp_json); + + cactus_destroy(model); + + return normal_ok + && mtp_ok + && normal_ids == mtp_ids + && json_number(mtp_json, "rounds") > 0.0 + && mtp_json.find("mtp") != std::string::npos; +} + +bool test_gemma4_mtp_multiturn_prefix_cache_invariant() { + if (!has_real_model_env()) { + std::cout << "Skipping: CACTUS_TEST_GEMMA4_TARGET and CACTUS_TEST_GEMMA4_ASSISTANT are not set\n"; + return true; + } + + cactus_model_t model = cactus_init(target_path(), nullptr, false); + if (!model) return false; + + const char* first_messages = R"([ + {"role": "user", "content": "what is your favorite ice cream flavor, be honest. I know even you have a preference"} + ])"; + + std::string mtp_options = "{" + "\"temperature\":0," + "\"top_p\":0," + "\"max_tokens\":24," + "\"telemetry_enabled\":false," + "\"mtp\":true," + "\"mtp_required\":true," + "\"mtp_cache_invariant_check\":true," + "\"mtp_max_draft_tokens\":2," + "\"mtp_assistant_path\":\"" + std::string(assistant_path()) + "\"" + "}"; + + std::vector first_ids; + std::string first_json; + bool first_ok = run_complete_collect_ids(model, first_messages, mtp_options.c_str(), first_ids, first_json); + std::string first_response = json_string(first_json, "response"); + + std::string second_messages = "[" + "{\"role\":\"user\",\"content\":\"what is your favorite ice cream flavor, be honest. I know even you have a preference\"}," + "{\"role\":\"assistant\",\"content\":\"" + escape_json(first_response) + "\"}," + "{\"role\":\"user\",\"content\":\"oooo I like that, and what of your favorite color\"}" + "]"; + + std::vector second_ids; + std::string second_json; + bool second_ok = first_ok && !first_response.empty() + && run_complete_collect_ids(model, second_messages.c_str(), mtp_options.c_str(), second_ids, second_json); + + cactus_destroy(model); + + return first_ok + && !first_response.empty() + && second_ok + && json_number(second_json, "rounds") > 0.0 + && second_json.find("\"enabled\":true") != std::string::npos; +} + +bool test_gemma4_mtp_low_acceptance_stays_enabled() { + if (!has_real_model_env()) { + std::cout << "Skipping: CACTUS_TEST_GEMMA4_TARGET and CACTUS_TEST_GEMMA4_ASSISTANT are not set\n"; + return true; + } + + cactus_model_t model = cactus_init(target_path(), nullptr, false); + if (!model) return false; + + const char* messages = R"([ + {"role": "user", "content": "Return a JSON object with keys name, role, and status."} + ])"; + + const char* normal_options = R"({ + "temperature": 0, + "top_p": 0, + "max_tokens": 64, + "telemetry_enabled": false + })"; + + std::string mtp_options = "{" + "\"temperature\":0," + "\"top_p\":0," + "\"max_tokens\":64," + "\"telemetry_enabled\":false," + "\"mtp\":true," + "\"mtp_required\":true," + "\"mtp_max_draft_tokens\":2," + "\"mtp_assistant_path\":\"" + std::string(assistant_path()) + "\"" + "}"; + + std::vector normal_ids; + std::vector mtp_ids; + std::string normal_json; + std::string mtp_json; + + bool normal_ok = run_complete_collect_ids(model, messages, normal_options, normal_ids, normal_json); + cactus_reset(model); + bool mtp_ok = run_complete_collect_ids(model, messages, mtp_options.c_str(), mtp_ids, mtp_json); + + cactus_destroy(model); + + return normal_ok + && mtp_ok + && normal_ids == mtp_ids + && json_number(mtp_json, "drafted_tokens") > 0.0 + && mtp_json.find("\"enabled\":true") != std::string::npos + && mtp_json.find("low_acceptance") == std::string::npos; +} + +bool test_gemma4_mtp_seeded_sampling_smoke() { + if (!has_real_model_env()) { + std::cout << "Skipping: CACTUS_TEST_GEMMA4_TARGET and CACTUS_TEST_GEMMA4_ASSISTANT are not set\n"; + return true; + } + + cactus_model_t model = cactus_init(target_path(), nullptr, false); + if (!model) return false; + + const char* messages = R"([ + {"role": "system", "content": "You are a concise assistant."}, + {"role": "user", "content": "Write one creative sentence about desert rain."} + ])"; + + std::string mtp_options = "{" + "\"temperature\":0.7," + "\"top_p\":0.95," + "\"top_k\":64," + "\"min_p\":0," + "\"max_tokens\":16," + "\"seed\":1234," + "\"telemetry_enabled\":false," + "\"mtp\":true," + "\"mtp_required\":true," + "\"mtp_max_draft_tokens\":2," + "\"mtp_assistant_path\":\"" + std::string(assistant_path()) + "\"" + "}"; + + std::vector first_ids; + std::vector second_ids; + std::string first_json; + std::string second_json; + + bool first_ok = run_complete_collect_ids(model, messages, mtp_options.c_str(), first_ids, first_json); + cactus_reset(model); + bool second_ok = run_complete_collect_ids(model, messages, mtp_options.c_str(), second_ids, second_json); + + cactus_destroy(model); + + return first_ok + && second_ok + && first_ids == second_ids + && first_json.find("mtp") != std::string::npos; +} + +bool test_gemma4_mtp_sampled_matches_manual_reference() { + if (!has_real_model_env()) { + std::cout << "Skipping: CACTUS_TEST_GEMMA4_TARGET and CACTUS_TEST_GEMMA4_ASSISTANT are not set\n"; + return true; + } + + cactus_model_t model = cactus_init(target_path(), nullptr, false); + if (!model) return false; + + const char* messages = R"([ + {"role": "system", "content": "You are a concise assistant."}, + {"role": "user", "content": "Write one creative sentence about desert rain."} + ])"; + + std::string mtp_options = "{" + "\"temperature\":0.7," + "\"top_p\":0.95," + "\"top_k\":64," + "\"min_p\":0," + "\"max_tokens\":16," + "\"seed\":1234," + "\"telemetry_enabled\":false," + "\"mtp\":true," + "\"mtp_required\":true," + "\"mtp_max_draft_tokens\":3," + "\"mtp_assistant_path\":\"" + std::string(assistant_path()) + "\"" + "}"; + + std::vector mtp_ids; + std::string mtp_json; + bool mtp_ok = run_complete_collect_ids(model, messages, mtp_options.c_str(), mtp_ids, mtp_json); + + cactus_reset(model); + std::vector manual_ids = run_manual_sampled_mtp_reference( + model, messages, assistant_path(), 16, 3, 1234); + + cactus_destroy(model); + + return mtp_ok + && mtp_ids == manual_ids + && json_number(mtp_json, "drafted_tokens") > 0.0 + && mtp_json.find("\"enabled\":true") != std::string::npos; +} + +bool test_gemma4_mtp_missing_required_assistant_errors() { + const char* target = target_path(); + if (!target || std::strlen(target) == 0) { + std::cout << "Skipping: CACTUS_TEST_GEMMA4_TARGET is not set\n"; + return true; + } + + cactus_model_t model = cactus_init(target, nullptr, false); + if (!model) return false; + + const char* messages = R"([ + {"role": "user", "content": "Say hello in three words."} + ])"; + + const char* options = R"({ + "temperature": 0, + "max_tokens": 8, + "telemetry_enabled": false, + "mtp": true, + "mtp_required": true, + "mtp_max_draft_tokens": 2, + "mtp_assistant_path": "/definitely/missing/gemma4-assistant" + })"; + + std::vector ids; + std::string json; + bool ok = run_complete_collect_ids(model, messages, options, ids, json); + + cactus_destroy(model); + + return !ok + || json.find("assistant") != std::string::npos + || json.find("mtp") != std::string::npos; +} + +int main() { + TestRunner runner("Gemma 4 MTP Fast Smoke"); + + runner.run_test("greedy_matches_standard", test_gemma4_mtp_greedy_matches_standard_decode()); + runner.run_test("short_chat_accepts_drafts", test_gemma4_mtp_short_chat_accepts_drafts()); + runner.run_test("cache_invariant", test_gemma4_mtp_cache_invariant()); + runner.run_test("multiturn_prefix_cache_invariant", test_gemma4_mtp_multiturn_prefix_cache_invariant()); + runner.run_test("low_acceptance_stays_enabled", test_gemma4_mtp_low_acceptance_stays_enabled()); + runner.run_test("seeded_sampling_smoke", test_gemma4_mtp_seeded_sampling_smoke()); + runner.run_test("sampled_manual_reference", test_gemma4_mtp_sampled_matches_manual_reference()); + runner.run_test("missing_required_assistant", test_gemma4_mtp_missing_required_assistant_errors()); + + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus-engine/tests/test_mtp_completion.cpp b/cactus-engine/tests/test_mtp_completion.cpp new file mode 100644 index 000000000..4ccabc475 --- /dev/null +++ b/cactus-engine/tests/test_mtp_completion.cpp @@ -0,0 +1,106 @@ +#include "test_utils.h" +#include "src/mtp_completion.h" + +#include +#include +#include + +using namespace TestUtils; + +bool test_streams_each_accepted_token_in_order() { + MtpCompletionAccumulator accumulator; + std::vector streamed_ids; + std::string streamed_text; + + accumulator.set_stream_callback([&](const AcceptedToken& token) { + streamed_ids.push_back(token.id); + streamed_text += token.text; + }); + + AcceptedTokenBatch batch{ + .tokens = {10, 11, 12}, + .texts = {"desert", " ", "rain"}, + }; + + accumulator.consume_accepted_batch(batch); + + return streamed_ids == std::vector({10, 11, 12}) + && streamed_text == "desert rain"; +} + +bool test_stop_sequence_checked_after_each_token() { + MtpCompletionAccumulator accumulator; + accumulator.set_stop_sequences({""}); + + AcceptedTokenBatch batch{ + .tokens = {1, 2, 3}, + .texts = {"hello", "", "SHOULD_NOT_STREAM"}, + }; + + MtpCompletionResult result = accumulator.consume_accepted_batch(batch); + + return result.stopped + && result.tokens_consumed == 2 + && accumulator.output_text() == "hello" + && accumulator.token_history() == std::vector({1, 2}); +} + +bool test_rejected_draft_tokens_do_not_enter_history() { + MtpCompletionAccumulator accumulator; + + AcceptedTokenBatch batch{ + .tokens = {4, 9}, + .texts = {"accepted", " replacement"}, + .drafted_tokens = 4, + .accepted_draft_tokens = 1, + .rejected_tokens = 3, + .rejected_draft_token_ids = {5, 6, 7}, + }; + + accumulator.consume_accepted_batch(batch); + + return accumulator.token_history() == std::vector({4, 9}); +} + +bool test_token_history_updates_once_per_accepted_token() { + MtpCompletionAccumulator accumulator; + + accumulator.consume_accepted_batch(AcceptedTokenBatch{ + .tokens = {1, 2}, + .texts = {"a", "b"}, + }); + accumulator.consume_accepted_batch(AcceptedTokenBatch{ + .tokens = {3}, + .texts = {"c"}, + }); + + return accumulator.token_history() == std::vector({1, 2, 3}); +} + +bool test_missing_assistant_records_fallback_reason() { + MtpCompletionAccumulator accumulator; + MtpFallbackDecision decision = decide_mtp_fallback(MtpAvailability{ + .requested = true, + .required = false, + .assistant_loaded = false, + .supports_prompt = true, + }); + + accumulator.record_fallback(decision); + + return decision.use_standard_decode + && accumulator.metrics().fallback_reason == "assistant_unavailable"; +} + +int main() { + TestRunner runner("MTP Completion Semantics Tests"); + + runner.run_test("streams_each_token", test_streams_each_accepted_token_in_order()); + runner.run_test("stop_after_each_token", test_stop_sequence_checked_after_each_token()); + runner.run_test("rejected_not_history", test_rejected_draft_tokens_do_not_enter_history()); + runner.run_test("history_once", test_token_history_updates_once_per_accepted_token()); + runner.run_test("fallback_reason", test_missing_assistant_records_fallback_reason()); + + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus-engine/tests/test_mtp_sampler.cpp b/cactus-engine/tests/test_mtp_sampler.cpp new file mode 100644 index 000000000..63d26bc62 --- /dev/null +++ b/cactus-engine/tests/test_mtp_sampler.cpp @@ -0,0 +1,186 @@ +#include "test_utils.h" +#include "src/mtp_sampler.h" + +#include +#include +#include + +using namespace TestUtils; + +static bool tokens_eq(const std::vector& actual, const std::vector& expected) { + return actual == expected; +} + +static bool approx_eq(float actual, float expected, float tolerance = 1e-5f) { + return std::abs(actual - expected) <= tolerance; +} + +static MtpDistribution dist(std::initializer_list probs) { + return MtpDistribution{std::vector(probs)}; +} + +bool test_greedy_accepts_matching_prefix_and_extra_target_token() { + MtpDraftBatch draft{ + .tokens = {1, 2, 3}, + .probabilities = {dist({0.05f, 0.90f, 0.03f, 0.02f}), + dist({0.04f, 0.03f, 0.88f, 0.05f}), + dist({0.01f, 0.02f, 0.03f, 0.94f})}, + }; + std::vector target = { + dist({0.01f, 0.97f, 0.01f, 0.01f}), + dist({0.01f, 0.01f, 0.97f, 0.01f}), + dist({0.01f, 0.01f, 0.01f, 0.97f}), + dist({0.02f, 0.03f, 0.90f, 0.05f}), + }; + + MtpVerificationResult result = verify_greedy_mtp_draft(draft, target); + + return result.accepted_draft_tokens == 3 + && !result.rejected + && tokens_eq(result.output_tokens, {1, 2, 3, 2}); +} + +bool test_greedy_replaces_first_mismatch_with_target_argmax() { + MtpDraftBatch draft{ + .tokens = {1, 2, 3}, + .probabilities = {dist({0.05f, 0.90f, 0.03f, 0.02f}), + dist({0.04f, 0.03f, 0.88f, 0.05f}), + dist({0.01f, 0.02f, 0.03f, 0.94f})}, + }; + std::vector target = { + dist({0.01f, 0.02f, 0.96f, 0.01f}), + dist({0.01f, 0.01f, 0.97f, 0.01f}), + dist({0.01f, 0.01f, 0.01f, 0.97f}), + dist({0.02f, 0.03f, 0.90f, 0.05f}), + }; + + MtpVerificationResult result = verify_greedy_mtp_draft(draft, target); + + return result.accepted_draft_tokens == 0 + && result.rejected + && tokens_eq(result.output_tokens, {2}); +} + +bool test_greedy_accepts_prefix_then_replaces_mismatch() { + MtpDraftBatch draft{ + .tokens = {1, 2, 3}, + .probabilities = {dist({0.05f, 0.90f, 0.03f, 0.02f}), + dist({0.04f, 0.03f, 0.88f, 0.05f}), + dist({0.01f, 0.02f, 0.03f, 0.94f})}, + }; + std::vector target = { + dist({0.01f, 0.97f, 0.01f, 0.01f}), + dist({0.01f, 0.01f, 0.97f, 0.01f}), + dist({0.90f, 0.02f, 0.03f, 0.05f}), + dist({0.02f, 0.03f, 0.90f, 0.05f}), + }; + + MtpVerificationResult result = verify_greedy_mtp_draft(draft, target); + + return result.accepted_draft_tokens == 2 + && result.rejected + && tokens_eq(result.output_tokens, {1, 2, 0}); +} + +bool test_rejection_adjusted_distribution_clamps_and_normalizes() { + MtpDistribution target = dist({0.30f, 0.30f, 0.40f}); + MtpDistribution assistant = dist({0.60f, 0.30f, 0.10f}); + + MtpDistribution adjusted = rejection_adjusted_distribution(target, assistant); + + return adjusted.probabilities.size() == 3 + && approx_eq(adjusted.probabilities[0], 0.0f) + && approx_eq(adjusted.probabilities[1], 0.0f) + && approx_eq(adjusted.probabilities[2], 1.0f); +} + +bool test_sparse_rejection_adjustment_uses_target_support() { + std::vector> target = { + {10, 0.30f}, + {11, 0.30f}, + {12, 0.40f}, + }; + std::vector> assistant = { + {10, 0.60f}, + {11, 0.30f}, + {99, 0.10f}, + }; + + auto adjusted = mtp_rejection_adjusted_sparse_distribution(target, assistant); + + return adjusted.size() == 1 + && adjusted[0].first == 12 + && approx_eq(adjusted[0].second, 1.0f) + && mtp_sparse_probability_at(adjusted, 10) == 0.0f + && mtp_sample_sparse(adjusted, 0.5f) == 12; +} + +bool test_sampled_accepts_when_target_probability_exceeds_assistant_probability() { + MtpDraftBatch draft{ + .tokens = {1}, + .probabilities = {dist({0.50f, 0.20f, 0.30f})}, + }; + std::vector target = { + dist({0.10f, 0.70f, 0.20f}), + dist({0.10f, 0.20f, 0.70f}), + }; + MtpDeterministicRng rng({0.99f, 0.99f}); + + MtpVerificationResult result = verify_sampled_mtp_draft(draft, target, rng); + + return result.accepted_draft_tokens == 1 + && !result.rejected + && tokens_eq(result.output_tokens, {1, 2}); +} + +bool test_sampled_rejects_and_samples_adjusted_replacement() { + MtpDraftBatch draft{ + .tokens = {0}, + .probabilities = {dist({0.60f, 0.30f, 0.10f})}, + }; + std::vector target = { + dist({0.30f, 0.30f, 0.40f}), + dist({0.20f, 0.20f, 0.60f}), + }; + MtpDeterministicRng rng({0.75f, 0.20f}); + + MtpVerificationResult result = verify_sampled_mtp_draft(draft, target, rng); + + return result.accepted_draft_tokens == 0 + && result.rejected + && tokens_eq(result.output_tokens, {2}); +} + +bool test_zero_assistant_probability_does_not_create_nan() { + MtpDraftBatch draft{ + .tokens = {1}, + .probabilities = {dist({0.50f, 0.0f, 0.50f})}, + }; + std::vector target = { + dist({0.20f, 0.30f, 0.50f}), + dist({0.20f, 0.20f, 0.60f}), + }; + MtpDeterministicRng rng({0.99f, 0.99f}); + + MtpVerificationResult result = verify_sampled_mtp_draft(draft, target, rng); + + return result.accepted_draft_tokens == 1 + && !result.rejected + && tokens_eq(result.output_tokens, {1, 2}); +} + +int main() { + TestRunner runner("MTP Sampler Tests"); + + runner.run_test("greedy_accepts_all", test_greedy_accepts_matching_prefix_and_extra_target_token()); + runner.run_test("greedy_rejects_first", test_greedy_replaces_first_mismatch_with_target_argmax()); + runner.run_test("greedy_rejects_after_prefix", test_greedy_accepts_prefix_then_replaces_mismatch()); + runner.run_test("rejection_adjustment", test_rejection_adjusted_distribution_clamps_and_normalizes()); + runner.run_test("sparse_rejection_adjustment", test_sparse_rejection_adjustment_uses_target_support()); + runner.run_test("sampled_accepts_p_ge_q", test_sampled_accepts_when_target_probability_exceeds_assistant_probability()); + runner.run_test("sampled_rejects_adjusted", test_sampled_rejects_and_samples_adjusted_replacement()); + runner.run_test("zero_q_no_nan", test_zero_assistant_probability_does_not_create_nan()); + + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus-graph/CMakeLists.txt b/cactus-graph/CMakeLists.txt index 975d3c73a..8aa748351 100644 --- a/cactus-graph/CMakeLists.txt +++ b/cactus-graph/CMakeLists.txt @@ -46,6 +46,7 @@ set(GRAPH_SOURCES src/ops_tensor.cpp src/ops_sample.cpp src/ops_cache.cpp + src/cache_transaction.cpp src/ops_dsp.cpp src/ops_image.cpp ) diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index 559e728bf..47e7e410c 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -4,10 +4,12 @@ #include "cactus_kernels.h" #include #include +#include #include #include #include #include +#include #include #include #include @@ -108,7 +110,7 @@ enum class OpType { STFT, ALTUP_PREDICT, ALTUP_CORRECT, GAUSSIAN_TOPK, MAXPOOL1D, BILSTM_SEQUENCE, LEAKY_RELU, CONV2D_K3S1P1, STATS_POOL, WEIGHTED_STATS_POOL, - KV_CACHE_STATE, KV_CACHE_APPEND, ATTENTION_CACHED, + KV_CACHE_STATE, KV_CACHE_APPEND, ATTENTION_CACHED, ATTENTION_CACHE_ONLY, CONV_CACHE_STATE, CONV_CACHE_APPEND, RFFT, IRFFT, MEL_FILTER_BANK, SPECTROGRAM, IMAGE_PREPROCESS, @@ -452,6 +454,23 @@ class CactusGraph { size_t node_id; }; + class KvCacheTransaction { + public: + KvCacheTransaction() = default; + KvCacheTransaction(CactusGraph* graph, std::vector> snapshots) + : graph_(graph), snapshots_(std::move(snapshots)) {} + + void rollback(); + void commit_all(); + void commit_prefix(size_t accepted_tokens); + void commit_selected(const std::vector& token_indices); + + private: + CactusGraph* graph_ = nullptr; + std::vector> snapshots_; + bool closed_ = false; + }; + void save(const std::string& path); static CactusGraph load(const std::string& path); @@ -559,6 +578,9 @@ class CactusGraph { size_t window_size = 0, size_t sink_size = 4); + KvCacheTransaction begin_kv_cache_transaction(const std::vector& cache_state_nodes); + void apply_pending_kv_cache_sequence_lengths(); + size_t attention_cached( size_t query, size_t key_new, @@ -569,6 +591,26 @@ class CactusGraph { size_t position_offset = 0, size_t window_size = 0, size_t v_head_dim = 0); + size_t attention_cached_masked( + size_t query, + size_t key_new, + size_t value_new, + size_t k_cache_state, + size_t v_cache_state, + size_t new_token_mask, + float scale, + size_t position_offset = 0, + size_t window_size = 0, + size_t v_head_dim = 0); + + size_t attention_cache_only( + size_t query, + size_t k_cache_state, + size_t v_cache_state, + float scale, + size_t position_offset = 0, + size_t window_size = 0, + size_t v_head_dim = 0); size_t conv_cache_state(size_t window_size, size_t hidden_dim); size_t conv_cache_append(size_t new_data, size_t cache_state_node); @@ -692,11 +734,14 @@ class CactusGraph { std::vector> nodes_; std::unordered_map node_index_map_; + std::vector> pending_kv_cache_sequence_lengths_; private: size_t binary_broadcast_op(OpType op, size_t input1, size_t input2); size_t reduction_op(OpType op, size_t input, int axis); static CactusGraph from_serialized(const GraphFile::SerializedGraph& serialized); + uint64_t kv_cache_sequence_length(size_t cache_state_node) const; + void set_kv_cache_sequence_length(size_t cache_state_node, uint64_t sequence_length); size_t next_node_id_; std::vector> mapped_files_; std::unordered_map weight_cache_; diff --git a/cactus-graph/src/builder.cpp b/cactus-graph/src/builder.cpp index c0e2f92b7..a711f2e1e 100644 --- a/cactus-graph/src/builder.cpp +++ b/cactus-graph/src/builder.cpp @@ -1548,3 +1548,50 @@ size_t CactusGraph::attention_cached(size_t query, size_t key_new, size_t value_ {query, key_new, value_new, k_cache_state, v_cache_state}, {batch, seq_len, num_q_heads, out_v_dim}, params); } + +size_t CactusGraph::attention_cached_masked(size_t query, size_t key_new, size_t value_new, + size_t k_cache_state, size_t v_cache_state, + size_t new_token_mask, float scale, size_t position_offset, + size_t window_size, size_t v_head_dim) { + const auto& q_shape = get_output_buffer(query).shape; + size_t batch = q_shape[0]; + size_t seq_len = q_shape[1]; + size_t num_q_heads = q_shape[2]; + size_t head_dim = q_shape[3]; + size_t out_v_dim = v_head_dim > 0 ? v_head_dim : head_dim; + + OpParams params{}; + params.scale = scale; + params.position_offset = position_offset; + params.window_size = window_size; + params.v_head_dim = v_head_dim; + params.output_precision = Precision::FP16; + return add_node(OpType::ATTENTION_CACHED, + {query, key_new, value_new, k_cache_state, v_cache_state, new_token_mask}, + {batch, seq_len, num_q_heads, out_v_dim}, params); +} + +size_t CactusGraph::attention_cache_only(size_t query, + size_t k_cache_state, + size_t v_cache_state, + float scale, + size_t position_offset, + size_t window_size, + size_t v_head_dim) { + const auto& q_shape = get_output_buffer(query).shape; + size_t batch = q_shape[0]; + size_t seq_len = q_shape[1]; + size_t num_q_heads = q_shape[2]; + size_t head_dim = q_shape[3]; + size_t out_v_dim = v_head_dim > 0 ? v_head_dim : head_dim; + + OpParams params{}; + params.scale = scale; + params.position_offset = position_offset; + params.window_size = window_size; + params.v_head_dim = v_head_dim; + params.output_precision = Precision::FP16; + return add_node(OpType::ATTENTION_CACHE_ONLY, + {query, k_cache_state, v_cache_state}, + {batch, seq_len, num_q_heads, out_v_dim}, params); +} diff --git a/cactus-graph/src/cache_transaction.cpp b/cactus-graph/src/cache_transaction.cpp new file mode 100644 index 000000000..8aefedee3 --- /dev/null +++ b/cactus-graph/src/cache_transaction.cpp @@ -0,0 +1,165 @@ +#include "../cactus_graph.h" +#include "cactus_kernels.h" + +#include +#include +#include + +namespace { + +struct KvCacheMetadata { + uint64_t current_seq_len; + uint64_t max_seq_len; + uint64_t num_kv_heads; + uint64_t head_dim; + uint64_t sink_size; + uint64_t reserved[3]; +}; + +static_assert(sizeof(KvCacheMetadata) == 64, "KvCacheMetadata must be 64 bytes"); + +KvCacheMetadata* kv_meta(BufferDesc& buffer) { + return static_cast(buffer.get_data()); +} + +const KvCacheMetadata* kv_meta(const BufferDesc& buffer) { + return static_cast(buffer.get_data()); +} + +int8_t* kv_int8_data(BufferDesc& buffer) { + return reinterpret_cast(static_cast(buffer.get_data()) + sizeof(KvCacheMetadata)); +} + +float* kv_scale_data(BufferDesc& buffer, size_t max_seq, size_t kv_heads, size_t head_dim) { + size_t int8_bytes = max_seq * kv_heads * head_dim; + return reinterpret_cast(static_cast(buffer.get_data()) + sizeof(KvCacheMetadata) + int8_bytes); +} + +void copy_kv_row(BufferDesc& buffer, uint64_t dst, uint64_t src) { + auto* meta = kv_meta(buffer); + const size_t max_seq = meta->max_seq_len; + const size_t kv_heads = meta->num_kv_heads; + const size_t head_dim = meta->head_dim; + const size_t num_groups = (head_dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; + if (dst >= max_seq || src >= max_seq) { + throw std::out_of_range("KV cache selected commit row out of range"); + } + + const size_t int8_stride = kv_heads * head_dim; + const size_t scale_stride = kv_heads * num_groups; + int8_t* int8_base = kv_int8_data(buffer); + float* scale_base = kv_scale_data(buffer, max_seq, kv_heads, head_dim); + std::memmove(int8_base + dst * int8_stride, + int8_base + src * int8_stride, + int8_stride); + std::memmove(scale_base + dst * scale_stride, + scale_base + src * scale_stride, + scale_stride * sizeof(float)); +} + +} // namespace + +uint64_t CactusGraph::kv_cache_sequence_length(size_t cache_state_node) const { + auto it = node_index_map_.find(cache_state_node); + if (it == node_index_map_.end()) { + throw std::invalid_argument("Unknown KV cache node"); + } + const auto& buffer = nodes_.at(it->second)->output_buffer; + if (!buffer.get_data()) { + return 0; + } + return kv_meta(buffer)->current_seq_len; +} + +void CactusGraph::set_kv_cache_sequence_length(size_t cache_state_node, uint64_t sequence_length) { + auto it = node_index_map_.find(cache_state_node); + if (it == node_index_map_.end()) { + throw std::invalid_argument("Unknown KV cache node"); + } + auto& buffer = nodes_.at(it->second)->output_buffer; + if (!buffer.get_data()) { + throw std::invalid_argument("KV cache node has no allocated buffer"); + } + auto* meta = kv_meta(buffer); + meta->current_seq_len = std::min(sequence_length, meta->max_seq_len); +} + +CactusGraph::KvCacheTransaction CactusGraph::begin_kv_cache_transaction(const std::vector& cache_state_nodes) { + std::vector> snapshots; + snapshots.reserve(cache_state_nodes.size()); + for (size_t node : cache_state_nodes) { + snapshots.emplace_back(node, kv_cache_sequence_length(node)); + } + return KvCacheTransaction(this, std::move(snapshots)); +} + +void CactusGraph::KvCacheTransaction::rollback() { + if (!graph_ || closed_) return; + for (const auto& [node, length] : snapshots_) { + graph_->set_kv_cache_sequence_length(node, length); + graph_->pending_kv_cache_sequence_lengths_.emplace_back(node, length); + } + closed_ = true; +} + +void CactusGraph::KvCacheTransaction::commit_all() { + if (!graph_ || closed_) return; + for (const auto& [node, _] : snapshots_) { + uint64_t committed_length = graph_->kv_cache_sequence_length(node); + graph_->pending_kv_cache_sequence_lengths_.emplace_back(node, committed_length); + } + closed_ = true; +} + +void CactusGraph::KvCacheTransaction::commit_prefix(size_t accepted_tokens) { + if (!graph_ || closed_) return; + for (const auto& [node, length] : snapshots_) { + uint64_t committed_length = length + accepted_tokens; + graph_->set_kv_cache_sequence_length(node, committed_length); + graph_->pending_kv_cache_sequence_lengths_.emplace_back(node, committed_length); + } + closed_ = true; +} + +void CactusGraph::KvCacheTransaction::commit_selected(const std::vector& token_indices) { + if (!graph_ || closed_) return; + size_t appended_tokens = 0; + for (size_t index : token_indices) { + appended_tokens = std::max(appended_tokens, index + 1); + } + for (const auto& [node, length] : snapshots_) { + auto it = graph_->node_index_map_.find(node); + if (it == graph_->node_index_map_.end()) { + throw std::invalid_argument("Unknown KV cache node"); + } + auto& buffer = graph_->nodes_.at(it->second)->output_buffer; + auto* meta = kv_meta(buffer); + if (appended_tokens > meta->max_seq_len) { + throw std::out_of_range("KV cache selected commit exceeds cache capacity"); + } + if (length + appended_tokens > meta->max_seq_len) { + throw std::runtime_error("KV cache selected commit after sliding-window eviction is unsupported"); + } + uint64_t append_start = length; + for (size_t dst = 0; dst < token_indices.size(); ++dst) { + const uint64_t src_row = append_start + token_indices[dst]; + const uint64_t dst_row = append_start + dst; + if (src_row != dst_row) { + copy_kv_row(buffer, dst_row, src_row); + } + } + uint64_t committed_length = append_start + token_indices.size(); + graph_->set_kv_cache_sequence_length(node, committed_length); + graph_->pending_kv_cache_sequence_lengths_.emplace_back(node, committed_length); + } + closed_ = true; +} + +void CactusGraph::apply_pending_kv_cache_sequence_lengths() { + if (pending_kv_cache_sequence_lengths_.empty()) return; + auto pending = std::move(pending_kv_cache_sequence_lengths_); + pending_kv_cache_sequence_lengths_.clear(); + for (const auto& [node, length] : pending) { + set_kv_cache_sequence_length(node, length); + } +} diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index f2f7bc471..76f9de085 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -74,6 +74,7 @@ DECLARE_COMPUTE(compute_persistent_node); DECLARE_COMPUTE(compute_kv_cache_state_node); DECLARE_COMPUTE(compute_kv_cache_append_node); DECLARE_COMPUTE(compute_attention_cached_node); +DECLARE_COMPUTE(compute_attention_cache_only_node); DECLARE_COMPUTE(compute_conv_cache_state_node); DECLARE_COMPUTE(compute_conv_cache_append_node); DECLARE_COMPUTE(compute_image_preprocess_node); @@ -172,6 +173,7 @@ static bool init_dispatch() { dispatch_flat[static_cast(OpType::KV_CACHE_STATE)] = compute_kv_cache_state_node; dispatch_flat[static_cast(OpType::KV_CACHE_APPEND)] = compute_kv_cache_append_node; dispatch_flat[static_cast(OpType::ATTENTION_CACHED)] = compute_attention_cached_node; + dispatch_flat[static_cast(OpType::ATTENTION_CACHE_ONLY)] = compute_attention_cache_only_node; dispatch_flat[static_cast(OpType::CONV_CACHE_STATE)] = compute_conv_cache_state_node; dispatch_flat[static_cast(OpType::CONV_CACHE_APPEND)] = compute_conv_cache_append_node; dispatch_flat[static_cast(OpType::IMAGE_PREPROCESS)] = compute_image_preprocess_node; @@ -217,7 +219,7 @@ static const char* op_type_names[] = { "STFT", "ALTUP_PREDICT", "ALTUP_CORRECT", "GAUSSIAN_TOPK", "MAXPOOL1D", "BILSTM_SEQUENCE", "LEAKY_RELU", "CONV2D_K3S1P1", "STATS_POOL", "WEIGHTED_STATS_POOL", - "KV_CACHE_STATE", "KV_CACHE_APPEND", "ATTENTION_CACHED", + "KV_CACHE_STATE", "KV_CACHE_APPEND", "ATTENTION_CACHED", "ATTENTION_CACHE_ONLY", "CONV_CACHE_STATE", "CONV_CACHE_APPEND", "RFFT", "IRFFT", "MEL_FILTER_BANK", "SPECTROGRAM", "IMAGE_PREPROCESS", "CLAMP", "DENSE_MLP_TQ_FUSED" @@ -318,6 +320,7 @@ void CactusGraph::execute(const std::string& profile_file) { populated_node_ids_.insert(node->id); } } + apply_pending_kv_cache_sequence_lengths(); return; } @@ -390,6 +393,8 @@ void CactusGraph::execute(const std::string& profile_file) { for (size_t node_idx = 0; node_idx < n; ++node_idx) { auto& node = nodes_[node_idx]; + if (node->op_type == OpType::INPUT) continue; + if (node->op_type != OpType::INPUT) { node->output_buffer.allocate_from_pool(pool); } @@ -671,6 +676,8 @@ void CactusGraph::execute(const std::string& profile_file) { profile_out.close(); } } + + apply_pending_kv_cache_sequence_lengths(); } void CactusGraph::hard_reset() { @@ -681,6 +688,7 @@ void CactusGraph::hard_reset() { next_node_id_ = 1; debug_nodes_.clear(); buffer_pool_.clear(); + pending_kv_cache_sequence_lengths_.clear(); } void CactusGraph::soft_reset() { diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 38830a6b6..acf79d0fe 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -2,6 +2,7 @@ #include "cactus_kernels.h" #include #include +#include namespace { @@ -158,12 +159,18 @@ void compute_attention_cached_node( GraphNode& node, const nodes_vector& nodes, const node_index_map_t& node_index_map) { + if (node.input_ids.size() != 5 && node.input_ids.size() != 6) { + throw std::runtime_error("ATTENTION_CACHED expects 5 or 6 inputs"); + } const auto& query_buf = get_input(node, 0, nodes, node_index_map); const auto& key_new_buf = get_input(node, 1, nodes, node_index_map); const auto& val_new_buf = get_input(node, 2, nodes, node_index_map); const auto& k_cache_buf = get_input(node, 3, nodes, node_index_map); const auto& v_cache_buf = get_input(node, 4, nodes, node_index_map); + const BufferDesc* mask_buf = node.input_ids.size() == 6 + ? &get_input(node, 5, nodes, node_index_map) + : nullptr; const auto* k_meta = get_meta(k_cache_buf); size_t cache_len = k_meta->current_seq_len; @@ -187,6 +194,17 @@ void compute_attention_cached_node( size_t new_seq_len = key_new_buf.total_size / (kv_heads * hdim); size_t history_len = (cache_len >= new_seq_len) ? cache_len - new_seq_len : 0; + const __fp16* mask_ptr = nullptr; + if (mask_buf) { + if (mask_buf->precision != Precision::FP16 + || mask_buf->shape.size() != 3 + || mask_buf->shape[0] != batch_size + || mask_buf->shape[1] != seq_len + || mask_buf->shape[2] != new_seq_len) { + throw std::runtime_error("ATTENTION_CACHED mask must be FP16 [B, T, new_T]"); + } + mask_ptr = mask_buf->data_as<__fp16>(); + } cactus_attention_hybrid_int8_fp16( query_buf.data_as<__fp16>(), @@ -204,6 +222,56 @@ void compute_attention_cached_node( true, node.params.window_size, KV_QUANT_GROUP_SIZE, + v_hdim, + mask_ptr); +} + +void compute_attention_cache_only_node( + GraphNode& node, + const nodes_vector& nodes, + const node_index_map_t& node_index_map) { + + const auto& query_buf = get_input(node, 0, nodes, node_index_map); + const auto& k_cache_buf = get_input(node, 1, nodes, node_index_map); + const auto& v_cache_buf = get_input(node, 2, nodes, node_index_map); + + const auto* k_meta = get_meta(k_cache_buf); + size_t cache_len = k_meta->current_seq_len; + size_t k_max = k_meta->max_seq_len; + size_t kv_heads = k_meta->num_kv_heads; + size_t hdim = k_meta->head_dim; + + const auto* v_meta = get_meta(v_cache_buf); + size_t v_hdim = node.params.v_head_dim > 0 ? node.params.v_head_dim : hdim; + size_t v_max = v_meta->max_seq_len; + + const int8_t* cached_keys = get_int8_data(k_cache_buf); + const float* k_scales = get_scales(k_cache_buf, k_max, kv_heads, hdim); + const int8_t* cached_values = get_int8_data(v_cache_buf); + const float* v_scales = get_scales(v_cache_buf, v_max, kv_heads, v_hdim); + + const auto& q_shape = query_buf.shape; + size_t batch_size = q_shape[0]; + size_t seq_len = q_shape[1]; + size_t num_q_heads = q_shape[2]; + + static const __fp16 empty_kv[1] = {static_cast<__fp16>(0.0f)}; + cactus_attention_hybrid_int8_fp16( + query_buf.data_as<__fp16>(), + cached_keys, + cached_values, + k_scales, + v_scales, + empty_kv, + empty_kv, + node.output_buffer.data_as<__fp16>(), + batch_size, seq_len, cache_len, 0, + num_q_heads, kv_heads, hdim, + node.params.scale, + node.params.position_offset, + true, + node.params.window_size, + KV_QUANT_GROUP_SIZE, v_hdim); } diff --git a/cactus-graph/tests/test_cache_transaction.cpp b/cactus-graph/tests/test_cache_transaction.cpp new file mode 100644 index 000000000..bdba54212 --- /dev/null +++ b/cactus-graph/tests/test_cache_transaction.cpp @@ -0,0 +1,148 @@ +#include "test_utils.h" + +#include +#include + +using namespace TestUtils; + +static uint64_t current_seq(CactusGraph& graph, size_t cache_node) { + auto* raw = static_cast(graph.get_output(cache_node)); + return *reinterpret_cast(raw); +} + +static bool append_tokens(CactusGraph& graph, size_t cache_node, size_t tokens, size_t kv_heads, size_t head_dim, float value) { + const size_t elements = tokens * kv_heads * head_dim; + size_t input = graph.input({elements}, Precision::FP16); + std::vector<__fp16> data(elements, static_cast<__fp16>(value)); + graph.set_input(input, data.data(), Precision::FP16); + graph.kv_cache_append(input, cache_node); + graph.execute(); + return current_seq(graph, cache_node) > 0; +} + +bool test_cache_transaction_rollback_restores_sequence_length() { + CactusGraph graph; + const size_t kv_heads = 2; + const size_t head_dim = 16; + size_t cache = graph.kv_cache_state(64, kv_heads, head_dim); + graph.execute(); + + if (!append_tokens(graph, cache, 4, kv_heads, head_dim, 1.0f)) return false; + if (current_seq(graph, cache) != 4) return false; + + auto txn = graph.begin_kv_cache_transaction({cache}); + + graph.soft_reset(); + if (!append_tokens(graph, cache, 3, kv_heads, head_dim, 2.0f)) return false; + if (current_seq(graph, cache) != 7) return false; + + txn.rollback(); + graph.execute(); + + return current_seq(graph, cache) == 4; +} + +bool test_cache_transaction_commit_all_keeps_draft_tokens() { + CactusGraph graph; + const size_t kv_heads = 2; + const size_t head_dim = 16; + size_t cache = graph.kv_cache_state(64, kv_heads, head_dim); + graph.execute(); + + if (!append_tokens(graph, cache, 4, kv_heads, head_dim, 1.0f)) return false; + + auto txn = graph.begin_kv_cache_transaction({cache}); + + graph.soft_reset(); + if (!append_tokens(graph, cache, 3, kv_heads, head_dim, 2.0f)) return false; + + txn.commit_all(); + graph.execute(); + + return current_seq(graph, cache) == 7; +} + +bool test_cache_transaction_commit_prefix_crops_rejected_suffix() { + CactusGraph graph; + const size_t kv_heads = 2; + const size_t head_dim = 16; + size_t cache = graph.kv_cache_state(64, kv_heads, head_dim); + graph.execute(); + + if (!append_tokens(graph, cache, 4, kv_heads, head_dim, 1.0f)) return false; + + auto txn = graph.begin_kv_cache_transaction({cache}); + + graph.soft_reset(); + if (!append_tokens(graph, cache, 6, kv_heads, head_dim, 2.0f)) return false; + + txn.commit_prefix(2); + graph.execute(); + + return current_seq(graph, cache) == 6; +} + +bool test_cache_transaction_tracks_multiple_layer_caches() { + CactusGraph graph; + const size_t kv_heads = 2; + const size_t head_dim = 16; + size_t k_cache = graph.kv_cache_state(64, kv_heads, head_dim); + size_t v_cache = graph.kv_cache_state(64, kv_heads, head_dim); + graph.execute(); + + if (!append_tokens(graph, k_cache, 5, kv_heads, head_dim, 1.0f)) return false; + graph.soft_reset(); + if (!append_tokens(graph, v_cache, 5, kv_heads, head_dim, 1.0f)) return false; + + auto txn = graph.begin_kv_cache_transaction({k_cache, v_cache}); + + graph.soft_reset(); + if (!append_tokens(graph, k_cache, 4, kv_heads, head_dim, 2.0f)) return false; + graph.soft_reset(); + if (!append_tokens(graph, v_cache, 4, kv_heads, head_dim, 2.0f)) return false; + + txn.rollback(); + graph.execute(); + + return current_seq(graph, k_cache) == 5 + && current_seq(graph, v_cache) == 5; +} + +bool test_cache_transaction_sliding_window_commit_prefix() { + CactusGraph graph; + const size_t kv_heads = 1; + const size_t head_dim = 16; + const size_t window = 8; + const size_t sink = 2; + size_t cache = graph.kv_cache_state(window, kv_heads, head_dim, window, sink); + graph.execute(); + + if (!append_tokens(graph, cache, 6, kv_heads, head_dim, 1.0f)) return false; + + auto txn = graph.begin_kv_cache_transaction({cache}); + + graph.soft_reset(); + size_t input = graph.input({4 * kv_heads * head_dim}, Precision::FP16); + std::vector<__fp16> data(4 * kv_heads * head_dim, static_cast<__fp16>(2.0f)); + graph.set_input(input, data.data(), Precision::FP16); + graph.kv_cache_append(input, cache, window, sink); + graph.execute(); + + txn.commit_prefix(1); + graph.execute(); + + return current_seq(graph, cache) == 7; +} + +int main() { + TestRunner runner("KV Cache Transaction Tests"); + + runner.run_test("rollback_restores_length", test_cache_transaction_rollback_restores_sequence_length()); + runner.run_test("commit_all_keeps_tokens", test_cache_transaction_commit_all_keeps_draft_tokens()); + runner.run_test("commit_prefix_crops_suffix", test_cache_transaction_commit_prefix_crops_rejected_suffix()); + runner.run_test("multiple_layer_caches", test_cache_transaction_tracks_multiple_layer_caches()); + runner.run_test("sliding_window_prefix", test_cache_transaction_sliding_window_commit_prefix()); + + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus-kernels/cactus_kernels.h b/cactus-kernels/cactus_kernels.h index ee06e2f47..d03ca86f1 100644 --- a/cactus-kernels/cactus_kernels.h +++ b/cactus-kernels/cactus_kernels.h @@ -442,7 +442,8 @@ void cactus_attention_hybrid_int8_fp16( bool is_causal = true, size_t window_size = 0, size_t group_size = KV_QUANT_GROUP_SIZE, - size_t v_head_dim = 0); + size_t v_head_dim = 0, + const __fp16* new_token_mask = nullptr); void cactus_conv1d_causal_depthwise_f16( diff --git a/cactus-kernels/src/attention_hybrid.cpp b/cactus-kernels/src/attention_hybrid.cpp index 044ff1408..b432242a3 100644 --- a/cactus-kernels/src/attention_hybrid.cpp +++ b/cactus-kernels/src/attention_hybrid.cpp @@ -5,8 +5,73 @@ #include #include #include +#include +#include +#include +#include #include +struct CactusAttentionTrace { + const char* path = "generic"; + size_t batch_size = 0; + size_t seq_len = 0; + size_t cache_len = 0; + size_t new_len = 0; + size_t num_q_heads = 0; + size_t num_kv_heads = 0; + size_t head_dim = 0; + size_t v_head_dim = 0; + bool masked = false; + bool enabled = false; + std::chrono::steady_clock::time_point start; + + CactusAttentionTrace( + size_t batch, size_t seq, size_t cache, size_t new_tokens, + size_t q_heads, size_t kv_heads, size_t hdim, size_t vdim, + bool has_mask) + : batch_size(batch), seq_len(seq), cache_len(cache), new_len(new_tokens), + num_q_heads(q_heads), num_kv_heads(kv_heads), head_dim(hdim), + v_head_dim(vdim), masked(has_mask) { + static const bool trace_enabled = [] { + const char* file = std::getenv("CACTUS_ATTENTION_TRACE_FILE"); + return file && file[0]; + }(); + enabled = trace_enabled; + if (enabled) start = std::chrono::steady_clock::now(); + } + + ~CactusAttentionTrace() { + if (!enabled) return; + const char* file = std::getenv("CACTUS_ATTENTION_TRACE_FILE"); + if (!file || !file[0]) return; + static std::mutex mutex; + std::lock_guard lock(mutex); + bool write_header = false; + { + std::ifstream in(file, std::ios::binary | std::ios::ate); + write_header = !in.good() || in.tellg() == 0; + } + std::ofstream out(file, std::ios::app); + if (!out) return; + if (write_header) { + out << "path,batch_size,seq_len,cache_len,new_len,num_q_heads,num_kv_heads,head_dim,v_head_dim,masked,ms\n"; + } + auto end = std::chrono::steady_clock::now(); + double ms = std::chrono::duration(end - start).count(); + out << path << "," + << batch_size << "," + << seq_len << "," + << cache_len << "," + << new_len << "," + << num_q_heads << "," + << num_kv_heads << "," + << head_dim << "," + << v_head_dim << "," + << (masked ? "true" : "false") << "," + << std::fixed << std::setprecision(3) << ms << "\n"; + } +}; + static void cactus_attention_hybrid_int8_fp16_decode_dot( const __fp16* queries, const int8_t* keys_cached, @@ -347,18 +412,25 @@ void cactus_attention_hybrid_int8_fp16( bool is_causal, size_t window_size, size_t quant_group_size, - size_t v_head_dim + size_t v_head_dim, + const __fp16* new_token_mask ) { if (v_head_dim == 0) v_head_dim = head_dim; if (scale == 0.0f) { scale = 1.0f / sqrtf(static_cast(head_dim)); } + CactusAttentionTrace trace( + batch_size, seq_len, cache_len, new_len, + num_q_heads, num_kv_heads, head_dim, v_head_dim, + new_token_mask != nullptr); if (seq_len == 1 && + new_token_mask == nullptr && head_dim == v_head_dim && head_dim <= 512 && head_dim % 32 == 0 && quant_group_size == 32) { + trace.path = "decode_dot"; cactus_attention_hybrid_int8_fp16_decode_dot( queries, keys_cached, values_cached, k_scales, v_scales, keys_new, values_new, output, @@ -390,6 +462,7 @@ void cactus_attention_hybrid_int8_fp16( const size_t k_seq_stride = num_kv_heads * head_dim; const size_t v_seq_stride = num_kv_heads * v_head_dim; const size_t o_seq_stride = num_q_heads * v_head_dim; + const size_t mask_batch_stride = seq_len * new_len; CactusThreading::parallel_for(batch_size * num_q_heads * seq_len, CactusThreading::Thresholds::ATTENTION, [=](size_t start_idx, size_t end_idx) { @@ -411,6 +484,7 @@ void cactus_attention_hybrid_int8_fp16( const int8_t* V_cached_base = values_cached + batch_idx * v_cached_batch_stride; const __fp16* K_new_base = keys_new + batch_idx * k_new_batch_stride; const __fp16* V_new_base = values_new + batch_idx * v_new_batch_stride; + const __fp16* mask_base = new_token_mask ? new_token_mask + batch_idx * mask_batch_stride : nullptr; __fp16* O_base = output + batch_idx * o_batch_stride; const __fp16* q_vec = Q_base + q_pos * q_seq_stride + q_head_idx * head_dim; @@ -458,7 +532,13 @@ void cactus_attention_hybrid_int8_fp16( } } - if ((is_causal && kv_pos > absolute_q_pos) || window_masked) { + bool tree_masked = false; + if (mask_base && kv_pos >= cache_len) { + const size_t new_pos = kv_pos - cache_len; + tree_masked = static_cast(mask_base[q_pos * new_len + new_pos]) == 0.0f; + } + + if ((is_causal && kv_pos > absolute_q_pos) || window_masked || tree_masked) { block_scores[kv_idx] = -std::numeric_limits::infinity(); continue; } diff --git a/cactus/build.sh b/cactus/build.sh index b139b23e0..281b459ed 100755 --- a/cactus/build.sh +++ b/cactus/build.sh @@ -38,7 +38,7 @@ mkdir -p build cd build cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null 2>&1 -make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) +cmake --build . --parallel ${CACTUS_BUILD_JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)} echo "Cactus library built successfully!" echo "Library location: $(pwd)/libcactus.a" diff --git a/python/cactus/cli/download.py b/python/cactus/cli/download.py index 353fdfb9d..f4d0d0ae5 100644 --- a/python/cactus/cli/download.py +++ b/python/cactus/cli/download.py @@ -423,6 +423,52 @@ def get_weights_dir(model_id: str) -> Path: return _PROJECT_ROOT / "weights" / get_model_dir_name(model_id) +_PACKAGED_ASSISTANTS = { + "gemma-4-e2b-it": { + "repo_id": "Cactus-Compute/gemma-4-e2b-it-cq", + "assistant_zip": "assistant-L4.zip", + "assistant_dir": "assistant", + }, +} + + +def ensure_packaged_artifacts(model_id: str, weights_dir: Path) -> bool: + """Download companion artifacts (e.g. MTP assistant weights) for known packaged models. + + Returns True if no extra artifacts are required or the download succeeds. + """ + package = _PACKAGED_ASSISTANTS.get(get_model_dir_name(model_id)) + if not package: + return True + + assistant_path = weights_dir / package["assistant_dir"] + if (assistant_path / "assistant_cactus_manifest.json").exists(): + return True + + try: + from huggingface_hub import hf_hub_download, list_repo_files + except ImportError: + print("huggingface_hub not installed - run: pip install huggingface_hub") + return False + + repo_id = package["repo_id"] + assistant_zip = package["assistant_zip"] + try: + if assistant_zip not in list_repo_files(repo_id, repo_type="model"): + print(f"Assistant weights not found in {repo_id}") + return False + + assistant_path.mkdir(parents=True, exist_ok=True) + print(f"Downloading {repo_id}/{assistant_zip} ...") + assistant_zip_path = Path(hf_hub_download(repo_id=repo_id, filename=assistant_zip, repo_type="model")) + print("Extracting assistant weights...") + _safe_zip_extract(assistant_zip_path, assistant_path) + return True + except Exception as exc: + print(f"Assistant download failed: {exc}") + return False + + def download_legacy_fp16_zip(model_id: str, weights_dir: Path) -> bool: """Download pre-converted FP16 weights zip from Cactus-Compute (legacy fallback). @@ -509,6 +555,8 @@ def cmd_download(args): shutil.rmtree(weights_dir) if weights_dir.exists() and (weights_dir / "config.txt").exists(): + if not ensure_packaged_artifacts(model_id, weights_dir): + return 1 print_color(GREEN, f"Model weights found at {weights_dir}") return 0 @@ -534,6 +582,8 @@ def cmd_download(args): size_text = f" ({resolution.archive.size / (1024 * 1024):.1f} MiB)" if resolution.archive.size else "" print(f" Downloading {resolution.archive.filename} [{combo_label(resolution.archive.combo)}]{size_text}") download_cq_archive(resolution, weights_dir, token=token, cache_dir=cache_dir) + if not ensure_packaged_artifacts(model_id, weights_dir): + return 1 print_color(GREEN, f"CQ model ready at {weights_dir}") return 0 except Exception as cq_exc: diff --git a/python/cactus/hf_gemma_assistant.py b/python/cactus/hf_gemma_assistant.py new file mode 100644 index 000000000..b2a9aa823 --- /dev/null +++ b/python/cactus/hf_gemma_assistant.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import argparse + +import torch +import transformers +from transformers import AutoModelForCausalLM, AutoProcessor, TextStreamer + + +TARGET_MODEL_ID = "google/gemma-4-E2B-it" +ASSISTANT_MODEL_ID = "google/gemma-4-E2B-it-assistant" + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Run Gemma 4 E2B IT with its Hugging Face MTP assistant." + ) + parser.add_argument("prompt", help="Single user prompt to generate one response for.") + return parser.parse_args() + + +def main(): + args = parse_args() + if not hasattr(transformers, "Gemma4AssistantForCausalLM"): + raise RuntimeError( + "Gemma 4 Assistant requires transformers with gemma4_assistant support. " + "Install with: pip install -U 'transformers>=5.8.0'" + ) + + processor = AutoProcessor.from_pretrained(TARGET_MODEL_ID, padding_side="left") + target_model = AutoModelForCausalLM.from_pretrained( + TARGET_MODEL_ID, + dtype=torch.bfloat16, + device_map="auto", + ) + assistant_model = AutoModelForCausalLM.from_pretrained( + ASSISTANT_MODEL_ID, + dtype=torch.bfloat16, + device_map="auto", + ) + + messages = [{"role": "user", "content": args.prompt}] + inputs = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + ).to(target_model.device) + streamer = TextStreamer( + processor.tokenizer, + skip_prompt=True, + skip_special_tokens=True, + ) + + target_model.generate( + **inputs, + assistant_model=assistant_model, + max_new_tokens=256, + do_sample=False, + streamer=streamer, + ) + + +if __name__ == "__main__": + main() diff --git a/python/cactus/hf_gemma_mtp_check.py b/python/cactus/hf_gemma_mtp_check.py new file mode 100644 index 000000000..bb231cbf7 --- /dev/null +++ b/python/cactus/hf_gemma_mtp_check.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python3 +import argparse +import copy +import inspect +import sys + +import torch +from transformers import AutoModelForCausalLM, AutoProcessor +from transformers.generation.candidate_generator import ( + _prepare_attention_mask, + _prepare_position_ids, + _prepare_token_type_ids, +) +from transformers.generation.logits_process import LogitsProcessorList +from transformers.generation.stopping_criteria import StoppingCriteriaList + + +TARGET_MODEL_ID = "google/gemma-4-E2B-it" +ASSISTANT_MODEL_ID = "google/gemma-4-E2B-it-assistant" +PROMPT_SUITE = [ + ("short_chat", "who are you"), + ("desert_sentence", "Write one short sentence about desert rain."), + ("json_object", "Return a JSON object with keys name, role, and status."), + ("python_function", "Write a short Python function that adds two numbers."), + ("count_1_to_100", "Count from 1 to 100, separated by commas."), +] + + +def parse_args(): + parser = argparse.ArgumentParser(description="Compare HF Gemma 4 MTP with a local manual MTP loop.") + parser.add_argument("prompt", nargs="?", default="Write one short sentence about speculative decoding.") + parser.add_argument("--max-new-tokens", type=int, default=16) + parser.add_argument("--seed", type=int, default=1234) + parser.add_argument("--top-p", type=float, default=None) + parser.add_argument("--top-k", type=int, default=None) + parser.add_argument("--trace", action="store_true", help="Print round-by-round manual MTP trace rows.") + parser.add_argument("--num-assistant-tokens", type=int, default=None) + parser.add_argument("--acceptance-suite", action="store_true", help="Print greedy acceptance totals for the prompt suite.") + return parser.parse_args() + + +def clone_inputs(inputs): + return {key: value.clone() if torch.is_tensor(value) else value for key, value in inputs.items()} + + +def prepare_generation(model, inputs, assistant_model, max_new_tokens, do_sample, temperature, top_p=None, top_k=None): + kwargs = clone_inputs(inputs) + kwargs.update( + { + "max_new_tokens": max_new_tokens, + "do_sample": do_sample, + } + ) + if temperature is not None: + kwargs["temperature"] = temperature + if top_p is not None: + kwargs["top_p"] = top_p + if top_k is not None: + kwargs["top_k"] = top_k + + generation_config, model_kwargs = model._prepare_generation_config(None, **kwargs) + logits_processor = LogitsProcessorList() + stopping_criteria = StoppingCriteriaList() + + accepts_attention_mask = "attention_mask" in set(inspect.signature(model.forward).parameters.keys()) + kwargs_has_attention_mask = model_kwargs.get("attention_mask", None) is not None + + inputs_tensor, model_input_name, model_kwargs = model._prepare_model_inputs( + None, generation_config.bos_token_id, model_kwargs + ) + batch_size = inputs_tensor.shape[0] + device = inputs_tensor.device + model._prepare_special_tokens(generation_config, kwargs_has_attention_mask, device=device) + + if not kwargs_has_attention_mask and not model.config.is_encoder_decoder and accepts_attention_mask: + model_kwargs["attention_mask"] = model._prepare_attention_mask_for_generation( + inputs_tensor, generation_config, model_kwargs + ) + elif kwargs_has_attention_mask: + if model_input_name == "input_ids" and len(model_kwargs["attention_mask"].shape) > 2: + raise ValueError("attention_mask passed to generate must be 2D.") + + kwargs_has_position_ids = model_kwargs.get("position_ids", None) is not None + accepts_position_ids = "position_ids" in set(inspect.signature(model.forward).parameters.keys()) + if not kwargs_has_position_ids and accepts_position_ids and not model.config.is_encoder_decoder: + model_kwargs["position_ids"] = model._prepare_position_ids_for_generation(inputs_tensor, model_kwargs) + + if model.config.is_encoder_decoder and "encoder_outputs" not in model_kwargs: + model_kwargs = model._prepare_encoder_decoder_kwargs_for_generation( + inputs_tensor, model_kwargs, model_input_name, generation_config + ) + + if model.config.is_encoder_decoder: + input_ids, model_kwargs = model._prepare_decoder_input_ids_for_generation( + batch_size=batch_size, + model_input_name=model_input_name, + model_kwargs=model_kwargs, + decoder_start_token_id=generation_config._decoder_start_token_tensor, + device=inputs_tensor.device, + ) + else: + input_ids = inputs_tensor if model_input_name == "input_ids" else model_kwargs.pop("input_ids") + + input_ids, model_kwargs = model._expand_inputs_for_generation( + input_ids=input_ids, + expand_size=max(generation_config.num_beams, generation_config.num_return_sequences), + is_encoder_decoder=model.config.is_encoder_decoder, + **model_kwargs, + ) + + input_ids_length = input_ids.shape[1] + generation_config = model._prepare_generated_length( + generation_config=generation_config, + has_default_max_length=False, + has_default_min_length=True, + model_input_name=model_input_name, + inputs_tensor=inputs_tensor, + input_ids_length=input_ids_length, + ) + + if model._supports_logits_to_keep() and "logits_to_keep" not in model_kwargs: + model_kwargs["logits_to_keep"] = 1 + model_kwargs["use_cache"] = generation_config.use_cache + + generation_mode = generation_config.get_generation_mode(assistant_model) + max_cache_length = generation_config.max_length - 1 + model._prepare_cache_for_generation( + generation_config, model_kwargs, generation_mode, batch_size, max_cache_length + ) + + logits_processor = model._get_logits_processor( + generation_config=generation_config, + input_ids_seq_length=input_ids_length, + encoder_input_ids=inputs_tensor, + prefix_allowed_tokens_fn=None, + logits_processor=logits_processor, + device=inputs_tensor.device, + model_kwargs=model_kwargs, + negative_prompt_ids=None, + negative_prompt_attention_mask=None, + ) + + return generation_config, input_ids, model_kwargs, logits_processor, stopping_criteria + + +def eos_token_ids(generation_config, assistant_generation_config, device): + ids = set() + for value in (generation_config.eos_token_id, assistant_generation_config.eos_token_id): + if value is None: + continue + if isinstance(value, int): + ids.add(value) + else: + ids.update(value) + return torch.tensor(list(ids), dtype=torch.long, device=device) if ids else None + + +def manual_draft(target_model, assistant_model, input_ids, model_kwargs, outputs, n_matches, generation_config): + max_new_tokens = min( + int(assistant_model.generation_config.num_assistant_tokens), + generation_config.max_length - input_ids.shape[1] - 1, + ) + if max_new_tokens <= 0: + return input_ids, None + + current_length = input_ids.shape[1] + last_hidden_state = outputs.hidden_states[-1][:, n_matches : n_matches + 1] + shared_kv_states = { + key: (value[0][:, :, :current_length, :], value[1][:, :, :current_length, :]) + for key, value in outputs.shared_kv_states.items() + } + last_token_id = input_ids[:, -1:] + position_ids = torch.tensor([[input_ids.shape[1] - 1]], dtype=torch.long, device=assistant_model.device) + stopped = torch.zeros(input_ids.shape[0], dtype=torch.bool, device=input_ids.device) + eos_ids = eos_token_ids(generation_config, assistant_model.generation_config, last_token_id.device) + + drafted_logits = [] + drafted_tokens = [] + for _ in range(max_new_tokens): + last_token_embedding = target_model.get_input_embeddings()(last_token_id) + inputs_embeds = torch.cat([last_token_embedding, last_hidden_state], dim=-1) + with torch.no_grad(): + assistant_outputs = assistant_model( + inputs_embeds=inputs_embeds, + attention_mask=model_kwargs.get("attention_mask"), + position_ids=position_ids, + shared_kv_states=shared_kv_states, + use_cache=False, + ) + + last_token_id = assistant_outputs.logits.argmax(dim=-1) + last_hidden_state = assistant_outputs.last_hidden_state + if stopped.any(): + stopped_expanded = stopped.unsqueeze(1) + last_token_id = torch.where(stopped_expanded, generation_config.pad_token_id, last_token_id) + drafted_logits.append( + torch.where(stopped_expanded.unsqueeze(-1), torch.zeros_like(assistant_outputs.logits), assistant_outputs.logits) + ) + else: + drafted_logits.append(assistant_outputs.logits) + + drafted_tokens.append(last_token_id) + if eos_ids is not None: + stopped = torch.logical_or(stopped, torch.isin(last_token_id.squeeze(1), eos_ids)) + if stopped.all(): + break + + return torch.cat([input_ids, torch.cat(drafted_tokens, dim=1)], dim=1), torch.cat(drafted_logits, dim=1) + + +def speculative_sample(candidate_input_ids, candidate_logits, candidate_length, new_logits, is_done_candidate): + candidate_new_ids = candidate_input_ids[:, -candidate_length:] + q = candidate_logits.softmax(dim=-1) + q_i = q[:, torch.arange(candidate_length), candidate_new_ids].squeeze(0, 1) + p = new_logits.softmax(dim=-1) + p_i = p[:, torch.arange(candidate_length), candidate_new_ids].squeeze(0, 1) + probability_ratio = p_i / q_i + is_accepted = torch.rand_like(probability_ratio) <= probability_ratio + n_matches = ((~is_accepted).cumsum(dim=-1) < 1).sum() + + if is_done_candidate and n_matches == candidate_length: + n_matches -= 1 + valid_tokens = candidate_new_ids[:, : n_matches + 1] + else: + gamma = candidate_logits.shape[1] + p_next = p[:, n_matches, :] + if n_matches < gamma: + q_next = q[:, n_matches, :] + p_prime = torch.clamp((p_next - q_next), min=0) + p_prime.div_(p_prime.sum()) + else: + p_prime = p_next + next_token = torch.multinomial(p_prime, num_samples=1).squeeze(1)[None, :] + valid_tokens = torch.cat((candidate_new_ids[:, :n_matches], next_token), dim=-1) if n_matches > 0 else next_token + + return valid_tokens, n_matches + + +def ids_for_trace(tensor): + if tensor is None: + return [] + if tensor.ndim == 2: + tensor = tensor[0] + return [int(x) for x in tensor.detach().cpu().tolist()] + + +def manual_generate(model, assistant_model, inputs, max_new_tokens, do_sample, temperature, + top_p=None, top_k=None, trace=False, stats=None): + generation_config, input_ids, model_kwargs, logits_processor, stopping_criteria = prepare_generation( + model, inputs, assistant_model, max_new_tokens, do_sample, temperature, top_p, top_k + ) + if not model_kwargs["use_cache"]: + raise ValueError("Manual assisted generation requires use_cache=True.") + + outputs = None + n_matches = 0 + is_first_iteration = True + + round_index = 0 + while input_ids.shape[1] < generation_config.max_length: + cur_len = input_ids.shape[1] + previous_n_matches = int(n_matches) + if is_first_iteration: + candidate_input_ids, candidate_logits = input_ids, None + else: + candidate_input_ids, candidate_logits = manual_draft( + model, assistant_model, input_ids, model_kwargs, outputs, n_matches, generation_config + ) + candidate_length = candidate_input_ids.shape[1] - input_ids.shape[1] + is_done_candidate = stopping_criteria(candidate_input_ids, None) + eos_ids = generation_config._eos_token_tensor + if eos_ids is not None and candidate_length > 0: + is_done_candidate = is_done_candidate or torch.isin( + candidate_input_ids[:, -1], eos_ids.to(candidate_input_ids.device) + ).all() + + candidate_kwargs = copy.copy(model_kwargs) + candidate_kwargs = _prepare_attention_mask( + candidate_kwargs, candidate_input_ids.shape[1], model.config.is_encoder_decoder + ) + candidate_kwargs = _prepare_token_type_ids(candidate_kwargs, candidate_input_ids.shape[1]) + if (position_ids := candidate_kwargs.get("position_ids")) is not None and candidate_length > 0: + new_length = candidate_length + position_ids.shape[-1] + candidate_kwargs = _prepare_position_ids(candidate_kwargs, new_length, model.config.is_encoder_decoder) + + next_sequence_length = candidate_length + 1 if not is_first_iteration else None + model_inputs = model.prepare_inputs_for_generation( + candidate_input_ids, + next_sequence_length=next_sequence_length, + is_first_iteration=is_first_iteration, + **candidate_kwargs, + ) + if "logits_to_keep" in model_inputs: + model_inputs["logits_to_keep"] = candidate_length + 1 + model_inputs["output_hidden_states"] = True + model_inputs["return_shared_kv_states"] = True + + with torch.no_grad(): + outputs = model(**model_inputs) + + new_logits = outputs.logits[:, -candidate_length - 1 :].to(dtype=torch.float32, device=input_ids.device) + for i in range(candidate_length + 1): + new_logits[:, i, :] = logits_processor(candidate_input_ids[:, : cur_len + i], new_logits[:, i, :]) + + if do_sample and candidate_logits is not None: + valid_tokens, n_matches = speculative_sample( + candidate_input_ids, candidate_logits.to(input_ids.device), candidate_length, new_logits, is_done_candidate + ) + else: + if do_sample: + probs = new_logits.softmax(dim=-1) + selected_tokens = torch.multinomial(probs[0, :, :], num_samples=1).squeeze(1)[None, :] + else: + selected_tokens = new_logits.argmax(dim=-1) + + candidate_new_tokens = candidate_input_ids[:, cur_len:] + n_matches = ((~(candidate_new_tokens == selected_tokens[:, :-1])).cumsum(dim=-1) < 1).sum() + if is_done_candidate and n_matches == candidate_length: + n_matches -= 1 + valid_tokens = selected_tokens[:, : n_matches + 1] + + input_ids = torch.cat((input_ids, valid_tokens), dim=-1) + if not is_first_iteration: + round_index += 1 + if stats is not None: + stats.append({ + "drafted": int(candidate_length), + "accepted": int(n_matches), + "rejected": int(n_matches) < int(candidate_length), + }) + if trace and not is_first_iteration: + print( + "[hf_trace] " + f"round={round_index} " + f"input_len={cur_len} " + f"previous_n_matches={previous_n_matches} " + f"assistant_position_id={cur_len - 1} " + f"assistant_input_token={int(candidate_input_ids[0, cur_len - 1])} " + f"assistant_draft_tokens={ids_for_trace(candidate_input_ids[:, cur_len:])} " + f"target_verify_input_tokens={ids_for_trace(candidate_input_ids[:, cur_len - 1:])} " + f"target_argmax_tokens={ids_for_trace(new_logits.argmax(dim=-1))} " + f"accepted_count={int(n_matches)} " + f"valid_committed_tokens={ids_for_trace(valid_tokens)} " + f"target_cache_crop_length={input_ids.shape[1] - 1}" + ) + outputs.past_key_values.crop(input_ids.shape[1] - 1) + model_kwargs = model._update_model_kwargs_for_generation( + outputs, + model_kwargs, + is_encoder_decoder=model.config.is_encoder_decoder, + num_new_tokens=n_matches + 1, + ) + is_first_iteration = False + + eos_ids = generation_config._eos_token_tensor + if eos_ids is not None and torch.isin(input_ids[:, -1], eos_ids.to(input_ids.device)).all(): + break + + return input_ids + + +def builtin_generate(model, assistant_model, inputs, max_new_tokens, do_sample, temperature, top_p=None, top_k=None): + kwargs = { + **clone_inputs(inputs), + "assistant_model": assistant_model, + "max_new_tokens": max_new_tokens, + "do_sample": do_sample, + } + if temperature is not None: + kwargs["temperature"] = temperature + if top_p is not None: + kwargs["top_p"] = top_p + if top_k is not None: + kwargs["top_k"] = top_k + return model.generate(**kwargs) + + +def run_case(label, model, assistant_model, processor, inputs, max_new_tokens, seed, + do_sample, temperature, top_p, top_k, trace): + torch.manual_seed(seed) + builtin_ids = builtin_generate(model, assistant_model, inputs, max_new_tokens, do_sample, temperature, top_p, top_k) + torch.manual_seed(seed) + manual_ids = manual_generate( + model, assistant_model, inputs, max_new_tokens, do_sample, temperature, top_p, top_k, trace) + + input_len = inputs["input_ids"].shape[-1] + builtin_new = builtin_ids[0, input_len:].tolist() + manual_new = manual_ids[0, input_len:].tolist() + print(f"\n{label}") + print(f"match: {builtin_new == manual_new}") + print(f"builtin ids: {builtin_new}") + print(f"manual ids: {manual_new}") + print(f"builtin: {processor.decode(builtin_ids[0, input_len:], skip_special_tokens=True)!r}") + print(f"manual: {processor.decode(manual_ids[0, input_len:], skip_special_tokens=True)!r}") + if builtin_new != manual_new: + raise AssertionError(f"{label} manual MTP output did not match built-in HF assisted generation") + + +def prompt_inputs(processor, model, prompt): + messages = [{"role": "user", "content": prompt}] + return processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + ).to(model.device) + + +def run_acceptance_suite(model, assistant_model, processor, max_new_tokens, seed): + print("source,prompt,draft,max_new_tokens,drafted,accepted,rejected,rounds,acceptance_rate") + for name, prompt in PROMPT_SUITE: + inputs = prompt_inputs(processor, model, prompt) + stats = [] + torch.manual_seed(seed) + manual_generate(model, assistant_model, inputs, max_new_tokens, False, None, None, None, False, stats) + drafted = sum(row["drafted"] for row in stats) + accepted = sum(row["accepted"] for row in stats) + rejected = sum(1 for row in stats if row["rejected"]) + rounds = len(stats) + rate = accepted / drafted if drafted else 0.0 + print( + f"hf,{name},{int(assistant_model.generation_config.num_assistant_tokens)}," + f"{max_new_tokens},{drafted},{accepted},{rejected},{rounds},{rate:.4f}" + ) + + +def main(): + args = parse_args() + print(f"target_model_id={TARGET_MODEL_ID}", file=sys.stderr) + print(f"assistant_model_id={ASSISTANT_MODEL_ID}", file=sys.stderr) + processor = AutoProcessor.from_pretrained(TARGET_MODEL_ID, padding_side="left") + model = AutoModelForCausalLM.from_pretrained(TARGET_MODEL_ID, dtype=torch.bfloat16, device_map="auto") + assistant_model = AutoModelForCausalLM.from_pretrained(ASSISTANT_MODEL_ID, dtype=torch.bfloat16, device_map="auto") + if args.num_assistant_tokens is not None: + assistant_model.generation_config.num_assistant_tokens = args.num_assistant_tokens + if args.acceptance_suite: + run_acceptance_suite(model, assistant_model, processor, args.max_new_tokens, args.seed) + return + messages = [{"role": "user", "content": args.prompt}] + inputs = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + ).to(model.device) + + run_case("temperature=0", model, assistant_model, processor, inputs, args.max_new_tokens, + args.seed, False, None, None, None, args.trace) + run_case("temperature=0.7", model, assistant_model, processor, inputs, args.max_new_tokens, + args.seed, True, 0.7, args.top_p, args.top_k, args.trace) + + +if __name__ == "__main__": + main() diff --git a/python/pyproject.toml b/python/pyproject.toml index 06d4b8c72..f75b1f17e 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -5,7 +5,8 @@ description = "Cactus Python tools" requires-python = ">=3.10" dependencies = [ "torch", - "transformers", + "transformers>=5.8.0", + "accelerate", "numpy", "huggingface-hub", ] @@ -16,6 +17,8 @@ lora = ["peft"] [project.scripts] cactus = "cactus.cli:main" +cactus-hf-gemma-assistant = "cactus.hf_gemma_assistant:main" +cactus-hf-gemma-mtp-check = "cactus.hf_gemma_mtp_check:main" [build-system] requires = ["setuptools>=61.0"] diff --git a/python/requirements.txt b/python/requirements.txt index f675d0773..21aab2c55 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,7 +1,8 @@ torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 -transformers==5.0.0 +transformers>=5.8.0 +accelerate pillow==11.3.0 num2words==0.5.12 huggingface_hub==1.4.1 diff --git a/tests/bench/README.md b/tests/bench/README.md new file mode 100644 index 000000000..235fa4008 --- /dev/null +++ b/tests/bench/README.md @@ -0,0 +1,42 @@ +# E2E Decode Benchmark + +Minimal Cactus-only decode speed comparison copied forward from the `int4-benchmark` branch. + +```bash +./tests/bench_e2e_decode.sh +``` + +The wrapper sources `./venv/bin/activate`, runs `cactus build`, compiles `tests/bench/e2e_cactus_decode.cpp`, then benchmarks each existing model path in `tests/bench/e2e_models.json`. Missing model directories are skipped. + +By default it runs a harder prompt suite that avoids trivial 100% acceptance cases: + +```text +json_object +python_function +explain_schrodinger +rank_tradeoffs +debug_plan +constrained_json +code_review +``` + +Use `--prompt-suite long_context` to measure decode after a roughly 1k-token natural prose prefill. That suite uses a hand-written launch-readiness memo with conflicting facts and a synthesis question so MTP behavior is not dominated by repetitive generated records. `--prompt-suite all` runs both suites. + +By default the CSV includes only the baseline (`0`) run with 2 warmups and 5 measured reps. Use `--mtp-max-drafts 0,2,3,4` for sweeps, or `--mtp-max-draft 3` for one setting. Sampled runs can be measured with `--temperature`, `--top-p`, `--top-k`, `--min-p`, and `--seed`. + +Useful overrides: + +```bash +./tests/bench_e2e_decode.sh --reps 5 --max-tokens 512 --model qwen3-0.6b +./tests/bench_e2e_decode.sh --config /path/to/models.json --output decode.csv +./tests/bench_e2e_decode.sh --prompt "who are you" --mtp-max-draft 0 +./tests/bench_e2e_decode.sh --model gemma-4-e2b-it --prompt-suite long_context --long-context-repeats 192 --max-tokens 256 --mtp-max-drafts 0,3 +./tests/bench_e2e_decode.sh --model gemma-4-e2b-it --prompt "Count from 1 to 100, separated by commas." --max-tokens 128 --mtp-max-drafts 0,2,3,4,5 +./tests/bench_e2e_decode.sh --model gemma-4-e2b-it --temperature 0.7 --top-p 0.95 --top-k 64 --min-p 0 --seed 1234 --mtp-max-draft 3 +``` + +CSV columns: + +```text +backend,model,prompt,mtp_max_draft,rep,prefill_tokens,decode_tokens,prefill_tps,decode_tps,ttft_ms,total_ms,mtp_requested,mtp_enabled,mtp_drafted_tokens,mtp_accepted_tokens,mtp_rejected_tokens,mtp_rounds,mtp_fallback_reason,assistant_draft_ms,target_verify_ms,sampling_or_argmax_ms,kv_transaction_ms,callback_stream_ms +``` diff --git a/tests/bench/analyze_mtp_cost_diagnostics.py b/tests/bench/analyze_mtp_cost_diagnostics.py new file mode 100644 index 000000000..65be2d19c --- /dev/null +++ b/tests/bench/analyze_mtp_cost_diagnostics.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +import argparse +import csv +import statistics +from collections import defaultdict +from pathlib import Path + + +SUMMARY_FIELDS = [ + "prompt_name", + "shape", + "mean_decode_tps", + "min_decode_tps", + "baseline_mean_decode_tps", + "mean_ratio_vs_baseline", + "min_ratio_vs_baseline", + "avg_target_forward_ms_per_forward", + "avg_assistant_ms_per_pass", + "avg_assistant_passes_per_forward", + "avg_assistant_total_ms_per_forward", + "avg_accepted_drafts_per_forward", + "avg_emitted_tokens_per_forward", + "avg_other_ms_per_forward", + "avg_total_ms_per_forward", + "avg_allocated_ms_per_token_after_first", + "cost_amortization_ratio", +] + + +def f(value): + return float(value or 0.0) + + +def read_csv(path): + with Path(path).open(newline="") as fh: + return list(csv.DictReader(fh)) + + +def mean(values): + return statistics.mean(values) if values else 0.0 + + +def write_csv(path, rows, fields): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="") as fh: + writer = csv.DictWriter(fh, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) + + +def load_bench_rows(paths): + rows = [] + for path in paths: + rows.extend(read_csv(path)) + return rows + + +def summarize(args): + bench_rows = load_bench_rows(args.bench_csv) + round_rows = read_csv(Path(args.trace_dir) / "round_trace.csv") + token_rows = read_csv(Path(args.trace_dir) / "token_trace.csv") + + bench_by_case = defaultdict(list) + for row in bench_rows: + prompt = row.get("prompt") or row.get("prompt_name") or "custom" + shape = row.get("shape") or ("baseline" if row.get("mtp_max_draft") == "0" else f"base_main{row.get('mtp_max_draft')}") + bench_by_case[(prompt, shape)].append(row) + + baseline_tps = {} + for (prompt, shape), rows in bench_by_case.items(): + if shape == "baseline": + baseline_tps[prompt] = mean([f(r["decode_tps"]) for r in rows]) + + rounds_by_case = defaultdict(list) + for row in round_rows: + rounds_by_case[(row["prompt_name"], row["shape"])].append(row) + + tokens_by_case = defaultdict(list) + for row in token_rows: + tokens_by_case[(row["prompt_name"], row["shape"])].append(row) + + baseline_ms_per_token = {} + for (prompt, shape), rows in tokens_by_case.items(): + if shape == "baseline": + baseline_ms_per_token[prompt] = mean([f(r["allocated_total_ms"]) for r in rows]) + + summary = [] + for key in sorted(set(bench_by_case) | set(rounds_by_case) | set(tokens_by_case)): + prompt, shape = key + benches = bench_by_case.get(key, []) + rounds = rounds_by_case.get(key, []) + tokens = tokens_by_case.get(key, []) + decode_tps = [f(r["decode_tps"]) for r in benches] + base_tps = baseline_tps.get(prompt, 0.0) + target_ms = [f(r["target_forward_ms"]) for r in rounds] + assistant_total_ms = [f(r["assistant_total_ms"]) for r in rounds] + assistant_passes = [f(r["assistant_pass_count"]) for r in rounds] + other_ms = [ + f(r["sampling_or_argmax_ms"]) + f(r["kv_transaction_ms"]) + f(r["callback_stream_ms"]) + f(r["loop_overhead_ms"]) + for r in rounds + ] + total_ms = [f(r["round_total_ms"]) for r in rounds] + emitted = [f(r["generated_tokens_emitted"]) for r in rounds] + accepted = [f(r["accepted_drafts"]) for r in rounds] + allocated = [f(r["allocated_total_ms"]) for r in tokens] + assistant_pass_sum = sum(assistant_passes) + avg_total = mean(total_ms) + base_ms = baseline_ms_per_token.get(prompt, 0.0) + cost_amortization = (base_ms * mean(emitted) / avg_total) if avg_total > 0.0 else 0.0 + summary.append({ + "prompt_name": prompt, + "shape": shape, + "mean_decode_tps": f"{mean(decode_tps):.3f}", + "min_decode_tps": f"{min(decode_tps) if decode_tps else 0.0:.3f}", + "baseline_mean_decode_tps": f"{base_tps:.3f}", + "mean_ratio_vs_baseline": f"{(mean(decode_tps) / base_tps) if base_tps else 0.0:.3f}", + "min_ratio_vs_baseline": f"{(min(decode_tps) / base_tps) if base_tps and decode_tps else 0.0:.3f}", + "avg_target_forward_ms_per_forward": f"{mean(target_ms):.3f}", + "avg_assistant_ms_per_pass": f"{(sum(assistant_total_ms) / assistant_pass_sum) if assistant_pass_sum else 0.0:.3f}", + "avg_assistant_passes_per_forward": f"{mean(assistant_passes):.3f}", + "avg_assistant_total_ms_per_forward": f"{mean(assistant_total_ms):.3f}", + "avg_accepted_drafts_per_forward": f"{mean(accepted):.3f}", + "avg_emitted_tokens_per_forward": f"{mean(emitted):.3f}", + "avg_other_ms_per_forward": f"{mean(other_ms):.3f}", + "avg_total_ms_per_forward": f"{avg_total:.3f}", + "avg_allocated_ms_per_token_after_first": f"{mean(allocated):.3f}", + "cost_amortization_ratio": f"{cost_amortization:.3f}", + }) + + write_csv(Path(args.output_dir) / "summary_by_prompt_shape.csv", summary, SUMMARY_FIELDS) + + best_rows = [] + by_prompt = defaultdict(list) + for row in summary: + by_prompt[row["prompt_name"]].append(row) + for prompt, rows in sorted(by_prompt.items()): + best = max(rows, key=lambda r: float(r["cost_amortization_ratio"])) + best_rows.append(best) + write_csv(Path(args.output_dir) / "best_shape_by_prompt.csv", best_rows, SUMMARY_FIELDS) + + lines = [ + "# Gemma 4 MTP Fixed-Shape Cost Diagnostic Summary", + "", + "The diagnostic unit is post-first-token decode work. Cost amortization above 1.0 means the measured emitted-token benefit offsets the target, assistant, and other per-forward costs.", + "", + "## Per-Prompt Findings", + "", + ] + for prompt, rows in sorted(by_prompt.items()): + baseline = next((r for r in rows if r["shape"] == "baseline"), None) + mtp_rows = [r for r in rows if r["shape"] != "baseline"] + if not mtp_rows: + continue + best = max(mtp_rows, key=lambda r: float(r["cost_amortization_ratio"])) + worst = min(mtp_rows, key=lambda r: float(r["cost_amortization_ratio"])) + base_ms = float(baseline["avg_allocated_ms_per_token_after_first"]) if baseline else 0.0 + lines.append( + f"- `{prompt}`: best `{best['shape']}` emits {best['avg_emitted_tokens_per_forward']} tokens/forward " + f"while paying {best['avg_target_forward_ms_per_forward']} ms target + " + f"{best['avg_assistant_total_ms_per_forward']} ms assistant + {best['avg_other_ms_per_forward']} ms other " + f"(amortization {best['cost_amortization_ratio']}, mean TPS ratio {best['mean_ratio_vs_baseline']}); " + f"worst `{worst['shape']}` emits {worst['avg_emitted_tokens_per_forward']} with amortization " + f"{worst['cost_amortization_ratio']} against a {base_ms:.3f} ms baseline token cost." + ) + + lines.extend(["", "## Shape Table", ""]) + lines.append("| prompt | shape | emitted/fwd | target ms | assistant ms | other ms | total ms | amortization | mean TPS ratio |") + lines.append("|---|---:|---:|---:|---:|---:|---:|---:|---:|") + for row in summary: + lines.append( + f"| {row['prompt_name']} | {row['shape']} | {row['avg_emitted_tokens_per_forward']} | " + f"{row['avg_target_forward_ms_per_forward']} | {row['avg_assistant_total_ms_per_forward']} | " + f"{row['avg_other_ms_per_forward']} | {row['avg_total_ms_per_forward']} | " + f"{row['cost_amortization_ratio']} | {row['mean_ratio_vs_baseline']} |" + ) + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + (Path(args.output_dir) / "diagnostic_summary.md").write_text("\n".join(lines) + "\n") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--bench-csv", nargs="+", required=True) + parser.add_argument("--trace-dir", required=True) + parser.add_argument("--output-dir", required=True) + summarize(parser.parse_args()) + + +if __name__ == "__main__": + main() diff --git a/tests/bench/e2e_cactus_decode.cpp b/tests/bench/e2e_cactus_decode.cpp new file mode 100644 index 000000000..810cb1092 --- /dev/null +++ b/tests/bench/e2e_cactus_decode.cpp @@ -0,0 +1,379 @@ +#if __has_include("cactus/cactus.h") +#include "cactus/cactus.h" +#elif __has_include("cactus_engine.h") +#include "cactus_engine.h" +#else +#error "No Cactus public header found" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct Args { + std::string model_path; + std::string model_name = "model"; + std::string prompt_name = "custom"; + std::string prompt = "Write a detailed explanation of how transformers work in deep learning."; + int max_tokens = 256; + int reps = 5; + int warmup = 2; + float temperature = 0.0f; + float top_p = 0.95f; + int top_k = 1; + float min_p = 0.15f; + int seed = 0; + int mtp_max_draft_tokens = 0; +}; + +struct Result { + int prefill_tokens = 0; + int decode_tokens = 0; + double prefill_tps = 0.0; + double decode_tps = 0.0; + double ttft_ms = 0.0; + double total_ms = 0.0; + bool mtp_requested = false; + bool mtp_enabled = false; + int mtp_drafted_tokens = 0; + int mtp_accepted_tokens = 0; + int mtp_rejected_tokens = 0; + int mtp_rounds = 0; + double mtp_assistant_draft_ms = 0.0; + double mtp_target_verify_ms = 0.0; + double mtp_sampling_or_argmax_ms = 0.0; + double mtp_kv_transaction_ms = 0.0; + double mtp_callback_stream_ms = 0.0; + std::string mtp_fallback_reason; +}; + +void usage(const char* argv0) { + std::cerr << "Usage: " << argv0 << " --model-path [options]\n" + << " --model-name Name written to CSV (default: model)\n" + << " --prompt-name Prompt label written to CSV (default: custom)\n" + << " --prompt Prompt to generate from\n" + << " --max-tokens Decode tokens per rep (default: 256)\n" + << " --reps Measured reps (default: 5)\n" + << " --warmup Warmup reps, not written to CSV (default: 2)\n" + << " --temperature Sampling temperature (default: 0)\n" + << " --top-p Top-p sampling threshold (default: 0.95)\n" + << " --top-k Top-k sampling threshold (default: 1)\n" + << " --min-p Min-p sampling threshold (default: 0.15)\n" + << " --seed Sampling seed; 0 uses random seed (default: 0)\n" + << " --mtp-max-draft Max draft tokens per MTP round; 0 disables MTP (default: 0)\n"; +} + +Args parse_args(int argc, char** argv) { + Args args; + for (int i = 1; i < argc; ++i) { + std::string arg = argv[i]; + auto need_value = [&](const char* name) -> const char* { + if (i + 1 >= argc) { + throw std::runtime_error(std::string("Missing value for ") + name); + } + return argv[++i]; + }; + + if (arg == "--model-path") { + args.model_path = need_value("--model-path"); + } else if (arg == "--model-name") { + args.model_name = need_value("--model-name"); + } else if (arg == "--prompt-name") { + args.prompt_name = need_value("--prompt-name"); + } else if (arg == "--prompt") { + args.prompt = need_value("--prompt"); + } else if (arg == "--max-tokens") { + args.max_tokens = std::stoi(need_value("--max-tokens")); + } else if (arg == "--reps") { + args.reps = std::stoi(need_value("--reps")); + } else if (arg == "--warmup") { + args.warmup = std::stoi(need_value("--warmup")); + } else if (arg == "--temperature") { + args.temperature = std::stof(need_value("--temperature")); + } else if (arg == "--top-p") { + args.top_p = std::stof(need_value("--top-p")); + } else if (arg == "--top-k") { + args.top_k = std::stoi(need_value("--top-k")); + } else if (arg == "--min-p") { + args.min_p = std::stof(need_value("--min-p")); + } else if (arg == "--seed") { + args.seed = std::stoi(need_value("--seed")); + } else if (arg == "--mtp-max-draft") { + args.mtp_max_draft_tokens = std::stoi(need_value("--mtp-max-draft")); + } else if (arg == "--help") { + usage(argv[0]); + std::exit(0); + } else { + throw std::runtime_error("Unknown argument: " + arg); + } + } + + if (args.model_path.empty()) { + throw std::runtime_error("Missing --model-path"); + } + if (args.max_tokens <= 0 || args.reps <= 0 || args.warmup < 0) { + throw std::runtime_error("--max-tokens and --reps must be positive, and --warmup must be non-negative"); + } + if (args.mtp_max_draft_tokens < 0) { + throw std::runtime_error("--mtp-max-draft must be non-negative"); + } + return args; +} + +std::string escape_json(const std::string& value) { + std::ostringstream out; + for (unsigned char c : value) { + switch (c) { + case '"': out << "\\\""; break; + case '\\': out << "\\\\"; break; + case '\b': out << "\\b"; break; + case '\f': out << "\\f"; break; + case '\n': out << "\\n"; break; + case '\r': out << "\\r"; break; + case '\t': out << "\\t"; break; + default: + if (c < 0x20) { + out << "\\u" << std::hex << std::setw(4) << std::setfill('0') << static_cast(c); + } else { + out << c; + } + } + } + return out.str(); +} + +std::string csv_escape(const std::string& value) { + bool quote = value.find_first_of(",\"\n\r") != std::string::npos; + if (!quote) return value; + + std::string out = "\""; + for (char c : value) { + if (c == '"') out += "\"\""; + else out += c; + } + out += "\""; + return out; +} + +double json_number(const char* json, const char* key) { + std::string needle = std::string("\"") + key + "\""; + const char* pos = std::strstr(json, needle.c_str()); + if (!pos) return 0.0; + pos += needle.size(); + while (*pos == ' ' || *pos == '\t' || *pos == ':' || *pos == '\n' || *pos == '\r') { + ++pos; + } + return std::strtod(pos, nullptr); +} + +bool json_bool(const char* json, const char* key) { + std::string needle = std::string("\"") + key + "\""; + const char* pos = std::strstr(json, needle.c_str()); + if (!pos) return false; + pos += needle.size(); + while (*pos == ' ' || *pos == '\t' || *pos == ':' || *pos == '\n' || *pos == '\r') { + ++pos; + } + return std::strncmp(pos, "true", 4) == 0; +} + +std::string json_string(const char* json, const char* key) { + std::string needle = std::string("\"") + key + "\""; + const char* pos = std::strstr(json, needle.c_str()); + if (!pos) return {}; + pos += needle.size(); + while (*pos == ' ' || *pos == '\t' || *pos == ':' || *pos == '\n' || *pos == '\r') { + ++pos; + } + if (*pos != '"') return {}; + ++pos; + + std::string result; + while (*pos && *pos != '"') { + if (*pos == '\\' && pos[1]) { + ++pos; + switch (*pos) { + case '"': result.push_back('"'); break; + case '\\': result.push_back('\\'); break; + case 'n': result.push_back('\n'); break; + case 'r': result.push_back('\r'); break; + case 't': result.push_back('\t'); break; + default: result.push_back(*pos); break; + } + } else { + result.push_back(*pos); + } + ++pos; + } + return result; +} + +std::string cactus_error(const char* fallback) { + const char* err = cactus_get_last_error(); + if (err && err[0]) return err; + return fallback; +} + +std::string diagnostic_shape(const Args& args) { + if (args.mtp_max_draft_tokens == 0) return "baseline"; + const char* tree_env = std::getenv("CACTUS_GEMMA4_MTP_TREE_MAIN_TOKENS"); + if (tree_env && tree_env[0]) { + char* end = nullptr; + long main_tokens = std::strtol(tree_env, &end, 10); + if (end != tree_env && *end == '\0' && main_tokens > 0 + && args.mtp_max_draft_tokens == main_tokens + 1) { + return "alt_main" + std::to_string(main_tokens); + } + } + return "base_main" + std::to_string(args.mtp_max_draft_tokens); +} + +void configure_diagnostic_trace_env(const Args& args, int rep, bool active) { + const char* trace_dir = std::getenv("CACTUS_GEMMA4_MTP_TRACE_CSV_DIR"); + if (active && trace_dir && trace_dir[0]) { + std::string shape = diagnostic_shape(args); + std::string rep_s = std::to_string(rep); + std::string run_id = args.model_name + "_" + args.prompt_name + "_" + shape + "_rep" + rep_s; + setenv("CACTUS_MTP_DIAG_RUN_ID", run_id.c_str(), 1); + setenv("CACTUS_MTP_DIAG_MODEL", args.model_name.c_str(), 1); + setenv("CACTUS_MTP_DIAG_PROMPT", args.prompt_name.c_str(), 1); + setenv("CACTUS_MTP_DIAG_SHAPE", shape.c_str(), 1); + setenv("CACTUS_MTP_DIAG_REP", rep_s.c_str(), 1); + } else { + unsetenv("CACTUS_MTP_DIAG_RUN_ID"); + unsetenv("CACTUS_MTP_DIAG_MODEL"); + unsetenv("CACTUS_MTP_DIAG_PROMPT"); + unsetenv("CACTUS_MTP_DIAG_SHAPE"); + unsetenv("CACTUS_MTP_DIAG_REP"); + } +} + +Result run_once(cactus_model_t model, const Args& args, int rep, bool trace_active) { + configure_diagnostic_trace_env(args, rep, trace_active); + cactus_reset(model); + + std::string messages = "[{\"role\":\"user\",\"content\":\"" + escape_json(args.prompt) + "\"}]"; + std::string options = "{\"temperature\":" + std::to_string(args.temperature) + + ",\"top_p\":" + std::to_string(args.top_p) + + ",\"top_k\":" + std::to_string(args.top_k) + + ",\"min_p\":" + std::to_string(args.min_p) + + ",\"max_tokens\":" + + std::to_string(args.max_tokens) + + ",\"auto_handoff\":false,\"confidence_threshold\":0.0"; + if (args.seed > 0) { + options += ",\"seed\":" + std::to_string(args.seed); + } + if (args.mtp_max_draft_tokens > 0) { + options += ",\"mtp\":true"; + options += ",\"mtp_max_draft_tokens\":" + std::to_string(args.mtp_max_draft_tokens); + } + options += ",\"stop_sequences\":[\"<|im_end|>\",\"\"]}"; + + size_t response_size = static_cast(args.max_tokens) * 512 + 65536; + std::vector response(response_size, 0); + + auto start = std::chrono::steady_clock::now(); + int rc = cactus_complete(model, messages.c_str(), response.data(), response.size(), + options.c_str(), nullptr, nullptr, nullptr, nullptr, 0); + auto end = std::chrono::steady_clock::now(); + + if (rc < 0) { + throw std::runtime_error(response.data()[0] ? response.data() : cactus_error("cactus_complete failed")); + } + + Result result; + result.prefill_tokens = static_cast(json_number(response.data(), "prefill_tokens")); + result.decode_tokens = static_cast(json_number(response.data(), "decode_tokens")); + result.prefill_tps = json_number(response.data(), "prefill_tps"); + result.decode_tps = json_number(response.data(), "decode_tps"); + result.ttft_ms = json_number(response.data(), "time_to_first_token_ms"); + result.total_ms = json_number(response.data(), "total_time_ms"); + result.mtp_requested = json_bool(response.data(), "requested"); + result.mtp_enabled = json_bool(response.data(), "enabled"); + result.mtp_drafted_tokens = static_cast(json_number(response.data(), "drafted_tokens")); + result.mtp_accepted_tokens = static_cast(json_number(response.data(), "accepted_tokens")); + result.mtp_rejected_tokens = static_cast(json_number(response.data(), "rejected_tokens")); + result.mtp_rounds = static_cast(json_number(response.data(), "rounds")); + result.mtp_assistant_draft_ms = json_number(response.data(), "assistant_draft_ms"); + result.mtp_target_verify_ms = json_number(response.data(), "target_verify_ms"); + result.mtp_sampling_or_argmax_ms = json_number(response.data(), "sampling_or_argmax_ms"); + result.mtp_kv_transaction_ms = json_number(response.data(), "kv_transaction_ms"); + result.mtp_callback_stream_ms = json_number(response.data(), "callback_stream_ms"); + result.mtp_fallback_reason = json_string(response.data(), "fallback_reason"); + if (result.total_ms <= 0.0) { + result.total_ms = std::chrono::duration(end - start).count(); + } + return result; +} + +void write_csv_row(const Args& args, int rep, const Result& result) { + std::cout << "cactus," + << csv_escape(args.model_name) << "," + << csv_escape(args.prompt_name) << "," + << diagnostic_shape(args) << "," + << args.mtp_max_draft_tokens << "," + << rep << "," + << result.prefill_tokens << "," + << result.decode_tokens << "," + << std::fixed << std::setprecision(2) + << result.prefill_tps << "," + << result.decode_tps << "," + << result.ttft_ms << "," + << result.total_ms << "," + << (result.mtp_requested ? "true" : "false") << "," + << (result.mtp_enabled ? "true" : "false") << "," + << result.mtp_drafted_tokens << "," + << result.mtp_accepted_tokens << "," + << result.mtp_rejected_tokens << "," + << result.mtp_rounds << "," + << csv_escape(result.mtp_fallback_reason) << "," + << result.mtp_assistant_draft_ms << "," + << result.mtp_target_verify_ms << "," + << result.mtp_sampling_or_argmax_ms << "," + << result.mtp_kv_transaction_ms << "," + << result.mtp_callback_stream_ms << "\n"; +} + +} // namespace + +int main(int argc, char** argv) { + try { + Args args = parse_args(argc, argv); + + std::cerr << "Loading " << args.model_name << " from " << args.model_path << "\n"; + cactus_model_t model = cactus_init(args.model_path.c_str(), nullptr, false); + if (!model) { + throw std::runtime_error(cactus_error("cactus_init failed")); + } + + for (int i = 0; i < args.warmup; ++i) { + Result warmup = run_once(model, args, i, false); + std::cerr << " warmup " << i << ": decode=" << std::fixed << std::setprecision(1) + << warmup.decode_tps << " tok/s\n"; + } + + for (int rep = 0; rep < args.reps; ++rep) { + Result result = run_once(model, args, rep, true); + std::cerr << " rep " << rep << ": decode=" << std::fixed << std::setprecision(1) + << result.decode_tps << " tok/s, prefill=" << result.prefill_tps + << " tok/s, total=" << result.total_ms << " ms\n"; + write_csv_row(args, rep, result); + } + + cactus_destroy(model); + } catch (const std::exception& e) { + std::cerr << "error: " << e.what() << "\n"; + return 1; + } + + return 0; +} diff --git a/tests/bench/e2e_models.json b/tests/bench/e2e_models.json new file mode 100644 index 000000000..791f59760 --- /dev/null +++ b/tests/bench/e2e_models.json @@ -0,0 +1,20 @@ +{ + "models": [ + { + "name": "gemma-4-e2b-it", + "path": "weights/gemma-4-e2b-it" + }, + { + "name": "gemma-4-e2b-it-cq4", + "path": "weights/gemma-4-e2b-it-cq4" + }, + { + "name": "qwen3-0.6b", + "path": "weights/qwen3-0.6b" + }, + { + "name": "gemma-3-270m-it", + "path": "weights/gemma-3-270m-it" + } + ] +} diff --git a/tests/bench_e2e_decode.sh b/tests/bench_e2e_decode.sh new file mode 100755 index 000000000..fbdfa03f3 --- /dev/null +++ b/tests/bench_e2e_decode.sh @@ -0,0 +1,429 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PYTHON="$ROOT/venv/bin/python3" +CACTUS="$ROOT/venv/bin/cactus" +CONFIG="$ROOT/tests/bench/e2e_models.json" +OUTFILE="$ROOT/e2e_decode_results.csv" +PROMPT="" +PROMPT_SUITE="mixed" +LONG_CONTEXT_REPEATS=24 +MAX_TOKENS=128 +REPS=5 +WARMUP=2 +MODEL_FILTER="" +TEMPERATURE=0 +TOP_P=0.95 +TOP_K=1 +MIN_P=0.15 +SEED=0 +MTP_MAX_DRAFTS="0" + +usage() { + cat <&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ ! -x "$PYTHON" ]]; then + echo "Missing venv Python at $PYTHON" >&2 + exit 1 +fi + +if [[ ! -x "$CACTUS" ]]; then + echo "Missing cactus CLI at $CACTUS" >&2 + exit 1 +fi + +if [[ ! -f "$CONFIG" ]]; then + echo "Model config not found: $CONFIG" >&2 + exit 1 +fi + +case "$PROMPT_SUITE" in + mixed|long_context|all) + ;; + *) + echo "--prompt-suite must be one of: mixed, long_context, all" >&2 + exit 1 + ;; +esac + +if ! [[ "$LONG_CONTEXT_REPEATS" =~ ^[0-9]+$ ]] || [[ "$LONG_CONTEXT_REPEATS" -le 0 ]]; then + echo "--long-context-repeats must be positive" >&2 + exit 1 +fi + +IFS=',' read -r -a MTP_DRAFT_RAW_VALUES <<< "$MTP_MAX_DRAFTS" +MTP_DRAFT_VALUES=() +for draft in "${MTP_DRAFT_RAW_VALUES[@]}"; do + draft="${draft//[[:space:]]/}" + if [[ -z "$draft" ]]; then + continue + fi + if ! [[ "$draft" =~ ^[0-9]+$ ]]; then + echo "--mtp-max-drafts must be a comma-separated list of non-negative integers" >&2 + exit 1 + fi + MTP_DRAFT_VALUES+=("$draft") +done + +if [[ "${#MTP_DRAFT_VALUES[@]}" -eq 0 ]]; then + echo "--mtp-max-drafts must include at least one draft value" >&2 + exit 1 +fi + +source "$ROOT/venv/bin/activate" +cactus build + +SRC="$ROOT/tests/bench/e2e_cactus_decode.cpp" +BIN="$ROOT/tests/build/e2e_cactus_decode" +LIB="$ROOT/cactus/build/libcactus.a" +mkdir -p "$ROOT/tests/build" + +if [[ ! -f "$LIB" ]]; then + echo "Cactus library not found after build: $LIB" >&2 + exit 1 +fi + +echo "Compiling e2e decode benchmark..." +if [[ "$(uname)" == "Darwin" ]]; then + CXX="${CXX:-clang++}" + if [[ -f "$ROOT/libs/curl/macos/libcurl.a" ]]; then + CURL_ARGS=("$ROOT/libs/curl/macos/libcurl.a") + elif [[ -f "$ROOT/cactus-engine/libs/curl/macos/libcurl.a" ]]; then + CURL_ARGS=("$ROOT/cactus-engine/libs/curl/macos/libcurl.a") + else + CURL_ARGS=("-lcurl") + fi + "$CXX" -std=c++20 -O3 \ + -I"$ROOT" \ + -I"$ROOT/cactus" \ + -I"$ROOT/cactus-engine" \ + -I"$ROOT/cactus-graph" \ + -I"$ROOT/cactus-kernels" \ + "$SRC" "$LIB" \ + -o "$BIN" \ + "${CURL_ARGS[@]}" \ + -framework Accelerate \ + -framework CoreML \ + -framework Foundation \ + -framework Security \ + -framework SystemConfiguration \ + -framework CFNetwork +else + CXX="${CXX:-g++}" + "$CXX" -std=c++20 -O3 \ + -I"$ROOT" \ + -I"$ROOT/cactus" \ + -I"$ROOT/cactus-engine" \ + -I"$ROOT/cactus-graph" \ + -I"$ROOT/cactus-kernels" \ + "$SRC" "$LIB" \ + -o "$BIN" \ + -lcurl \ + -pthread +fi + +PROMPT_NAMES=() +PROMPTS=() +if [[ -n "$PROMPT" ]]; then + PROMPT_NAMES=("custom") + PROMPTS=("$PROMPT") +else + mapfile -t PROMPT_ROWS < <("$PYTHON" - "$PROMPT_SUITE" "$LONG_CONTEXT_REPEATS" <<'PY' +import sys + +suite = sys.argv[1] +_repeat = int(sys.argv[2]) + +mixed = [ + ("json_object", "Return a JSON object with keys name, role, and status."), + ("python_function", "Write a short Python function that adds two numbers."), + ("explain_schrodinger", "Simply explain Schrodinger's cat."), + ("rank_tradeoffs", "Compare memory bandwidth, arithmetic intensity, and cache locality for a small-batch transformer decode step. Rank which bottleneck matters most and justify the ranking."), + ("debug_plan", "A speculative decoder is fast on repetitive prompts but slower on open-ended reasoning prompts. Give a concrete debugging plan with measurements, hypotheses, and expected failure modes."), + ("constrained_json", "Return a compact JSON object with keys decision, confidence, risks, and next_steps for whether to enable speculative decoding by default on mixed workloads."), + ("code_review", "Review this C++ snippet for correctness risks and performance traps: for (int i = 0; i <= n; ++i) { total += values[i] * scale; if (total > limit) break; }."), +] + +long_context_brief = " ".join([ + "You are reviewing a private launch-readiness memo for a mobile speech product that uses on-device decoding, speculative drafting, and an optional cloud fallback.", + "The memo covers three weeks of dogfood reports from support agents, QA engineers, and two external pilot customers.", + "It is intentionally messy: some notes are measurements, some are opinions, and some contradict each other because the teams tested different builds.", + "Do not assume that the most recent paragraph is the most accurate one.", + "In week one, the iOS team reported that ordinary dictation felt faster after speculative decoding was enabled, especially for short messages and repeated corrections.", + "Their dashboard showed median end-to-end latency down by 18 percent, but the same dashboard excluded sessions that switched languages mid-sentence.", + "The Android team saw a smaller 7 percent latency gain and warned that battery drain rose during long meetings.", + "A QA note says the Android sample was skewed toward older phones with thermal throttling already active.", + "The platform lead argued that the battery signal should not block the launch because the device was already under stress before the decoder started.", + "A support lead disagreed and wrote that users do not care whether the extra heat was caused by the decoder or by background sync.", + "The assistant draft model improved speed on templated emails, calendar edits, and short search queries.", + "It did poorly on legal dictation, medical terminology, code snippets, and speech with named entities that appeared only once.", + "One pilot customer, Northstar Clinics, said the model often accepted the first two words of a phrase and then rejected the third, causing a subtle rhythm change that made clinicians pause.", + "Another pilot, Harbor Logistics, praised the same rhythm because warehouse notes often reused stock phrases.", + "The product manager wants one default policy, but the field team wants a policy that depends on context length, domain, and recent rejection rate.", + "A kernel engineer traced the first multi-token verify path and found that target verification was dominated by a small-batch quantized matmul kernel, not attention.", + "After a focused optimization, the M equals 2 verifier improved enough that one-draft speculation became profitable on mixed internal prompts.", + "However, the benchmark suite still contained repetitive prompts that gave unrealistically high acceptance, including simple counting and identity questions.", + "The benchmark owner removed those prompts and added harder tasks involving JSON structure, debugging plans, code review, and tradeoff ranking.", + "Those harder prompts reduced acceptance but still showed a modest average speedup.", + "The cloud fallback classifier is not ready for launch.", + "It catches some low-confidence sessions, but it also sends too many private utterances to the cloud when users dictate names, addresses, and support case numbers.", + "Legal approved the fallback only if it is opt-in for external pilots and disabled for regulated healthcare accounts.", + "The privacy reviewer also asked for clearer logs explaining why a session was handed off.", + "The model quality team found three correctness risks.", + "First, speculative acceptance was too high on boilerplate where the assistant copied a phrase but missed a negation later in the sentence.", + "Second, sampled decoding occasionally diverged after a rejected draft because a cache rollback bug had survived in an experimental branch.", + "Third, benchmark summaries sometimes mixed prefill throughput with decode throughput and made the launch look safer than it was.", + "The cache rollback bug is fixed on main, but the team has not rerun every long-context smoke test since the fix.", + "The release manager is deciding between three options.", + "Option A ships MTP enabled by default for all non-healthcare customers, with cloud fallback disabled.", + "Option B ships MTP disabled by default while allowing pilots to enable a fixed draft limit after local validation.", + "Option C delays MTP and ships only the baseline decoder plus extra telemetry.", + "Marketing prefers option A because it gives the clearest speed story.", + "Support prefers option B because it gives them a reasoned answer when customers report odd pauses.", + "The infrastructure team prefers option C because telemetry dashboards are still hard to interpret.", + "A senior engineer wrote that option B is the only plan that respects both the kernel win and the remaining product risk.", + "The same engineer warned that unclear rollout criteria can hide regressions, so dashboards must report both enabled time and accepted-token rate.", + "A customer success manager added that pilots will forgive a small speedup if the product behaves consistently, but they will escalate quickly if the cursor appears to hesitate after rare names or compliance terms.", + "The QA lead wants one more run that separates long-context dictation from short command-and-control tasks because the current aggregate hides the exact category where users notice pauses.", + "The engineering manager agreed, but said the launch decision cannot wait for a perfect benchmark and must rely on the best risk-adjusted evidence available this week.", + "Your task: using only this memo, recommend one launch option, identify the two strongest technical reasons for that option, identify the strongest objection to it, and specify the first three measurements that must appear on the launch dashboard.", +]) + +long_context = [ + ( + "long_context_1k_triage", + long_context_brief, + ), +] + +if suite == "mixed": + selected = mixed +elif suite == "long_context": + selected = long_context +elif suite == "all": + selected = mixed + long_context +else: + raise SystemExit(f"Unknown prompt suite: {suite}") + +for name, prompt in selected: + if "\t" in name or "\t" in prompt or "\n" in prompt: + raise SystemExit("Prompt names and prompts must be single-line TSV-safe strings") + print(f"{name}\t{prompt}") +PY +) + for row in "${PROMPT_ROWS[@]}"; do + IFS=$'\t' read -r prompt_name prompt_text <<< "$row" + PROMPT_NAMES+=("$prompt_name") + PROMPTS+=("$prompt_text") + done +fi + +if [[ "${#PROMPTS[@]}" -eq 0 ]]; then + echo "No prompts selected" >&2 + exit 1 +fi + +echo "backend,model,prompt,shape,mtp_max_draft,rep,prefill_tokens,decode_tokens,prefill_tps,decode_tps,ttft_ms,total_ms,mtp_requested,mtp_enabled,mtp_drafted_tokens,mtp_accepted_tokens,mtp_rejected_tokens,mtp_rounds,mtp_fallback_reason,assistant_draft_ms,target_verify_ms,sampling_or_argmax_ms,kv_transaction_ms,callback_stream_ms" > "$OUTFILE" + +mapfile -t MODELS < <("$PYTHON" - "$ROOT" "$CONFIG" "$MODEL_FILTER" <<'PY' +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]).resolve() +config_path = Path(sys.argv[2]).expanduser() +model_filter = sys.argv[3] + +if not config_path.is_absolute(): + config_path = (root / config_path).resolve() + +with config_path.open("r", encoding="utf-8") as f: + config = json.load(f) + +for model in config.get("models", []): + name = model.get("name", "") + if model_filter and name != model_filter: + continue + path = model.get("path") + if path is None: + variants = model.get("variants", {}) + path = variants.get("cactus") + if not name or not path: + continue + path = Path(path).expanduser() + if not path.is_absolute(): + path = (root / path).resolve() + print(f"{name}\t{path}") +PY +) + +ran=0 +for entry in "${MODELS[@]}"; do + IFS=$'\t' read -r model_name model_path <<< "$entry" + if [[ ! -d "$model_path" ]]; then + echo "Skipping $model_name; missing $model_path" + continue + fi + + for idx in "${!PROMPTS[@]}"; do + prompt_name="${PROMPT_NAMES[$idx]}" + prompt_text="${PROMPTS[$idx]}" + for draft in "${MTP_DRAFT_VALUES[@]}"; do + echo "" + echo "Benchmarking $model_name / $prompt_name / draft $draft" + "$BIN" \ + --model-name "$model_name" \ + --model-path "$model_path" \ + --prompt-name "$prompt_name" \ + --prompt "$prompt_text" \ + --max-tokens "$MAX_TOKENS" \ + --reps "$REPS" \ + --warmup "$WARMUP" \ + --temperature "$TEMPERATURE" \ + --top-p "$TOP_P" \ + --top-k "$TOP_K" \ + --min-p "$MIN_P" \ + --seed "$SEED" \ + --mtp-max-draft "$draft" >> "$OUTFILE" + done + done + ran=$((ran + 1)) +done + +if [[ "$ran" -eq 0 ]]; then + echo "No configured model paths were found. Edit $CONFIG or pass --config." >&2 + exit 1 +fi + +echo "" +echo "Results saved to $OUTFILE" +"$PYTHON" - "$OUTFILE" <<'PY' +import csv +import statistics +import sys +from collections import defaultdict + +rows = list(csv.DictReader(open(sys.argv[1], newline=""))) +by_case = defaultdict(list) +for row in rows: + by_case[(row["model"], row.get("prompt", "custom"), row.get("shape") or row.get("mtp_max_draft", ""))].append(row) + +print() +print(f"{'Model':<20} {'Prompt':<18} {'Shape':>10} {'Mean TPS':>10} {'Min TPS':>9} {'+/-':>8} {'Accept %':>9} {'Rejects':>8} {'TTFT ms':>10} {'Fallback':<18}") +print("-" * 130) +for (model, prompt, shape), vals in by_case.items(): + decode = [float(v["decode_tps"]) for v in vals] + ttft = [float(v["ttft_ms"]) for v in vals] + drafted = sum(float(v.get("mtp_drafted_tokens", 0) or 0) for v in vals) + accepted = sum(float(v.get("mtp_accepted_tokens", 0) or 0) for v in vals) + rejected = sum(float(v.get("mtp_rejected_tokens", 0) or 0) for v in vals) + d_mean = statistics.mean(decode) + d_min = min(decode) + d_std = statistics.stdev(decode) if len(decode) > 1 else 0.0 + t_mean = statistics.mean(ttft) + accept_rate = (accepted / drafted * 100.0) if drafted else 0.0 + fallbacks = sorted({v.get("mtp_fallback_reason", "") for v in vals if v.get("mtp_fallback_reason", "")}) + fallback = "|".join(fallbacks)[:18] + print(f"{model:<20} {prompt:<18} {shape:>10} {d_mean:>10.1f} {d_min:>9.1f} {d_std:>8.1f} {accept_rate:>8.1f}% {rejected:>8.0f} {t_mean:>10.1f} {fallback:<18}") +PY diff --git a/tests/mtp/gemma4_mtp_cases.json b/tests/mtp/gemma4_mtp_cases.json new file mode 100644 index 000000000..78cce14b2 --- /dev/null +++ b/tests/mtp/gemma4_mtp_cases.json @@ -0,0 +1,41 @@ +{ + "max_runtime_seconds": 180, + "model_env": { + "target": "CACTUS_TEST_GEMMA4_TARGET", + "assistant": "CACTUS_TEST_GEMMA4_ASSISTANT" + }, + "cases": [ + { + "name": "greedy_parity", + "temperature": 0.0, + "max_tokens": 24, + "prompt": "Write one short sentence about desert rain.", + "assertions": [ + "mtp token ids equal standard greedy token ids", + "mtp metrics are present" + ] + }, + { + "name": "seeded_sampling_smoke", + "temperature": 0.7, + "top_k": 64, + "top_p": 0.95, + "seed": 1234, + "max_tokens": 16, + "prompt": "Write one creative sentence about desert rain.", + "assertions": [ + "fixed seed produces deterministic token ids", + "mtp metrics are present" + ] + }, + { + "name": "missing_required_assistant", + "temperature": 0.0, + "max_tokens": 8, + "prompt": "Say hello in three words.", + "assertions": [ + "mtp_required with a missing assistant does not silently fall back" + ] + } + ] +} diff --git a/tests/mtp/run_fast_gemma4_mtp.sh b/tests/mtp/run_fast_gemma4_mtp.sh new file mode 100755 index 000000000..4200fca35 --- /dev/null +++ b/tests/mtp/run_fast_gemma4_mtp.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +TIMEOUT_SECONDS=180 +MAX_TOKENS=24 + +usage() { + cat <&2 + usage >&2 + exit 1 + ;; + esac +done + +source "$ROOT/venv/bin/activate" +export CACTUS_TEST_GEMMA4_MAX_TOKENS="$MAX_TOKENS" +cactus build + +JOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)" + +mkdir -p "$ROOT/cactus-engine/tests/build" +cmake -S "$ROOT/cactus-engine/tests" -B "$ROOT/cactus-engine/tests/build" \ + -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF >/dev/null +cmake --build "$ROOT/cactus-engine/tests/build" -j "$JOBS" + +mkdir -p "$ROOT/cactus-graph/build" +cmake -S "$ROOT/cactus-graph" -B "$ROOT/cactus-graph/build" \ + -DCACTUS_BUILD_TESTS=ON -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF >/dev/null +cmake --build "$ROOT/cactus-graph/build" -j "$JOBS" + +run_with_timeout() { + local exe="$1" + if [[ ! -x "$exe" ]]; then + echo "Missing test executable: $exe" >&2 + exit 1 + fi + + if command -v timeout >/dev/null 2>&1; then + timeout "$TIMEOUT_SECONDS" "$exe" + else + "$exe" + fi +} + +ENGINE_BUILD="$ROOT/cactus-engine/tests/build" +GRAPH_BUILD="$ROOT/cactus-graph/build" + +echo "Running Gemma 4 MTP fast test suite with max_tokens=$MAX_TOKENS" + +run_with_timeout "$ENGINE_BUILD/test_mtp_sampler" +run_with_timeout "$GRAPH_BUILD/test_cache_transaction" +run_with_timeout "$ENGINE_BUILD/test_decode_strategy" +run_with_timeout "$ENGINE_BUILD/test_mtp_completion" +run_with_timeout "$ENGINE_BUILD/test_gemma4_mtp_fast" From 95272c26f26774df5738a877355c3a26d2044906 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 15 May 2026 01:24:08 -0700 Subject: [PATCH 2/2] Refactor MTP decode primitives --- cactus-engine/CMakeLists.txt | 1 - .../models/gemma4/gemma4_mtp_assistant.cpp | 292 +++++----- .../models/gemma4/gemma4_mtp_assistant.h | 22 + cactus-engine/models/gemma4/model_gemma4.cpp | 510 +++++++++++++----- cactus-engine/models/gemma4/model_gemma4.h | 18 + .../models/gemma4/model_gemma4_mm.cpp | 4 + cactus-engine/src/complete.cpp | 226 +------- cactus-engine/src/decode_strategy.h | 134 ----- cactus-engine/src/engine.h | 14 + cactus-engine/src/gemma4_mtp_decode.cpp | 411 -------------- cactus-engine/src/gemma4_mtp_decode.h | 111 ---- cactus-engine/src/mtp_completion.h | 189 ++++++- cactus-engine/src/mtp_decode.h | 206 +++++++ cactus-engine/src/mtp_sampler.h | 248 ++++++--- cactus-engine/src/utils.h | 2 +- cactus-engine/tests/test_decode_strategy.cpp | 165 ------ .../test_gemma4_mtp_assistant_config.cpp | 55 ++ cactus-engine/tests/test_mtp_completion.cpp | 15 + cactus-engine/tests/test_mtp_sampler.cpp | 71 ++- cactus-graph/cactus_graph.h | 5 + cactus-graph/src/cache_transaction.cpp | 24 + cactus-graph/tests/test_cache_transaction.cpp | 23 + tests/mtp/run_fast_gemma4_mtp.sh | 1 - 23 files changed, 1337 insertions(+), 1410 deletions(-) delete mode 100644 cactus-engine/src/decode_strategy.h delete mode 100644 cactus-engine/src/gemma4_mtp_decode.cpp delete mode 100644 cactus-engine/src/gemma4_mtp_decode.h create mode 100644 cactus-engine/src/mtp_decode.h delete mode 100644 cactus-engine/tests/test_decode_strategy.cpp create mode 100644 cactus-engine/tests/test_gemma4_mtp_assistant_config.cpp diff --git a/cactus-engine/CMakeLists.txt b/cactus-engine/CMakeLists.txt index fb078ce81..4a98c470a 100644 --- a/cactus-engine/CMakeLists.txt +++ b/cactus-engine/CMakeLists.txt @@ -69,7 +69,6 @@ set(ENGINE_SOURCES src/init.cpp src/embed.cpp src/complete.cpp - src/gemma4_mtp_decode.cpp src/transcribe.cpp src/tokenizer.cpp src/npu.cpp diff --git a/cactus-engine/models/gemma4/gemma4_mtp_assistant.cpp b/cactus-engine/models/gemma4/gemma4_mtp_assistant.cpp index dbb700e93..4f94afe36 100644 --- a/cactus-engine/models/gemma4/gemma4_mtp_assistant.cpp +++ b/cactus-engine/models/gemma4/gemma4_mtp_assistant.cpp @@ -1,4 +1,5 @@ #include "gemma4_mtp_assistant.h" +#include "src/mtp_sampler.h" #include #include @@ -16,81 +17,6 @@ namespace engine { namespace { -constexpr size_t kTargetHiddenDim = 1536; -constexpr size_t kCentroidCount = 2048; -constexpr size_t kTopCentroidCount = 32; -constexpr size_t kTokensPerCentroid = 128; -constexpr size_t kTokenOrderingSize = kCentroidCount * kTokensPerCentroid; - -std::vector probs_from_logits(std::vector logits, float temperature, float top_p, size_t top_k, float min_p) { - if (logits.empty()) return {}; - if (temperature == 0.0f) { - std::vector probs(logits.size(), 0.0f); - auto it = std::max_element(logits.begin(), logits.end()); - probs[std::distance(logits.begin(), it)] = 1.0f; - return probs; - } - if (temperature > 0.0f) { - for (float& value : logits) value /= temperature; - } - if (top_k > 0 && top_k < logits.size()) { - std::vector sorted = logits; - std::nth_element(sorted.begin(), sorted.begin() + top_k - 1, sorted.end(), std::greater()); - float kth = sorted[top_k - 1]; - for (float& value : logits) { - if (value < kth) value = -std::numeric_limits::infinity(); - } - } - auto normalize = [](const std::vector& values) { - std::vector probs(values.size(), 0.0f); - float max_value = -std::numeric_limits::infinity(); - for (float value : values) max_value = std::max(max_value, value); - if (!std::isfinite(max_value)) return probs; - double sum = 0.0; - for (size_t i = 0; i < values.size(); ++i) { - if (std::isfinite(values[i])) { - probs[i] = std::exp(static_cast(values[i] - max_value)); - sum += probs[i]; - } - } - if (sum <= 0.0 || !std::isfinite(sum)) return probs; - for (float& prob : probs) prob = static_cast(prob / sum); - return probs; - }; - if (min_p > 0.0f) { - auto probs = normalize(logits); - float threshold = (*std::max_element(probs.begin(), probs.end())) * min_p; - for (size_t i = 0; i < logits.size(); ++i) { - if (probs[i] < threshold) logits[i] = -std::numeric_limits::infinity(); - } - } - if (top_p > 0.0f && top_p < 1.0f) { - std::vector> sorted; - sorted.reserve(logits.size()); - for (size_t i = 0; i < logits.size(); ++i) { - if (std::isfinite(logits[i])) sorted.emplace_back(logits[i], i); - } - std::sort(sorted.begin(), sorted.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); - std::vector sorted_logits; - sorted_logits.reserve(sorted.size()); - for (const auto& item : sorted) sorted_logits.push_back(item.first); - auto sorted_probs = normalize(sorted_logits); - float cumulative = 0.0f; - bool remove = false; - for (size_t i = 0; i < sorted.size(); ++i) { - cumulative += sorted_probs[i]; - if (remove) logits[sorted[i].second] = -std::numeric_limits::infinity(); - if (cumulative > top_p) remove = true; - } - } - auto probs = normalize(logits); - float sum = std::accumulate(probs.begin(), probs.end(), 0.0f); - if (sum <= 0.0f || !std::isfinite(sum)) { - std::fill(probs.begin(), probs.end(), 1.0f / static_cast(probs.size())); - } - return probs; -} - std::vector> sparse_probs_from_logits_row( const void* logits_ptr, Precision precision, size_t row_offset, size_t vocab_size, float temperature, float top_p, size_t top_k, float min_p) { @@ -123,68 +49,118 @@ std::vector> sparse_probs_from_logits_row( logits.emplace_back(top.top().second, top.top().first); top.pop(); } - if (temperature > 0.0f) { - for (auto& item : logits) item.second /= temperature; - } - std::sort(logits.begin(), logits.end(), [](const auto& a, const auto& b) { - return a.second > b.second; + return mtp_sparse_distribution_from_logits(std::move(logits), MtpSamplingOptions{ + .temperature = temperature, + .top_p = top_p, + .top_k = top_k, + .min_p = min_p, }); +} - auto normalize = [](const std::vector>& values) { - std::vector> probs; - probs.reserve(values.size()); - float max_value = -std::numeric_limits::infinity(); - for (const auto& item : values) max_value = std::max(max_value, item.second); - if (!std::isfinite(max_value)) return probs; - double sum = 0.0; - for (const auto& item : values) { - float prob = std::exp(static_cast(item.second - max_value)); - probs.emplace_back(item.first, prob); - sum += prob; - } - if (sum <= 0.0 || !std::isfinite(sum)) return std::vector>{}; - for (auto& item : probs) item.second = static_cast(item.second / sum); - return probs; - }; +size_t json_size_field(const std::string& json, const std::string& key, size_t fallback) { + size_t pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return fallback; + pos = json.find(':', pos); + if (pos == std::string::npos) return fallback; + char* end = nullptr; + unsigned long value = std::strtoul(json.c_str() + pos + 1, &end, 10); + return end == json.c_str() + pos + 1 ? fallback : static_cast(value); +} + +float json_float_field(const std::string& json, const std::string& key, float fallback) { + size_t pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return fallback; + pos = json.find(':', pos); + if (pos == std::string::npos) return fallback; + char* end = nullptr; + float value = std::strtof(json.c_str() + pos + 1, &end); + return end == json.c_str() + pos + 1 ? fallback : value; +} - if (min_p > 0.0f) { - auto probs = normalize(logits); - if (!probs.empty()) { - float max_prob = 0.0f; - for (const auto& item : probs) max_prob = std::max(max_prob, item.second); - float threshold = max_prob * min_p; - std::vector> filtered; - filtered.reserve(logits.size()); - for (size_t i = 0; i < logits.size(); ++i) { - if (probs[i].second >= threshold) filtered.push_back(logits[i]); +bool json_bool_field(const std::string& json, const std::string& key, bool fallback) { + size_t pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return fallback; + pos = json.find(':', pos); + if (pos == std::string::npos) return fallback; + while (++pos < json.size() && std::isspace(static_cast(json[pos]))) {} + if (json.compare(pos, 4, "true") == 0) return true; + if (json.compare(pos, 5, "false") == 0) return false; + return fallback; +} + +std::vector json_object_array(const std::string& json, const std::string& key) { + std::vector objects; + size_t pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return objects; + size_t array_start = json.find('[', pos); + if (array_start == std::string::npos) return objects; + int depth = 0; + size_t object_start = std::string::npos; + for (size_t i = array_start + 1; i < json.size(); ++i) { + if (json[i] == '{') { + if (depth == 0) object_start = i; + depth++; + } else if (json[i] == '}') { + depth--; + if (depth == 0 && object_start != std::string::npos) { + objects.push_back(json.substr(object_start, i - object_start + 1)); + object_start = std::string::npos; } - logits = std::move(filtered); - } - } - if (top_p > 0.0f && top_p < 1.0f) { - auto probs = normalize(logits); - std::vector> filtered; - filtered.reserve(logits.size()); - float cumulative = 0.0f; - bool remove = false; - for (size_t i = 0; i < logits.size(); ++i) { - cumulative += i < probs.size() ? probs[i].second : 0.0f; - if (!remove) filtered.push_back(logits[i]); - if (cumulative > top_p) remove = true; + } else if (json[i] == ']' && depth == 0) { + break; } - logits = std::move(filtered); } + return objects; +} - auto probs = normalize(logits); - float sum = 0.0f; - for (const auto& item : probs) sum += item.second; - if (sum <= 0.0f || !std::isfinite(sum)) { - float uniform = probs.empty() ? 0.0f : 1.0f / static_cast(probs.size()); - for (auto& item : probs) item.second = uniform; - } - return probs; } +Gemma4MtpAssistantConfig default_gemma4_mtp_assistant_config() { + Gemma4MtpAssistantConfig config; + config.layers = { + Gemma4MtpAssistantLayerConfig{}, + Gemma4MtpAssistantLayerConfig{}, + Gemma4MtpAssistantLayerConfig{}, + Gemma4MtpAssistantLayerConfig{ + .head_dim = 512, + .num_heads = 4, + .rot_dim = 128, + .rope_freq = 1000000.0f, + .window = 0, + .full_attention = true, + }, + }; + return config; +} + +Gemma4MtpAssistantConfig parse_gemma4_mtp_assistant_config(const std::string& manifest_json) { + Gemma4MtpAssistantConfig config = default_gemma4_mtp_assistant_config(); + config.target_hidden_dim = json_size_field(manifest_json, "target_hidden_dim", config.target_hidden_dim); + config.centroid_count = json_size_field(manifest_json, "centroid_count", config.centroid_count); + config.top_centroid_count = json_size_field(manifest_json, "top_centroid_count", config.top_centroid_count); + config.tokens_per_centroid = json_size_field(manifest_json, "tokens_per_centroid", config.tokens_per_centroid); + + auto layer_objects = json_object_array(manifest_json, "layers"); + if (!layer_objects.empty()) { + auto defaults = default_gemma4_mtp_assistant_config().layers; + config.layers.clear(); + config.layers.reserve(layer_objects.size()); + for (size_t i = 0; i < layer_objects.size(); ++i) { + Gemma4MtpAssistantLayerConfig layer = i < defaults.size() + ? defaults[i] + : Gemma4MtpAssistantLayerConfig{}; + const auto& object = layer_objects[i]; + layer.head_dim = json_size_field(object, "head_dim", layer.head_dim); + layer.num_heads = json_size_field(object, "num_heads", layer.num_heads); + layer.rot_dim = json_size_field(object, "rot_dim", layer.rot_dim); + layer.rope_freq = json_float_field(object, "rope_freq", layer.rope_freq); + layer.window = json_size_field(object, "window", layer.window); + layer.full_attention = json_bool_field(object, "full_attention", layer.full_attention); + config.layers.push_back(layer); + } + } + + return config; } bool Gemma4MtpAssistant::init(CactusGraph* graph, const std::string& assistant_path) { @@ -209,13 +185,17 @@ bool Gemma4MtpAssistant::init(CactusGraph* graph, const std::string& assistant_p graph_ = graph; path_ = resolved_path.string(); + std::ifstream manifest_file(path_ + "/assistant_cactus_manifest.json"); + std::string manifest_json((std::istreambuf_iterator(manifest_file)), std::istreambuf_iterator()); + config_ = parse_gemma4_mtp_assistant_config(manifest_json); + token_embeddings_ = graph_->mmap_embeddings(path_ + "/token_embeddings.weights"); masked_centroids_ = graph_->mmap_weights(path_ + "/masked_embedding_centroids.weights"); pre_projection_ = graph_->mmap_weights(path_ + "/pre_projection.weights"); post_projection_ = graph_->mmap_weights(path_ + "/post_projection.weights"); output_norm_ = graph_->mmap_weights(path_ + "/output_norm.weights"); - layers_.resize(4); + layers_.resize(config_.layers.size()); for (uint32_t i = 0; i < layers_.size(); ++i) { auto& layer = layers_[i]; std::string prefix = path_ + "/layer_" + std::to_string(i) + "_"; @@ -241,7 +221,8 @@ bool Gemma4MtpAssistant::init(CactusGraph* graph, const std::string& assistant_p throw std::runtime_error("Invalid masked_embedding_token_ordering.json"); } token_ordering_.clear(); - token_ordering_.reserve(kTokenOrderingSize); + const size_t expected_ordering_size = config_.centroid_count * config_.tokens_per_centroid; + token_ordering_.reserve(expected_ordering_size); size_t pos = array_start + 1; while (pos < array_end) { while (pos < array_end @@ -257,7 +238,7 @@ bool Gemma4MtpAssistant::init(CactusGraph* graph, const std::string& assistant_p token_ordering_.push_back(static_cast(value)); pos = static_cast(end_ptr - ordering_json.c_str()); } - if (token_ordering_.size() != kTokenOrderingSize) { + if (token_ordering_.size() != expected_ordering_size) { throw std::runtime_error("Unexpected Gemma 4 assistant token ordering size"); } @@ -292,14 +273,14 @@ size_t Gemma4MtpAssistant::apply_rope(CactusGraph* gb, size_t tensor, size_t hea size_t Gemma4MtpAssistant::build_layer(CactusGraph* gb, size_t hidden, uint32_t layer_idx, const Gemma4Model::SharedCacheNodes& cache_nodes, size_t position) { const auto& layer = layers_.at(layer_idx); - const bool is_full = layer_idx == 3; - const size_t head_dim = is_full ? 512 : 256; - const size_t num_heads = 4; - const size_t rot_dim = is_full ? 128 : head_dim; - const float rope_freq = is_full ? 1000000.0f : 10000.0f; - const size_t window = is_full ? 0 : 512; - const size_t cache_k = is_full ? cache_nodes.full_k : cache_nodes.sliding_k; - const size_t cache_v = is_full ? cache_nodes.full_v : cache_nodes.sliding_v; + const auto& layer_config = config_.layers.at(layer_idx); + const size_t head_dim = layer_config.head_dim; + const size_t num_heads = layer_config.num_heads; + const size_t rot_dim = layer_config.rot_dim; + const float rope_freq = layer_config.rope_freq; + const size_t window = layer_config.window; + const size_t cache_k = layer_config.full_attention ? cache_nodes.full_k : cache_nodes.sliding_k; + const size_t cache_v = layer_config.full_attention ? cache_nodes.full_v : cache_nodes.sliding_v; if (cache_k == 0 || cache_v == 0) { throw std::runtime_error("Gemma 4 MTP target shared KV cache is unavailable"); } @@ -330,7 +311,7 @@ std::vector Gemma4MtpAssistant::candidate_tokens_from_centroids(Cactus const auto& centroid_buf = gb->get_output_buffer(centroid_logits_node); void* centroid_ptr = gb->get_output(centroid_logits_node); const size_t centroid_count = centroid_buf.total_size; - if (centroid_count == 0 || centroid_count * kTokensPerCentroid > token_ordering_.size()) { + if (centroid_count == 0 || centroid_count * config_.tokens_per_centroid > token_ordering_.size()) { throw std::runtime_error("Gemma 4 MTP assistant centroid/token ordering shape mismatch"); } @@ -347,15 +328,15 @@ std::vector Gemma4MtpAssistant::candidate_tokens_from_centroids(Cactus } centroids.emplace_back(value, i); } - const size_t top_count = std::min(kTopCentroidCount, centroids.size()); + const size_t top_count = std::min(config_.top_centroid_count, centroids.size()); std::partial_sort(centroids.begin(), centroids.begin() + top_count, centroids.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); std::vector candidates; - candidates.reserve(top_count * kTokensPerCentroid); + candidates.reserve(top_count * config_.tokens_per_centroid); for (size_t c = 0; c < top_count; ++c) { - size_t offset = static_cast(centroids[c].second) * kTokensPerCentroid; - for (size_t i = 0; i < kTokensPerCentroid; ++i) { + size_t offset = static_cast(centroids[c].second) * config_.tokens_per_centroid; + for (size_t i = 0; i < config_.tokens_per_centroid; ++i) { candidates.push_back(token_ordering_[offset + i]); } } @@ -464,13 +445,13 @@ Gemma4MtpAssistant::StepResult Gemma4MtpAssistant::draft_one( if (!initialized_ || !graph_) { throw std::runtime_error("Gemma4MtpAssistant is not initialized"); } - if (target_hidden.size() != kTargetHiddenDim) { - throw std::runtime_error("Gemma4MtpAssistant expects 1536-wide target hidden vectors"); + if (target_hidden.size() != config_.target_hidden_dim) { + throw std::runtime_error("Gemma4MtpAssistant target hidden vector width mismatch"); } auto* gb = graph_; gb->soft_reset_keep_pool(); - float embedding_scale = std::sqrt(static_cast(kTargetHiddenDim)) * 16.0f; + float embedding_scale = std::sqrt(static_cast(config_.target_hidden_dim)) * 16.0f; if (const char* value = std::getenv("CACTUS_GEMMA4_MTP_EMBED_SCALE")) { embedding_scale *= std::strtof(value, nullptr); } @@ -482,7 +463,7 @@ Gemma4MtpAssistant::StepResult Gemma4MtpAssistant::draft_one( gb->set_input(token_input, &token_value, Precision::FP32); auto target_embedding = gb->scalar_multiply(gb->embedding(target_embedding_node, token_input), embedding_scale); - auto hidden_input = gb->input({1, kTargetHiddenDim}, Precision::FP16); + auto hidden_input = gb->input({1, config_.target_hidden_dim}, Precision::FP16); gb->set_input(hidden_input, target_hidden.data(), Precision::FP16); if (hidden_scale != 1.0f) { hidden_input = gb->scalar_multiply(hidden_input, hidden_scale); @@ -546,22 +527,27 @@ Gemma4MtpAssistant::StepResult Gemma4MtpAssistant::draft_one( const int8_t* src = static_cast(logits_ptr) + row_offset; for (size_t i = 0; i < vocab_size; ++i) logits_values[i] = static_cast(src[i]); } - result.probabilities = probs_from_logits(std::move(logits_values), temperature, top_p, top_k, min_p); + result.probabilities = mtp_distribution_from_logits(std::move(logits_values), MtpSamplingOptions{ + .temperature = temperature, + .top_p = top_p, + .top_k = top_k, + .min_p = min_p, + }).probabilities; } } if (return_hidden) { - result.hidden.resize(kTargetHiddenDim); + result.hidden.resize(config_.target_hidden_dim); const auto& projected_buf = gb->get_output_buffer(projected); void* projected_ptr = gb->get_output(projected); if (projected_buf.precision == Precision::FP16) { const __fp16* src = static_cast(projected_ptr); - std::copy(src, src + kTargetHiddenDim, result.hidden.begin()); + std::copy(src, src + config_.target_hidden_dim, result.hidden.begin()); } else if (projected_buf.precision == Precision::FP32) { const float* src = static_cast(projected_ptr); - for (size_t i = 0; i < kTargetHiddenDim; ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + for (size_t i = 0; i < config_.target_hidden_dim; ++i) result.hidden[i] = static_cast<__fp16>(src[i]); } else { const int8_t* src = static_cast(projected_ptr); - for (size_t i = 0; i < kTargetHiddenDim; ++i) result.hidden[i] = static_cast<__fp16>(src[i]); + for (size_t i = 0; i < config_.target_hidden_dim; ++i) result.hidden[i] = static_cast<__fp16>(src[i]); } } return result; diff --git a/cactus-engine/models/gemma4/gemma4_mtp_assistant.h b/cactus-engine/models/gemma4/gemma4_mtp_assistant.h index 47ae3c933..12e6f5d58 100644 --- a/cactus-engine/models/gemma4/gemma4_mtp_assistant.h +++ b/cactus-engine/models/gemma4/gemma4_mtp_assistant.h @@ -10,6 +10,26 @@ namespace cactus { namespace engine { +struct Gemma4MtpAssistantLayerConfig { + size_t head_dim = 256; + size_t num_heads = 4; + size_t rot_dim = 256; + float rope_freq = 10000.0f; + size_t window = 512; + bool full_attention = false; +}; + +struct Gemma4MtpAssistantConfig { + size_t target_hidden_dim = 1536; + size_t centroid_count = 2048; + size_t top_centroid_count = 32; + size_t tokens_per_centroid = 128; + std::vector layers; +}; + +Gemma4MtpAssistantConfig default_gemma4_mtp_assistant_config(); +Gemma4MtpAssistantConfig parse_gemma4_mtp_assistant_config(const std::string& manifest_json); + class Gemma4MtpAssistant { public: struct StepResult { @@ -23,6 +43,7 @@ class Gemma4MtpAssistant { bool init(CactusGraph* graph, const std::string& assistant_path); bool initialized() const { return initialized_; } + const Gemma4MtpAssistantConfig& config() const { return config_; } StepResult draft_one(uint32_t target_token, size_t target_embedding_node, @@ -70,6 +91,7 @@ class Gemma4MtpAssistant { size_t pre_projection_ = 0; size_t post_projection_ = 0; size_t output_norm_ = 0; + Gemma4MtpAssistantConfig config_; std::vector layers_; std::vector token_ordering_; }; diff --git a/cactus-engine/models/gemma4/model_gemma4.cpp b/cactus-engine/models/gemma4/model_gemma4.cpp index e4d374e29..25bc7a9b9 100644 --- a/cactus-engine/models/gemma4/model_gemma4.cpp +++ b/cactus-engine/models/gemma4/model_gemma4.cpp @@ -1,6 +1,9 @@ #include "model_gemma4.h" +#include "gemma4_mtp_assistant.h" #include "cactus_graph.h" +#include "src/mtp_sampler.h" #include +#include #include #include #include @@ -18,6 +21,360 @@ Gemma4Model::Gemma4Model(const Config& config) : Model(config) { weight_nodes_.layers.resize(config.num_layers); } +class Gemma4MtpCapability final : public SpeculativeDecodeCapability { +public: + Gemma4MtpCapability(Model& owner, Gemma4Model& target) + : owner_(owner), target_(target) {} + + std::string strategy_name() const override { return "gemma4_mtp"; } + + std::string default_assistant_path() const override { + return (std::filesystem::path(owner_.get_model_folder_path()) / "assistant").string(); + } + + Model* target_model() override { return &target_; } + +private: + Model& owner_; + Gemma4Model& target_; +}; + +SpeculativeDecodeCapability* Gemma4Model::speculative_decode_capability() { + if (!speculative_capability_) { + speculative_capability_ = std::make_unique(*this, *this); + } + return speculative_capability_.get(); +} + +namespace { + +std::vector<__fp16> gemma4_mtp_hidden_row(const Gemma4Model::GreedyBatchWithHidden& batch, size_t row) { + if (batch.hidden_dim == 0 || row >= batch.hidden.size() / batch.hidden_dim) { + return {}; + } + return std::vector<__fp16>( + batch.hidden.begin() + row * batch.hidden_dim, + batch.hidden.begin() + (row + 1) * batch.hidden_dim); +} + +std::vector<__fp16> gemma4_mtp_tree_mask(size_t token_count) { + std::vector<__fp16> mask(token_count * token_count, static_cast<__fp16>(0)); + for (size_t row = 0; row < token_count; ++row) { + mask[row * token_count] = static_cast<__fp16>(1); + if (row == 1 || row >= 3) mask[row * token_count + 1] = static_cast<__fp16>(1); + if (row == 2) mask[row * token_count + 2] = static_cast<__fp16>(1); + for (size_t col = 3; col <= row; ++col) { + mask[row * token_count + col] = static_cast<__fp16>(1); + } + } + return mask; +} + +size_t gemma4_mtp_tree_main_tokens_for_draft_limit(size_t draft_limit) { + const char* env = std::getenv("CACTUS_GEMMA4_MTP_TREE_MAIN_TOKENS"); + if (env && env[0] != '\0') { + char* end = nullptr; + unsigned long value = std::strtoul(env, &end, 10); + if (end != env && *end == '\0' && value >= 1 && value <= 3 && draft_limit == value + 1) { + return static_cast(value); + } + return 0; + } + return 0; +} + +MtpDraftBatch gemma4_mtp_draft_tokens( + Gemma4Model& target, + Gemma4MtpAssistant& assistant, + uint32_t input_token, + const std::vector<__fp16>& previous_hidden, + const Gemma4Model::SharedCacheNodes& cache_nodes, + size_t assistant_position, + size_t draft_limit, + const MtpSamplingOptions& options, + bool sparse_sampled, + std::mt19937& rng) { + const size_t tree_main_tokens = gemma4_mtp_tree_main_tokens_for_draft_limit(draft_limit); + const bool use_tree = options.temperature == 0.0f + && tree_main_tokens > 0 + && target.can_use_mtp_tree_attention(tree_main_tokens + 2); + const size_t main_draft_limit = use_tree ? tree_main_tokens : draft_limit; + + return mtp_draft_tokens( + input_token, + previous_hidden, + main_draft_limit, + options, + sparse_sampled, + rng, + [&](uint32_t assistant_input_token, + const std::vector<__fp16>& assistant_hidden, + size_t i, + bool needs_hidden) { + auto step = assistant.draft_one( + assistant_input_token, + target.token_embedding_node_for_mtp(), + assistant_hidden, + cache_nodes, + assistant_position, + i == 0 ? (1.0f / 16.0f) : 1.0f, + options.temperature, + options.top_p, + options.top_k, + options.min_p, + needs_hidden, + use_tree && i == 0); + + MtpDraftStep result; + result.token = step.token; + if (use_tree && i == 0 && step.has_second_token) { + result.alt_first_token = step.second_token; + result.has_alt_first_token = true; + } + if (options.temperature > 0.0f) { + if (sparse_sampled) { + result.sparse_probabilities = std::move(step.sparse_probabilities); + } else { + result.probabilities = std::move(step.probabilities); + } + } + result.hidden = std::move(step.hidden); + return result; + }); +} + +MtpVerificationResult gemma4_mtp_verify_and_commit( + Gemma4Model& target, + uint32_t input_token, + const MtpDraftBatch& draft, + size_t remaining_tokens, + const MtpSamplingOptions& options, + bool sparse_sampled, + std::mt19937& rng) { + MtpVerificationResult result; + result.verify_tokens.reserve(draft.tokens.size() + 1); + result.verify_tokens.push_back(input_token); + const bool use_tree = options.temperature == 0.0f + && draft.has_alt_first_token + && !draft.tokens.empty() + && draft.tokens.size() <= 3 + && target.can_use_mtp_tree_attention(draft.tokens.size() + 2); + if (use_tree) { + result.verify_tokens.push_back(draft.tokens[0]); + result.verify_tokens.push_back(draft.alt_first_token); + result.verify_tokens.insert(result.verify_tokens.end(), draft.tokens.begin() + 1, draft.tokens.end()); + } else { + result.verify_tokens.insert(result.verify_tokens.end(), draft.tokens.begin(), draft.tokens.end()); + } + + auto* graph = static_cast(target.graph_handle_); + size_t verifier_cache_start = target.cache_position(); + auto kv_start = std::chrono::steady_clock::now(); + auto cache_txn = graph->begin_kv_cache_transaction(target.cache_state_nodes_for_mtp()); + auto kv_end = std::chrono::steady_clock::now(); + result.kv_transaction_ms += mtp_elapsed_ms(kv_start, kv_end); + + Gemma4Model::GreedyBatchWithHidden target_batch; + auto verify_start = std::chrono::steady_clock::now(); + if (use_tree) { + target_batch = target.decode_greedy_tokens_with_hidden_tree( + result.verify_tokens, gemma4_mtp_tree_mask(result.verify_tokens.size())); + } else if (options.temperature == 0.0f) { + target_batch = target.decode_greedy_tokens_with_hidden_early_stop( + result.verify_tokens, draft.tokens); + } else if (sparse_sampled) { + target_batch = target.decode_tokens_with_hidden_and_sparse_probs( + result.verify_tokens, options.temperature, options.top_p, options.top_k, options.min_p); + } else { + target_batch = target.decode_tokens_with_hidden_and_probs( + result.verify_tokens, options.temperature, options.top_p, options.top_k, options.min_p); + } + auto verify_end = std::chrono::steady_clock::now(); + result.target_verify_ms += mtp_elapsed_ms(verify_start, verify_end); + result.target_tokens = target_batch.tokens; + + auto sample_start = std::chrono::steady_clock::now(); + if (options.temperature == 0.0f) { + if (use_tree) { + if (target_batch.tokens.size() != result.verify_tokens.size()) { + throw std::runtime_error("Gemma 4 tree verifier target row count mismatch"); + } + auto main_row = [](size_t draft_index) { + return draft_index == 0 ? size_t{1} : draft_index + 2; + }; + auto prediction_row = [&](size_t draft_index) { + return draft_index == 0 ? size_t{0} : main_row(draft_index - 1); + }; + result.committed_verify_indices.push_back(0); + result.next_hidden_row = 0; + + for (size_t i = 0; i < draft.tokens.size(); ++i) { + const size_t pred_row = prediction_row(i); + if (target_batch.tokens[pred_row] != draft.tokens[i]) { + if (i == 0 && target_batch.tokens[0] == draft.alt_first_token) { + result.output_tokens.push_back(draft.alt_first_token); + result.accepted_draft_tokens = 1; + result.next_hidden_row = 2; + if (result.output_tokens.size() < remaining_tokens) { + result.output_tokens.push_back(target_batch.tokens[2]); + result.emitted_extra_target_token = true; + } + result.committed_verify_indices = {0, 2}; + } else { + result.output_tokens.push_back(target_batch.tokens[pred_row]); + result.rejected = true; + } + break; + } + result.output_tokens.push_back(draft.tokens[i]); + result.accepted_draft_tokens = i + 1; + result.next_hidden_row = main_row(i); + result.committed_verify_indices.push_back(result.next_hidden_row); + } + + if (!result.rejected + && !result.emitted_extra_target_token + && result.accepted_draft_tokens == draft.tokens.size() + && result.output_tokens.size() < remaining_tokens) { + result.output_tokens.push_back(target_batch.tokens[result.next_hidden_row]); + result.emitted_extra_target_token = true; + } + } else { + const size_t produced = target_batch.tokens.size(); + const size_t draft_size = draft.tokens.size(); + for (; result.accepted_draft_tokens < draft_size && result.accepted_draft_tokens < produced; ++result.accepted_draft_tokens) { + if (draft.tokens[result.accepted_draft_tokens] != target_batch.tokens[result.accepted_draft_tokens]) { + result.output_tokens.push_back(target_batch.tokens[result.accepted_draft_tokens]); + result.rejected = true; + break; + } + result.output_tokens.push_back(draft.tokens[result.accepted_draft_tokens]); + } + if (!result.rejected && result.output_tokens.size() < remaining_tokens + && produced > draft_size) { + result.output_tokens.push_back(target_batch.tokens[draft_size]); + result.emitted_extra_target_token = true; + } + } + } else { + for (; result.accepted_draft_tokens < draft.tokens.size(); ++result.accepted_draft_tokens) { + uint32_t y = draft.tokens[result.accepted_draft_tokens]; + float p_y = 0.0f; + float q_y = 0.0f; + if (sparse_sampled) { + p_y = mtp_sparse_probability_at(target_batch.sparse_probabilities[result.accepted_draft_tokens], y); + q_y = mtp_sparse_probability_at(draft.sparse_probabilities[result.accepted_draft_tokens], y); + } else { + const auto& p = target_batch.probabilities[result.accepted_draft_tokens]; + const auto& q = draft.probabilities[result.accepted_draft_tokens].probabilities; + p_y = y < p.size() ? p[y] : 0.0f; + q_y = y < q.size() ? q[y] : 0.0f; + } + + float accept_probability = q_y <= 0.0f ? 1.0f : std::min(1.0f, p_y / q_y); + std::uniform_real_distribution dist(0.0f, 1.0f); + if (dist(rng) <= accept_probability) { + result.output_tokens.push_back(y); + continue; + } + + if (sparse_sampled) { + auto adjusted = mtp_rejection_adjusted_sparse_distribution( + target_batch.sparse_probabilities[result.accepted_draft_tokens], + draft.sparse_probabilities[result.accepted_draft_tokens]); + result.output_tokens.push_back(mtp_sample_sparse_distribution(adjusted, rng)); + } else { + auto adjusted = rejection_adjusted_distribution( + MtpDistribution{target_batch.probabilities[result.accepted_draft_tokens]}, + draft.probabilities[result.accepted_draft_tokens]); + result.output_tokens.push_back(mtp_sample_distribution(adjusted.probabilities, rng)); + } + result.rejected = true; + break; + } + if (!result.rejected && result.output_tokens.size() < remaining_tokens) { + if (sparse_sampled) { + result.output_tokens.push_back(mtp_sample_sparse_distribution( + target_batch.sparse_probabilities[draft.tokens.size()], rng)); + } else { + result.output_tokens.push_back(mtp_sample_distribution( + target_batch.probabilities[draft.tokens.size()], rng)); + } + result.emitted_extra_target_token = true; + } + } + auto sample_end = std::chrono::steady_clock::now(); + result.sampling_or_argmax_ms += mtp_elapsed_ms(sample_start, sample_end); + + if (use_tree) { + for (uint32_t token : result.output_tokens) { + target.record_sampled_token(token); + } + } + + if (!result.output_tokens.empty()) { + size_t row = use_tree + ? result.next_hidden_row + : (options.temperature == 0.0f ? result.accepted_draft_tokens : result.output_tokens.size() - 1); + result.next_hidden = gemma4_mtp_hidden_row(target_batch, row); + } + + size_t committed_cache_tokens = result.output_tokens.size(); + result.target_cache_crop_length = verifier_cache_start + committed_cache_tokens; + kv_start = std::chrono::steady_clock::now(); + if (use_tree) { + cache_txn.commit_selected(result.committed_verify_indices); + graph->apply_pending_kv_cache_sequence_lengths(); + target.set_cache_position_for_mtp(result.target_cache_crop_length); + } else if (committed_cache_tokens < result.verify_tokens.size()) { + cache_txn.commit_prefix(committed_cache_tokens); + graph->apply_pending_kv_cache_sequence_lengths(); + target.set_cache_position_for_mtp(result.target_cache_crop_length); + } else { + cache_txn.commit_all(); + graph->apply_pending_kv_cache_sequence_lengths(); + } + kv_end = std::chrono::steady_clock::now(); + result.kv_transaction_ms += mtp_elapsed_ms(kv_start, kv_end); + + return result; +} + +} + +MtpRoundResult Gemma4Model::decode_mtp_round(Gemma4MtpAssistant& assistant, + uint32_t input_token, + const std::vector<__fp16>& previous_hidden, + const SharedCacheNodes& cache_nodes, + size_t assistant_position, + size_t draft_limit, + size_t remaining_tokens, + const MtpSamplingOptions& options, + bool sparse_sampled, + std::mt19937& rng) { + MtpRoundResult round; + round.draft = gemma4_mtp_draft_tokens( + *this, + assistant, + input_token, + previous_hidden, + cache_nodes, + assistant_position, + draft_limit, + options, + sparse_sampled, + rng); + round.verification = gemma4_mtp_verify_and_commit( + *this, + input_token, + round.draft, + remaining_tokens, + options, + sparse_sampled, + rng); + return round; +} + static uint32_t gemma4_argmax_logits(CactusGraph* gb, size_t logits_node) { const auto& logits_buf = gb->get_output_buffer(logits_node); if (logits_buf.shape.empty()) { @@ -123,145 +480,6 @@ static bool gemma4_argmax_output_projection_rows(CactusGraph* gb, size_t hidden_ return false; } -static std::vector gemma4_probs_from_logits(std::vector logits, float temperature, - float top_p, size_t top_k, float min_p) { - if (logits.empty()) return {}; - if (temperature == 0.0f) { - std::vector probs(logits.size(), 0.0f); - auto it = std::max_element(logits.begin(), logits.end()); - probs[std::distance(logits.begin(), it)] = 1.0f; - return probs; - } - if (temperature > 0.0f) { - for (float& value : logits) value /= temperature; - } - if (top_k > 0 && top_k < logits.size()) { - std::vector sorted = logits; - std::nth_element(sorted.begin(), sorted.begin() + top_k - 1, sorted.end(), std::greater()); - float kth = sorted[top_k - 1]; - for (float& value : logits) { - if (value < kth) value = -std::numeric_limits::infinity(); - } - } - auto normalize = [](const std::vector& values) { - std::vector probs(values.size(), 0.0f); - float max_value = -std::numeric_limits::infinity(); - for (float value : values) max_value = std::max(max_value, value); - if (!std::isfinite(max_value)) return probs; - double sum = 0.0; - for (size_t i = 0; i < values.size(); ++i) { - if (std::isfinite(values[i])) { - probs[i] = std::exp(static_cast(values[i] - max_value)); - sum += probs[i]; - } - } - if (sum <= 0.0 || !std::isfinite(sum)) return probs; - for (float& prob : probs) prob = static_cast(prob / sum); - return probs; - }; - if (min_p > 0.0f) { - auto probs = normalize(logits); - float max_prob = *std::max_element(probs.begin(), probs.end()); - float threshold = max_prob * min_p; - for (size_t i = 0; i < logits.size(); ++i) { - if (probs[i] < threshold) logits[i] = -std::numeric_limits::infinity(); - } - } - if (top_p > 0.0f && top_p < 1.0f) { - std::vector> sorted; - sorted.reserve(logits.size()); - for (size_t i = 0; i < logits.size(); ++i) { - if (std::isfinite(logits[i])) sorted.emplace_back(logits[i], i); - } - std::sort(sorted.begin(), sorted.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); - std::vector sorted_logits; - sorted_logits.reserve(sorted.size()); - for (const auto& item : sorted) sorted_logits.push_back(item.first); - auto sorted_probs = normalize(sorted_logits); - float cumulative = 0.0f; - bool remove = false; - for (size_t i = 0; i < sorted.size(); ++i) { - cumulative += sorted_probs[i]; - if (remove) { - logits[sorted[i].second] = -std::numeric_limits::infinity(); - } - if (cumulative > top_p) remove = true; - } - } - auto probs = normalize(logits); - float sum = std::accumulate(probs.begin(), probs.end(), 0.0f); - if (sum <= 0.0f || !std::isfinite(sum)) { - std::fill(probs.begin(), probs.end(), 1.0f / static_cast(probs.size())); - } - return probs; -} - -static std::vector> gemma4_sparse_probs_from_top_logits( - std::vector> logits, float temperature, float top_p, float min_p) { - if (logits.empty()) return {}; - if (temperature > 0.0f) { - for (auto& item : logits) item.second /= temperature; - } - std::sort(logits.begin(), logits.end(), [](const auto& a, const auto& b) { - return a.second > b.second; - }); - - auto normalize = [](const std::vector>& values) { - std::vector> probs; - probs.reserve(values.size()); - float max_value = -std::numeric_limits::infinity(); - for (const auto& item : values) max_value = std::max(max_value, item.second); - if (!std::isfinite(max_value)) return probs; - double sum = 0.0; - for (const auto& item : values) { - float prob = std::exp(static_cast(item.second - max_value)); - probs.emplace_back(item.first, prob); - sum += prob; - } - if (sum <= 0.0 || !std::isfinite(sum)) return std::vector>{}; - for (auto& item : probs) item.second = static_cast(item.second / sum); - return probs; - }; - - if (min_p > 0.0f) { - auto probs = normalize(logits); - if (!probs.empty()) { - float max_prob = 0.0f; - for (const auto& item : probs) max_prob = std::max(max_prob, item.second); - float threshold = max_prob * min_p; - std::vector> filtered; - filtered.reserve(logits.size()); - for (size_t i = 0; i < logits.size(); ++i) { - if (probs[i].second >= threshold) filtered.push_back(logits[i]); - } - logits = std::move(filtered); - } - } - - if (top_p > 0.0f && top_p < 1.0f) { - auto probs = normalize(logits); - std::vector> filtered; - filtered.reserve(logits.size()); - float cumulative = 0.0f; - bool remove = false; - for (size_t i = 0; i < logits.size(); ++i) { - cumulative += i < probs.size() ? probs[i].second : 0.0f; - if (!remove) filtered.push_back(logits[i]); - if (cumulative > top_p) remove = true; - } - logits = std::move(filtered); - } - - auto probs = normalize(logits); - float sum = 0.0f; - for (const auto& item : probs) sum += item.second; - if (sum <= 0.0f || !std::isfinite(sum)) { - float uniform = probs.empty() ? 0.0f : 1.0f / static_cast(probs.size()); - for (auto& item : probs) item.second = uniform; - } - return probs; -} - static std::vector> gemma4_sparse_probs_from_logits_row( const void* logits_ptr, Precision precision, size_t row_offset, size_t vocab_size, float temperature, float top_p, size_t top_k, float min_p) { @@ -294,7 +512,12 @@ static std::vector> gemma4_sparse_probs_from_logits_r logits.emplace_back(top.top().second, top.top().first); top.pop(); } - return gemma4_sparse_probs_from_top_logits(std::move(logits), temperature, top_p, min_p); + return mtp_sparse_distribution_from_logits(std::move(logits), MtpSamplingOptions{ + .temperature = temperature, + .top_p = top_p, + .top_k = top_k, + .min_p = min_p, + }); } static std::vector> gemma4_prob_rows(CactusGraph* gb, size_t logits_node, @@ -321,7 +544,12 @@ static std::vector> gemma4_prob_rows(CactusGraph* gb, size_t const int8_t* src = static_cast(logits_ptr) + row * vocab_size; for (size_t i = 0; i < vocab_size; ++i) logits[i] = static_cast(src[i]); } - result.push_back(gemma4_probs_from_logits(std::move(logits), temperature, top_p, top_k, min_p)); + result.push_back(mtp_distribution_from_logits(std::move(logits), MtpSamplingOptions{ + .temperature = temperature, + .top_p = top_p, + .top_k = top_k, + .min_p = min_p, + }).probabilities); } return result; } diff --git a/cactus-engine/models/gemma4/model_gemma4.h b/cactus-engine/models/gemma4/model_gemma4.h index bb8a04ab6..97a7f1603 100644 --- a/cactus-engine/models/gemma4/model_gemma4.h +++ b/cactus-engine/models/gemma4/model_gemma4.h @@ -1,7 +1,10 @@ #pragma once #include "../model.h" +#include "src/mtp_decode.h" +#include +#include #include bool test_gemma4_vision(bool expect_npu); @@ -10,12 +13,15 @@ bool test_gemma4_audio(bool expect_npu); namespace cactus { namespace engine { +class Gemma4MtpAssistant; + class Gemma4Model : public Model { friend class Gemma4MmModel; public: Gemma4Model(); explicit Gemma4Model(const Config& config); ~Gemma4Model() override = default; + SpeculativeDecodeCapability* speculative_decode_capability() override; struct GreedyStepWithHidden { uint32_t token = 0; @@ -60,6 +66,16 @@ class Gemma4Model : public Model { void set_cache_position_for_mtp(size_t position) { cache_total_seq_len_ = position; } std::vector cache_state_nodes_for_mtp() const; bool can_use_mtp_tree_attention(size_t token_count) const; + MtpRoundResult decode_mtp_round(Gemma4MtpAssistant& assistant, + uint32_t input_token, + const std::vector<__fp16>& previous_hidden, + const SharedCacheNodes& cache_nodes, + size_t assistant_position, + size_t draft_limit, + size_t remaining_tokens, + const MtpSamplingOptions& options, + bool sparse_sampled, + std::mt19937& rng); protected: size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, @@ -162,6 +178,7 @@ class Gemma4Model : public Model { std::vector shared_v_nodes_; size_t active_attention_mask_node_ = 0; std::vector active_position_deltas_; + std::unique_ptr speculative_capability_; std::vector<__fp16> v_norm_ones_weight_; size_t v_norm_ones_node_ = 0; @@ -400,6 +417,7 @@ class Gemma4MmModel : public Model { void reset_cache() override; size_t get_cache_size() const override; + SpeculativeDecodeCapability* speculative_decode_capability() override; std::vector get_image_embeddings(const std::string& image_path) override; std::vector get_audio_embeddings(const std::vector& audio_features) override; void compact_kv_cache() override; diff --git a/cactus-engine/models/gemma4/model_gemma4_mm.cpp b/cactus-engine/models/gemma4/model_gemma4_mm.cpp index 1c242093c..718148dd0 100644 --- a/cactus-engine/models/gemma4/model_gemma4_mm.cpp +++ b/cactus-engine/models/gemma4/model_gemma4_mm.cpp @@ -61,6 +61,10 @@ size_t Gemma4MmModel::get_cache_size() const { return language_model_.get_cache_size(); } +SpeculativeDecodeCapability* Gemma4MmModel::speculative_decode_capability() { + return language_model_.speculative_decode_capability(); +} + void Gemma4MmModel::compact_kv_cache() { language_model_.compact_kv_cache(); } diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 06df887f3..0692e1d85 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -1,6 +1,6 @@ #include "../cactus_engine.h" #include "cloud.h" -#include "gemma4_mtp_decode.h" +#include "mtp_completion.h" #include "utils.h" #include "telemetry.h" #include "cactus_kernels.h" @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -32,24 +31,6 @@ namespace { std::vector> extract_schema_property_types(const std::string& schema); std::vector extract_schema_required(const std::string& schema); -Gemma4Model* gemma4_language_model(Model* model) { - if (auto* gemma4 = dynamic_cast(model)) { - return gemma4; - } - if (auto* mm = dynamic_cast(model)) { - return &mm->language_model(); - } - return nullptr; -} - -std::string default_gemma4_mtp_assistant_path(Model* model) { - if (!model) { - return {}; - } - std::filesystem::path assistant_path = std::filesystem::path(model->get_model_folder_path()) / "assistant"; - return assistant_path.string(); -} - std::string token_list_for_trace(const std::vector& tokens) { std::ostringstream out; out << "["; @@ -61,168 +42,6 @@ std::string token_list_for_trace(const std::vector& tokens) { return out.str(); } -std::string env_string(const char* key, const std::string& fallback = "") { - const char* value = std::getenv(key); - return value && value[0] ? std::string(value) : fallback; -} - -std::string csv_escape_field(const std::string& value) { - if (value.find_first_of(",\"\n\r") == std::string::npos) return value; - std::string out = "\""; - for (char c : value) { - if (c == '"') out += "\"\""; - else out += c; - } - out += "\""; - return out; -} - -std::string bool_field(bool value) { - return value ? "true" : "false"; -} - -std::string double_field(double value) { - std::ostringstream out; - out << std::fixed << std::setprecision(3) << value; - return out.str(); -} - -std::string join_ms(const std::vector& values) { - std::ostringstream out; - out << std::fixed << std::setprecision(3); - for (size_t i = 0; i < values.size(); ++i) { - if (i > 0) out << ";"; - out << values[i]; - } - return out.str(); -} - -void append_csv_line(const std::filesystem::path& path, - const std::string& header, - const std::vector& fields) { - bool write_header = !std::filesystem::exists(path) || std::filesystem::file_size(path) == 0; - std::ofstream out(path, std::ios::app); - if (!out) { - throw std::runtime_error("Unable to open diagnostic trace CSV: " + path.string()); - } - if (write_header) out << header << "\n"; - for (size_t i = 0; i < fields.size(); ++i) { - if (i > 0) out << ","; - out << csv_escape_field(fields[i]); - } - out << "\n"; -} - -struct DiagnosticTraceContext { - bool enabled = false; - std::filesystem::path dir; - std::string run_id; - std::string model; - std::string prompt_name; - std::string shape; - std::string rep; - - static DiagnosticTraceContext from_env() { - DiagnosticTraceContext ctx; - std::string dir = env_string("CACTUS_GEMMA4_MTP_TRACE_CSV_DIR"); - if (dir.empty()) return ctx; - ctx.run_id = env_string("CACTUS_MTP_DIAG_RUN_ID"); - if (ctx.run_id.empty()) return ctx; - ctx.enabled = true; - ctx.dir = dir; - std::filesystem::create_directories(ctx.dir); - ctx.model = env_string("CACTUS_MTP_DIAG_MODEL", "model"); - ctx.prompt_name = env_string("CACTUS_MTP_DIAG_PROMPT", "prompt"); - ctx.shape = env_string("CACTUS_MTP_DIAG_SHAPE", "unknown"); - ctx.rep = env_string("CACTUS_MTP_DIAG_REP", "0"); - return ctx; - } - - void write_round(size_t round_index, - size_t generated_start, - size_t generated_end, - size_t target_batch_m, - size_t assistant_pass_count, - size_t drafted_tokens, - size_t accepted_drafts, - bool rejected, - bool alt_branch_accepted, - bool emitted_extra_target_token, - double target_forward_ms, - double assistant_total_ms, - const std::vector& assistant_step_ms, - double sampling_or_argmax_ms, - double kv_transaction_ms, - double callback_stream_ms, - double loop_overhead_ms, - double round_total_ms) const { - if (!enabled || generated_end <= generated_start) return; - static const std::string header = - "run_id,model,prompt_name,shape,rep,round_index,generated_token_start,generated_token_end," - "generated_tokens_emitted,target_batch_m,assistant_pass_count,drafted_tokens,accepted_drafts," - "rejected,alt_branch_accepted,emitted_extra_target_token,target_forward_ms,assistant_total_ms," - "assistant_step_ms,sampling_or_argmax_ms,kv_transaction_ms,callback_stream_ms,loop_overhead_ms,round_total_ms"; - append_csv_line(dir / "round_trace.csv", header, { - run_id, model, prompt_name, shape, rep, - std::to_string(round_index), - std::to_string(generated_start), - std::to_string(generated_end), - std::to_string(generated_end - generated_start), - std::to_string(target_batch_m), - std::to_string(assistant_pass_count), - std::to_string(drafted_tokens), - std::to_string(accepted_drafts), - bool_field(rejected), - bool_field(alt_branch_accepted), - bool_field(emitted_extra_target_token), - double_field(target_forward_ms), - double_field(assistant_total_ms), - join_ms(assistant_step_ms), - double_field(sampling_or_argmax_ms), - double_field(kv_transaction_ms), - double_field(callback_stream_ms), - double_field(loop_overhead_ms), - double_field(round_total_ms), - }); - } - - void write_token(size_t token_position, - uint32_t token_id, - size_t round_index, - size_t token_index_in_round, - const std::string& source, - size_t round_emitted, - double target_forward_ms, - double assistant_total_ms, - double other_ms, - double round_total_ms) const { - if (!enabled || round_emitted == 0) return; - static const std::string header = - "run_id,model,prompt_name,shape,rep,token_position,token_id,round_index,token_index_in_round," - "source,round_generated_tokens_emitted,round_target_forward_ms,round_assistant_total_ms," - "round_other_ms,round_total_ms,allocated_target_forward_ms,allocated_assistant_ms," - "allocated_other_ms,allocated_total_ms"; - double denom = static_cast(round_emitted); - append_csv_line(dir / "token_trace.csv", header, { - run_id, model, prompt_name, shape, rep, - std::to_string(token_position), - std::to_string(token_id), - std::to_string(round_index), - std::to_string(token_index_in_round), - source, - std::to_string(round_emitted), - double_field(target_forward_ms), - double_field(assistant_total_ms), - double_field(other_ms), - double_field(round_total_ms), - double_field(target_forward_ms / denom), - double_field(assistant_total_ms / denom), - double_field(other_ms / denom), - double_field(round_total_ms / denom), - }); - } -}; - std::string extract_last_user_query(const std::vector& messages) { for (auto it = messages.rbegin(); it != messages.rend(); ++it) { if (it->role == "user") { @@ -863,8 +682,9 @@ int cactus_complete( handle->should_stop = false; auto* tokenizer = handle->model->get_tokenizer(); auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, true, true, pcm_buffer, pcm_buffer_size); - if (prompt.options.mtp && prompt.options.mtp_assistant_path.empty()) { - prompt.options.mtp_assistant_path = default_gemma4_mtp_assistant_path(handle->model.get()); + SpeculativeDecodeCapability* mtp_capability = handle->model->speculative_decode_capability(); + if (prompt.options.mtp && prompt.options.mtp_assistant_path.empty() && mtp_capability) { + prompt.options.mtp_assistant_path = mtp_capability->default_assistant_path(); } bool mtp_assistant_available = false; @@ -881,7 +701,9 @@ int cactus_complete( bool has_images = prompt.has_images(); bool has_audio = prompt.has_audio(); - Gemma4Model* mtp_target = gemma4_language_model(handle->model.get()); + Gemma4Model* mtp_target = mtp_capability + ? dynamic_cast(mtp_capability->target_model()) + : nullptr; Gemma4MtpAssistant mtp_assistant; bool mtp_enabled = false; size_t mtp_drafted_tokens = 0; @@ -902,9 +724,9 @@ int cactus_complete( && prompt.options.max_tokens <= 32; if (short_completion_gate) { mtp_fallback_reason = "short_completion"; - } else if (!mtp_target) { - mtp_fallback_reason = "target_not_gemma4_text"; - } else if (!text_only) { + } else if (!mtp_capability || !mtp_target) { + mtp_fallback_reason = "unsupported_target"; + } else if (!text_only || !mtp_capability->supports_prompt(has_images, has_audio, !prompt.tools.empty())) { mtp_fallback_reason = "only_text_supported"; } else if (!mtp_assistant_available) { mtp_fallback_reason = "assistant_unavailable"; @@ -1009,8 +831,8 @@ int cactus_complete( first_input, prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, prompt.options.min_p); next_token = mtp_sparse_sampled - ? gemma4_mtp_sample_sparse_distribution(first.sparse_probabilities.back(), mtp_rng) - : gemma4_mtp_sample_distribution(first.probabilities.back(), mtp_rng); + ? mtp_sample_sparse_distribution(first.sparse_probabilities.back(), mtp_rng) + : mtp_sample_distribution(first.probabilities.back(), mtp_rng); size_t hidden_row = first.hidden_dim == 0 ? 0 : first.hidden.size() / first.hidden_dim - 1; mtp_prev_hidden.assign( first.hidden.begin() + hidden_row * first.hidden_dim, @@ -1266,13 +1088,12 @@ int cactus_complete( size_t produced = 1; bool stopped = false; size_t mtp_previous_n_matches = 0; - Gemma4MtpSamplingOptions mtp_sampling_options{ + MtpSamplingOptions mtp_sampling_options{ .temperature = prompt.options.temperature, .top_p = prompt.options.top_p, .top_k = prompt.options.top_k, .min_p = prompt.options.min_p, }; - Gemma4MtpDecodeStrategy mtp_strategy(*mtp_target, mtp_assistant); while (produced < prompt.options.max_tokens && !handle->should_stop) { size_t remaining = prompt.options.max_tokens - produced; if (remaining == 0) break; @@ -1285,7 +1106,8 @@ int cactus_complete( auto cache_nodes = mtp_target->shared_cache_nodes_for_mtp(); const size_t assistant_position = handle->processed_tokens.empty() ? 0 : handle->processed_tokens.size() - 1; - auto round = mtp_strategy.decode_round( + auto round = mtp_target->decode_mtp_round( + mtp_assistant, next_token, mtp_prev_hidden, cache_nodes, @@ -1299,7 +1121,7 @@ int cactus_complete( auto& draft = round.draft; auto& verified = round.verification; mtp_drafted_tokens += draft.tokens.size(); - mtp_accepted_tokens += verified.accepted; + mtp_accepted_tokens += verified.accepted_draft_tokens; mtp_assistant_draft_ms += draft.assistant_draft_ms; mtp_target_verify_ms += verified.target_verify_ms; mtp_sampling_or_argmax_ms += draft.sampling_or_argmax_ms + verified.sampling_or_argmax_ms; @@ -1315,7 +1137,7 @@ int cactus_complete( << " assistant_draft_tokens=" << token_list_for_trace(draft.tokens) << " target_verify_input_tokens=" << token_list_for_trace(verified.verify_tokens) << " target_argmax_tokens=" << token_list_for_trace(verified.target_tokens) - << " accepted_count=" << verified.accepted + << " accepted_count=" << verified.accepted_draft_tokens << " valid_committed_tokens=" << token_list_for_trace(verified.output_tokens) << " target_cache_crop_length=" << verified.target_cache_crop_length << " assistant_draft_ms=" << draft.assistant_draft_ms @@ -1324,7 +1146,7 @@ int cactus_complete( << " kv_transaction_ms=" << verified.kv_transaction_ms << "\n"; } - mtp_previous_n_matches = verified.accepted; + mtp_previous_n_matches = verified.accepted_draft_tokens; if (!verified.next_hidden.empty()) { mtp_prev_hidden = std::move(verified.next_hidden); @@ -1370,7 +1192,7 @@ int cactus_complete( verified.verify_tokens.size(), draft.assistant_step_ms.size(), draft.tokens.size() + (draft.has_alt_first_token ? 1 : 0), - verified.accepted, + verified.accepted_draft_tokens, verified.rejected, alt_branch_accepted, verified.emitted_extra_target_token, @@ -1392,7 +1214,7 @@ int cactus_complete( } } else { for (size_t i = 0; i < emitted_count; ++i) { - if (i < verified.accepted) { + if (i < verified.accepted_draft_tokens) { sources.push_back("accepted_main_draft"); } else if (verified.rejected) { sources.push_back("rejection_target"); @@ -1652,12 +1474,12 @@ int cactus_prefill( handle->processed_images = prompt.images; auto end_time = std::chrono::high_resolution_clock::now(); - double elapsed_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - double prefill_tps = (prefill_result.prefilled_count > 0 && elapsed_ms > 0.0) - ? (static_cast(prefill_result.prefilled_count) * 1000.0) / elapsed_ms + double mtp_elapsed_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + double prefill_tps = (prefill_result.prefilled_count > 0 && mtp_elapsed_ms > 0.0) + ? (static_cast(prefill_result.prefilled_count) * 1000.0) / mtp_elapsed_ms : 0.0; - std::string result = construct_prefill_response_json(true, nullptr, prefill_result.prefilled_count, prefill_tps, elapsed_ms); + std::string result = construct_prefill_response_json(true, nullptr, prefill_result.prefilled_count, prefill_tps, mtp_elapsed_ms); if (result.size() >= buffer_size) { std::string error_msg = "Response buffer too small"; std::string error_json = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); diff --git a/cactus-engine/src/decode_strategy.h b/cactus-engine/src/decode_strategy.h deleted file mode 100644 index 79e661725..000000000 --- a/cactus-engine/src/decode_strategy.h +++ /dev/null @@ -1,134 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -struct DecodeOptions { - bool mtp_enabled = false; - bool mtp_required = false; - float temperature = -1.0f; - float top_p = -1.0f; - float min_p = 0.15f; - float repetition_penalty = 1.1f; - size_t top_k = 0; - size_t max_draft_tokens = 6; -}; - -struct DecodeStepResult { - uint32_t token = 0; - float entropy = 0.0f; -}; - -struct DraftBatch { - std::vector tokens; -}; - -struct TargetDecodeState { - std::vector input_tokens; -}; - -struct AcceptedTokenBatch { - std::vector tokens; - std::vector texts; - std::vector entropies; - size_t drafted_tokens = 0; - size_t accepted_draft_tokens = 0; - size_t rejected_tokens = 0; - std::vector rejected_draft_token_ids; -}; - -class MissingAssistantError : public std::runtime_error { -public: - explicit MissingAssistantError(const std::string& message) : std::runtime_error(message) {} -}; - -class TargetDecoder { -public: - virtual ~TargetDecoder() = default; - virtual DecodeStepResult decode_one(const std::vector& input_tokens, - const DecodeOptions& options) = 0; -}; - -class AssistantDraftProvider { -public: - virtual ~AssistantDraftProvider() = default; - virtual DraftBatch draft(const TargetDecodeState& state, size_t max_draft_tokens) = 0; -}; - -class SpeculativeVerifier { -public: - virtual ~SpeculativeVerifier() = default; - virtual AcceptedTokenBatch verify(const DraftBatch& draft, - const TargetDecodeState& state, - const DecodeOptions& options) = 0; -}; - -class DecodeStrategy { -public: - virtual ~DecodeStrategy() = default; - virtual AcceptedTokenBatch decode_next(const std::vector& input_tokens, - const DecodeOptions& options) = 0; - virtual std::string name() const = 0; -}; - -class StandardDecodeStrategy final : public DecodeStrategy { -public: - explicit StandardDecodeStrategy(TargetDecoder& target) : target_(target) {} - - AcceptedTokenBatch decode_next(const std::vector& input_tokens, - const DecodeOptions& options) override { - DecodeStepResult step = target_.decode_one(input_tokens, options); - return AcceptedTokenBatch{ - .tokens = {step.token}, - .entropies = {step.entropy}, - }; - } - - std::string name() const override { return "standard"; } - -private: - TargetDecoder& target_; -}; - -class Gemma4MtpDecodeStrategy final : public DecodeStrategy { -public: - Gemma4MtpDecodeStrategy(TargetDecoder& target, - AssistantDraftProvider& assistant, - SpeculativeVerifier& verifier) - : target_(target), assistant_(assistant), verifier_(verifier) {} - - AcceptedTokenBatch decode_next(const std::vector& input_tokens, - const DecodeOptions& options) override { - (void)target_; - TargetDecodeState state{.input_tokens = input_tokens}; - DraftBatch draft = assistant_.draft(state, options.max_draft_tokens); - return verifier_.verify(draft, state, options); - } - - std::string name() const override { return "gemma4_mtp"; } - -private: - TargetDecoder& target_; - AssistantDraftProvider& assistant_; - SpeculativeVerifier& verifier_; -}; - -inline std::unique_ptr make_decode_strategy(TargetDecoder& target, - AssistantDraftProvider* assistant, - SpeculativeVerifier* verifier, - const DecodeOptions& options) { - if (!options.mtp_enabled) { - return std::make_unique(target); - } - if (!assistant || !verifier) { - if (options.mtp_required) { - throw MissingAssistantError("Gemma 4 MTP assistant is unavailable"); - } - return std::make_unique(target); - } - return std::make_unique(target, *assistant, *verifier); -} diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 782baa16f..e0caa74a5 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -552,6 +552,19 @@ class ToolCallConstrainer { void init_common_tokens(); }; +class Model; + +class SpeculativeDecodeCapability { +public: + virtual ~SpeculativeDecodeCapability() = default; + virtual std::string strategy_name() const = 0; + virtual std::string default_assistant_path() const = 0; + virtual bool supports_prompt(bool has_images, bool has_audio, bool has_tools) const { + return !has_images && !has_audio && !has_tools; + } + virtual Model* target_model() = 0; +}; + class Model { public: struct DebugNode { @@ -567,6 +580,7 @@ class Model { const Config& get_config() const { return config_; } Tokenizer* get_tokenizer() const { return tokenizer_.get(); } const std::string& get_model_folder_path() const { return model_folder_path_; } + virtual SpeculativeDecodeCapability* speculative_decode_capability() { return nullptr; } const std::vector& get_debug_nodes() const; virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); diff --git a/cactus-engine/src/gemma4_mtp_decode.cpp b/cactus-engine/src/gemma4_mtp_decode.cpp deleted file mode 100644 index 0c1dd5430..000000000 --- a/cactus-engine/src/gemma4_mtp_decode.cpp +++ /dev/null @@ -1,411 +0,0 @@ -#include "gemma4_mtp_decode.h" - -#include "mtp_sampler.h" -#include "cactus_graph.h" - -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -namespace { - -std::vector rejection_adjusted_distribution(const std::vector& target, - const std::vector& assistant) { - std::vector adjusted(target.size(), 0.0f); - float sum = 0.0f; - for (size_t i = 0; i < target.size() && i < assistant.size(); ++i) { - float value = std::max(target[i] - assistant[i], 0.0f); - adjusted[i] = value; - sum += value; - } - if (sum <= 0.0f || !std::isfinite(sum)) { - adjusted = target; - sum = std::accumulate(adjusted.begin(), adjusted.end(), 0.0f); - } - if (sum <= 0.0f || !std::isfinite(sum)) { - std::fill(adjusted.begin(), adjusted.end(), adjusted.empty() ? 0.0f : 1.0f / static_cast(adjusted.size())); - return adjusted; - } - for (float& value : adjusted) value /= sum; - return adjusted; -} - -std::vector<__fp16> hidden_row(const Gemma4Model::GreedyBatchWithHidden& batch, size_t row) { - if (batch.hidden_dim == 0 || row >= batch.hidden.size() / batch.hidden_dim) { - return {}; - } - return std::vector<__fp16>( - batch.hidden.begin() + row * batch.hidden_dim, - batch.hidden.begin() + (row + 1) * batch.hidden_dim); -} - -double elapsed_ms(std::chrono::steady_clock::time_point start, - std::chrono::steady_clock::time_point end) { - return std::chrono::duration(end - start).count(); -} - -std::vector<__fp16> tree_mask(size_t token_count) { - std::vector<__fp16> mask(token_count * token_count, static_cast<__fp16>(0)); - for (size_t row = 0; row < token_count; ++row) { - mask[row * token_count] = static_cast<__fp16>(1); - if (row == 1 || row >= 3) mask[row * token_count + 1] = static_cast<__fp16>(1); - if (row == 2) mask[row * token_count + 2] = static_cast<__fp16>(1); - for (size_t col = 3; col <= row; ++col) { - mask[row * token_count + col] = static_cast<__fp16>(1); - } - } - return mask; -} - -size_t tree_main_tokens_for_draft_limit(size_t draft_limit) { - const char* env = std::getenv("CACTUS_GEMMA4_MTP_TREE_MAIN_TOKENS"); - if (env && env[0] != '\0') { - char* end = nullptr; - unsigned long value = std::strtoul(env, &end, 10); - if (end != env && *end == '\0' && value >= 1 && value <= 3 && draft_limit == value + 1) { - return static_cast(value); - } - return 0; - } - return 0; -} - -} - -uint32_t gemma4_mtp_sample_distribution(const std::vector& probabilities, std::mt19937& rng) { - if (probabilities.empty()) return 0; - std::uniform_real_distribution dist(0.0f, 1.0f); - float sample = dist(rng); - float cumulative = 0.0f; - for (size_t i = 0; i < probabilities.size(); ++i) { - cumulative += std::max(0.0f, probabilities[i]); - if (sample <= cumulative) return static_cast(i); - } - return static_cast(probabilities.size() - 1); -} - -uint32_t gemma4_mtp_sample_sparse_distribution( - const std::vector>& probabilities, - std::mt19937& rng) { - if (probabilities.empty()) return 0; - std::uniform_real_distribution dist(0.0f, 1.0f); - return mtp_sample_sparse(probabilities, dist(rng)); -} - -Gemma4AssistantDraftProvider::Gemma4AssistantDraftProvider(Gemma4Model& target, - Gemma4MtpAssistant& assistant) - : target_(target), assistant_(assistant) {} - -Gemma4MtpDraftBatch Gemma4AssistantDraftProvider::draft( - uint32_t input_token, - const std::vector<__fp16>& previous_hidden, - const Gemma4Model::SharedCacheNodes& cache_nodes, - size_t assistant_position, - size_t draft_limit, - const Gemma4MtpSamplingOptions& options, - bool sparse_sampled, - std::mt19937& rng) { - Gemma4MtpDraftBatch batch; - batch.tokens.reserve(draft_limit); - if (options.temperature > 0.0f) { - if (sparse_sampled) { - batch.sparse_probabilities.reserve(draft_limit); - } else { - batch.probabilities.reserve(draft_limit); - } - } - - std::vector<__fp16> assistant_hidden = previous_hidden; - uint32_t assistant_input_token = input_token; - const size_t tree_main_tokens = tree_main_tokens_for_draft_limit(draft_limit); - const bool use_tree = options.temperature == 0.0f - && tree_main_tokens > 0 - && target_.can_use_mtp_tree_attention(tree_main_tokens + 2); - const size_t main_draft_limit = use_tree ? tree_main_tokens : draft_limit; - batch.assistant_step_ms.reserve(main_draft_limit); - for (size_t i = 0; i < main_draft_limit; ++i) { - auto draft_start = std::chrono::steady_clock::now(); - auto step = assistant_.draft_one( - assistant_input_token, - target_.token_embedding_node_for_mtp(), - assistant_hidden, - cache_nodes, - assistant_position, - i == 0 ? (1.0f / 16.0f) : 1.0f, - options.temperature, - options.top_p, - options.top_k, - options.min_p, - i + 1 < main_draft_limit, - use_tree && i == 0); - auto draft_end = std::chrono::steady_clock::now(); - double step_ms = elapsed_ms(draft_start, draft_end); - batch.assistant_draft_ms += step_ms; - batch.assistant_step_ms.push_back(step_ms); - - uint32_t draft_token = step.token; - if (use_tree && i == 0 && step.has_second_token) { - batch.alt_first_token = step.second_token; - batch.has_alt_first_token = true; - } - if (options.temperature > 0.0f) { - auto sample_start = std::chrono::steady_clock::now(); - if (sparse_sampled) { - draft_token = gemma4_mtp_sample_sparse_distribution(step.sparse_probabilities, rng); - batch.sparse_probabilities.push_back(std::move(step.sparse_probabilities)); - } else { - draft_token = gemma4_mtp_sample_distribution(step.probabilities, rng); - batch.probabilities.push_back(std::move(step.probabilities)); - } - auto sample_end = std::chrono::steady_clock::now(); - batch.sampling_or_argmax_ms += elapsed_ms(sample_start, sample_end); - } - - batch.tokens.push_back(draft_token); - if (i + 1 < main_draft_limit) { - assistant_hidden = std::move(step.hidden); - } - assistant_input_token = draft_token; - } - return batch; -} - -Gemma4SpeculativeVerifier::Gemma4SpeculativeVerifier(Gemma4Model& target) : target_(target) {} - -Gemma4MtpVerifyResult Gemma4SpeculativeVerifier::verify_and_commit( - uint32_t input_token, - const Gemma4MtpDraftBatch& draft, - size_t remaining_tokens, - const Gemma4MtpSamplingOptions& options, - bool sparse_sampled, - std::mt19937& rng) { - Gemma4MtpVerifyResult result; - result.verify_tokens.reserve(draft.tokens.size() + 1); - result.verify_tokens.push_back(input_token); - const bool use_tree = options.temperature == 0.0f - && draft.has_alt_first_token - && !draft.tokens.empty() - && draft.tokens.size() <= 3 - && target_.can_use_mtp_tree_attention(draft.tokens.size() + 2); - if (use_tree) { - result.verify_tokens.push_back(draft.tokens[0]); - result.verify_tokens.push_back(draft.alt_first_token); - result.verify_tokens.insert(result.verify_tokens.end(), draft.tokens.begin() + 1, draft.tokens.end()); - } else { - result.verify_tokens.insert(result.verify_tokens.end(), draft.tokens.begin(), draft.tokens.end()); - } - - auto* graph = static_cast(target_.graph_handle_); - size_t verifier_cache_start = target_.cache_position(); - auto kv_start = std::chrono::steady_clock::now(); - auto cache_txn = graph->begin_kv_cache_transaction(target_.cache_state_nodes_for_mtp()); - auto kv_end = std::chrono::steady_clock::now(); - result.kv_transaction_ms += elapsed_ms(kv_start, kv_end); - - Gemma4Model::GreedyBatchWithHidden target_batch; - auto verify_start = std::chrono::steady_clock::now(); - if (use_tree) { - target_batch = target_.decode_greedy_tokens_with_hidden_tree( - result.verify_tokens, tree_mask(result.verify_tokens.size())); - } else if (options.temperature == 0.0f) { - target_batch = target_.decode_greedy_tokens_with_hidden_early_stop( - result.verify_tokens, draft.tokens); - } else if (sparse_sampled) { - target_batch = target_.decode_tokens_with_hidden_and_sparse_probs( - result.verify_tokens, options.temperature, options.top_p, options.top_k, options.min_p); - } else { - target_batch = target_.decode_tokens_with_hidden_and_probs( - result.verify_tokens, options.temperature, options.top_p, options.top_k, options.min_p); - } - auto verify_end = std::chrono::steady_clock::now(); - result.target_verify_ms += elapsed_ms(verify_start, verify_end); - result.target_tokens = target_batch.tokens; - - auto sample_start = std::chrono::steady_clock::now(); - if (options.temperature == 0.0f) { - if (use_tree) { - if (target_batch.tokens.size() != result.verify_tokens.size()) { - throw std::runtime_error("Gemma 4 tree verifier target row count mismatch"); - } - auto main_row = [](size_t draft_index) { - return draft_index == 0 ? size_t{1} : draft_index + 2; - }; - auto prediction_row = [&](size_t draft_index) { - return draft_index == 0 ? size_t{0} : main_row(draft_index - 1); - }; - result.committed_verify_indices.push_back(0); - result.next_hidden_row = 0; - - for (size_t i = 0; i < draft.tokens.size(); ++i) { - const size_t pred_row = prediction_row(i); - if (target_batch.tokens[pred_row] != draft.tokens[i]) { - if (i == 0 && target_batch.tokens[0] == draft.alt_first_token) { - result.output_tokens.push_back(draft.alt_first_token); - result.accepted = 1; - result.next_hidden_row = 2; - if (result.output_tokens.size() < remaining_tokens) { - result.output_tokens.push_back(target_batch.tokens[2]); - result.emitted_extra_target_token = true; - } - result.committed_verify_indices = {0, 2}; - } else { - result.output_tokens.push_back(target_batch.tokens[pred_row]); - result.rejected = true; - } - break; - } - result.output_tokens.push_back(draft.tokens[i]); - result.accepted = i + 1; - result.next_hidden_row = main_row(i); - result.committed_verify_indices.push_back(result.next_hidden_row); - } - - if (!result.rejected - && !result.emitted_extra_target_token - && result.accepted == draft.tokens.size() - && result.output_tokens.size() < remaining_tokens) { - result.output_tokens.push_back(target_batch.tokens[result.next_hidden_row]); - result.emitted_extra_target_token = true; - } - } else { - const size_t produced = target_batch.tokens.size(); - const size_t draft_size = draft.tokens.size(); - for (; result.accepted < draft_size && result.accepted < produced; ++result.accepted) { - if (draft.tokens[result.accepted] != target_batch.tokens[result.accepted]) { - result.output_tokens.push_back(target_batch.tokens[result.accepted]); - result.rejected = true; - break; - } - result.output_tokens.push_back(draft.tokens[result.accepted]); - } - if (!result.rejected && result.output_tokens.size() < remaining_tokens - && produced > draft_size) { - result.output_tokens.push_back(target_batch.tokens[draft_size]); - result.emitted_extra_target_token = true; - } - } - } else { - for (; result.accepted < draft.tokens.size(); ++result.accepted) { - uint32_t y = draft.tokens[result.accepted]; - float p_y = 0.0f; - float q_y = 0.0f; - if (sparse_sampled) { - p_y = mtp_sparse_probability_at(target_batch.sparse_probabilities[result.accepted], y); - q_y = mtp_sparse_probability_at(draft.sparse_probabilities[result.accepted], y); - } else { - const auto& p = target_batch.probabilities[result.accepted]; - const auto& q = draft.probabilities[result.accepted]; - p_y = y < p.size() ? p[y] : 0.0f; - q_y = y < q.size() ? q[y] : 0.0f; - } - - float accept_probability = q_y <= 0.0f ? 1.0f : std::min(1.0f, p_y / q_y); - std::uniform_real_distribution dist(0.0f, 1.0f); - if (dist(rng) <= accept_probability) { - result.output_tokens.push_back(y); - continue; - } - - if (sparse_sampled) { - auto adjusted = mtp_rejection_adjusted_sparse_distribution( - target_batch.sparse_probabilities[result.accepted], - draft.sparse_probabilities[result.accepted]); - result.output_tokens.push_back(gemma4_mtp_sample_sparse_distribution(adjusted, rng)); - } else { - auto adjusted = rejection_adjusted_distribution( - target_batch.probabilities[result.accepted], - draft.probabilities[result.accepted]); - result.output_tokens.push_back(gemma4_mtp_sample_distribution(adjusted, rng)); - } - result.rejected = true; - break; - } - if (!result.rejected && result.output_tokens.size() < remaining_tokens) { - if (sparse_sampled) { - result.output_tokens.push_back(gemma4_mtp_sample_sparse_distribution( - target_batch.sparse_probabilities[draft.tokens.size()], rng)); - } else { - result.output_tokens.push_back(gemma4_mtp_sample_distribution( - target_batch.probabilities[draft.tokens.size()], rng)); - } - result.emitted_extra_target_token = true; - } - } - auto sample_end = std::chrono::steady_clock::now(); - result.sampling_or_argmax_ms += elapsed_ms(sample_start, sample_end); - - if (use_tree) { - for (uint32_t token : result.output_tokens) { - target_.record_sampled_token(token); - } - } - - if (!result.output_tokens.empty()) { - size_t row = use_tree - ? result.next_hidden_row - : (options.temperature == 0.0f ? result.accepted : result.output_tokens.size() - 1); - result.next_hidden = hidden_row(target_batch, row); - } - - size_t committed_cache_tokens = result.output_tokens.size(); - result.target_cache_crop_length = verifier_cache_start + committed_cache_tokens; - kv_start = std::chrono::steady_clock::now(); - if (use_tree) { - cache_txn.commit_selected(result.committed_verify_indices); - graph->apply_pending_kv_cache_sequence_lengths(); - target_.set_cache_position_for_mtp(result.target_cache_crop_length); - } else if (committed_cache_tokens < result.verify_tokens.size()) { - cache_txn.commit_prefix(committed_cache_tokens); - graph->apply_pending_kv_cache_sequence_lengths(); - target_.set_cache_position_for_mtp(result.target_cache_crop_length); - } else { - cache_txn.commit_all(); - graph->apply_pending_kv_cache_sequence_lengths(); - } - kv_end = std::chrono::steady_clock::now(); - result.kv_transaction_ms += elapsed_ms(kv_start, kv_end); - - return result; -} - -Gemma4MtpDecodeStrategy::Gemma4MtpDecodeStrategy(Gemma4Model& target, - Gemma4MtpAssistant& assistant) - : draft_provider_(target, assistant), verifier_(target) {} - -Gemma4MtpRoundResult Gemma4MtpDecodeStrategy::decode_round( - uint32_t input_token, - const std::vector<__fp16>& previous_hidden, - const Gemma4Model::SharedCacheNodes& cache_nodes, - size_t assistant_position, - size_t draft_limit, - size_t remaining_tokens, - const Gemma4MtpSamplingOptions& options, - bool sparse_sampled, - std::mt19937& rng) { - Gemma4MtpRoundResult round; - round.draft = draft_provider_.draft( - input_token, - previous_hidden, - cache_nodes, - assistant_position, - draft_limit, - options, - sparse_sampled, - rng); - round.verification = verifier_.verify_and_commit( - input_token, - round.draft, - remaining_tokens, - options, - sparse_sampled, - rng); - return round; -} - -} -} diff --git a/cactus-engine/src/gemma4_mtp_decode.h b/cactus-engine/src/gemma4_mtp_decode.h deleted file mode 100644 index 1723b29c3..000000000 --- a/cactus-engine/src/gemma4_mtp_decode.h +++ /dev/null @@ -1,111 +0,0 @@ -#pragma once - -#include "../models/gemma4/gemma4_mtp_assistant.h" -#include "../models/gemma4/model_gemma4.h" - -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -struct Gemma4MtpSamplingOptions { - float temperature = 0.0f; - float top_p = 0.0f; - size_t top_k = 0; - float min_p = 0.0f; -}; - -struct Gemma4MtpDraftBatch { - std::vector tokens; - uint32_t alt_first_token = 0; - bool has_alt_first_token = false; - std::vector> probabilities; - std::vector>> sparse_probabilities; - std::vector assistant_step_ms; - double assistant_draft_ms = 0.0; - double sampling_or_argmax_ms = 0.0; -}; - -struct Gemma4MtpVerifyResult { - std::vector output_tokens; - std::vector verify_tokens; - std::vector target_tokens; - std::vector committed_verify_indices; - std::vector<__fp16> next_hidden; - size_t next_hidden_row = 0; - size_t accepted = 0; - bool rejected = false; - bool emitted_extra_target_token = false; - size_t target_cache_crop_length = 0; - double target_verify_ms = 0.0; - double sampling_or_argmax_ms = 0.0; - double kv_transaction_ms = 0.0; -}; - -struct Gemma4MtpRoundResult { - Gemma4MtpDraftBatch draft; - Gemma4MtpVerifyResult verification; -}; - -uint32_t gemma4_mtp_sample_distribution(const std::vector& probabilities, std::mt19937& rng); -uint32_t gemma4_mtp_sample_sparse_distribution( - const std::vector>& probabilities, - std::mt19937& rng); - -class Gemma4AssistantDraftProvider { -public: - Gemma4AssistantDraftProvider(Gemma4Model& target, Gemma4MtpAssistant& assistant); - - Gemma4MtpDraftBatch draft(uint32_t input_token, - const std::vector<__fp16>& previous_hidden, - const Gemma4Model::SharedCacheNodes& cache_nodes, - size_t assistant_position, - size_t draft_limit, - const Gemma4MtpSamplingOptions& options, - bool sparse_sampled, - std::mt19937& rng); - -private: - Gemma4Model& target_; - Gemma4MtpAssistant& assistant_; -}; - -class Gemma4SpeculativeVerifier { -public: - explicit Gemma4SpeculativeVerifier(Gemma4Model& target); - - Gemma4MtpVerifyResult verify_and_commit(uint32_t input_token, - const Gemma4MtpDraftBatch& draft, - size_t remaining_tokens, - const Gemma4MtpSamplingOptions& options, - bool sparse_sampled, - std::mt19937& rng); - -private: - Gemma4Model& target_; -}; - -class Gemma4MtpDecodeStrategy { -public: - Gemma4MtpDecodeStrategy(Gemma4Model& target, Gemma4MtpAssistant& assistant); - - Gemma4MtpRoundResult decode_round(uint32_t input_token, - const std::vector<__fp16>& previous_hidden, - const Gemma4Model::SharedCacheNodes& cache_nodes, - size_t assistant_position, - size_t draft_limit, - size_t remaining_tokens, - const Gemma4MtpSamplingOptions& options, - bool sparse_sampled, - std::mt19937& rng); - -private: - Gemma4AssistantDraftProvider draft_provider_; - Gemma4SpeculativeVerifier verifier_; -}; - -} -} diff --git a/cactus-engine/src/mtp_completion.h b/cactus-engine/src/mtp_completion.h index 443403d4e..3c73bacd5 100644 --- a/cactus-engine/src/mtp_completion.h +++ b/cactus-engine/src/mtp_completion.h @@ -1,10 +1,15 @@ #pragma once -#include "decode_strategy.h" - +#include #include +#include +#include #include +#include +#include +#include #include +#include #include struct AcceptedToken { @@ -12,6 +17,16 @@ struct AcceptedToken { std::string text; }; +struct AcceptedTokenBatch { + std::vector tokens; + std::vector texts; + std::vector entropies; + size_t drafted_tokens = 0; + size_t accepted_draft_tokens = 0; + size_t rejected_tokens = 0; + std::vector rejected_draft_token_ids; +}; + struct MtpCompletionResult { bool stopped = false; size_t tokens_consumed = 0; @@ -30,6 +45,7 @@ struct MtpAvailability { bool required = false; bool assistant_loaded = false; bool supports_prompt = true; + bool supports_target = true; }; struct MtpFallbackDecision { @@ -42,6 +58,13 @@ inline MtpFallbackDecision decide_mtp_fallback(const MtpAvailability& availabili if (!availability.requested) { return MtpFallbackDecision{.use_standard_decode = true, .reason = "mtp_disabled"}; } + if (!availability.supports_target) { + return MtpFallbackDecision{ + .use_standard_decode = !availability.required, + .error = availability.required, + .reason = "unsupported_target", + }; + } if (!availability.supports_prompt) { return MtpFallbackDecision{ .use_standard_decode = !availability.required, @@ -123,3 +146,165 @@ class MtpCompletionAccumulator { std::vector token_history_; MtpCompletionMetrics metrics_; }; + +inline std::string mtp_env_string(const char* key, const std::string& fallback = "") { + const char* value = std::getenv(key); + return value && value[0] ? std::string(value) : fallback; +} + +inline std::string mtp_csv_escape_field(const std::string& value) { + if (value.find_first_of(",\"\n\r") == std::string::npos) return value; + std::string out = "\""; + for (char c : value) { + if (c == '"') out += "\"\""; + else out += c; + } + out += "\""; + return out; +} + +inline std::string mtp_bool_field(bool value) { + return value ? "true" : "false"; +} + +inline std::string mtp_double_field(double value) { + std::ostringstream out; + out << std::fixed << std::setprecision(3) << value; + return out.str(); +} + +inline std::string mtp_join_ms(const std::vector& values) { + std::ostringstream out; + out << std::fixed << std::setprecision(3); + for (size_t i = 0; i < values.size(); ++i) { + if (i > 0) out << ";"; + out << values[i]; + } + return out.str(); +} + +inline void mtp_append_csv_line(const std::filesystem::path& path, + const std::string& header, + const std::vector& fields) { + bool write_header = !std::filesystem::exists(path) || std::filesystem::file_size(path) == 0; + std::ofstream out(path, std::ios::app); + if (!out) { + throw std::runtime_error("Unable to open diagnostic trace CSV: " + path.string()); + } + if (write_header) out << header << "\n"; + for (size_t i = 0; i < fields.size(); ++i) { + if (i > 0) out << ","; + out << mtp_csv_escape_field(fields[i]); + } + out << "\n"; +} + +struct DiagnosticTraceContext { + bool enabled = false; + std::filesystem::path dir; + std::string run_id; + std::string model; + std::string prompt_name; + std::string shape; + std::string rep; + + static DiagnosticTraceContext from_env() { + DiagnosticTraceContext ctx; + std::string dir = mtp_env_string("CACTUS_GEMMA4_MTP_TRACE_CSV_DIR"); + if (dir.empty()) return ctx; + ctx.run_id = mtp_env_string("CACTUS_MTP_DIAG_RUN_ID"); + if (ctx.run_id.empty()) return ctx; + ctx.enabled = true; + ctx.dir = dir; + std::filesystem::create_directories(ctx.dir); + ctx.model = mtp_env_string("CACTUS_MTP_DIAG_MODEL", "model"); + ctx.prompt_name = mtp_env_string("CACTUS_MTP_DIAG_PROMPT", "prompt"); + ctx.shape = mtp_env_string("CACTUS_MTP_DIAG_SHAPE", "unknown"); + ctx.rep = mtp_env_string("CACTUS_MTP_DIAG_REP", "0"); + return ctx; + } + + void write_round(size_t round_index, + size_t generated_start, + size_t generated_end, + size_t target_batch_m, + size_t assistant_pass_count, + size_t drafted_tokens, + size_t accepted_drafts, + bool rejected, + bool alt_branch_accepted, + bool emitted_extra_target_token, + double target_forward_ms, + double assistant_total_ms, + const std::vector& assistant_step_ms, + double sampling_or_argmax_ms, + double kv_transaction_ms, + double callback_stream_ms, + double loop_overhead_ms, + double round_total_ms) const { + if (!enabled || generated_end <= generated_start) return; + static const std::string header = + "run_id,model,prompt_name,shape,rep,round_index,generated_token_start,generated_token_end," + "generated_tokens_emitted,target_batch_m,assistant_pass_count,drafted_tokens,accepted_drafts," + "rejected,alt_branch_accepted,emitted_extra_target_token,target_forward_ms,assistant_total_ms," + "assistant_step_ms,sampling_or_argmax_ms,kv_transaction_ms,callback_stream_ms,loop_overhead_ms,round_total_ms"; + mtp_append_csv_line(dir / "round_trace.csv", header, { + run_id, model, prompt_name, shape, rep, + std::to_string(round_index), + std::to_string(generated_start), + std::to_string(generated_end), + std::to_string(generated_end - generated_start), + std::to_string(target_batch_m), + std::to_string(assistant_pass_count), + std::to_string(drafted_tokens), + std::to_string(accepted_drafts), + mtp_bool_field(rejected), + mtp_bool_field(alt_branch_accepted), + mtp_bool_field(emitted_extra_target_token), + mtp_double_field(target_forward_ms), + mtp_double_field(assistant_total_ms), + mtp_join_ms(assistant_step_ms), + mtp_double_field(sampling_or_argmax_ms), + mtp_double_field(kv_transaction_ms), + mtp_double_field(callback_stream_ms), + mtp_double_field(loop_overhead_ms), + mtp_double_field(round_total_ms), + }); + } + + void write_token(size_t token_position, + uint32_t token_id, + size_t round_index, + size_t token_index_in_round, + const std::string& source, + size_t round_emitted, + double target_forward_ms, + double assistant_total_ms, + double other_ms, + double round_total_ms) const { + if (!enabled || round_emitted == 0) return; + static const std::string header = + "run_id,model,prompt_name,shape,rep,token_position,token_id,round_index,token_index_in_round," + "source,round_generated_tokens_emitted,round_target_forward_ms,round_assistant_total_ms," + "round_other_ms,round_total_ms,allocated_target_forward_ms,allocated_assistant_ms," + "allocated_other_ms,allocated_total_ms"; + double denom = static_cast(round_emitted); + mtp_append_csv_line(dir / "token_trace.csv", header, { + run_id, model, prompt_name, shape, rep, + std::to_string(token_position), + std::to_string(token_id), + std::to_string(round_index), + std::to_string(token_index_in_round), + source, + std::to_string(round_emitted), + mtp_double_field(target_forward_ms), + mtp_double_field(assistant_total_ms), + mtp_double_field(other_ms), + mtp_double_field(round_total_ms), + mtp_double_field(target_forward_ms / denom), + mtp_double_field(assistant_total_ms / denom), + mtp_double_field(other_ms / denom), + mtp_double_field(round_total_ms / denom), + }); + } +}; diff --git a/cactus-engine/src/mtp_decode.h b/cactus-engine/src/mtp_decode.h new file mode 100644 index 000000000..6d06b7797 --- /dev/null +++ b/cactus-engine/src/mtp_decode.h @@ -0,0 +1,206 @@ +#pragma once + +#include "mtp_sampler.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +struct MtpDraftStep { + uint32_t token = 0; + uint32_t alt_first_token = 0; + bool has_alt_first_token = false; + std::vector probabilities; + std::vector> sparse_probabilities; + std::vector<__fp16> hidden; +}; + +struct MtpDraftBatch { + std::vector tokens; + uint32_t alt_first_token = 0; + bool has_alt_first_token = false; + std::vector probabilities; + std::vector>> sparse_probabilities; + std::vector assistant_step_ms; + double assistant_draft_ms = 0.0; + double sampling_or_argmax_ms = 0.0; +}; + +struct MtpVerificationResult { + std::vector output_tokens; + std::vector verify_tokens; + std::vector target_tokens; + std::vector committed_verify_indices; + std::vector<__fp16> next_hidden; + size_t next_hidden_row = 0; + size_t accepted_draft_tokens = 0; + bool rejected = false; + bool emitted_extra_target_token = false; + size_t target_cache_crop_length = 0; + double target_verify_ms = 0.0; + double sampling_or_argmax_ms = 0.0; + double kv_transaction_ms = 0.0; +}; + +struct MtpRoundResult { + MtpDraftBatch draft; + MtpVerificationResult verification; +}; + +class MtpDeterministicRng { +public: + explicit MtpDeterministicRng(std::initializer_list values) : values_(values) {} + + float uniform() { + if (values_.empty()) return 0.0f; + float value = values_[index_ % values_.size()]; + index_++; + return std::clamp(value, 0.0f, 1.0f); + } + +private: + std::vector values_; + size_t index_ = 0; +}; + +inline double mtp_elapsed_ms(std::chrono::steady_clock::time_point start, + std::chrono::steady_clock::time_point end) { + return std::chrono::duration(end - start).count(); +} + +inline uint32_t mtp_sample_distribution(const std::vector& probabilities, std::mt19937& rng) { + if (probabilities.empty()) return 0; + std::uniform_real_distribution dist(0.0f, 1.0f); + return mtp_sample(MtpDistribution{probabilities}, dist(rng)); +} + +inline uint32_t mtp_sample_sparse_distribution(const std::vector>& probabilities, + std::mt19937& rng) { + if (probabilities.empty()) return 0; + std::uniform_real_distribution dist(0.0f, 1.0f); + return mtp_sample_sparse(probabilities, dist(rng)); +} + +template +MtpDraftBatch mtp_draft_tokens(uint32_t input_token, + const std::vector<__fp16>& previous_hidden, + size_t draft_limit, + const MtpSamplingOptions& options, + bool sparse_sampled, + std::mt19937& rng, + DraftOne&& draft_one) { + MtpDraftBatch batch; + batch.tokens.reserve(draft_limit); + batch.assistant_step_ms.reserve(draft_limit); + if (options.temperature > 0.0f) { + if (sparse_sampled) { + batch.sparse_probabilities.reserve(draft_limit); + } else { + batch.probabilities.reserve(draft_limit); + } + } + + std::vector<__fp16> assistant_hidden = previous_hidden; + uint32_t assistant_input_token = input_token; + for (size_t i = 0; i < draft_limit; ++i) { + auto draft_start = std::chrono::steady_clock::now(); + MtpDraftStep step = draft_one(assistant_input_token, assistant_hidden, i, i + 1 < draft_limit); + auto draft_end = std::chrono::steady_clock::now(); + double step_ms = mtp_elapsed_ms(draft_start, draft_end); + batch.assistant_draft_ms += step_ms; + batch.assistant_step_ms.push_back(step_ms); + + uint32_t draft_token = step.token; + if (i == 0 && step.has_alt_first_token) { + batch.alt_first_token = step.alt_first_token; + batch.has_alt_first_token = true; + } + if (options.temperature > 0.0f) { + auto sample_start = std::chrono::steady_clock::now(); + if (sparse_sampled) { + draft_token = mtp_sample_sparse_distribution(step.sparse_probabilities, rng); + batch.sparse_probabilities.push_back(std::move(step.sparse_probabilities)); + } else { + draft_token = mtp_sample_distribution(step.probabilities, rng); + batch.probabilities.push_back(MtpDistribution{std::move(step.probabilities)}); + } + auto sample_end = std::chrono::steady_clock::now(); + batch.sampling_or_argmax_ms += mtp_elapsed_ms(sample_start, sample_end); + } + + batch.tokens.push_back(draft_token); + if (i + 1 < draft_limit) { + assistant_hidden = std::move(step.hidden); + } + assistant_input_token = draft_token; + } + return batch; +} + +inline MtpVerificationResult verify_greedy_mtp_draft(const MtpDraftBatch& draft, + const std::vector& target) { + if (target.size() < draft.tokens.size() + 1) { + throw std::invalid_argument("MTP target distributions must include one extra next-token distribution"); + } + + MtpVerificationResult result; + for (size_t i = 0; i < draft.tokens.size(); ++i) { + uint32_t target_token = mtp_argmax(target[i]); + if (draft.tokens[i] != target_token) { + result.output_tokens.push_back(target_token); + result.rejected = true; + return result; + } + result.output_tokens.push_back(draft.tokens[i]); + result.accepted_draft_tokens++; + } + + result.output_tokens.push_back(mtp_argmax(target[draft.tokens.size()])); + result.emitted_extra_target_token = true; + return result; +} + +inline MtpVerificationResult verify_sampled_mtp_draft(const MtpDraftBatch& draft, + const std::vector& target, + MtpDeterministicRng& rng) { + if (target.size() < draft.tokens.size() + 1) { + throw std::invalid_argument("MTP target distributions must include one extra next-token distribution"); + } + if (draft.probabilities.size() < draft.tokens.size()) { + throw std::invalid_argument("MTP draft must include assistant probabilities for each token"); + } + + MtpVerificationResult result; + for (size_t i = 0; i < draft.tokens.size(); ++i) { + uint32_t y = draft.tokens[i]; + const auto& q = draft.probabilities[i]; + const auto& p = target[i]; + if (y >= q.probabilities.size() || y >= p.probabilities.size()) { + throw std::out_of_range("MTP draft token outside distribution vocabulary"); + } + + float q_y = q.probabilities[y]; + float p_y = p.probabilities[y]; + float accept_probability = q_y <= 0.0f ? 1.0f : std::min(1.0f, p_y / q_y); + + if (rng.uniform() <= accept_probability) { + result.output_tokens.push_back(y); + result.accepted_draft_tokens++; + continue; + } + + MtpDistribution adjusted = rejection_adjusted_distribution(p, q); + result.output_tokens.push_back(mtp_sample(adjusted, rng.uniform())); + result.rejected = true; + return result; + } + + result.output_tokens.push_back(mtp_sample(target[draft.tokens.size()], rng.uniform())); + result.emitted_extra_target_token = true; + return result; +} diff --git a/cactus-engine/src/mtp_sampler.h b/cactus-engine/src/mtp_sampler.h index 78bcce2d8..0ea8c43f9 100644 --- a/cactus-engine/src/mtp_sampler.h +++ b/cactus-engine/src/mtp_sampler.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -13,31 +13,11 @@ struct MtpDistribution { std::vector probabilities; }; -struct MtpDraftBatch { - std::vector tokens; - std::vector probabilities; -}; - -struct MtpVerificationResult { - std::vector output_tokens; - size_t accepted_draft_tokens = 0; - bool rejected = false; -}; - -class MtpDeterministicRng { -public: - explicit MtpDeterministicRng(std::initializer_list values) : values_(values) {} - - float uniform() { - if (values_.empty()) return 0.0f; - float value = values_[index_ % values_.size()]; - index_++; - return std::clamp(value, 0.0f, 1.0f); - } - -private: - std::vector values_; - size_t index_ = 0; +struct MtpSamplingOptions { + float temperature = 0.0f; + float top_p = 0.0f; + size_t top_k = 0; + float min_p = 0.0f; }; inline uint32_t mtp_argmax(const MtpDistribution& dist) { @@ -85,6 +65,161 @@ inline float mtp_sparse_probability_at(const std::vector mtp_normalize_logits(const std::vector& logits) { + std::vector probs(logits.size(), 0.0f); + float max_value = -std::numeric_limits::infinity(); + for (float value : logits) max_value = std::max(max_value, value); + if (!std::isfinite(max_value)) return probs; + + double sum = 0.0; + for (size_t i = 0; i < logits.size(); ++i) { + if (std::isfinite(logits[i])) { + probs[i] = static_cast(std::exp(static_cast(logits[i] - max_value))); + sum += probs[i]; + } + } + if (sum <= 0.0 || !std::isfinite(sum)) return probs; + + for (float& prob : probs) prob = static_cast(prob / sum); + return probs; +} + +inline MtpDistribution mtp_distribution_from_logits(std::vector logits, + const MtpSamplingOptions& options) { + MtpDistribution result; + if (logits.empty()) return result; + + if (options.temperature == 0.0f) { + result.probabilities.assign(logits.size(), 0.0f); + auto it = std::max_element(logits.begin(), logits.end()); + result.probabilities[std::distance(logits.begin(), it)] = 1.0f; + return result; + } + + if (options.temperature > 0.0f) { + for (float& value : logits) value /= options.temperature; + } + + if (options.top_k > 0 && options.top_k < logits.size()) { + std::vector sorted = logits; + std::nth_element(sorted.begin(), sorted.begin() + options.top_k - 1, sorted.end(), std::greater()); + float kth = sorted[options.top_k - 1]; + for (float& value : logits) { + if (value < kth) value = -std::numeric_limits::infinity(); + } + } + + if (options.min_p > 0.0f) { + auto probs = mtp_normalize_logits(logits); + if (!probs.empty()) { + float threshold = (*std::max_element(probs.begin(), probs.end())) * options.min_p; + for (size_t i = 0; i < logits.size(); ++i) { + if (probs[i] < threshold) logits[i] = -std::numeric_limits::infinity(); + } + } + } + + if (options.top_p > 0.0f && options.top_p < 1.0f) { + std::vector> sorted; + sorted.reserve(logits.size()); + for (size_t i = 0; i < logits.size(); ++i) { + if (std::isfinite(logits[i])) sorted.emplace_back(logits[i], i); + } + std::sort(sorted.begin(), sorted.end(), [](const auto& a, const auto& b) { + return a.first > b.first; + }); + + std::vector sorted_logits; + sorted_logits.reserve(sorted.size()); + for (const auto& item : sorted) sorted_logits.push_back(item.first); + auto sorted_probs = mtp_normalize_logits(sorted_logits); + + float cumulative = 0.0f; + bool remove = false; + for (size_t i = 0; i < sorted.size(); ++i) { + cumulative += sorted_probs[i]; + if (remove) logits[sorted[i].second] = -std::numeric_limits::infinity(); + if (cumulative > options.top_p) remove = true; + } + } + + result.probabilities = mtp_normalize_logits(logits); + float sum = std::accumulate(result.probabilities.begin(), result.probabilities.end(), 0.0f); + if (sum <= 0.0f || !std::isfinite(sum)) { + float uniform = result.probabilities.empty() ? 0.0f : 1.0f / static_cast(result.probabilities.size()); + std::fill(result.probabilities.begin(), result.probabilities.end(), uniform); + } + return result; +} + +inline std::vector> mtp_sparse_distribution_from_logits( + std::vector> logits, + const MtpSamplingOptions& options) { + if (logits.empty()) return {}; + + if (options.temperature > 0.0f) { + for (auto& item : logits) item.second /= options.temperature; + } + std::sort(logits.begin(), logits.end(), [](const auto& a, const auto& b) { + return a.second > b.second; + }); + + auto normalize = [](const std::vector>& values) { + std::vector> probs; + probs.reserve(values.size()); + float max_value = -std::numeric_limits::infinity(); + for (const auto& item : values) max_value = std::max(max_value, item.second); + if (!std::isfinite(max_value)) return probs; + double sum = 0.0; + for (const auto& item : values) { + float prob = static_cast(std::exp(static_cast(item.second - max_value))); + probs.emplace_back(item.first, prob); + sum += prob; + } + if (sum <= 0.0 || !std::isfinite(sum)) return std::vector>{}; + for (auto& item : probs) item.second = static_cast(item.second / sum); + return probs; + }; + + if (options.min_p > 0.0f) { + auto probs = normalize(logits); + if (!probs.empty()) { + float max_prob = 0.0f; + for (const auto& item : probs) max_prob = std::max(max_prob, item.second); + float threshold = max_prob * options.min_p; + std::vector> filtered; + filtered.reserve(logits.size()); + for (size_t i = 0; i < logits.size(); ++i) { + if (probs[i].second >= threshold) filtered.push_back(logits[i]); + } + logits = std::move(filtered); + } + } + + if (options.top_p > 0.0f && options.top_p < 1.0f) { + auto probs = normalize(logits); + std::vector> filtered; + filtered.reserve(logits.size()); + float cumulative = 0.0f; + bool remove = false; + for (size_t i = 0; i < logits.size(); ++i) { + cumulative += i < probs.size() ? probs[i].second : 0.0f; + if (!remove) filtered.push_back(logits[i]); + if (cumulative > options.top_p) remove = true; + } + logits = std::move(filtered); + } + + auto probs = normalize(logits); + float sum = 0.0f; + for (const auto& item : probs) sum += item.second; + if (sum <= 0.0f || !std::isfinite(sum)) { + float uniform = probs.empty() ? 0.0f : 1.0f / static_cast(probs.size()); + for (auto& item : probs) item.second = uniform; + } + return probs; +} + inline MtpDistribution rejection_adjusted_distribution(const MtpDistribution& target, const MtpDistribution& assistant) { if (target.probabilities.size() != assistant.probabilities.size()) { @@ -136,64 +271,3 @@ inline std::vector> mtp_rejection_adjusted_sparse_dis for (auto& item : adjusted) item.second /= sum; return adjusted; } - -inline MtpVerificationResult verify_greedy_mtp_draft(const MtpDraftBatch& draft, - const std::vector& target) { - if (target.size() < draft.tokens.size() + 1) { - throw std::invalid_argument("MTP target distributions must include one extra next-token distribution"); - } - - MtpVerificationResult result; - for (size_t i = 0; i < draft.tokens.size(); ++i) { - uint32_t target_token = mtp_argmax(target[i]); - if (draft.tokens[i] != target_token) { - result.output_tokens.push_back(target_token); - result.rejected = true; - return result; - } - result.output_tokens.push_back(draft.tokens[i]); - result.accepted_draft_tokens++; - } - - result.output_tokens.push_back(mtp_argmax(target[draft.tokens.size()])); - return result; -} - -inline MtpVerificationResult verify_sampled_mtp_draft(const MtpDraftBatch& draft, - const std::vector& target, - MtpDeterministicRng& rng) { - if (target.size() < draft.tokens.size() + 1) { - throw std::invalid_argument("MTP target distributions must include one extra next-token distribution"); - } - if (draft.probabilities.size() < draft.tokens.size()) { - throw std::invalid_argument("MTP draft must include assistant probabilities for each token"); - } - - MtpVerificationResult result; - for (size_t i = 0; i < draft.tokens.size(); ++i) { - uint32_t y = draft.tokens[i]; - const auto& q = draft.probabilities[i]; - const auto& p = target[i]; - if (y >= q.probabilities.size() || y >= p.probabilities.size()) { - throw std::out_of_range("MTP draft token outside distribution vocabulary"); - } - - float q_y = q.probabilities[y]; - float p_y = p.probabilities[y]; - float accept_probability = q_y <= 0.0f ? 1.0f : std::min(1.0f, p_y / q_y); - - if (rng.uniform() <= accept_probability) { - result.output_tokens.push_back(y); - result.accepted_draft_tokens++; - continue; - } - - MtpDistribution adjusted = rejection_adjusted_distribution(p, q); - result.output_tokens.push_back(mtp_sample(adjusted, rng.uniform())); - result.rejected = true; - return result; - } - - result.output_tokens.push_back(mtp_sample(target[draft.tokens.size()], rng.uniform())); - return result; -} diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index 0eb6bf736..504fdf01b 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -443,7 +443,7 @@ struct InferenceOptions { float temperature = 0.0f; float top_p = 0.0f; float min_p = 0.15f; - float repetition_penalty = 1.0f; + float repetition_penalty = 1.1f; float confidence_threshold = -1.0f; size_t top_k = 0; size_t max_tokens = 100; diff --git a/cactus-engine/tests/test_decode_strategy.cpp b/cactus-engine/tests/test_decode_strategy.cpp deleted file mode 100644 index bc3ba39b0..000000000 --- a/cactus-engine/tests/test_decode_strategy.cpp +++ /dev/null @@ -1,165 +0,0 @@ -#include "test_utils.h" -#include "src/decode_strategy.h" - -#include -#include -#include -#include - -using namespace TestUtils; - -class FakeTargetDecoder final : public TargetDecoder { -public: - explicit FakeTargetDecoder(std::vector tokens) : tokens_(std::move(tokens)) {} - - DecodeStepResult decode_one(const std::vector& input_tokens, - const DecodeOptions& options) override { - seen_inputs.push_back(input_tokens); - seen_options.push_back(options); - uint32_t token = tokens_.at(index_++); - return DecodeStepResult{.token = token, .entropy = 0.0f}; - } - - std::vector> seen_inputs; - std::vector seen_options; - -private: - std::vector tokens_; - size_t index_ = 0; -}; - -class FakeAssistantDraftProvider final : public AssistantDraftProvider { -public: - explicit FakeAssistantDraftProvider(std::vector drafts) : drafts_(std::move(drafts)) {} - - DraftBatch draft(const TargetDecodeState& state, size_t max_draft_tokens) override { - seen_states.push_back(state); - seen_limits.push_back(max_draft_tokens); - return drafts_.at(index_++); - } - - std::vector seen_states; - std::vector seen_limits; - -private: - std::vector drafts_; - size_t index_ = 0; -}; - -class FakeVerifier final : public SpeculativeVerifier { -public: - explicit FakeVerifier(std::vector batches) : batches_(std::move(batches)) {} - - AcceptedTokenBatch verify(const DraftBatch& draft, - const TargetDecodeState& state, - const DecodeOptions& options) override { - seen_drafts.push_back(draft); - seen_states.push_back(state); - seen_options.push_back(options); - return batches_.at(index_++); - } - - std::vector seen_drafts; - std::vector seen_states; - std::vector seen_options; - -private: - std::vector batches_; - size_t index_ = 0; -}; - -bool test_standard_strategy_is_null_object_fallback() { - FakeTargetDecoder target({42}); - StandardDecodeStrategy strategy(target); - - DecodeOptions options; - options.temperature = 0.0f; - AcceptedTokenBatch batch = strategy.decode_next({7, 8}, options); - - return batch.tokens == std::vector{42} - && batch.entropies.size() == 1 - && target.seen_inputs == std::vector>{{7, 8}}; -} - -bool test_mtp_strategy_returns_accepted_token_batch() { - FakeTargetDecoder target({99}); - FakeAssistantDraftProvider assistant({ - DraftBatch{.tokens = {10, 11, 12}}, - }); - FakeVerifier verifier({ - AcceptedTokenBatch{ - .tokens = {10, 11, 7}, - .entropies = {0.1f, 0.2f, 0.3f}, - .drafted_tokens = 3, - .accepted_draft_tokens = 2, - .rejected_tokens = 1, - }, - }); - Gemma4MtpDecodeStrategy strategy(target, assistant, verifier); - - DecodeOptions options; - options.temperature = 0.0f; - options.max_draft_tokens = 3; - AcceptedTokenBatch batch = strategy.decode_next({1, 2, 3}, options); - - return batch.tokens == std::vector({10, 11, 7}) - && batch.drafted_tokens == 3 - && batch.accepted_draft_tokens == 2 - && batch.rejected_tokens == 1 - && assistant.seen_limits == std::vector{3} - && verifier.seen_drafts.size() == 1; -} - -bool test_strategy_factory_selects_standard_when_mtp_disabled() { - FakeTargetDecoder target({5}); - FakeAssistantDraftProvider assistant({}); - FakeVerifier verifier({}); - - DecodeOptions options; - options.mtp_enabled = false; - std::unique_ptr strategy = make_decode_strategy(target, &assistant, &verifier, options); - - return strategy->name() == std::string("standard"); -} - -bool test_strategy_factory_selects_mtp_when_available() { - FakeTargetDecoder target({5}); - FakeAssistantDraftProvider assistant({}); - FakeVerifier verifier({}); - - DecodeOptions options; - options.mtp_enabled = true; - options.max_draft_tokens = 6; - std::unique_ptr strategy = make_decode_strategy(target, &assistant, &verifier, options); - - return strategy->name() == std::string("gemma4_mtp"); -} - -bool test_strategy_factory_errors_when_mtp_required_but_unavailable() { - FakeTargetDecoder target({5}); - - DecodeOptions options; - options.mtp_enabled = true; - options.mtp_required = true; - - try { - (void)make_decode_strategy(target, nullptr, nullptr, options); - } catch (const MissingAssistantError& err) { - return std::string(err.what()).find("assistant") != std::string::npos; - } - - return false; -} - -int main() { - TestRunner runner("Decode Strategy Tests"); - - runner.run_test("standard_null_object", test_standard_strategy_is_null_object_fallback()); - runner.run_test("mtp_returns_batch", test_mtp_strategy_returns_accepted_token_batch()); - runner.run_test("factory_standard", test_strategy_factory_selects_standard_when_mtp_disabled()); - runner.run_test("factory_mtp", test_strategy_factory_selects_mtp_when_available()); - runner.run_test("factory_missing_required", test_strategy_factory_errors_when_mtp_required_but_unavailable()); - - runner.print_summary(); - return runner.all_passed() ? 0 : 1; -} diff --git a/cactus-engine/tests/test_gemma4_mtp_assistant_config.cpp b/cactus-engine/tests/test_gemma4_mtp_assistant_config.cpp new file mode 100644 index 000000000..0aa4d3b60 --- /dev/null +++ b/cactus-engine/tests/test_gemma4_mtp_assistant_config.cpp @@ -0,0 +1,55 @@ +#include "test_utils.h" +#include "models/gemma4/gemma4_mtp_assistant.h" + +#include + +using namespace TestUtils; +using namespace cactus::engine; + +bool test_assistant_config_defaults_match_existing_e2b_layout() { + Gemma4MtpAssistantConfig config = parse_gemma4_mtp_assistant_config("{}"); + + return config.target_hidden_dim == 1536 + && config.centroid_count == 2048 + && config.top_centroid_count == 32 + && config.tokens_per_centroid == 128 + && config.layers.size() == 4 + && config.layers[0].head_dim == 256 + && config.layers[3].head_dim == 512; +} + +bool test_assistant_config_reads_manifest_overrides() { + const std::string manifest = R"({ + "target_hidden_dim": 2048, + "centroid_count": 4096, + "top_centroid_count": 16, + "tokens_per_centroid": 64, + "layers": [ + {"head_dim": 128, "num_heads": 8, "rot_dim": 128, "rope_freq": 10000.0, "window": 1024, "full_attention": false}, + {"head_dim": 256, "num_heads": 8, "rot_dim": 64, "rope_freq": 1000000.0, "window": 0, "full_attention": true} + ] + })"; + + Gemma4MtpAssistantConfig config = parse_gemma4_mtp_assistant_config(manifest); + + return config.target_hidden_dim == 2048 + && config.centroid_count == 4096 + && config.top_centroid_count == 16 + && config.tokens_per_centroid == 64 + && config.layers.size() == 2 + && config.layers[0].window == 1024 + && !config.layers[0].full_attention + && config.layers[1].head_dim == 256 + && config.layers[1].rot_dim == 64 + && config.layers[1].full_attention; +} + +int main() { + TestRunner runner("Gemma 4 MTP Assistant Config Tests"); + + runner.run_test("defaults", test_assistant_config_defaults_match_existing_e2b_layout()); + runner.run_test("manifest_overrides", test_assistant_config_reads_manifest_overrides()); + + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus-engine/tests/test_mtp_completion.cpp b/cactus-engine/tests/test_mtp_completion.cpp index 4ccabc475..ced6a5a65 100644 --- a/cactus-engine/tests/test_mtp_completion.cpp +++ b/cactus-engine/tests/test_mtp_completion.cpp @@ -92,6 +92,20 @@ bool test_missing_assistant_records_fallback_reason() { && accumulator.metrics().fallback_reason == "assistant_unavailable"; } +bool test_non_gemma_target_reports_generic_unsupported_reason() { + MtpFallbackDecision decision = decide_mtp_fallback(MtpAvailability{ + .requested = true, + .required = false, + .assistant_loaded = false, + .supports_prompt = false, + .supports_target = false, + }); + + return decision.use_standard_decode + && !decision.error + && decision.reason == "unsupported_target"; +} + int main() { TestRunner runner("MTP Completion Semantics Tests"); @@ -100,6 +114,7 @@ int main() { runner.run_test("rejected_not_history", test_rejected_draft_tokens_do_not_enter_history()); runner.run_test("history_once", test_token_history_updates_once_per_accepted_token()); runner.run_test("fallback_reason", test_missing_assistant_records_fallback_reason()); + runner.run_test("unsupported_target", test_non_gemma_target_reports_generic_unsupported_reason()); runner.print_summary(); return runner.all_passed() ? 0 : 1; diff --git a/cactus-engine/tests/test_mtp_sampler.cpp b/cactus-engine/tests/test_mtp_sampler.cpp index 63d26bc62..31a4aebfc 100644 --- a/cactus-engine/tests/test_mtp_sampler.cpp +++ b/cactus-engine/tests/test_mtp_sampler.cpp @@ -1,8 +1,9 @@ #include "test_utils.h" -#include "src/mtp_sampler.h" +#include "src/mtp_decode.h" #include #include +#include #include using namespace TestUtils; @@ -115,6 +116,40 @@ bool test_sparse_rejection_adjustment_uses_target_support() { && mtp_sample_sparse(adjusted, 0.5f) == 12; } +bool test_dense_logits_to_probs_applies_temperature_top_k_top_p_min_p() { + MtpSamplingOptions options{ + .temperature = 1.0f, + .top_p = 0.80f, + .top_k = 3, + .min_p = 0.0f, + }; + + MtpDistribution probs = mtp_distribution_from_logits({1.0f, 4.0f, 3.0f, 2.0f}, options); + + return probs.probabilities.size() == 4 + && probs.probabilities[0] == 0.0f + && probs.probabilities[3] == 0.0f + && probs.probabilities[1] > probs.probabilities[2] + && approx_eq(probs.probabilities[1] + probs.probabilities[2], 1.0f); +} + +bool test_sparse_logits_to_probs_matches_dense_on_selected_support() { + MtpSamplingOptions options{ + .temperature = 1.0f, + .top_p = 1.0f, + .top_k = 2, + .min_p = 0.0f, + }; + + auto sparse = mtp_sparse_distribution_from_logits({{10, 4.0f}, {11, 2.0f}}, options); + + return sparse.size() == 2 + && sparse[0].first == 10 + && sparse[1].first == 11 + && sparse[0].second > sparse[1].second + && approx_eq(sparse[0].second + sparse[1].second, 1.0f); +} + bool test_sampled_accepts_when_target_probability_exceeds_assistant_probability() { MtpDraftBatch draft{ .tokens = {1}, @@ -169,6 +204,37 @@ bool test_zero_assistant_probability_does_not_create_nan() { && tokens_eq(result.output_tokens, {1, 2}); } +bool test_draft_loop_advances_token_and_hidden_state() { + MtpSamplingOptions options; + std::mt19937 rng(7); + std::vector inputs; + std::vector hidden_sizes; + std::vector needs_hidden_values; + + MtpDraftBatch batch = mtp_draft_tokens( + 5, + {static_cast<__fp16>(1)}, + 3, + options, + false, + rng, + [&](uint32_t input_token, const std::vector<__fp16>& hidden, size_t i, bool needs_hidden) { + inputs.push_back(input_token); + hidden_sizes.push_back(hidden.size()); + needs_hidden_values.push_back(needs_hidden); + MtpDraftStep step; + step.token = input_token + 1; + step.hidden = {static_cast<__fp16>(10 + i)}; + return step; + }); + + return tokens_eq(batch.tokens, {6, 7, 8}) + && tokens_eq(inputs, {5, 6, 7}) + && hidden_sizes == std::vector({1, 1, 1}) + && needs_hidden_values == std::vector({true, true, false}) + && batch.assistant_step_ms.size() == 3; +} + int main() { TestRunner runner("MTP Sampler Tests"); @@ -177,9 +243,12 @@ int main() { runner.run_test("greedy_rejects_after_prefix", test_greedy_accepts_prefix_then_replaces_mismatch()); runner.run_test("rejection_adjustment", test_rejection_adjusted_distribution_clamps_and_normalizes()); runner.run_test("sparse_rejection_adjustment", test_sparse_rejection_adjustment_uses_target_support()); + runner.run_test("dense_logits_to_probs", test_dense_logits_to_probs_applies_temperature_top_k_top_p_min_p()); + runner.run_test("sparse_logits_to_probs", test_sparse_logits_to_probs_matches_dense_on_selected_support()); runner.run_test("sampled_accepts_p_ge_q", test_sampled_accepts_when_target_probability_exceeds_assistant_probability()); runner.run_test("sampled_rejects_adjusted", test_sampled_rejects_and_samples_adjusted_replacement()); runner.run_test("zero_q_no_nan", test_zero_assistant_probability_does_not_create_nan()); + runner.run_test("draft_loop_state", test_draft_loop_advances_token_and_hidden_state()); runner.print_summary(); return runner.all_passed() ? 0 : 1; diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index 47e7e410c..e7edcc642 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -459,6 +459,11 @@ class CactusGraph { KvCacheTransaction() = default; KvCacheTransaction(CactusGraph* graph, std::vector> snapshots) : graph_(graph), snapshots_(std::move(snapshots)) {} + ~KvCacheTransaction(); + KvCacheTransaction(const KvCacheTransaction&) = delete; + KvCacheTransaction& operator=(const KvCacheTransaction&) = delete; + KvCacheTransaction(KvCacheTransaction&& other) noexcept; + KvCacheTransaction& operator=(KvCacheTransaction&& other) noexcept; void rollback(); void commit_all(); diff --git a/cactus-graph/src/cache_transaction.cpp b/cactus-graph/src/cache_transaction.cpp index 8aefedee3..8477a3707 100644 --- a/cactus-graph/src/cache_transaction.cpp +++ b/cactus-graph/src/cache_transaction.cpp @@ -93,6 +93,30 @@ CactusGraph::KvCacheTransaction CactusGraph::begin_kv_cache_transaction(const st return KvCacheTransaction(this, std::move(snapshots)); } +CactusGraph::KvCacheTransaction::~KvCacheTransaction() { + rollback(); +} + +CactusGraph::KvCacheTransaction::KvCacheTransaction(KvCacheTransaction&& other) noexcept + : graph_(other.graph_), + snapshots_(std::move(other.snapshots_)), + closed_(other.closed_) { + other.graph_ = nullptr; + other.closed_ = true; +} + +CactusGraph::KvCacheTransaction& CactusGraph::KvCacheTransaction::operator=(KvCacheTransaction&& other) noexcept { + if (this != &other) { + rollback(); + graph_ = other.graph_; + snapshots_ = std::move(other.snapshots_); + closed_ = other.closed_; + other.graph_ = nullptr; + other.closed_ = true; + } + return *this; +} + void CactusGraph::KvCacheTransaction::rollback() { if (!graph_ || closed_) return; for (const auto& [node, length] : snapshots_) { diff --git a/cactus-graph/tests/test_cache_transaction.cpp b/cactus-graph/tests/test_cache_transaction.cpp index bdba54212..1eeb1c96e 100644 --- a/cactus-graph/tests/test_cache_transaction.cpp +++ b/cactus-graph/tests/test_cache_transaction.cpp @@ -42,6 +42,28 @@ bool test_cache_transaction_rollback_restores_sequence_length() { return current_seq(graph, cache) == 4; } +bool test_cache_transaction_auto_rolls_back_when_abandoned() { + CactusGraph graph; + const size_t kv_heads = 2; + const size_t head_dim = 16; + size_t cache = graph.kv_cache_state(64, kv_heads, head_dim); + graph.execute(); + + if (!append_tokens(graph, cache, 4, kv_heads, head_dim, 1.0f)) return false; + + { + auto txn = graph.begin_kv_cache_transaction({cache}); + (void)txn; + + graph.soft_reset(); + if (!append_tokens(graph, cache, 5, kv_heads, head_dim, 2.0f)) return false; + if (current_seq(graph, cache) != 9) return false; + } + graph.execute(); + + return current_seq(graph, cache) == 4; +} + bool test_cache_transaction_commit_all_keeps_draft_tokens() { CactusGraph graph; const size_t kv_heads = 2; @@ -138,6 +160,7 @@ int main() { TestRunner runner("KV Cache Transaction Tests"); runner.run_test("rollback_restores_length", test_cache_transaction_rollback_restores_sequence_length()); + runner.run_test("auto_rollback", test_cache_transaction_auto_rolls_back_when_abandoned()); runner.run_test("commit_all_keeps_tokens", test_cache_transaction_commit_all_keeps_draft_tokens()); runner.run_test("commit_prefix_crops_suffix", test_cache_transaction_commit_prefix_crops_rejected_suffix()); runner.run_test("multiple_layer_caches", test_cache_transaction_tracks_multiple_layer_caches()); diff --git a/tests/mtp/run_fast_gemma4_mtp.sh b/tests/mtp/run_fast_gemma4_mtp.sh index 4200fca35..b867ca6f2 100755 --- a/tests/mtp/run_fast_gemma4_mtp.sh +++ b/tests/mtp/run_fast_gemma4_mtp.sh @@ -79,6 +79,5 @@ echo "Running Gemma 4 MTP fast test suite with max_tokens=$MAX_TOKENS" run_with_timeout "$ENGINE_BUILD/test_mtp_sampler" run_with_timeout "$GRAPH_BUILD/test_cache_transaction" -run_with_timeout "$ENGINE_BUILD/test_decode_strategy" run_with_timeout "$ENGINE_BUILD/test_mtp_completion" run_with_timeout "$ENGINE_BUILD/test_gemma4_mtp_fast"