diff --git a/cactus-engine/src/cloud.cpp b/cactus-engine/src/cloud.cpp index 84049cf38..d5e094c94 100644 --- a/cactus-engine/src/cloud.cpp +++ b/cactus-engine/src/cloud.cpp @@ -197,6 +197,8 @@ static std::string call_cloud_endpoint(const std::string& url, curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_cb); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_body); curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, timeout_ms); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, std::min(timeout_ms, 2000L)); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); if (!env_flag_enabled("CACTUS_CLOUD_STRICT_SSL")) { curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index d62cb14bb..a6c20e65b 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -6,6 +6,7 @@ #include "wav.h" #include #include +#include #include #include #include @@ -456,8 +457,14 @@ PreparedPrompt prepare_prompt( prompt.model_type = handle->model->get_config().model_type; if (prompt.options.confidence_threshold < 0.0f) { - float model_default = handle->model->get_config().default_cloud_handoff_threshold; - prompt.options.confidence_threshold = (model_default > 0.0f) ? model_default : 0.7f; + if (handle->model->has_handoff_probe()) { + // The Gemma4 probe returns p_wrong; confidence is 1 - p_wrong. + // Route when the probe is less than 50% confident in local output. + prompt.options.confidence_threshold = 0.50f; + } else { + float model_default = handle->model->get_config().default_cloud_handoff_threshold; + prompt.options.confidence_threshold = (model_default > 0.0f) ? model_default : 0.7f; + } } if (prompt.model_type == Config::ModelType::GEMMA4) { @@ -694,6 +701,83 @@ int cactus_complete( bool has_images = prompt.has_images(); bool has_audio = prompt.has_audio(); + const bool cloud_disabled = env_flag_enabled("CACTUS_DISABLE_CLOUD_HANDOFF"); + const bool cloud_eligible = !cloud_disabled && + prompt.options.auto_handoff && (!has_images || prompt.options.handoff_with_images); + handle->model->reset_handoff_probe_rollout(); + const bool defer_local_stream_until_probe = cloud_eligible && handle->model->has_handoff_probe(); + bool pre_generation_cloud_attempted = false; + + auto make_cloud_request = [&](const std::string& local_output_hint, + const std::vector& local_calls_hint) { + CloudCompletionRequest request; + request.messages = prompt.messages; + request.tools = prompt.tools; + request.local_output = local_output_hint; + request.local_function_calls = local_calls_hint; + request.has_images = has_images; + request.has_audio = has_audio; + if (has_audio && pcm_buffer != nullptr && pcm_buffer_size > 0) { + request.audio_pcm.assign(pcm_buffer, pcm_buffer + pcm_buffer_size); + } + request.cloud_key = resolve_cloud_api_key(nullptr); + return request; + }; + + auto return_cloud_completion = [&](const CloudCompletionResult& cloud_result, + double ttft_ms, + double total_ms, + float confidence, + size_t prompt_token_count) { + std::string cloud_response = cloud_result.response; + std::vector cloud_calls = cloud_result.function_calls; + if (callback && !cloud_response.empty()) { + callback(cloud_response.c_str(), 0, user_data); + } + std::string result = construct_response_json(cloud_response, cloud_calls, ttft_ms, + total_ms, 0.0, 0.0, prompt_token_count, + 0, confidence, true, ""); + if (result.length() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + return -1; + } + std::strcpy(response_buffer, result.c_str()); + + cactus::telemetry::CompletionMetrics metrics{}; + metrics.success = true; + metrics.cloud_handoff = true; + metrics.ttft_ms = ttft_ms; + metrics.prefill_tps = 0.0; + metrics.decode_tps = 0.0; + metrics.response_time_ms = total_ms; + metrics.confidence = confidence; + metrics.ram_usage_mb = get_ram_usage_mb(); + metrics.prefill_tokens = prompt_token_count; + metrics.decode_tokens = 0; + metrics.error_message = nullptr; + metrics.function_calls_json = nullptr; + cactus::telemetry::recordCompletion(handle->model_name.c_str(), metrics); + return static_cast(result.length()); + }; + + if (cloud_eligible && prompt.options.confidence_threshold >= 1.0f) { + pre_generation_cloud_attempted = true; + CACTUS_LOG_WARN("cloud_handoff", "Cloud handoff triggered before local generation; waiting up to " + << prompt.options.cloud_timeout_ms << " ms before falling back"); + auto cloud_result = cloud_complete_request( + make_cloud_request("", {}), + static_cast(prompt.options.cloud_timeout_ms)); + auto now = std::chrono::high_resolution_clock::now(); + double elapsed_ms = std::chrono::duration_cast(now - start_time).count() / 1000.0; + if (cloud_result.ok && (!cloud_result.response.empty() || !cloud_result.function_calls.empty())) { + return return_cloud_completion(cloud_result, elapsed_ms, elapsed_ms, 0.0f, prompt.tokens.size()); + } + std::string cloud_error = cloud_result.error.empty() ? "cloud completion failed" : cloud_result.error; + CACTUS_LOG_WARN("cloud_handoff", "Cloud completion failed before local generation: " << cloud_error); + handle_error_response(("cloud handoff failed before local generation: " + cloud_error).c_str(), + response_buffer, buffer_size); + return -1; + } auto stop_token_sequences = build_stop_sequences(tokenizer, prompt.options.stop_sequences, prompt.model_type, !prompt.tools.empty()); @@ -737,40 +821,7 @@ int cactus_complete( time_to_first_token = std::chrono::duration_cast(token_end - start_time).count() / 1000.0; float confidence = 1.0f - first_token_entropy; - bool cloud_used = false; std::string cloud_error; - std::future cloud_future; - bool cloud_future_started = false; - const bool cloud_disabled = env_flag_enabled("CACTUS_DISABLE_CLOUD_HANDOFF"); - const bool cloud_eligible = !cloud_disabled && - prompt.options.auto_handoff && (!has_images || prompt.options.handoff_with_images); - - auto maybe_start_cloud_handoff = [&](const std::string& local_output_hint, - const std::vector& local_calls_hint) { - if (!cloud_eligible || cloud_future_started) { - return; - } - CloudCompletionRequest request; - request.messages = prompt.messages; - request.tools = prompt.tools; - request.local_output = local_output_hint; - request.local_function_calls = local_calls_hint; - request.has_images = has_images; - request.has_audio = has_audio; - if (has_audio && pcm_buffer != nullptr && pcm_buffer_size > 0) { - request.audio_pcm.assign(pcm_buffer, pcm_buffer + pcm_buffer_size); - } - request.cloud_key = resolve_cloud_api_key(nullptr); - - cloud_future_started = true; - cloud_future = std::async(std::launch::async, [request, &prompt]() { - return cloud_complete_request(request, static_cast(prompt.options.cloud_timeout_ms)); - }); - }; - - if (confidence < prompt.options.confidence_threshold) { - maybe_start_cloud_handoff("", {}); - } generated_tokens.push_back(next_token); handle->processed_tokens.push_back(next_token); @@ -787,7 +838,27 @@ int cactus_complete( entropy.add(first_token_entropy); if (!matches_stop_sequence(generated_tokens, stop_token_sequences)) { - if (callback) { + if (!defer_local_stream_until_probe + && !pre_generation_cloud_attempted + && confidence < prompt.options.confidence_threshold) { + CACTUS_LOG_WARN("cloud_handoff", "Cloud handoff triggered before local streaming; waiting up to " + << prompt.options.cloud_timeout_ms << " ms before falling back"); + CloudCompletionResult cloud_result = cloud_complete_request( + make_cloud_request("", {}), + static_cast(prompt.options.cloud_timeout_ms)); + auto now = std::chrono::high_resolution_clock::now(); + double elapsed_ms = std::chrono::duration_cast(now - start_time).count() / 1000.0; + if (cloud_result.ok && (!cloud_result.response.empty() || !cloud_result.function_calls.empty())) { + if (prompt.options.force_tools && !prompt.tools.empty()) { + handle->model->clear_tool_constraints(); + } + return return_cloud_completion(cloud_result, elapsed_ms, elapsed_ms, confidence, prompt_tokens); + } + cloud_error = cloud_result.error.empty() ? "cloud completion failed" : cloud_result.error; + CACTUS_LOG_WARN("cloud_handoff", "Cloud completion failed before local streaming, falling back to local output: " << cloud_error); + } + + if (callback && !defer_local_stream_until_probe) { std::string new_text = tokenizer->decode({next_token}); callback(new_text.c_str(), next_token, user_data); } @@ -813,7 +884,6 @@ int cactus_complete( if (entropy.rolling_confidence() < prompt.options.confidence_threshold) { entropy.spike_handoff = true; - maybe_start_cloud_handoff("", {}); } if (prompt.options.force_tools && !prompt.tools.empty()) { @@ -825,7 +895,7 @@ int cactus_complete( break; } - if (callback) { + if (callback && !defer_local_stream_until_probe) { std::string new_text = tokenizer->decode({next_token}); callback(new_text.c_str(), next_token, user_data); } @@ -835,6 +905,14 @@ int cactus_complete( } confidence = entropy.mean_confidence(); + if (defer_local_stream_until_probe && handle->model->has_handoff_probe_rollout()) { + float wrong_probability = handle->model->handoff_probe_wrong_probability(); + if (std::isfinite(wrong_probability)) { + confidence = std::max(0.0f, std::min(1.0f, 1.0f - wrong_probability)); + CACTUS_LOG_DEBUG("cloud_handoff", "Gemma4 handoff probe p_wrong=" + << wrong_probability << " confidence=" << confidence); + } + } if (prompt.options.force_tools && !prompt.tools.empty()) { handle->model->clear_tool_constraints(); @@ -868,10 +946,6 @@ int cactus_complete( } } - if (confidence < prompt.options.confidence_threshold) { - maybe_start_cloud_handoff(regular_response, function_calls); - } - std::string local_completion = regular_response; if (local_completion.empty() && function_calls.empty()) { local_completion = response_text; @@ -879,29 +953,32 @@ int cactus_complete( std::string primary_response = local_completion; std::vector primary_function_calls = function_calls; - if (cloud_future_started) { - auto status = cloud_future.wait_for(std::chrono::milliseconds(prompt.options.cloud_timeout_ms)); - if (status == std::future_status::ready) { - CloudCompletionResult cloud_result = cloud_future.get(); - if (cloud_result.ok && (!cloud_result.response.empty() || !cloud_result.function_calls.empty())) { - cloud_used = true; - if (!cloud_result.response.empty()) { - primary_response = cloud_result.response; - } - if (!cloud_result.function_calls.empty()) { - primary_function_calls = cloud_result.function_calls; - } - } else { - cloud_error = cloud_result.error.empty() ? "cloud completion failed" : cloud_result.error; - CACTUS_LOG_WARN("cloud_handoff", "Cloud completion failed, falling back to local output: " << cloud_error); + bool handoff_succeeded = false; + if (defer_local_stream_until_probe && !pre_generation_cloud_attempted + && confidence < prompt.options.confidence_threshold) { + CACTUS_LOG_WARN("cloud_handoff", "Cloud handoff triggered by Gemma4 probe: p_wrong=" + << (1.0f - confidence) << " confidence=" << confidence + << "; waiting up to " << prompt.options.cloud_timeout_ms << " ms"); + CloudCompletionResult cloud_result = cloud_complete_request( + make_cloud_request(local_completion, function_calls), + static_cast(prompt.options.cloud_timeout_ms)); + auto now = std::chrono::high_resolution_clock::now(); + double elapsed_ms = std::chrono::duration_cast(now - start_time).count() / 1000.0; + if (cloud_result.ok && (!cloud_result.response.empty() || !cloud_result.function_calls.empty())) { + if (prompt.options.force_tools && !prompt.tools.empty()) { + handle->model->clear_tool_constraints(); } - } else { - cloud_error = "timeout"; - CACTUS_LOG_WARN("cloud_handoff", "Cloud completion timed out, falling back to local output: " << cloud_error); + return return_cloud_completion(cloud_result, elapsed_ms, elapsed_ms, confidence, prompt_tokens); } + cloud_error = cloud_result.error.empty() ? "cloud completion failed" : cloud_result.error; + CACTUS_LOG_WARN("cloud_handoff", "Cloud completion failed after probe handoff, falling back to local output: " + << cloud_error); + } + + if (callback && defer_local_stream_until_probe && !primary_response.empty()) { + callback(primary_response.c_str(), 0, user_data); } - const bool handoff_succeeded = cloud_used; std::string result = construct_response_json(primary_response, primary_function_calls, time_to_first_token, total_time_ms, prefill_tps, decode_tps, prompt_tokens, completion_tokens, confidence, handoff_succeeded, diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index af5c4d48f..bfac47540 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -53,7 +53,6 @@ bool is_npu_available(); } // namespace npu namespace engine { - struct Config { uint32_t vocab_size = 151936; uint32_t bos_token_id = 151643; @@ -631,6 +630,11 @@ class Model { bool has_vocab_bias() const { return !vocab_bias_.empty(); } const std::unordered_map& get_vocab_bias() const { return vocab_bias_; } + bool has_handoff_probe() const { return handoff_probe_loaded_; } + void reset_handoff_probe_rollout() { handoff_probe_hidden_.clear(); } + bool has_handoff_probe_rollout() const; + float handoff_probe_wrong_probability() const; + private: struct Binding { int node_id = -1; @@ -688,13 +692,16 @@ class Model { ChunkedPrefillResult run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, bool prepare_decode); void run_full_context_text(); - uint32_t argmax_component_logits(Component& comp, size_t logit_row = std::numeric_limits::max()); + uint32_t argmax_component_logits(Component& comp, size_t logit_row = std::numeric_limits::max(), + float* out_uncertainty = nullptr); void write_int_input(Component& comp, const std::string& name, int64_t value); void write_int_input_at(Component& comp, const std::string& name, size_t index, int64_t value); void write_bytes_input(Component& comp, const std::string& name, const void* data, size_t byte_size); int input_index(const Component& comp, const std::string& name) const; int output_index(const Component& comp, const std::string& name) const; - uint32_t argmax_last_logits(); + uint32_t argmax_last_logits(float* out_uncertainty = nullptr); + bool load_handoff_probe(); + void maybe_capture_handoff_probe_hidden(const Component& comp); void run_vision_encoder(const std::string& image_path); void run_audio_encoder(const std::vector& audio_features); void run_audio_encoder_messages(const std::vector>& audio_features_per_message); @@ -754,6 +761,24 @@ class Model { ToolCallConstrainer tool_constrainer_; std::unordered_map vocab_bias_; + bool handoff_probe_loaded_ = false; + uint32_t handoff_probe_feat_dim_ = 0; + uint32_t handoff_probe_t_h_ = 0; + uint32_t handoff_probe_h1_ = 0; + uint32_t handoff_probe_h2_ = 0; + std::vector handoff_probe_norm_weight_; + std::vector handoff_probe_norm_bias_; + std::vector handoff_probe_proj_weight_; + std::vector handoff_probe_proj_bias_; + std::vector handoff_probe_attn_query_; + std::vector handoff_probe_head0_weight_; + std::vector handoff_probe_head0_bias_; + std::vector handoff_probe_head2_weight_; + std::vector handoff_probe_head2_bias_; + std::vector handoff_probe_head4_weight_; + std::vector handoff_probe_head4_bias_; + std::vector handoff_probe_hidden_; + mutable std::vector debug_nodes_; }; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 7bca073a5..8301f0c82 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -188,6 +188,171 @@ Model::Model(const Config& config) : config_(config) {} Model::~Model() = default; +namespace { + +bool read_exact(std::ifstream& in, void* data, size_t bytes) { + in.read(static_cast(data), static_cast(bytes)); + return static_cast(in.gcount()) == bytes; +} + +bool read_float_vector(std::ifstream& in, std::vector& out, size_t count) { + out.resize(count); + return read_exact(in, out.data(), count * sizeof(float)); +} + +float relu(float x) { + return x > 0.0f ? x : 0.0f; +} + +} // namespace + +bool Model::load_handoff_probe() { + fs::path path = fs::path(bundle_dir_) / "handoff_probe.bin"; + if (!fs::exists(path)) return false; + + std::ifstream in(path, std::ios::binary); + if (!in.is_open()) return false; + + char magic[8] = {}; + uint32_t version = 0; + if (!read_exact(in, magic, sizeof(magic)) || std::string(magic, sizeof(magic)) != std::string("CHP10P6\0", 8)) { + CACTUS_LOG_WARN("cloud_handoff", "Ignoring invalid handoff probe header at " << path); + return false; + } + if (!read_exact(in, &version, sizeof(version)) + || !read_exact(in, &handoff_probe_feat_dim_, sizeof(handoff_probe_feat_dim_)) + || !read_exact(in, &handoff_probe_t_h_, sizeof(handoff_probe_t_h_)) + || !read_exact(in, &handoff_probe_h1_, sizeof(handoff_probe_h1_)) + || !read_exact(in, &handoff_probe_h2_, sizeof(handoff_probe_h2_))) { + CACTUS_LOG_WARN("cloud_handoff", "Ignoring truncated handoff probe header at " << path); + return false; + } + if (version != 1 || handoff_probe_feat_dim_ == 0 || handoff_probe_t_h_ == 0 + || handoff_probe_h1_ == 0 || handoff_probe_h2_ == 0) { + CACTUS_LOG_WARN("cloud_handoff", "Ignoring unsupported handoff probe metadata at " << path); + return false; + } + + const size_t feat = handoff_probe_feat_dim_; + const size_t th = handoff_probe_t_h_; + const size_t h1 = handoff_probe_h1_; + const size_t h2 = handoff_probe_h2_; + bool ok = true; + ok = ok && read_float_vector(in, handoff_probe_norm_weight_, feat); + ok = ok && read_float_vector(in, handoff_probe_norm_bias_, feat); + ok = ok && read_float_vector(in, handoff_probe_proj_weight_, th * feat); + ok = ok && read_float_vector(in, handoff_probe_proj_bias_, th); + ok = ok && read_float_vector(in, handoff_probe_attn_query_, th); + ok = ok && read_float_vector(in, handoff_probe_head0_weight_, h1 * th); + ok = ok && read_float_vector(in, handoff_probe_head0_bias_, h1); + ok = ok && read_float_vector(in, handoff_probe_head2_weight_, h2 * h1); + ok = ok && read_float_vector(in, handoff_probe_head2_bias_, h2); + ok = ok && read_float_vector(in, handoff_probe_head4_weight_, h2); + ok = ok && read_float_vector(in, handoff_probe_head4_bias_, 1); + if (!ok) { + CACTUS_LOG_WARN("cloud_handoff", "Ignoring truncated handoff probe weights at " << path); + return false; + } + + handoff_probe_hidden_.clear(); + handoff_probe_loaded_ = true; + CACTUS_LOG_INFO("cloud_handoff", "Loaded Gemma4 v10p6 handoff probe from " << path); + return true; +} + +bool Model::has_handoff_probe_rollout() const { + return handoff_probe_loaded_ + && handoff_probe_feat_dim_ > 0 + && handoff_probe_hidden_.size() >= static_cast(handoff_probe_feat_dim_); +} + +void Model::maybe_capture_handoff_probe_hidden(const Component& comp) { + if (!handoff_probe_loaded_ || handoff_probe_feat_dim_ == 0) return; + int idx = output_index(comp, "probe_hidden"); + if (idx < 0 || static_cast(idx) >= comp.output_node_ids.size()) return; + size_t node = static_cast(comp.output_node_ids[idx]); + const auto& desc = comp.graph->get_output_buffer(node); + if (desc.total_size < handoff_probe_feat_dim_) return; + size_t rows = desc.total_size / handoff_probe_feat_dim_; + if (rows == 0) return; + size_t row = rows - 1; + const auto* data = static_cast(comp.graph->get_output(node)); + if (!data) return; + size_t base = row * static_cast(handoff_probe_feat_dim_); + for (size_t i = 0; i < handoff_probe_feat_dim_; ++i) { + handoff_probe_hidden_.push_back(read_scalar_value(desc.precision, data, base + i)); + } +} + +float Model::handoff_probe_wrong_probability() const { + if (!has_handoff_probe_rollout()) return std::numeric_limits::quiet_NaN(); + + const size_t feat = handoff_probe_feat_dim_; + const size_t th = handoff_probe_t_h_; + const size_t h1 = handoff_probe_h1_; + const size_t h2 = handoff_probe_h2_; + const size_t tokens = std::min(handoff_probe_hidden_.size() / feat, 1024); + if (tokens == 0) return std::numeric_limits::quiet_NaN(); + + std::vector projected(tokens * th); + std::vector scores(tokens); + for (size_t t = 0; t < tokens; ++t) { + const float* x = handoff_probe_hidden_.data() + t * feat; + double mean = 0.0; + for (size_t i = 0; i < feat; ++i) mean += x[i]; + mean /= static_cast(feat); + double var = 0.0; + for (size_t i = 0; i < feat; ++i) { + double d = static_cast(x[i]) - mean; + var += d * d; + } + var /= static_cast(feat); + float inv_std = static_cast(1.0 / std::sqrt(var + 1e-5)); + + for (size_t j = 0; j < th; ++j) { + double acc = handoff_probe_proj_bias_[j]; + const float* w = handoff_probe_proj_weight_.data() + j * feat; + for (size_t i = 0; i < feat; ++i) { + float xn = (x[i] - static_cast(mean)) * inv_std; + xn = xn * handoff_probe_norm_weight_[i] + handoff_probe_norm_bias_[i]; + acc += static_cast(w[i]) * xn; + } + float u = relu(static_cast(acc)); + projected[t * th + j] = u; + scores[t] += u * handoff_probe_attn_query_[j]; + } + scores[t] /= std::sqrt(static_cast(th)); + } + + float max_score = *std::max_element(scores.begin(), scores.end()); + double denom = 0.0; + for (float s : scores) denom += std::exp(static_cast(s - max_score)); + std::vector pooled(th, 0.0f); + for (size_t t = 0; t < tokens; ++t) { + float alpha = static_cast(std::exp(static_cast(scores[t] - max_score)) / denom); + const float* u = projected.data() + t * th; + for (size_t j = 0; j < th; ++j) pooled[j] += alpha * u[j]; + } + + std::vector y1(h1); + for (size_t i = 0; i < h1; ++i) { + double acc = handoff_probe_head0_bias_[i]; + const float* w = handoff_probe_head0_weight_.data() + i * th; + for (size_t j = 0; j < th; ++j) acc += static_cast(w[j]) * pooled[j]; + y1[i] = relu(static_cast(acc)); + } + std::vector y2(h2); + for (size_t i = 0; i < h2; ++i) { + double acc = handoff_probe_head2_bias_[i]; + const float* w = handoff_probe_head2_weight_.data() + i * h1; + for (size_t j = 0; j < h1; ++j) acc += static_cast(w[j]) * y1[j]; + y2[i] = relu(static_cast(acc)); + } + double logit = handoff_probe_head4_bias_[0]; + for (size_t j = 0; j < h2; ++j) logit += static_cast(handoff_probe_head4_weight_[j]) * y2[j]; + return static_cast(1.0 / (1.0 + std::exp(-logit))); +} + bool Model::init(const std::string& bundle_dir, size_t context_size, const std::string& /*system_prompt*/, bool /*do_warmup*/) { if (initialized_) return true; @@ -334,6 +499,11 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, CACTUS_LOG_WARN("model", "NPU vision encoder load failed for " << full_path << "; falling back to CPU"); } } + if (load_handoff_probe() && decoder_ && output_index(*decoder_, "probe_hidden") < 0) { + CACTUS_LOG_WARN("cloud_handoff", "Handoff probe is packaged, but decoder_step does not expose probe_hidden; " + "reconvert Gemma4 to enable probe-based handoff"); + handoff_probe_loaded_ = false; + } initialized_ = true; return true; @@ -554,11 +724,13 @@ void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { write_int_input(*decoder_, "input_ids", static_cast(token_id)); write_int_input(*decoder_, "position_ids", static_cast(position)); decoder_->graph->execute(); + maybe_capture_handoff_probe_hidden(*decoder_); return; } run_encoder_step(token_id, position); copy_component_outputs_to_inputs(*encoder_, *decoder_); decoder_->graph->execute(); + maybe_capture_handoff_probe_hidden(*decoder_); } void Model::run_encoder_step(uint32_t token_id, size_t position) { @@ -1208,7 +1380,7 @@ void Model::run_audio_encoder(const std::vector& audio_features) { } } -uint32_t Model::argmax_component_logits(Component& comp, size_t logit_row) { +uint32_t Model::argmax_component_logits(Component& comp, size_t logit_row, float* out_uncertainty) { size_t out_node = static_cast(comp.output_node_ids.empty() ? 0 : comp.output_node_ids[0]); const auto& desc = comp.graph->get_output_buffer(out_node); void* ptr = comp.graph->get_output(out_node); @@ -1223,24 +1395,39 @@ uint32_t Model::argmax_component_logits(Component& comp, size_t logit_row) { size_t row_off = row * vocab; uint32_t best = 0; float best_v = -std::numeric_limits::infinity(); + float second_v = -std::numeric_limits::infinity(); + auto observe_logit = [&](size_t i, float v) { + if (v > best_v) { + second_v = best_v; + best_v = v; + best = static_cast(i); + } else if (v > second_v) { + second_v = v; + } + }; if (desc.precision == Precision::FP32) { float* p = static_cast(ptr) + row_off; - for (size_t i = 0; i < vocab; ++i) if (p[i] > best_v) { best_v = p[i]; best = static_cast(i); } + for (size_t i = 0; i < vocab; ++i) observe_logit(i, p[i]); } else if (desc.precision == Precision::FP16) { __fp16* p = static_cast<__fp16*>(ptr) + row_off; - for (size_t i = 0; i < vocab; ++i) { - float v = static_cast(p[i]); - if (v > best_v) { best_v = v; best = static_cast(i); } - } + for (size_t i = 0; i < vocab; ++i) observe_logit(i, static_cast(p[i])); } else { int8_t* p = static_cast(ptr) + row_off; - for (size_t i = 0; i < vocab; ++i) if (p[i] > best_v) { best_v = static_cast(p[i]); best = static_cast(i); } + for (size_t i = 0; i < vocab; ++i) observe_logit(i, static_cast(p[i])); + } + if (out_uncertainty) { + float confidence = 1.0f; + if (std::isfinite(best_v) && std::isfinite(second_v)) { + float margin = std::max(-60.0f, std::min(60.0f, best_v - second_v)); + confidence = 1.0f / (1.0f + std::exp(-margin)); + } + *out_uncertainty = std::max(0.0f, std::min(1.0f, 1.0f - confidence)); } return best; } -uint32_t Model::argmax_last_logits() { - return argmax_component_logits(*decoder_); +uint32_t Model::argmax_last_logits(float* out_uncertainty) { + return argmax_component_logits(*decoder_, std::numeric_limits::max(), out_uncertainty); } bool Model::prefill_and_sample_first_token(const std::vector& tokens, uint32_t& out_token) { @@ -2059,8 +2246,7 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); run_full_context_text(); cache_total_seq_len_ = context_tokens_.size(); - if (out_entropy) *out_entropy = 0.0f; - uint32_t result = argmax_last_logits(); + uint32_t result = argmax_last_logits(out_entropy); record_sampled_token(result); return result; } @@ -2069,8 +2255,7 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* } run_step(tokens.back(), cache_total_seq_len_ + tokens.size() - 1, /*read_logits=*/true); cache_total_seq_len_ += tokens.size(); - if (out_entropy) *out_entropy = 0.0f; - uint32_t result = argmax_last_logits(); + uint32_t result = argmax_last_logits(out_entropy); record_sampled_token(result); return result; } diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index 2648274c2..7e9e8aef6 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -812,7 +812,7 @@ inline void handle_error_response(const std::string& error_message, char* respon json << "\"cloud_handoff\":false,"; json << "\"response\":null,"; json << "\"function_calls\":[],"; - json << "\"confidence\":0.0,"; + json << "\"confidence\":null,"; json << "\"time_to_first_token_ms\":0.0,"; json << "\"total_time_ms\":0.0,"; json << "\"prefill_tps\":0.0,"; diff --git a/cactus-engine/tests/chat.cpp b/cactus-engine/tests/chat.cpp index 594c75de2..b578d5b16 100644 --- a/cactus-engine/tests/chat.cpp +++ b/cactus-engine/tests/chat.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -202,16 +203,31 @@ struct TokenPrinter { ++count; } - void print_stats(double ram_mb) const { + void print_stats(double ram_mb, double confidence, bool cloud_handoff, + int reported_tokens = -1, double reported_decode_tps = -1.0, + double reported_ttft_ms = -1.0, double reported_total_ms = -1.0) const { auto end = std::chrono::steady_clock::now(); double total_s = std::chrono::duration(end - start).count(); double ttft_s = saw_first ? std::chrono::duration(first - start).count() : 0.0; double decode_s = saw_first ? std::chrono::duration(end - first).count() : total_s; + int display_tokens = reported_tokens >= 0 ? reported_tokens : count; double tps = (count > 1 && decode_s > 0.0) ? (count - 1) / decode_s : (total_s > 0.0 ? count / total_s : 0.0); - std::cout << "\n[" << count << " tokens | latency: " + if (reported_decode_tps >= 0.0) tps = reported_decode_tps; + if (reported_ttft_ms >= 0.0) ttft_s = reported_ttft_ms / 1000.0; + if (reported_total_ms >= 0.0) total_s = reported_total_ms / 1000.0; + std::cout << "\n[" << display_tokens << " tokens | latency: " << std::fixed << std::setprecision(3) << ttft_s << "s | total: " << total_s << "s | " << std::setprecision(1) << tps << " tok/s"; + if (confidence >= 0.0) { + double handoff_pct = std::max(0.0, std::min(100.0, (1.0 - confidence) * 100.0)); + std::cout << " | handoff: " << handoff_pct << "%" + << " | confidence: " << std::max(0.0, std::min(100.0, confidence * 100.0)) << "%"; + } else { + std::cout << " | handoff: forced" + << " | confidence: n/a"; + } + std::cout << " | cloud: " << (cloud_handoff ? "yes" : "no"); if (ram_mb > 0.0) { std::cout << " | RAM: " << ram_mb << " MB"; } @@ -325,15 +341,30 @@ std::string json_string_value(const std::string& json, const std::string& key) { return unescape_json(json.substr(start, end - start)); } -double json_number_value(const std::string& json, const std::string& key) { +double json_number_value(const std::string& json, const std::string& key, double fallback = 0.0) { std::string needle = "\"" + key + "\":"; size_t start = json.find(needle); - if (start == std::string::npos) return 0.0; + if (start == std::string::npos) return fallback; start += needle.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) { + ++start; + } + if (json.compare(start, 4, "null") == 0) return fallback; char* end = nullptr; return std::strtod(json.c_str() + start, &end); } +bool json_bool_value(const std::string& json, const std::string& key) { + std::string needle = "\"" + key + "\":"; + size_t start = json.find(needle); + if (start == std::string::npos) return false; + start += needle.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) { + ++start; + } + return json.compare(start, 4, "true") == 0; +} + std::string build_messages(const std::string& system_prompt, const std::vector>& history, const std::string& image, @@ -365,12 +396,16 @@ void print_usage(const char* argv0) { std::cerr << "Usage: " << argv0 << " [--system ] [--image ] [--audio ]" << " [--prompt ] [--input-ids ] [--max-new-tokens ]" - << " [--result-json ] [--thinking]\n"; + << " [--result-json ] [--thinking] [--no-cloud-handoff]" + << " [--confidence-threshold ] [--cloud-timeout-ms ]\n"; } } // namespace int main(int argc, char** argv) { + std::cout << std::unitbuf; + std::cerr << std::unitbuf; + if (argc < 2) { print_usage(argv[0]); return 1; @@ -385,6 +420,9 @@ int main(int argc, char** argv) { std::string result_json; int max_new_tokens = kMaxTokens; bool thinking = false; + bool auto_handoff = true; + double confidence_threshold = -1.0; + int cloud_timeout_ms = 15000; for (int i = 2; i < argc; ++i) { std::string arg = argv[i]; @@ -404,6 +442,12 @@ int main(int argc, char** argv) { result_json = argv[++i]; } else if (arg == "--thinking") { thinking = true; + } else if (arg == "--no-cloud-handoff") { + auto_handoff = false; + } else if (arg == "--confidence-threshold" && i + 1 < argc) { + confidence_threshold = std::atof(argv[++i]); + } else if (arg == "--cloud-timeout-ms" && i + 1 < argc) { + cloud_timeout_ms = std::max(0, std::atoi(argv[++i])); } } @@ -556,7 +600,9 @@ int main(int argc, char** argv) { std::string options = "{\"temperature\":0.7,\"top_p\":0.95,\"top_k\":40,\"max_tokens\":" + std::to_string(max_new_tokens) + ",\"enable_thinking_if_supported\":" + (thinking ? "true" : "false") - + ",\"auto_handoff\":false,\"confidence_threshold\":0.0" + + ",\"auto_handoff\":" + (auto_handoff ? "true" : "false") + + ",\"confidence_threshold\":" + std::to_string(confidence_threshold) + + ",\"cloud_timeout_ms\":" + std::to_string(cloud_timeout_ms) + ",\"stop_sequences\":[\"<|im_end|>\",\"\"]}"; if (!current_image.empty()) std::cout << "[image: " << current_image << "]\n"; @@ -584,12 +630,18 @@ int main(int argc, char** argv) { if (!result_json.empty() && !write_text_file(result_json, response_json)) { std::cerr << "Failed to write result JSON: " << result_json << "\n"; } + bool cloud_handoff = json_bool_value(response_json, "cloud_handoff"); + double confidence = json_number_value(response_json, "confidence", -1.0); double ram_mb = json_number_value(response_json, "ram_usage_mb"); - printer.print_stats(ram_mb); + int decode_tokens = static_cast(json_number_value(response_json, "decode_tokens", -1.0)); + double decode_tps = json_number_value(response_json, "decode_tps", -1.0); + double ttft_ms = json_number_value(response_json, "time_to_first_token_ms", -1.0); + double total_ms = json_number_value(response_json, "total_time_ms", -1.0); + printer.print_stats(ram_mb, confidence, cloud_handoff, decode_tokens, decode_tps, ttft_ms, total_ms); std::cout << "\n"; if (rc < 0) { - std::cerr << "Error: " << response.data() << "\n"; + std::cout << "Error: " << response.data() << "\n"; history.pop_back(); continue; } diff --git a/probe.pt b/probe.pt new file mode 100644 index 000000000..434ec3750 Binary files /dev/null and b/probe.pt differ diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 40b742411..9d603c220 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -193,6 +193,12 @@ def create_parser(): help="Optional path to save transpiled bundle results as JSON") run_parser.add_argument("--thinking", action="store_true", help="Enable thinking/reasoning for models that support it") + run_parser.add_argument("--no-cloud-handoff", action="store_true", + help="Disable automatic cloud handoff for this run") + run_parser.add_argument("--confidence-threshold", type=float, default=None, + help="Confidence threshold below which local completions may hand off to cloud") + run_parser.add_argument("--cloud-timeout-ms", type=int, default=None, + help="Maximum time to wait for cloud handoff before falling back locally") # ── transcribe ──────────────────────────────────────────────────── transcribe_parser = subparsers.add_parser("transcribe", help="Download ASR model and run transcription", diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index f5e90ac94..decdb5c6c 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -329,5 +329,13 @@ def ensure_bundle(model_id, *, bits=4, token=None, if rc != 0: raise RuntimeError(f"Transpilation failed for {model_id}") + try: + from cactus.convert.handoff_probe import export_gemma4_handoff_probe + + if export_gemma4_handoff_probe(output_dir, model_id=model_id): + print_color(GREEN, f"Gemma4 cloud handoff probe packaged into {output_dir}") + except Exception as e: + print_color(YELLOW, f"Warning: failed to package Gemma4 cloud handoff probe: {e}") + print_color(GREEN, f"Model converted and transpiled to {output_dir}") return output_dir diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index 165c8d2c6..bcc049aaf 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -46,11 +46,15 @@ def cmd_run(args): ("--input-ids", args.input_ids), ("--max-new-tokens", args.max_new_tokens), ("--result-json", args.result_json), + ("--confidence-threshold", args.confidence_threshold), + ("--cloud-timeout-ms", args.cloud_timeout_ms), ): if value is not None: cmd.extend([flag, str(value)]) if args.thinking: cmd.append("--thinking") + if args.no_cloud_handoff: + cmd.append("--no-cloud-handoff") print_color(GREEN, f"Starting Cactus Chat with model: {bundle_dir}") print() diff --git a/python/cactus/convert/handoff_probe.py b/python/cactus/convert/handoff_probe.py new file mode 100644 index 000000000..8fc45058e --- /dev/null +++ b/python/cactus/convert/handoff_probe.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import io +import hashlib +import json +import struct +import zipfile +from pathlib import Path +from typing import Any + + +_PROBE_MAGIC = b"CHP10P6\0" +_ORDERED_KEYS = ( + "norm.weight", + "norm.bias", + "proj.weight", + "proj.bias", + "attn_query", + "head.0.weight", + "head.0.bias", + "head.2.weight", + "head.2.bias", + "head.4.weight", + "head.4.bias", +) + + +def _candidate_probe_files(output_dir: Path) -> list[Path]: + cwd = Path.cwd() + return [ + output_dir / "probe.pt", + output_dir / "global_attn_probe_v10p6.pt", + cwd / "probe.pt", + cwd / "v10p6_probe_release" / "global_attn_probe_v10p6.pt", + Path.home() / "Downloads" / "probe.pt", + Path.home() / "Downloads" / "v10p6_probe_release" / "global_attn_probe_v10p6.pt", + ] + + +def _candidate_probe_zips(output_dir: Path) -> list[Path]: + cwd = Path.cwd() + return [ + output_dir / "v10p6_probe_release.zip", + cwd / "v10p6_probe_release.zip", + Path.home() / "Downloads" / "v10p6_probe_release.zip", + ] + + +def _load_checkpoint_from_zip(zip_path: Path) -> Any: + import torch + + with zipfile.ZipFile(zip_path) as zf: + names = set(zf.namelist()) + for name in ( + "v10p6_probe_release/global_attn_probe_v10p6.pt", + "global_attn_probe_v10p6.pt", + "probe.pt", + ): + if name in names: + with zf.open(name) as f: + return torch.load(io.BytesIO(f.read()), map_location="cpu") + raise FileNotFoundError(f"no probe checkpoint found in {zip_path}") + + +def _state_dict_from_checkpoint(checkpoint: Any) -> dict[str, Any]: + if isinstance(checkpoint, dict): + for key in ("state_dict", "model_state"): + value = checkpoint.get(key) + if isinstance(value, dict): + return value + return checkpoint + + +def _load_checkpoint(output_dir: Path) -> tuple[Any, str] | tuple[None, None]: + import torch + + for path in _candidate_probe_files(output_dir): + if path.exists(): + return torch.load(path, map_location="cpu"), str(path) + for path in _candidate_probe_zips(output_dir): + if path.exists(): + return _load_checkpoint_from_zip(path), str(path) + return None, None + + +def _tensor_hash(state: dict[str, Any]) -> str: + digest = hashlib.sha256() + for key in sorted(_ORDERED_KEYS): + tensor = state[key].detach().cpu().contiguous().float() + digest.update(key.encode("utf-8")) + digest.update(str(tuple(tensor.shape)).encode("utf-8")) + digest.update(tensor.numpy().tobytes(order="C")) + return digest.hexdigest() + + +def export_gemma4_handoff_probe(output_dir: str | Path, *, model_id: str | None = None) -> bool: + """Package the Gemma4 v10p6 cloud-handoff probe into a C++-readable bundle file.""" + model_key = (model_id or "").lower() + if model_key and "gemma-4" not in model_key and "gemma4" not in model_key: + return False + + out_dir = Path(output_dir) + checkpoint, source = _load_checkpoint(out_dir) + if checkpoint is None: + return False + + state = _state_dict_from_checkpoint(checkpoint) + missing = [key for key in _ORDERED_KEYS if key not in state] + if missing: + raise RuntimeError(f"handoff probe checkpoint missing tensors: {', '.join(missing)}") + + out_dir.mkdir(parents=True, exist_ok=True) + probe_path = out_dir / "handoff_probe.bin" + with probe_path.open("wb") as f: + f.write(_PROBE_MAGIC) + f.write(struct.pack(" torch.Tensor: + capture_layer_index: int | None = None, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: hidden_states = inputs_embeds + captured_hidden: torch.Tensor | None = None layer_types = tuple(dict.fromkeys(getattr(backbone.config, "layer_types", ()))) position_embeddings = { layer_type: backbone.rotary_emb(hidden_states, position_ids, layer_type) @@ -557,8 +559,15 @@ def _gemma4_text_backbone_forward( use_cache=False, shared_kv_states=shared_kv_states, ) + if capture_layer_index is not None and layer_index == int(capture_layer_index): + captured_hidden = hidden_states - return backbone.norm(hidden_states) if apply_norm else hidden_states + output_hidden = backbone.norm(hidden_states) if apply_norm else hidden_states + if capture_layer_index is None: + return output_hidden + if captured_hidden is None: + captured_hidden = hidden_states + return output_hidden, captured_hidden def _gemma4_strip_audio_padding(audio_output: object) -> torch.Tensor: @@ -3130,7 +3139,39 @@ def get_transpile_metadata(self): class Gemma4DecoderStepAdapter(Gemma4DecoderAdapter): - pass + def forward( + self, + inputs_embeds: torch.Tensor, + per_layer_inputs: torch.Tensor, + position_ids: torch.LongTensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + normalized_per_layer_inputs = per_layer_inputs + if normalized_per_layer_inputs.numel() == 0: + normalized_per_layer_inputs = None + attention_mask = torch.ones( + position_ids.shape, + dtype=torch.long, + device=position_ids.device, + ) + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) + hidden_states, probe_hidden = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + capture_layer_index=28, + ) + logits = self.model.lm_head(hidden_states[:, -1:, :]) + logits = _gemma4_apply_final_logit_softcapping(self.model, logits) + return logits, probe_hidden[:, -1:, :] class Gemma4DecoderPrefillChunkAdapter(Gemma4DecoderAdapter): @@ -3530,7 +3571,7 @@ def _require(component: str) -> None: module=decoder_step, example_inputs=tuple(decoder_step_inputs), input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, - output_keys=("logits",), + output_keys=("logits", "probe_hidden"), graph_meta={ **common_graph_meta, "component": "decoder_step",