diff --git a/cactus/engine/engine.h b/cactus/engine/engine.h index ee611981d..afebfeb07 100644 --- a/cactus/engine/engine.h +++ b/cactus/engine/engine.h @@ -665,15 +665,11 @@ class ToolCallConstrainer { int needle_arguments_depth_ = 0; int needle_nesting_depth_ = 0; bool needle_in_string_value_ = false; - bool needle_between_pairs_ = false; bool needle_prev_char_escape_ = false; std::unique_ptr needle_name_trie_; std::unordered_map> needle_param_tries_; - std::unordered_map> needle_required_params_; - std::unordered_set needle_seen_arg_keys_; std::vector needle_token_strings_; std::unordered_map> needle_token_index_; - std::unordered_set needle_arg_close_tokens_; std::unordered_set tool_start_tokens_; std::unordered_set tool_end_tokens_; @@ -713,7 +709,6 @@ class ToolCallConstrainer { void needle_insert_word(NeedleTrieNode* root, const std::string& word); const NeedleTrieNode* needle_get_trie_node(const NeedleTrieNode* root, const std::string& prefix) const; bool needle_check_token_valid(const std::string& token_text, const NeedleTrieNode* trie_node) const; - bool needle_has_unseen_completion(const NeedleTrieNode* node, std::string& partial) const; }; class Model { diff --git a/cactus/engine/engine_constraints.cpp b/cactus/engine/engine_constraints.cpp index eb865db5f..cb03cef2f 100644 --- a/cactus/engine/engine_constraints.cpp +++ b/cactus/engine/engine_constraints.cpp @@ -4,7 +4,8 @@ namespace cactus { namespace engine { constexpr float FORCE_BIAS = 500.0f; -constexpr float BLOCK_BIAS = -500.0f; +constexpr float BLOCK_BIAS = -500.0f; +constexpr float NEEDLE_BLOCK_BIAS = -1e9f; void ToolCallConstrainer::add_tokens_for_string(const std::string& str, std::unordered_set& token_set) { if (!tokenizer_) return; @@ -87,20 +88,6 @@ const ToolCallConstrainer::NeedleTrieNode* ToolCallConstrainer::needle_get_trie_ return node; } -bool ToolCallConstrainer::needle_has_unseen_completion(const NeedleTrieNode* node, std::string& partial) const { - if (!node) return false; - if (node->is_terminal && !needle_seen_arg_keys_.count(partial)) return true; - for (const auto& [ch, child] : node->children) { - partial.push_back(ch); - if (needle_has_unseen_completion(child.get(), partial)) { - partial.pop_back(); - return true; - } - partial.pop_back(); - } - return false; -} - bool ToolCallConstrainer::needle_check_token_valid(const std::string& token_text, const NeedleTrieNode* trie_node) const { const NeedleTrieNode* node = trie_node; @@ -124,12 +111,10 @@ void ToolCallConstrainer::reset_needle_constraints() { needle_buffer_.clear(); needle_constrained_buf_.clear(); needle_current_function_.clear(); - needle_seen_arg_keys_.clear(); needle_in_arguments_ = false; needle_arguments_depth_ = 0; needle_nesting_depth_ = 0; needle_in_string_value_ = false; - needle_between_pairs_ = false; needle_prev_char_escape_ = false; } @@ -157,9 +142,6 @@ void ToolCallConstrainer::feed_needle_char(char ch) { if (ch == '"') { if (needle_json_state_ == NeedleJsonState::IN_NAME) { needle_current_function_ = needle_constrained_buf_; - needle_seen_arg_keys_.clear(); - } else { - needle_seen_arg_keys_.insert(needle_constrained_buf_); } needle_constrained_buf_.clear(); needle_json_state_ = NeedleJsonState::FREE; @@ -183,9 +165,6 @@ void ToolCallConstrainer::feed_needle_char(char ch) { } if (ch == '"') { needle_in_string_value_ = false; - if (needle_in_arguments_ && needle_nesting_depth_ == needle_arguments_depth_) { - needle_between_pairs_ = true; - } } return; } @@ -197,8 +176,6 @@ void ToolCallConstrainer::feed_needle_char(char ch) { if (ch == '}' && needle_in_arguments_ && needle_nesting_depth_ < needle_arguments_depth_) { needle_in_arguments_ = false; } - } else if (ch == ',' && needle_in_arguments_ && needle_nesting_depth_ == needle_arguments_depth_) { - needle_between_pairs_ = true; } if (needle_buffer_.size() >= 8 && @@ -215,10 +192,11 @@ void ToolCallConstrainer::feed_needle_char(char ch) { return; } - if (needle_in_arguments_ && needle_at_arg_key_start()) { + if (needle_in_arguments_ && + needle_nesting_depth_ == needle_arguments_depth_ && + needle_at_arg_key_start()) { needle_json_state_ = NeedleJsonState::IN_ARG_KEY; needle_constrained_buf_.clear(); - needle_between_pairs_ = false; return; } @@ -238,7 +216,6 @@ void ToolCallConstrainer::init_needle_constraints() { needle_name_trie_ = std::make_unique(); needle_param_tries_.clear(); - needle_required_params_.clear(); for (const auto& tool : tool_specs_) { if (!tool.name.empty()) { needle_insert_word(needle_name_trie_.get(), tool.name); @@ -251,42 +228,26 @@ void ToolCallConstrainer::init_needle_constraints() { } } needle_param_tries_[tool.name] = std::move(param_root); - needle_required_params_[tool.name] = tool.required_parameter_names; } const uint32_t vocab_size = tokenizer_ ? tokenizer_->get_vocab_size() : 0; if (needle_token_strings_.size() != vocab_size) { needle_token_strings_.assign(vocab_size, ""); needle_token_index_.clear(); - needle_arg_close_tokens_.clear(); - - auto would_close_args = [](const std::string& tok, bool start_in_str) -> bool { - bool in_str = start_in_str; - bool esc = false; - int depth = 0; - for (char ch : tok) { - if (in_str) { - if (esc) { esc = false; } - else if (ch == '\\') { esc = true; } - else if (ch == '"') { in_str = false; } - continue; - } - if (ch == '"') { in_str = true; continue; } - if (ch == '{' || ch == '[') { depth++; continue; } - if (ch == '}') { if (depth == 0) return true; depth--; } - else if (ch == ']' && depth > 0) depth--; - } - return false; - }; + + const uint32_t eos = tokenizer_->get_eos_token(); + const uint32_t bos = tokenizer_->get_bos_token(); + const uint32_t unk = tokenizer_->get_unk_token(); + constexpr uint32_t pad = 0; for (uint32_t token_id = 0; token_id < vocab_size; ++token_id) { + if (token_id == pad || token_id == eos || token_id == bos || token_id == unk) { + continue; + } std::string decoded = tokenizer_->decode({token_id}); needle_token_strings_[token_id] = decoded; if (decoded.empty()) continue; needle_token_index_[decoded.front()].push_back(token_id); - if (would_close_args(decoded, false) || would_close_args(decoded, true)) { - needle_arg_close_tokens_.insert(token_id); - } } } } @@ -699,8 +660,10 @@ void ToolCallConstrainer::compute_bias() { if (!active_) return; - for (uint32_t t : backtick_tokens_) { - current_bias_[t] = BLOCK_BIAS; + if (!is_needle()) { + for (uint32_t t : backtick_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } } if (model_type_ == Config::ModelType::LFM2) { @@ -923,30 +886,6 @@ void ToolCallConstrainer::compute_bias() { default: if (needle_json_state_ == NeedleJsonState::FREE) { - if (needle_in_arguments_ && needle_nesting_depth_ == needle_arguments_depth_) { - auto param_it = needle_param_tries_.find(needle_current_function_); - if (param_it != needle_param_tries_.end()) { - std::string partial; - bool all_seen = !needle_has_unseen_completion(param_it->second.get(), partial); - if (all_seen && needle_between_pairs_) { - for (uint32_t t : needle_arg_close_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - } else if (!all_seen && (needle_between_pairs_ || needle_in_string_value_)) { - auto req_it = needle_required_params_.find(needle_current_function_); - if (req_it != needle_required_params_.end()) { - for (const auto& req : req_it->second) { - if (!needle_seen_arg_keys_.count(req)) { - for (uint32_t t : needle_arg_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - } - } - } - } - } - } break; } @@ -964,57 +903,40 @@ void ToolCallConstrainer::compute_bias() { break; } - std::vector valid_tokens(needle_token_strings_.size(), false); - bool has_valid = false; - auto mark_valid_tokens = [&](char first_char) { - auto idx_it = needle_token_index_.find(first_char); - if (idx_it == needle_token_index_.end()) { - return; - } - for (uint32_t token_id : idx_it->second) { - if (!valid_tokens[token_id] && - needle_check_token_valid(needle_token_strings_[token_id], trie_node)) { - valid_tokens[token_id] = true; - has_valid = true; + { + std::vector valid_tokens(needle_token_strings_.size(), false); + bool has_valid = false; + auto mark_valid_tokens = [&](char first_char) { + auto idx_it = needle_token_index_.find(first_char); + if (idx_it == needle_token_index_.end()) { + return; } - } - }; + for (uint32_t token_id : idx_it->second) { + if (!valid_tokens[token_id] && + needle_check_token_valid(needle_token_strings_[token_id], trie_node)) { + valid_tokens[token_id] = true; + has_valid = true; + } + } + }; - for (const auto& [first_char, _] : trie_node->children) { - mark_valid_tokens(first_char); - } - if (trie_node->is_terminal) { - mark_valid_tokens('"'); - } + for (const auto& [first_char, _] : trie_node->children) { + mark_valid_tokens(first_char); + } + if (trie_node->is_terminal) { + mark_valid_tokens('"'); + } - if (!has_valid) { - break; - } + if (!has_valid) { + break; + } - if (needle_json_state_ == NeedleJsonState::IN_ARG_KEY && !needle_seen_arg_keys_.empty()) { for (size_t token_id = 0; token_id < valid_tokens.size(); ++token_id) { - if (!valid_tokens[token_id]) continue; - const std::string& tok = needle_token_strings_[token_id]; - size_t quote_pos = tok.find('"'); - if (quote_pos != std::string::npos) { - if (needle_seen_arg_keys_.count(needle_constrained_buf_ + tok.substr(0, quote_pos))) { - valid_tokens[token_id] = false; - } - } else { - std::string partial = needle_constrained_buf_ + tok; - const NeedleTrieNode* new_node = needle_get_trie_node(trie_node, tok); - if (!needle_has_unseen_completion(new_node, partial)) { - valid_tokens[token_id] = false; - } + if (!valid_tokens[token_id]) { + current_bias_[static_cast(token_id)] = NEEDLE_BLOCK_BIAS; } } } - - for (size_t token_id = 0; token_id < valid_tokens.size(); ++token_id) { - if (!valid_tokens[token_id]) { - current_bias_[static_cast(token_id)] = BLOCK_BIAS; - } - } break; } } else { diff --git a/cactus/ffi/cactus_complete.cpp b/cactus/ffi/cactus_complete.cpp index c979c5a31..461e8b8ff 100644 --- a/cactus/ffi/cactus_complete.cpp +++ b/cactus/ffi/cactus_complete.cpp @@ -1,6 +1,7 @@ #include "cactus_ffi.h" #include "cactus_cloud.h" #include "cactus_utils.h" +#include "needle_tools.h" #include "telemetry/telemetry.h" #include "../../libs/audio/wav.h" #include @@ -8,6 +9,7 @@ #include #include #include +#include #include using namespace cactus::engine; @@ -101,10 +103,20 @@ void strip_thinking_from_cache(CactusModelHandle* handle, } void setup_tool_constraints(CactusModelHandle* handle, const std::vector& tools, - bool force_tools, float& temperature) { + bool force_tools, float& temperature, + const std::unordered_map& needle_name_map = {}) { if (!force_tools || tools.empty()) return; - handle->model->set_tool_constraints(build_tool_constraint_specs(tools)); + auto specs = build_tool_constraint_specs(tools); + + if (!needle_name_map.empty()) { + for (auto& spec : specs) { + std::string snake = needle::to_snake_case(spec.name); + spec.name = snake; + } + } + + handle->model->set_tool_constraints(specs); if (temperature == 0.0f) { temperature = 0.01f; @@ -237,9 +249,114 @@ std::vector extract_schema_required(const std::string& schema) { return required; } -std::string serialize_needle_tools_json(const std::vector& tools, const char* raw_tools_json) { +std::string serialize_needle_tools(const std::vector& tools, + const char* raw_tools_json, + std::unordered_map& name_map) { + name_map.clear(); + + if (raw_tools_json && std::strlen(raw_tools_json) > 2) { + std::string raw(raw_tools_json); + + bool is_needle_format = (raw.find("\"parameters\"") != std::string::npos && + raw.find("\"schema\"") == std::string::npos && + raw.find("\"type\":\"function\"") == std::string::npos); + + if (is_needle_format) { + std::string result; + result.reserve(raw.size()); + const std::string name_key = "\"name\":"; + size_t pos = 0; + + while (pos < raw.size()) { + size_t found = raw.find(name_key, pos); + if (found == std::string::npos) { + result.append(raw, pos, raw.size() - pos); + break; + } + result.append(raw, pos, found + name_key.size() - pos); + pos = found + name_key.size(); + + while (pos < raw.size() && (raw[pos] == ' ' || raw[pos] == '\t' || + raw[pos] == '\n' || raw[pos] == '\r')) { + result += raw[pos]; + pos++; + } + if (pos >= raw.size() || raw[pos] != '"') continue; + result += '"'; + pos++; + + size_t name_start = pos; + bool escaped = false; + while (pos < raw.size()) { + if (escaped) { escaped = false; pos++; continue; } + if (raw[pos] == '\\') { escaped = true; pos++; continue; } + if (raw[pos] == '"') break; + pos++; + } + std::string orig_name = raw.substr(name_start, pos - name_start); + std::string snake_name = needle::to_snake_case(orig_name); + name_map[snake_name] = orig_name; + + result += snake_name; + } + + std::string compact; + compact.reserve(result.size()); + bool in_str = false; + bool esc = false; + size_t ri = 0; + while (ri < result.size()) { + unsigned char ch = static_cast(result[ri]); + if (in_str) { + if (esc) { + compact += static_cast(ch); + esc = false; + } else if (ch == '\\') { + compact += '\\'; + esc = true; + } else if (ch == '"') { + compact += '"'; + in_str = false; + } else if (ch >= 0x80) { + uint32_t cp = 0; + int extra = 0; + if ((ch & 0xE0) == 0xC0) { cp = ch & 0x1F; extra = 1; } + else if ((ch & 0xF0) == 0xE0) { cp = ch & 0x0F; extra = 2; } + else if ((ch & 0xF8) == 0xF0) { cp = ch & 0x07; extra = 3; } + else { compact += static_cast(ch); ri++; continue; } + for (int e = 0; e < extra && ri + 1 < result.size(); e++) { + ri++; + cp = (cp << 6) | (static_cast(result[ri]) & 0x3F); + } + if (cp <= 0xFFFF) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", cp); + compact += buf; + } else { + cp -= 0x10000; + char buf[14]; + std::snprintf(buf, sizeof(buf), "\\u%04x\\u%04x", + 0xD800 + (cp >> 10), 0xDC00 + (cp & 0x3FF)); + compact += buf; + } + } else { + compact += static_cast(ch); + } + } else { + if (ch == '"') { in_str = true; compact += '"'; } + else if (ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r') { + compact += static_cast(ch); + } + } + ri++; + } + + return compact; + } + } + if (tools.empty()) { - return (raw_tools_json && std::strlen(raw_tools_json) > 0) ? raw_tools_json : "[]"; + return "[]"; } std::ostringstream oss; @@ -249,7 +366,10 @@ std::string serialize_needle_tools_json(const std::vector& tools, oss << ","; } - oss << "{\"name\":\"" << escape_json_string(tools[i].name) << "\""; + std::string snake_name = needle::to_snake_case(tools[i].name); + name_map[snake_name] = tools[i].name; + + oss << "{\"name\":\"" << escape_json_string(snake_name) << "\""; if (!tools[i].description.empty()) { oss << ",\"description\":\"" << escape_json_string(tools[i].description) << "\""; } @@ -260,16 +380,49 @@ std::string serialize_needle_tools_json(const std::vector& tools, oss << "{}"; } else { auto properties = extract_schema_property_types(schema_it->second); + auto required = extract_schema_required(schema_it->second); + std::unordered_set required_set(required.begin(), required.end()); + if (properties.empty()) { - oss << schema_it->second; + oss << "{}"; } else { + std::string properties_object = extract_json_object_field(schema_it->second, "properties"); + oss << "{"; for (size_t p = 0; p < properties.size(); ++p) { if (p > 0) { oss << ","; } - oss << "\"" << escape_json_string(properties[p].first) << "\":" - << "\"" << escape_json_string(properties[p].second) << "\""; + const std::string& param_name = properties[p].first; + const std::string& param_type = properties[p].second; + bool is_required = required_set.count(param_name) > 0; + + std::string description; + std::string prop_obj = extract_json_object_field(properties_object, param_name); + if (!prop_obj.empty()) { + std::string desc_pattern = "\"description\":\""; + size_t desc_pos = prop_obj.find(desc_pattern); + if (desc_pos != std::string::npos) { + size_t desc_start = desc_pos + desc_pattern.size(); + size_t desc_end = desc_start; + bool escaped = false; + while (desc_end < prop_obj.size()) { + if (escaped) { escaped = false; desc_end++; continue; } + if (prop_obj[desc_end] == '\\') { escaped = true; desc_end++; continue; } + if (prop_obj[desc_end] == '"') break; + desc_end++; + } + description = prop_obj.substr(desc_start, desc_end - desc_start); + } + } + + oss << "\"" << escape_json_string(param_name) << "\":{"; + oss << "\"type\":\"" << escape_json_string(param_type) << "\""; + if (!description.empty()) { + oss << ",\"description\":\"" << description << "\""; + } + oss << ",\"required\":" << (is_required ? "true" : "false"); + oss << "}"; } oss << "}"; } @@ -388,6 +541,8 @@ struct PreparedPrompt { std::vector audio_features; size_t audio_num_frames = 0; + std::unordered_map needle_name_map; + bool has_images() const { return std::any_of(images.begin(), images.end(), [](const auto& msg_imgs) { return !msg_imgs.empty(); }); @@ -509,7 +664,7 @@ PreparedPrompt prepare_prompt( prompt.options.force_tools = true; } - if (apply_tool_constraints) { + if (apply_tool_constraints && prompt.model_type != Config::ModelType::NEEDLE) { setup_tool_constraints(handle, prompt.tools, prompt.options.force_tools, prompt.options.temperature); } @@ -565,7 +720,7 @@ PreparedPrompt prepare_prompt( if (Config::is_gemma_family(prompt.model_type)) { formatted_tools = gemma::format_tools(prompt.tools, prompt.model_type == Config::ModelType::GEMMA4); } else if (prompt.model_type == Config::ModelType::NEEDLE) { - formatted_tools = serialize_needle_tools_json(prompt.tools, tools_json); + formatted_tools = serialize_needle_tools(prompt.tools, tools_json, prompt.needle_name_map); } else if (prompt.model_type == Config::ModelType::QWEN || prompt.model_type == Config::ModelType::QWEN3P5) { formatted_tools = serialize_tools_for_template(prompt.tools); } else { @@ -573,6 +728,11 @@ PreparedPrompt prepare_prompt( } if (prompt.model_type == Config::ModelType::NEEDLE) { + if (apply_tool_constraints) { + setup_tool_constraints(handle, prompt.tools, prompt.options.force_tools, + prompt.options.temperature, prompt.needle_name_map); + } + std::string query_text = cactus::engine::format_needle_query_text(prompt.messages); prompt.tokens = tokenizer->encode(query_text); auto suffix_tokens = encode_needle_tools_suffix(tokenizer, formatted_tools); @@ -928,6 +1088,10 @@ int cactus_complete( std::vector function_calls; parse_function_calls_from_response(response_text, regular_response, function_calls); + if (prompt.model_type == Config::ModelType::NEEDLE) { + needle::restore_tool_names(function_calls, prompt.needle_name_map); + } + std::string thinking_text; if (prompt.model_type == Config::ModelType::GEMMA4 || prompt.options.enable_thinking_if_supported) { std::string stripped_content; diff --git a/cactus/ffi/needle_tools.h b/cactus/ffi/needle_tools.h new file mode 100644 index 000000000..4330cff51 --- /dev/null +++ b/cactus/ffi/needle_tools.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace needle { + +inline std::string to_snake_case(const std::string& name) { + std::string s; + s.reserve(name.size() * 2); + for (char c : name) { + if (std::isalnum(static_cast(c)) || c == '_') { + s += c; + } else { + if (s.empty() || s.back() != '_') s += '_'; + } + } + + std::string s2; + s2.reserve(s.size() * 2); + for (size_t i = 0; i < s.size(); i++) { + char c = s[i]; + if (i > 0 && std::isupper(static_cast(c))) { + char prev = s[i - 1]; + if (std::islower(static_cast(prev)) || std::isdigit(static_cast(prev))) { + s2 += '_'; + } + } + s2 += c; + } + + std::string s3; + s3.reserve(s2.size() * 2); + for (size_t i = 0; i < s2.size(); i++) { + s3 += s2[i]; + if (i + 1 < s2.size() && + std::isupper(static_cast(s2[i])) && + std::isupper(static_cast(s2[i + 1]))) { + if (i + 2 < s2.size() && std::islower(static_cast(s2[i + 2]))) { + s3 += '_'; + } + } + } + + std::string result; + result.reserve(s3.size()); + bool prev_underscore = false; + for (char c : s3) { + if (c == '_') { + if (!prev_underscore) result += '_'; + prev_underscore = true; + } else { + result += static_cast(std::tolower(static_cast(c))); + prev_underscore = false; + } + } + + size_t start = result.find_first_not_of('_'); + if (start == std::string::npos) return result; + size_t end = result.find_last_not_of('_'); + return result.substr(start, end - start + 1); +} + +inline void restore_tool_names(std::vector& function_calls, + const std::unordered_map& name_map) { + if (name_map.empty()) return; + for (auto& call : function_calls) { + for (const auto& [snake, orig] : name_map) { + std::string from = "\"name\":\"" + snake + "\""; + size_t pos = call.find(from); + if (pos == std::string::npos) { + from = "\"name\": \"" + snake + "\""; + pos = call.find(from); + } + if (pos != std::string::npos) { + std::string to = from.substr(0, from.size() - snake.size() - 1) + orig + "\""; + call.replace(pos, from.size(), to); + break; + } + } + } +} + +} // namespace needle