From 670052ef0e734fd68dada9a1670b2b86beb6f70d Mon Sep 17 00:00:00 2001 From: ayushmk7 Date: Mon, 14 Sep 2026 17:43:56 -0400 Subject: [PATCH] Fix SP vocab loading for tokens containing tabs vocab.txt is written as IDtoken or IDtokenscore. The loader split on the first tab inside the token, so Gemma tokens such as "\t\t\t" were parsed as a score and init failed with a std::stof exception. Detect the scored format from the first line and split the score on the last tab only when scores are present. Fixes #577 Signed-off-by: ayushmk7 --- cactus-engine/src/sp.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cactus-engine/src/sp.cpp b/cactus-engine/src/sp.cpp index 89e536ece..1523b82b1 100644 --- a/cactus-engine/src/sp.cpp +++ b/cactus-engine/src/sp.cpp @@ -53,9 +53,11 @@ bool SPTokenizer::load_vocabulary_with_config(const std::string& vocab_file, con vocab_stream.seekg(0); bool is_id_token_format = false; + bool has_scores = false; if (!first_line.empty()) { is_id_token_format = (std::isdigit(first_line[0]) && first_line.find('\t') != std::string::npos); + has_scores = first_line.find('\t') != first_line.rfind('\t'); } if (is_id_token_format) { @@ -87,11 +89,10 @@ bool SPTokenizer::load_vocabulary_with_config(const std::string& vocab_file, con if (!token.empty() && id != UINT32_MAX) { float score = -static_cast(id); - size_t tab_in_token = token.find('\t'); - if (tab_in_token != std::string::npos) { - std::string score_str = token.substr(tab_in_token + 1); - if (!score_str.empty()) score = std::stof(score_str); - token = token.substr(0, tab_in_token); + size_t score_tab = has_scores ? token.rfind('\t') : std::string::npos; + if (score_tab != std::string::npos) { + score = std::stof(token.substr(score_tab + 1)); + token.resize(score_tab); } token_to_id_[token] = id; if (id >= id_to_token_.size()) {