From 2f40478c1cc89a8f18b98c787c5357a3efd88e41 Mon Sep 17 00:00:00 2001 From: ayushmk7 Date: Mon, 14 Sep 2026 17:47:01 -0400 Subject: [PATCH] Decode byte-fallback pieces in metaspace BPE tokenizers Gemma-style BPE vocabularies with byte_fallback emit <0xHH> pieces for codepoints missing from the vocab. The replace_metaspace decode path appended them verbatim, so Korean and Polish output contained literal <0xEA><0xB9><0xB0> instead of the decoded character. Map <0xHH> pieces back to raw bytes when byte_fallback is enabled, as SPTokenizer::decode already does. Fixes #684 Signed-off-by: ayushmk7 --- cactus-engine/src/bpe.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cactus-engine/src/bpe.cpp b/cactus-engine/src/bpe.cpp index 41d49c795..a0b6fc0ac 100644 --- a/cactus-engine/src/bpe.cpp +++ b/cactus-engine/src/bpe.cpp @@ -526,7 +526,14 @@ std::string BPETokenizer::decode(const std::vector& tokens) const { for (uint32_t token_id : tokens) { if (token_id >= id_to_token_.size()) continue; - result += id_to_token_[token_id]; + const std::string& piece = id_to_token_[token_id]; + unsigned int byte_val; + if (runtime_config_.byte_fallback && piece.size() == 6 && + std::sscanf(piece.c_str(), "<0x%02X>", &byte_val) == 1) { + result.push_back(static_cast(byte_val)); + } else { + result += piece; + } } size_t pos = 0;