diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 253426e6b..5e0672ca4 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -28,9 +28,11 @@ jobs: python-version: "3.12" - name: Install dependencies - run: pip install -e "python/[dev]" + run: pip install -e "python/[dev,serve]" - name: Run tests run: | cd python - pytest tests/ -v --ignore=tests/test_model.py + pytest tests/ -v \ + --ignore=tests/test_model.py \ + --ignore=tests/test_server_live.py diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 7bca073a5..58aade437 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -205,6 +205,18 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, CACTUS_LOG_ERROR("model", "Tokenizer init failed for bundle: " << bundle_dir); return false; } + const bool is_text_embedding = + components_.count("text_embedding") + && !components_.count("decoder") + && !components_.count("decoder_step") + && !components_.count("lm_encoder_step"); + if (is_text_embedding) { + // Embedding-only bundle: no decode route; get_embeddings loads the + // text_embedding component on demand. + cache_max_seq_len_ = context_size; + initialized_ = true; + return true; + } std::string encoder_name; std::string decoder_name; std::unordered_set required_components; @@ -1055,22 +1067,94 @@ void Model::run_media_step(size_t position, const uint8_t* feature_row, size_t f decoder_->graph->execute(); } +namespace { +void write_typed_buffer(std::vector& buf, Precision dst_prec, + const void* src_data, size_t src_bytes, Precision src_prec); +} // namespace + void Model::run_vision_encoder(const std::string& image_path) { if (!vision_encoder_) return; if (!load_component_graph(*vision_encoder_)) { throw std::runtime_error("failed to load vision_encoder"); } - Gemma4ImagePreprocessed prep = preprocess_gemma4_image(image_path, config_); - if (has_npu_vision_encoder() && vision_encode_via_npu(prep.pixel_values)) { - return; - } - if (!load_component_graph(*vision_encoder_)) { - throw std::runtime_error("failed to load vision_encoder"); + + auto write_int_buffer_typed = [&](int idx, const int64_t* src, size_t src_count) { + auto& buf = vision_encoder_->input_buffers[idx]; + size_t node = static_cast(vision_encoder_->runtime_input_node_ids[idx]); + const auto& desc = vision_encoder_->graph->get_output_buffer(node); + const size_t elem = PrecisionTraits::size_of(desc.precision); + const size_t cap = elem ? buf.size() / elem : 0; + const size_t n = std::min(cap, src_count); + for (size_t i = 0; i < n; ++i) { + int64_t v = src[i]; + switch (desc.precision) { + case Precision::FP32: reinterpret_cast(buf.data())[i] = static_cast(v); break; + case Precision::FP16: reinterpret_cast<__fp16*>(buf.data())[i] = static_cast<__fp16>(v); break; + case Precision::INT8: reinterpret_cast(buf.data())[i] = static_cast(v); break; + default: + if (elem == 8) reinterpret_cast(buf.data())[i] = v; + else if (elem == 4) reinterpret_cast(buf.data())[i] = static_cast(v); + break; + } + } + if (n < cap) std::memset(buf.data() + n * elem, 0, (cap - n) * elem); + }; + + if (family_ == "lfm2_vl") { + Lfm2VlImagePreprocessed prep = preprocess_lfm2_vl_image(image_path, config_); + if (has_npu_vision_encoder() && vision_encode_via_npu(prep.pixel_values)) { + return; + } + int pv_idx = input_index(*vision_encoder_, "pixel_values"); + if (pv_idx >= 0) { + auto& pv_buf = vision_encoder_->input_buffers[pv_idx]; + size_t pv_node = static_cast(vision_encoder_->runtime_input_node_ids[pv_idx]); + const auto& pv_desc = vision_encoder_->graph->get_output_buffer(pv_node); + write_typed_buffer(pv_buf, pv_desc.precision, + prep.pixel_values.data(), + prep.pixel_values.size() * sizeof(float), + Precision::FP32); + } + int pm_idx = input_index(*vision_encoder_, "pixel_attention_mask"); + if (pm_idx >= 0) { + write_int_buffer_typed(pm_idx, prep.pixel_attention_mask.data(), + prep.pixel_attention_mask.size()); + } + } else if (family_ == "qwen3_5" || family_ == "qwen3_vl" || config_.model_type == Config::ModelType::QWEN) { + Qwen3VlImagePreprocessed prep = preprocess_qwen3_vl_image(image_path, config_); + int pv_idx = input_index(*vision_encoder_, "pixel_values"); + if (pv_idx < 0) { + throw std::runtime_error("Qwen3-VL vision_encoder missing pixel_values input"); + } + auto& pv_buf = vision_encoder_->input_buffers[pv_idx]; + size_t pv_node = static_cast(vision_encoder_->runtime_input_node_ids[pv_idx]); + const auto& pv_desc = vision_encoder_->graph->get_output_buffer(pv_node); + write_typed_buffer(pv_buf, pv_desc.precision, + prep.pixel_values.data(), + prep.pixel_values.size() * sizeof(float), + Precision::FP32); + } else { + Gemma4ImagePreprocessed prep = preprocess_gemma4_image(image_path, config_); + if (has_npu_vision_encoder() && vision_encode_via_npu(prep.pixel_values)) { + return; + } + int pv_idx = input_index(*vision_encoder_, "pixel_values"); + if (pv_idx >= 0) { + auto& pv_buf = vision_encoder_->input_buffers[pv_idx]; + size_t pv_node = static_cast(vision_encoder_->runtime_input_node_ids[pv_idx]); + const auto& pv_desc = vision_encoder_->graph->get_output_buffer(pv_node); + write_typed_buffer(pv_buf, pv_desc.precision, + prep.pixel_values.data(), + prep.pixel_values.size() * sizeof(float), + Precision::FP32); + } + int pp_idx = input_index(*vision_encoder_, "pixel_position_ids"); + if (pp_idx >= 0) { + write_int_buffer_typed(pp_idx, prep.pixel_position_ids.data(), + prep.pixel_position_ids.size()); + } } - write_bytes_input(*vision_encoder_, "pixel_values", prep.pixel_values.data(), - prep.pixel_values.size() * sizeof(float)); - write_bytes_input(*vision_encoder_, "pixel_position_ids", prep.pixel_position_ids.data(), - prep.pixel_position_ids.size() * sizeof(int64_t)); + vision_encoder_->graph->execute(); for (size_t i = 0; i < vision_encoder_->output_node_ids.size() && i < vision_encoder_->logical_outputs.size(); ++i) { const std::string& name = vision_encoder_->logical_outputs[i]; @@ -2445,12 +2529,126 @@ uint32_t Model::decode_with_images(const std::vector& tokens, const st return decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); } -std::vector Model::get_image_embeddings(const std::string& /*image_path*/) { - throw std::runtime_error("Image embeddings not wired up for transpiled bundles yet"); +namespace { + +std::vector pool_and_normalize_media_feature( + const std::vector& bytes, + const std::vector& shape, + Precision precision, + const std::string& source +) { + const size_t elem_size = PrecisionTraits::size_of(precision); + if (elem_size == 0 || bytes.empty() || shape.empty()) { + throw std::runtime_error(source + " produced empty feature output"); + } + const size_t total_elems = bytes.size() / elem_size; + const size_t hidden_dim = shape.back(); + if (hidden_dim == 0 || total_elems == 0 || total_elems % hidden_dim != 0) { + throw std::runtime_error(source + " feature shape inconsistent with hidden_dim"); + } + + std::vector fp32(total_elems); + switch (precision) { + case Precision::FP32: + std::memcpy(fp32.data(), bytes.data(), total_elems * sizeof(float)); + break; + case Precision::FP16: + Quantization::fp16_to_fp32(reinterpret_cast(bytes.data()), fp32.data(), total_elems); + break; + case Precision::INT8: + Quantization::int8_to_fp32(reinterpret_cast(bytes.data()), fp32.data(), total_elems, 1.0f); + break; + default: + throw std::runtime_error(source + " feature precision not supported for embeddings"); + } + + const size_t n_rows = total_elems / hidden_dim; + std::vector pooled(hidden_dim, 0.0f); + for (size_t r = 0; r < n_rows; ++r) { + const float* src = fp32.data() + r * hidden_dim; + for (size_t d = 0; d < hidden_dim; ++d) pooled[d] += src[d]; + } + const float inv = 1.0f / static_cast(n_rows); + for (float& v : pooled) v *= inv; + + float norm_sq = 0.0f; + for (float v : pooled) norm_sq += v * v; + if (norm_sq > 1e-12f) { + const float inv_norm = 1.0f / std::sqrt(norm_sq); + for (float& v : pooled) v *= inv_norm; + } + return pooled; } -std::vector Model::get_audio_embeddings(const std::vector& /*mel_bins*/) { - throw std::runtime_error("Audio embeddings not wired up for transpiled bundles yet"); +} // namespace + +std::vector Model::get_image_embeddings(const std::string& image_path) { + if (!vision_encoder_) { + throw std::runtime_error("Model has no vision_encoder component"); + } + if (vision_encoder_->logical_outputs.empty()) { + throw std::runtime_error("vision_encoder has no logical outputs"); + } + const std::string output_name = vision_encoder_->logical_outputs[0]; + + run_vision_encoder(image_path); + + auto bytes_it = media_features_.find(output_name); + auto shape_it = media_feature_shapes_.find(output_name); + auto prec_it = media_feature_precisions_.find(output_name); + if (bytes_it == media_features_.end() || shape_it == media_feature_shapes_.end() + || prec_it == media_feature_precisions_.end()) { + throw std::runtime_error("vision_encoder produced no output for '" + output_name + "'"); + } + + std::vector embedding = pool_and_normalize_media_feature( + bytes_it->second, shape_it->second, prec_it->second, "vision_encoder"); + + for (const std::string& name : vision_encoder_->logical_outputs) { + media_features_.erase(name); + media_feature_shapes_.erase(name); + media_feature_precisions_.erase(name); + } + // run_vision_encoder unloads the graph; restore so subsequent paths that + // assume the encoder is loaded (e.g. transcribe_*) keep working. + load_component_graph(*vision_encoder_); + return embedding; +} + +std::vector Model::get_audio_embeddings(const std::vector& mel_bins) { + if (!audio_encoder_) { + throw std::runtime_error("Model has no audio_encoder component"); + } + if (mel_bins.empty()) { + throw std::runtime_error("Empty audio features"); + } + if (audio_encoder_->logical_outputs.empty()) { + throw std::runtime_error("audio_encoder has no logical outputs"); + } + const std::string output_name = audio_encoder_->logical_outputs[0]; + + run_audio_encoder_messages({mel_bins}); + + auto bytes_it = media_features_.find(output_name); + auto shape_it = media_feature_shapes_.find(output_name); + auto prec_it = media_feature_precisions_.find(output_name); + if (bytes_it == media_features_.end() || shape_it == media_feature_shapes_.end() + || prec_it == media_feature_precisions_.end()) { + throw std::runtime_error("audio_encoder produced no output for '" + output_name + "'"); + } + + std::vector embedding = pool_and_normalize_media_feature( + bytes_it->second, shape_it->second, prec_it->second, "audio_encoder"); + + for (const std::string& name : audio_encoder_->logical_outputs) { + media_features_.erase(name); + media_feature_shapes_.erase(name); + media_feature_precisions_.erase(name); + } + // run_audio_encoder_messages unloads the graph; restore so subsequent + // transcribe_* paths (which assume the encoder stays loaded) work. + load_component_graph(*audio_encoder_); + return embedding; } void Model::reset_cache() { @@ -2557,9 +2755,89 @@ void Model::remove_thinking_tokens(const std::vector>& } } -std::vector Model::get_embeddings(const std::vector& /*tokens*/, bool /*pooled*/, - bool /*normalize*/, const std::string& /*profile_file*/) { - return {}; +std::vector Model::get_embeddings(const std::vector& tokens, bool pooled, + bool normalize, const std::string& /*profile_file*/) { + if (!components_.count("text_embedding")) { + throw std::runtime_error("get_embeddings: bundle has no text_embedding component"); + } + if (tokens.empty()) { + throw std::runtime_error("get_embeddings: empty token sequence"); + } + Component* comp = &components_.at("text_embedding"); + if (!load_component_graph(*comp)) { + throw std::runtime_error("get_embeddings: failed to load embedding component graph"); + } + + // Embedding encoders (nomic / XLM-R) wrap the sequence with BOS/EOS, matching + // the reference tokenizer's add_special_tokens behavior. + std::vector wrapped; + wrapped.reserve(tokens.size() + 2); + if (tokenizer_) wrapped.push_back(tokenizer_->get_bos_token()); + wrapped.insert(wrapped.end(), tokens.begin(), tokens.end()); + if (tokenizer_) wrapped.push_back(tokenizer_->get_eos_token()); + + int ids_idx = input_index(*comp, "input_ids"); + if (ids_idx < 0) { + throw std::runtime_error("get_embeddings: embedding component missing input_ids"); + } + auto& ids_buf = comp->input_buffers[ids_idx]; + size_t ids_node = static_cast(comp->runtime_input_node_ids[ids_idx]); + const auto& ids_desc = comp->graph->get_output_buffer(ids_node); + size_t capacity = PrecisionTraits::size_of(ids_desc.precision) + ? ids_buf.size() / PrecisionTraits::size_of(ids_desc.precision) : wrapped.size(); + size_t n_real = std::min(capacity, wrapped.size()); + write_tokens_buffer(ids_buf, ids_desc.precision, wrapped, 0); + + int mask_idx = input_index(*comp, "attention_mask"); + if (mask_idx >= 0) { + auto& mb = comp->input_buffers[mask_idx]; + size_t mnode = static_cast(comp->runtime_input_node_ids[mask_idx]); + const auto& mdesc = comp->graph->get_output_buffer(mnode); + fill_int_buffer(mb, mdesc.precision, 1, n_real); + } + + comp->graph->execute(); + + if (comp->output_node_ids.empty()) { + throw std::runtime_error("get_embeddings: embedding component produced no outputs"); + } + size_t out_node = static_cast(comp->output_node_ids[0]); + const auto& desc = comp->graph->get_output_buffer(out_node); + void* ptr = comp->graph->get_output(out_node); + size_t hidden = desc.shape.empty() ? 0 : desc.shape.back(); + size_t seq = (desc.shape.size() >= 2) ? desc.shape[desc.shape.size() - 2] : 1; + if (hidden == 0) { + throw std::runtime_error("get_embeddings: embedding output has zero hidden dim"); + } + + const bool is_fp16 = desc.precision == Precision::FP16; + auto read_at = [&](size_t i) -> float { + return is_fp16 ? static_cast(reinterpret_cast(ptr)[i]) + : reinterpret_cast(ptr)[i]; + }; + + std::vector result(hidden, 0.0f); + if (pooled) { + size_t pool_rows = std::min(seq, std::max(1, n_real)); + for (size_t t = 0; t < pool_rows; ++t) { + for (size_t h = 0; h < hidden; ++h) result[h] += read_at(t * hidden + h); + } + for (size_t h = 0; h < hidden; ++h) result[h] /= static_cast(pool_rows); + } else { + for (size_t h = 0; h < hidden; ++h) result[h] = read_at(h); + } + + if (normalize) { + double norm = 0.0; + for (float v : result) norm += static_cast(v) * v; + float inv = static_cast(1.0 / std::max(std::sqrt(norm), 1e-12)); + for (float& v : result) v *= inv; + } + + comp->graph->release_runtime_buffers(); + comp->graph->release_all_weight_pages(); + unload_component_graph(*comp); + return result; } bool Config::from_json(const std::string& config_path) { @@ -2651,6 +2929,7 @@ bool Config::from_json(const std::string& config_path) { else if (mt == "parakeet_tdt" || mt == "parakeet-tdt") model_type = ModelType::PARAKEET_TDT; else if (mt == "youtu") model_type = ModelType::YOUTU; else if (mt == "needle") model_type = ModelType::NEEDLE; + else if (mt == "bert" || mt == "nomic") model_type = ModelType::NOMIC; else model_type = ModelType::GEMMA4; } else if (key == "model_variant") { diff --git a/cactus-engine/src/rag.cpp b/cactus-engine/src/rag.cpp index 77d9cff7c..dcf821b99 100644 --- a/cactus-engine/src/rag.cpp +++ b/cactus-engine/src/rag.cpp @@ -124,7 +124,13 @@ std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& q std::vector query_tokens = tokenizer->encode(query); if (query_tokens.empty()) return ""; - std::vector query_embedding = handle->model->get_embeddings(query_tokens, true, true); + std::vector query_embedding; + try { + query_embedding = handle->model->get_embeddings(query_tokens, true, true); + } catch (const std::exception& e) { + CACTUS_LOG_WARN("rag", "get_embeddings unavailable, skipping RAG context: " << e.what()); + return ""; + } if (query_embedding.size() != handle->corpus_embedding_dim) { CACTUS_LOG_WARN("rag", "Query embedding dimension mismatch"); return ""; @@ -262,8 +268,15 @@ std::vector select_relevant_tools( std::vector tokens = tokenizer->encode(text); if (!tokens.empty()) { - std::vector emb = handle->model->get_embeddings(tokens, true, true); - handle->tool_embeddings.push_back(std::move(emb)); + try { + std::vector emb = handle->model->get_embeddings(tokens, true, true); + handle->tool_embeddings.push_back(std::move(emb)); + } catch (const std::exception& e) { + CACTUS_LOG_WARN("tool_rag", "get_embeddings unavailable, returning all tools: " << e.what()); + handle->tool_texts.clear(); + handle->tool_embeddings.clear(); + return all_tools; + } } else { handle->tool_embeddings.push_back({}); } @@ -277,7 +290,13 @@ std::vector select_relevant_tools( return all_tools; } - std::vector query_embedding = handle->model->get_embeddings(query_tokens, true, true); + std::vector query_embedding; + try { + query_embedding = handle->model->get_embeddings(query_tokens, true, true); + } catch (const std::exception& e) { + CACTUS_LOG_WARN("tool_rag", "get_embeddings unavailable, returning all tools: " << e.what()); + return all_tools; + } if (query_embedding.empty()) { CACTUS_LOG_WARN("tool_rag", "Failed to get query embedding, returning all tools"); return all_tools; diff --git a/cactus-engine/src/sp.cpp b/cactus-engine/src/sp.cpp index 0bf5fb18c..89e536ece 100644 --- a/cactus-engine/src/sp.cpp +++ b/cactus-engine/src/sp.cpp @@ -235,8 +235,11 @@ std::string SPTokenizer::preprocess_text(const std::string& text) const { } std::string processed = ""; + if (sp_add_dummy_prefix_) processed += "\xE2\x96\x81"; - for (size_t i = text.find_first_not_of(" "); i < text.length(); i++) { + size_t start = text.find_first_not_of(" "); + if (start == std::string::npos) return processed; + for (size_t i = start; i < text.length(); i++) { char c = text[i]; if (c == ' ') { processed += "▁"; diff --git a/python/cactus/bindings/cactus.py b/python/cactus/bindings/cactus.py index 50b555b2a..0b9eff8bb 100644 --- a/python/cactus/bindings/cactus.py +++ b/python/cactus/bindings/cactus.py @@ -1061,7 +1061,7 @@ def cactus_embed(model, text, normalize=True): """ buf = (ctypes.c_float * 4096)() dim = ctypes.c_size_t() - rc = _lib.cactus_embed(model, _enc(text), buf, 4096, ctypes.byref(dim), normalize) + rc = _lib.cactus_embed(model, _enc(text), buf, ctypes.sizeof(buf), ctypes.byref(dim), normalize) if rc < 0: raise RuntimeError(_err("Embedding failed")) return list(buf[:dim.value]) @@ -1071,7 +1071,7 @@ def cactus_image_embed(model, image_path): """Compute an image embedding. Returns a list of floats.""" buf = (ctypes.c_float * 4096)() dim = ctypes.c_size_t() - rc = _lib.cactus_image_embed(model, _enc(image_path), buf, 4096, ctypes.byref(dim)) + rc = _lib.cactus_image_embed(model, _enc(image_path), buf, ctypes.sizeof(buf), ctypes.byref(dim)) if rc < 0: raise RuntimeError(_err("Image embedding failed")) return list(buf[:dim.value]) @@ -1081,7 +1081,7 @@ def cactus_audio_embed(model, audio_path): """Compute an audio embedding. Returns a list of floats.""" buf = (ctypes.c_float * 4096)() dim = ctypes.c_size_t() - rc = _lib.cactus_audio_embed(model, _enc(audio_path), buf, 4096, ctypes.byref(dim)) + rc = _lib.cactus_audio_embed(model, _enc(audio_path), buf, ctypes.sizeof(buf), ctypes.byref(dim)) if rc < 0: raise RuntimeError(_err("Audio embedding failed")) return list(buf[:dim.value]) diff --git a/python/cactus/convert/cactus_adapters/tokenizer.py b/python/cactus/convert/cactus_adapters/tokenizer.py index 2ec17713b..9e302fd47 100644 --- a/python/cactus/convert/cactus_adapters/tokenizer.py +++ b/python/cactus/convert/cactus_adapters/tokenizer.py @@ -143,7 +143,16 @@ def convert_hf_tokenizer(tokenizer, output_dir, token=None, model_id=None, label tokenizer_model = tokenizer_json_data.get("model", {}) if tokenizer_json_data else {} tokenizer_model_type = str(tokenizer_model.get("type", "")).upper() - is_sentencepiece = tokenizer_model_type != "BPE" and model_type in SENTENCEPIECE_MODEL_TYPES + is_unigram = tokenizer_model_type == "UNIGRAM" + is_sentencepiece = is_unigram or (tokenizer_model_type != "BPE" and model_type in SENTENCEPIECE_MODEL_TYPES) + + # Unigram (e.g. XLM-RoBERTa, used by nomic-embed) carries per-token scores that + # the SentencePiece runtime needs for Viterbi segmentation. + unigram_scores: dict[int, float] = {} + if is_unigram and isinstance(tokenizer_model.get("vocab"), list): + for token_id, entry in enumerate(tokenizer_model["vocab"]): + if isinstance(entry, (list, tuple)) and len(entry) == 2: + unigram_scores[token_id] = float(entry[1]) if tokenizer_model_type == "BPE" and tokenizer_model.get("vocab"): vocab = tokenizer_model["vocab"] @@ -376,8 +385,12 @@ def write_merges_file(merges_list): with open(vocab_output, 'w', encoding='utf-8') as f: for token_id, token_str in enumerate(id_to_token): if token_str: - f.write(f"{token_id}\t{token_str}\n") - print(f" Saved tokenizer vocabulary (ID\\ttoken format)") + if unigram_scores: + f.write(f"{token_id}\t{token_str}\t{unigram_scores.get(token_id, 0.0)}\n") + else: + f.write(f"{token_id}\t{token_str}\n") + vocab_fmt = "ID\\ttoken\\tscore" if unigram_scores else "ID\\ttoken" + print(f" Saved tokenizer vocabulary ({vocab_fmt} format)") special_tokens_output = output_dir / "special_tokens.json" with open(special_tokens_output, 'w', encoding='utf-8') as f: @@ -410,6 +423,9 @@ def write_merges_file(merges_list): decoder = "byte_level" elif is_sentencepiece: tokenizer_type = "sentencepiece" + if is_unigram: + normalizer = "metaspace" + decoder = "replace_metaspace" tokenizer_config_output = output_dir / "tokenizer_config.txt" with open(tokenizer_config_output, 'w') as f: @@ -422,6 +438,10 @@ def write_merges_file(merges_list): f.write(f"normalizer={normalizer}\n") f.write(f"decoder={decoder}\n") f.write(f"byte_fallback={'true' if byte_fallback else 'false'}\n") + if is_unigram: + f.write("sp_model_type=unigram\n") + f.write("sp_add_dummy_prefix=true\n") + f.write("sp_byte_fallback=false\n") if chat_template_data: f.write("has_chat_template=true\n") diff --git a/python/cactus/convert/model_adapters/adapters.py b/python/cactus/convert/model_adapters/adapters.py index 24d74715f..fbc032aa0 100644 --- a/python/cactus/convert/model_adapters/adapters.py +++ b/python/cactus/convert/model_adapters/adapters.py @@ -349,9 +349,22 @@ def name_tensor(self, source_name: str, tensor: Any, num_layers: int | None) -> norm2 = _nomic_layer_suffix(source_name, ".norm2.weight") if norm2 is not None: return NameMatch(source_name, f"layer_{norm2}_norm2.weights", "language", True, hf_name=source_name, adapter_name=source_name) + for suffix, template, transpose in ( + (".attn.Wqkv.weight", "layer_{i}_attn_qkv.weights", False), + (".attn.Wqkv.bias", "layer_{i}_attn_qkv.bias", False), + (".mlp.experts.mlp.w1", "layer_{i}_mlp_experts_w1.weights", False), + (".mlp.experts.mlp.w2", "layer_{i}_mlp_experts_w2.weights", True), + ): + layer = _nomic_layer_suffix(source_name, suffix) + if layer is not None: + return NameMatch(source_name, template.format(i=layer), "language", True, transpose, hf_name=source_name, adapter_name=source_name) return cactus_name_for_tensor(source_name, self.family, num_layers) def policy(self, match: NameMatch, shape: tuple[int, ...], requested_bits: int) -> TensorPolicy: + # The MoE router is tiny ([num_experts, hidden]) but decides expert selection; + # 4-bit quantizing it corrupts routing, so keep it in FP16. + if ".mlp.router.layer.weight" in match.source_name: + return TensorPolicy("fallback", "FP16", None, match.component, False, "none", "moe router precision-sensitive") policy = super().policy(match, shape, requested_bits) if policy.use_gptq and ".mlp.experts.mlp." in match.source_name: return replace(policy, use_gptq=False) diff --git a/python/cactus/convert/tests/test_lfm2_adapter.py b/python/cactus/convert/tests/test_lfm2_adapter.py new file mode 100644 index 000000000..49d5798c9 --- /dev/null +++ b/python/cactus/convert/tests/test_lfm2_adapter.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from cactus.convert.model_adapters.adapters import adapter_for_family + + +def test_lfm2_vl_adapter_selects_runtime_safe_model_class(): + from transformers import Lfm2VlForConditionalGeneration + + adapter = adapter_for_family("lfm2") + cfg = {"model_type": "lfm2", "architectures": ["Lfm2VlForConditionalGeneration"]} + assert adapter.model_class(cfg) is Lfm2VlForConditionalGeneration + + +def test_lfm2_processor_fallback_handles_tokenizers_backend(tmp_path): + import json + + from tokenizers import Tokenizer + from tokenizers.models import WordLevel + from tokenizers.pre_tokenizers import Whitespace + from transformers import Lfm2VlProcessor + + tokenizer = Tokenizer(WordLevel({"<|pad|>": 0, "<|startoftext|>": 1, "<|im_end|>": 2, "": 3, "hello": 4}, unk_token="<|pad|>")) + tokenizer.pre_tokenizer = Whitespace() + tokenizer.save(str(tmp_path / "tokenizer.json")) + (tmp_path / "tokenizer_config.json").write_text( + json.dumps( + { + "tokenizer_class": "TokenizersBackend", + "bos_token": "<|startoftext|>", + "eos_token": "<|im_end|>", + "pad_token": "<|pad|>", + "image_token": "", + "image_start_token": "", + "image_end_token": "", + "image_thumbnail": "", + } + ), + encoding="utf-8", + ) + (tmp_path / "preprocessor_config.json").write_text( + json.dumps( + { + "image_processor_type": "Lfm2VlImageProcessorFast", + "do_resize": True, + "size": {"height": 512, "width": 512}, + "do_rescale": True, + "rescale_factor": 1 / 255, + "do_normalize": True, + "image_mean": [0.5, 0.5, 0.5], + "image_std": [0.5, 0.5, 0.5], + "do_pad": True, + "data_format": "channels_first", + } + ), + encoding="utf-8", + ) + + processor = adapter_for_family("lfm2").load_processor(str(tmp_path)) + assert isinstance(processor, Lfm2VlProcessor) + assert processor.image_token == "" + assert processor.image_token_id == 3 diff --git a/python/cactus/convert/tests/test_naming_qdq.py b/python/cactus/convert/tests/test_naming_qdq.py index d1be06723..860c8ffd4 100644 --- a/python/cactus/convert/tests/test_naming_qdq.py +++ b/python/cactus/convert/tests/test_naming_qdq.py @@ -7,6 +7,7 @@ from cactus.convert.cactus_adapters.tensor_io import save_tensor_with_header from cactus.convert.export.qdq import convert_qdq +from cactus.convert.model_adapters.adapters import adapter_for_family from cactus.convert.model_adapters.naming import cactus_name_for_tensor, restore_hf_key_for_family @@ -289,3 +290,115 @@ def test_parakeet_batchnorm_tracking_tensors_are_ignored(): match = cactus_name_for_tensor("encoder.layers.0.conv.norm.num_batches_tracked", "parakeet", 24) assert match.recognized assert match.output_name is None + + +def test_nomic_normalizes_global_tensors(): + adapter = adapter_for_family("nomic") + state = { + "embeddings.word_embeddings.weight": torch.ones(4, 3), + "embeddings.token_type_embeddings.weight": torch.full((1, 3), 2.0), + "emb_ln.weight": torch.arange(3.0), + "emb_ln.bias": torch.arange(3.0) + 10, + } + normalized = adapter.normalize_state_dict(state) + assert set(normalized.state_dict) == {"token_embeddings", "embedding_layernorm.weight", "embedding_layernorm.bias"} + assert torch.equal(normalized.state_dict["token_embeddings"], torch.full((4, 3), 3.0)) + assert normalized.provenance["token_embeddings"].source_names == [ + "embeddings.word_embeddings.weight", + "embeddings.token_type_embeddings.weight", + ] + assert normalized.provenance["token_embeddings"].qdq_restore == "adapter_key" + assert adapter.name_tensor("token_embeddings", normalized.state_dict["token_embeddings"], 12).output_name == "token_embeddings.weights" + assert adapter.name_tensor("embedding_layernorm.weight", normalized.state_dict["embedding_layernorm.weight"], 12).output_name == "embedding_layernorm.weight" + + +def test_nomic_norm2_weight_uses_runtime_name(): + adapter = adapter_for_family("nomic") + match = adapter.name_tensor("encoder.layers.3.norm2.weight", torch.ones(768), 12) + assert match.recognized + assert match.output_name == "layer_3_norm2.weights" + + +def test_nomic_keeps_qkv_and_moe_experts_fused(): + # The v2 transpile path binds graph weights by their HF parameter name, so the + # converter emits one fused tensor per HF parameter (no q/k/v or per-expert split). + # w2 is stored transposed so the second expert matmul can consume it as a direct + # linear weight in the transpiled graph. + adapter = adapter_for_family("nomic") + adapter.num_experts = 8 + + qkv = torch.arange(2304 * 2, dtype=torch.float32).reshape(2304, 2) + match = adapter.name_tensor("encoder.layers.0.attn.Wqkv.weight", qkv, 12) + emissions = adapter.expand_tensor(match, qkv) + assert [e.output_name for e in emissions] == ["layer_0_attn_qkv.weights"] + assert tuple(emissions[0].tensor.shape) == (2304, 2) + + w1 = torch.empty(24576, 2) + match = adapter.name_tensor("encoder.layers.1.mlp.experts.mlp.w1", w1, 12) + emissions = adapter.expand_tensor(match, w1) + assert [e.output_name for e in emissions] == ["layer_1_mlp_experts_w1.weights"] + assert tuple(emissions[0].tensor.shape) == (24576, 2) + + w2 = torch.empty(24576, 2) + match = adapter.name_tensor("encoder.layers.1.mlp.experts.mlp.w2", w2, 12) + emissions = adapter.expand_tensor(match, w2) + assert [e.output_name for e in emissions] == ["layer_1_mlp_experts_w2.weights"] + assert tuple(emissions[0].tensor.shape) == (2, 24576) + + +def test_nomic_qdq_runtime_keys_are_unique(tmp_path): + cactus = tmp_path / "cactus" + out = tmp_path / "qdq" + cactus.mkdir() + save_tensor_with_header(torch.ones(2, 3), cactus / "layer_0_attn_q.weights", precision="FP16") + save_tensor_with_header(torch.ones(2, 3) * 2, cactus / "layer_0_attn_k.weights", precision="FP16") + (cactus / "conversion_manifest.json").write_text( + """[ + { + "source_name": "encoder.layers.0.attn.Wqkv.weight", + "hf_name": "encoder.layers.0.attn.Wqkv.weight", + "adapter_name": "encoder.layers.0.attn.Wqkv.weight", + "output_file": "layer_0_attn_q.weights", + "shape": [2, 3], + "dtype": "torch.float32", + "component": "language", + "policy": "fallback", + "precision": "FP16", + "status": "fallback", + "required": true, + "qdq_restore": "runtime_key", + "scale_factor": 1.0 + }, + { + "source_name": "encoder.layers.0.attn.Wqkv.weight", + "hf_name": "encoder.layers.0.attn.Wqkv.weight", + "adapter_name": "encoder.layers.0.attn.Wqkv.weight", + "output_file": "layer_0_attn_k.weights", + "shape": [2, 3], + "dtype": "torch.float32", + "component": "language", + "policy": "fallback", + "precision": "FP16", + "status": "fallback", + "required": true, + "qdq_restore": "runtime_key", + "scale_factor": 1.0 + } +]""", + encoding="utf-8", + ) + report = convert_qdq( + SimpleNamespace( + input=cactus, + out=out, + dtype="float16", + model_family="nomic", + shard_size_gb=1.0, + row_batch_size=64, + tmp_dir=None, + force=True, + ) + ) + tensors = load_file(out / "model.safetensors") + assert report["written_count"] == 2 + assert set(tensors) == {"layer_0_attn_q", "layer_0_attn_k"} diff --git a/python/cactus/convert/tests/test_nomic_adapter.py b/python/cactus/convert/tests/test_nomic_adapter.py deleted file mode 100644 index 4c8c3ab93..000000000 --- a/python/cactus/convert/tests/test_nomic_adapter.py +++ /dev/null @@ -1,226 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace - -import torch -from safetensors.torch import load_file - -from cactus.convert.cactus_adapters.tensor_io import save_tensor_with_header -from cactus.convert.export.qdq import convert_qdq -from cactus.convert.model_adapters.adapters import adapter_for_family -from cactus.convert.model_adapters.detection import detect_family - - -def _cfg() -> dict[str, object]: - return { - "model_type": "nomic_bert", - "vocab_size": 250048, - "n_embd": 768, - "n_head": 12, - "n_layer": 12, - "n_inner": 3072, - "n_positions": 2048, - "num_experts": 8, - "moe_top_k": 2, - "moe_every_n_layers": 2, - } - - -def test_nomic_detection_and_runtime_config(): - adapter = adapter_for_family("nomic") - assert detect_family(_cfg(), "auto") == "nomic" - assert adapter.runtime_model_type() == "bert" - runtime = adapter.runtime_config(_cfg()) - assert runtime["num_layers"] == 12 - assert runtime["hidden_dim"] == 768 - assert runtime["attention_heads"] == 12 - assert runtime["num_experts"] == 8 - assert runtime["num_experts_per_tok"] == 2 - assert runtime["moe_every_n_layers"] == 2 - - -def test_lfm2_vl_adapter_selects_runtime_safe_model_class(): - from transformers import Lfm2VlForConditionalGeneration - - adapter = adapter_for_family("lfm2") - cfg = {"model_type": "lfm2", "architectures": ["Lfm2VlForConditionalGeneration"]} - assert adapter.model_class(cfg) is Lfm2VlForConditionalGeneration - - -def test_lfm2_processor_fallback_handles_tokenizers_backend(tmp_path): - import json - - from tokenizers import Tokenizer - from tokenizers.models import WordLevel - from tokenizers.pre_tokenizers import Whitespace - from transformers import Lfm2VlProcessor - - tokenizer = Tokenizer(WordLevel({"<|pad|>": 0, "<|startoftext|>": 1, "<|im_end|>": 2, "": 3, "hello": 4}, unk_token="<|pad|>")) - tokenizer.pre_tokenizer = Whitespace() - tokenizer.save(str(tmp_path / "tokenizer.json")) - (tmp_path / "tokenizer_config.json").write_text( - json.dumps( - { - "tokenizer_class": "TokenizersBackend", - "bos_token": "<|startoftext|>", - "eos_token": "<|im_end|>", - "pad_token": "<|pad|>", - "image_token": "", - "image_start_token": "", - "image_end_token": "", - "image_thumbnail": "", - } - ), - encoding="utf-8", - ) - (tmp_path / "preprocessor_config.json").write_text( - json.dumps( - { - "image_processor_type": "Lfm2VlImageProcessorFast", - "do_resize": True, - "size": {"height": 512, "width": 512}, - "do_rescale": True, - "rescale_factor": 1 / 255, - "do_normalize": True, - "image_mean": [0.5, 0.5, 0.5], - "image_std": [0.5, 0.5, 0.5], - "do_pad": True, - "data_format": "channels_first", - } - ), - encoding="utf-8", - ) - - processor = adapter_for_family("lfm2").load_processor(str(tmp_path)) - assert isinstance(processor, Lfm2VlProcessor) - assert processor.image_token == "" - assert processor.image_token_id == 3 - - -def test_nomic_normalizes_global_tensors(): - adapter = adapter_for_family("nomic") - state = { - "embeddings.word_embeddings.weight": torch.ones(4, 3), - "embeddings.token_type_embeddings.weight": torch.full((1, 3), 2.0), - "emb_ln.weight": torch.arange(3.0), - "emb_ln.bias": torch.arange(3.0) + 10, - } - normalized = adapter.normalize_state_dict(state) - assert set(normalized.state_dict) == {"token_embeddings", "embedding_layernorm.weight", "embedding_layernorm.bias"} - assert torch.equal(normalized.state_dict["token_embeddings"], torch.full((4, 3), 3.0)) - assert normalized.provenance["token_embeddings"].source_names == [ - "embeddings.word_embeddings.weight", - "embeddings.token_type_embeddings.weight", - ] - assert normalized.provenance["token_embeddings"].qdq_restore == "adapter_key" - assert adapter.name_tensor("token_embeddings", normalized.state_dict["token_embeddings"], 12).output_name == "token_embeddings.weights" - assert adapter.name_tensor("embedding_layernorm.weight", normalized.state_dict["embedding_layernorm.weight"], 12).output_name == "embedding_layernorm.weight" - - -def test_nomic_norm2_weight_uses_runtime_name(): - adapter = adapter_for_family("nomic") - match = adapter.name_tensor("encoder.layers.3.norm2.weight", torch.ones(768), 12) - assert match.recognized - assert match.output_name == "layer_3_norm2.weights" - - -def test_nomic_expands_qkv_and_moe_runtime_channels(): - adapter = adapter_for_family("nomic") - adapter.runtime_config(_cfg()) - - qkv = torch.arange(2304 * 2, dtype=torch.float32).reshape(2304, 2) - match = adapter.name_tensor("encoder.layers.0.attn.Wqkv.weight", qkv, 12) - emissions = adapter.expand_tensor(match, qkv) - assert [e.output_name for e in emissions] == [ - "layer_0_attn_q.weights", - "layer_0_attn_k.weights", - "layer_0_attn_v.weights", - ] - assert [tuple(e.tensor.shape) for e in emissions] == [(768, 2), (768, 2), (768, 2)] - assert {e.qdq_restore for e in emissions} == {"runtime_key"} - - w1 = torch.empty(24576, 2) - match = adapter.name_tensor("encoder.layers.1.mlp.experts.mlp.w1", w1, 12) - emissions = adapter.expand_tensor(match, w1) - assert len(emissions) == 8 - assert emissions[0].output_name == "layer_1_mlp_expert_0.mlp1.weights" - assert emissions[-1].output_name == "layer_1_mlp_expert_7.mlp1.weights" - assert tuple(emissions[0].tensor.shape) == (3072, 2) - - w2 = torch.empty(24576, 2) - match = adapter.name_tensor("encoder.layers.1.mlp.experts.mlp.w2", w2, 12) - emissions = adapter.expand_tensor(match, w2) - assert len(emissions) == 8 - assert emissions[0].output_name == "layer_1_mlp_expert_0.mlp2.weights" - assert tuple(emissions[0].tensor.shape) == (2, 3072) - - -def test_nomic_uses_gptq_only_for_hookable_linear_modules(): - adapter = adapter_for_family("nomic") - qkv_match = adapter.name_tensor("encoder.layers.0.attn.Wqkv.weight", torch.empty(2304, 768), 12) - qkv_policy = adapter.policy(qkv_match, (768, 768), 4) - assert qkv_policy.use_gptq - assert adapter.module_target_name("encoder.layers.0.attn.Wqkv.weight", None) == "encoder.layers.0.attn.Wqkv" - - expert_match = adapter.name_tensor("encoder.layers.1.mlp.experts.mlp.w1", torch.empty(24576, 768), 12) - expert_policy = adapter.policy(expert_match, (3072, 768), 4) - assert not expert_policy.use_gptq - assert adapter.module_target_name("encoder.layers.1.mlp.experts.mlp.w1", None) is None - - -def test_nomic_qdq_runtime_keys_are_unique(tmp_path): - cactus = tmp_path / "cactus" - out = tmp_path / "qdq" - cactus.mkdir() - save_tensor_with_header(torch.ones(2, 3), cactus / "layer_0_attn_q.weights", precision="FP16") - save_tensor_with_header(torch.ones(2, 3) * 2, cactus / "layer_0_attn_k.weights", precision="FP16") - (cactus / "conversion_manifest.json").write_text( - """[ - { - "source_name": "encoder.layers.0.attn.Wqkv.weight", - "hf_name": "encoder.layers.0.attn.Wqkv.weight", - "adapter_name": "encoder.layers.0.attn.Wqkv.weight", - "output_file": "layer_0_attn_q.weights", - "shape": [2, 3], - "dtype": "torch.float32", - "component": "language", - "policy": "fallback", - "precision": "FP16", - "status": "fallback", - "required": true, - "qdq_restore": "runtime_key", - "scale_factor": 1.0 - }, - { - "source_name": "encoder.layers.0.attn.Wqkv.weight", - "hf_name": "encoder.layers.0.attn.Wqkv.weight", - "adapter_name": "encoder.layers.0.attn.Wqkv.weight", - "output_file": "layer_0_attn_k.weights", - "shape": [2, 3], - "dtype": "torch.float32", - "component": "language", - "policy": "fallback", - "precision": "FP16", - "status": "fallback", - "required": true, - "qdq_restore": "runtime_key", - "scale_factor": 1.0 - } -]""", - encoding="utf-8", - ) - report = convert_qdq( - SimpleNamespace( - input=cactus, - out=out, - dtype="float16", - model_family="nomic", - shard_size_gb=1.0, - row_batch_size=64, - tmp_dir=None, - force=True, - ) - ) - tensors = load_file(out / "model.safetensors") - assert report["written_count"] == 2 - assert set(tensors) == {"layer_0_attn_q", "layer_0_attn_k"} diff --git a/python/cactus/convert/tests/test_policy.py b/python/cactus/convert/tests/test_policy.py index 03c568a3e..9a6aeaa8b 100644 --- a/python/cactus/convert/tests/test_policy.py +++ b/python/cactus/convert/tests/test_policy.py @@ -250,3 +250,50 @@ def test_parakeet_adapter_conv_transform_is_scoped(): untouched, transform = adapter.transform_tensor(other, torch.randn(4, 4)) assert transform == "none" assert untouched.shape == (4, 4) + + +def _nomic_cfg() -> dict[str, object]: + return { + "model_type": "nomic_bert", + "vocab_size": 250048, + "n_embd": 768, + "n_head": 12, + "n_layer": 12, + "n_inner": 3072, + "n_positions": 2048, + "num_experts": 8, + "moe_top_k": 2, + "moe_every_n_layers": 2, + } + + +def test_nomic_detection_and_runtime_config(): + adapter = adapter_for_family("nomic") + assert detect_family(_nomic_cfg(), "auto") == "nomic" + assert adapter.runtime_model_type() == "bert" + runtime = adapter.runtime_config(_nomic_cfg()) + assert runtime["num_layers"] == 12 + assert runtime["hidden_dim"] == 768 + assert runtime["attention_heads"] == 12 + assert runtime["num_experts"] == 8 + assert runtime["num_experts_per_tok"] == 2 + assert runtime["moe_every_n_layers"] == 2 + + +def test_nomic_uses_gptq_only_for_hookable_linear_modules(): + adapter = adapter_for_family("nomic") + qkv_match = adapter.name_tensor("encoder.layers.0.attn.Wqkv.weight", torch.empty(2304, 768), 12) + qkv_policy = adapter.policy(qkv_match, (768, 768), 4) + assert qkv_policy.use_gptq + assert adapter.module_target_name("encoder.layers.0.attn.Wqkv.weight", None) == "encoder.layers.0.attn.Wqkv" + + expert_match = adapter.name_tensor("encoder.layers.1.mlp.experts.mlp.w1", torch.empty(24576, 768), 12) + expert_policy = adapter.policy(expert_match, (3072, 768), 4) + assert not expert_policy.use_gptq + assert adapter.module_target_name("encoder.layers.1.mlp.experts.mlp.w1", None) is None + + +def test_nomic_router_stays_fp16(): + adapter = adapter_for_family("nomic") + match = adapter.name_tensor("encoder.layers.1.mlp.router.layer.weight", torch.empty(8, 768), 12) + assert adapter.policy(match, (8, 768), 4).precision == "FP16" diff --git a/python/cactus/server.py b/python/cactus/server.py index 7d5c9ad05..cfab62ca2 100644 --- a/python/cactus/server.py +++ b/python/cactus/server.py @@ -18,6 +18,7 @@ from .bindings.cactus import ( cactus_complete, cactus_destroy, + cactus_embed, cactus_get_last_error, cactus_init, cactus_reset, @@ -30,6 +31,7 @@ LLM_MODEL_TYPES = {"gemma", "gemma3n", "gemma4", "lfm2", "qwen", "qwen3p5", "needle", "youtu"} STT_MODEL_TYPES = {"whisper", "parakeet_tdt", "parakeet-tdt"} +EMBED_MODEL_TYPES = {"bert", "nomic"} @dataclass(frozen=True) @@ -253,6 +255,11 @@ class ChatRequest(Permissive): tool_choice: str | ToolChoiceObject | None = None +class EmbeddingRequest(Permissive): + model: str + input: str | list[str] + + def _flatten_message(msg: ChatMessage) -> dict[str, Any]: out: dict[str, Any] = {"role": msg.role} if isinstance(msg.content, list): @@ -460,7 +467,19 @@ def create_app( PROJECT_ROOT / "weights" if is_repo_checkout() else Path.home() / ".cache" / "cactus" / "weights" ) registry = ModelRegistry(root, extra_model=model_path) - selected = registry.default_llm(default_model) + if default_model is not None: + selected = registry.models.get(default_model) + if selected is None: + raise RuntimeError(f"Requested model '{default_model}' is not a valid v2 Cactus bundle") + else: + try: + selected = registry.default_llm() + except RuntimeError: + # No LLM available — allow serving a non-LLM bundle (e.g. an embedding model). + available = sorted(registry.models.values(), key=lambda info: info.id) + if not available: + raise + selected = available[0] manager = ModelManager(registry, max_warm=max_warm) @asynccontextmanager @@ -581,4 +600,41 @@ async def create_transcription( } return {"text": text} + @app.post("/v1/embeddings") + async def create_embeddings(request: Request, req: EmbeddingRequest): + reg: ModelRegistry = request.app.state.registry + info = reg.require(req.model) + if info.model_type not in EMBED_MODEL_TYPES: + raise HTTPException(status_code=400, detail=f"Model '{req.model}' is not an embedding model") + inputs = [req.input] if isinstance(req.input, str) else list(req.input) + if not inputs: + raise HTTPException(status_code=400, detail="'input' must not be empty") + mgr: ModelManager = request.app.state.manager + + def _embed_all(handle) -> list[list[float]]: + vectors = [] + for text in inputs: + cactus_reset(handle) + vectors.append(cactus_embed(handle, text, True)) + return vectors + + async with mgr.acquire(req.model) as slot: + async with slot.lock: + try: + vectors = await asyncio.get_running_loop().run_in_executor( + None, lambda: _embed_all(slot.handle) + ) + except Exception as exc: + raise HTTPException(status_code=500, detail=str(exc)) + data = [ + {"object": "embedding", "index": i, "embedding": vector} + for i, vector in enumerate(vectors) + ] + return { + "object": "list", + "data": data, + "model": req.model, + "usage": {"prompt_tokens": 0, "total_tokens": 0}, + } + return app diff --git a/python/cactus/transpile/canonicalize/cleanup.py b/python/cactus/transpile/canonicalize/cleanup.py index 3ab1e5d26..b7f4de2aa 100644 --- a/python/cactus/transpile/canonicalize/cleanup.py +++ b/python/cactus/transpile/canonicalize/cleanup.py @@ -90,7 +90,9 @@ } FP32_SUPPORTED_INPUT_INDICES = { - "embedding": {1}, + # The embedding op dequantizes its weight (input 0) directly (CQ or FP16 when + # bound), and indices (input 1) stay integral — neither needs fp16 legalization. + "embedding": {0, 1}, } FP16_ONLY_OUTPUT_OPS = { diff --git a/python/cactus/transpile/component_plan.py b/python/cactus/transpile/component_plan.py index 6b58d7cc4..5c57ac9f3 100644 --- a/python/cactus/transpile/component_plan.py +++ b/python/cactus/transpile/component_plan.py @@ -146,6 +146,13 @@ def infer_component_plan_from_config( if profile_plan is not None: return profile_plan + if "nomic" in model_type or "nomic-embed" in lowered_id or any("nomicbert" in value for value in architectures): + return ComponentPlan( + task="text_embedding", + components=("text_embedding",), + force_component_pipeline=True, + ) + if _is_tdt_config(config, model_type, lowered_id): return ComponentPlan( task="tdt_transcription", diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index f9648e0fc..a21e73d2b 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -311,6 +311,7 @@ def _write_component_bundle( for component in ( "audio_encoder", "vision_encoder", + "text_embedding", "lm_encoder", "lm_encoder_text_chunk", "decoder_prefill_chunk", @@ -1170,6 +1171,8 @@ def _infer_task_from_config(model_id_or_path: str) -> str: if isinstance(loss_cfg, dict) and str(loss_cfg.get("loss_name", "") or "").lower() == "tdt": return "tdt_transcription" + if "nomic" in model_type or any("NomicBert" in value for value in architectures): + return "text_embedding" if any("CausalLM" in value for value in architectures): return "causal_lm_logits" if any("CTC" in value for value in architectures): @@ -1180,6 +1183,8 @@ def _infer_task_from_config(model_id_or_path: str) -> str: return "seq2seq_transcription" lowered_id = model_id_or_path.lower() + if "nomic-embed" in lowered_id: + return "text_embedding" if "parakeet-tdt" in lowered_id: return "tdt_transcription" if "whisper" in lowered_id: @@ -1743,6 +1748,43 @@ def _prepare_text_inputs( ) +_NOMIC_EMBED_SEQ_LEN = 256 + + +def _prepare_text_embedding_inputs( + tokenizer: object, + *, + prompt: str, + input_ids_text: str | None, + seq_len: int = _NOMIC_EMBED_SEQ_LEN, +) -> PreparedInputs: + base = _prepare_text_inputs( + tokenizer, + prompt=prompt, + input_ids_text=input_ids_text, + max_new_tokens=0, + ) + prompt_input_ids = base.tensors[0] + prompt_token_count = int(prompt_input_ids.shape[1]) + padding_token_id = int(base.metadata.get("padding_token_id", 0)) + target = max(1, int(seq_len)) + input_ids = torch.full((1, target), padding_token_id, dtype=torch.long) + attention_mask = torch.zeros((1, target), dtype=torch.long) + keep = min(prompt_token_count, target) + input_ids[:, :keep] = prompt_input_ids[:, :keep] + attention_mask[:, :keep] = 1 + return PreparedInputs( + names=("input_ids", "attention_mask"), + tensors=(input_ids, attention_mask), + metadata={ + "prompt": prompt, + "prompt_token_count": prompt_token_count, + "target_token_count": target, + "padding_token_id": padding_token_id, + }, + ) + + def _add_multimodal_generation_headroom( prepared: PreparedInputs, *, @@ -2549,6 +2591,33 @@ def _load_transformers_bundle( model = load_tdt_local_model(model_source, torch_dtype=torch_dtype).eval() return model_source, None, model, config + if task == "text_embedding": + # nomic-bert weights are produced from the model's remote modeling code + # (fused Wqkv / experts.mlp.w1,w2); the transformers-native nomic_bert has a + # different architecture, so the remote code is required to match the bundle. + remote_kwargs = {**common_kwargs, "trust_remote_code": True} + tokenizer = None + tokenizer_errors: list[str] = [] + for source in source_candidates: + try: + tokenizer = AutoTokenizer.from_pretrained(source, **remote_kwargs) + break + except Exception as exc: + tokenizer_errors.append(f"{source}: {exc}") + if tokenizer is None: + raise RuntimeError( + f"Could not load tokenizer for {model_id}:\n" + + "\n".join(tokenizer_errors) + ) + model = AutoModel.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **remote_kwargs, + ).eval() + return model_source, tokenizer, model, config + if task == "causal_lm_logits": tokenizer = None tokenizer_errors: list[str] = [] @@ -2860,6 +2929,7 @@ def main() -> int: "encoder_hidden_states", "seq2seq_transcription", "tdt_transcription", + "text_embedding", ), help="Transpile task. Use auto to infer from config/model id.", ) @@ -3071,6 +3141,19 @@ def main() -> int: weights_dir=weights_dir, inputs_metadata=prepared.metadata, ) + elif task == "text_embedding": + prepared = _prepare_text_embedding_inputs( + processor_or_tokenizer, + prompt=args.prompt, + input_ids_text=args.input_ids.strip() or None, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + weights_dir=weights_dir, + inputs_metadata=prepared.metadata, + ) elif task == "multimodal_causal_lm_logits": config_model_type = str(model_config.get("model_type", "") or "").lower() if config_model_type == "lfm2_vl": diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index 47f8f73fb..d43a3ca38 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -4446,6 +4446,120 @@ def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: return _extract_tensor_output(outputs, preferred_field="last_hidden_state") +def _nomic_rotate_half(x: torch.Tensor) -> torch.Tensor: + x1, x2 = x.chunk(2, dim=-1) + return torch.cat([-x2, x1], dim=-1) + + +class NomicTextEmbeddingAdapter(torch.nn.Module): + """Export-friendly nomic-bert encoder that returns last_hidden_state. + + Reuses the HF submodules (embeddings, layernorms, attention/MLP weights) but + reimplements the forward to avoid the HF rotary cache and the data-dependent + MoE dispatch (`torch.where`/`index_add_`) which do not survive torch.export. + Pooling and L2 normalization are applied downstream in the C++ engine. + """ + + def __init__(self, model: torch.nn.Module) -> None: + super().__init__() + for name in ("embeddings", "emb_ln", "encoder"): + if not isinstance(getattr(model, name, None), torch.nn.Module): + raise NotImplementedError(f"{type(model).__name__} is not a nomic-bert model") + self.embeddings = model.embeddings + self.emb_ln = model.emb_ln + self.encoder = model.encoder + config = model.config + self.n_heads = int(config.n_head) + self.head_dim = int(config.n_embd) // int(config.n_head) + self.hidden_dim = int(config.n_embd) + self.n_experts = int(getattr(config, "num_experts", 0) or 0) + self.top_k = int(getattr(config, "moe_top_k", 0) or 0) + self.ffn_dim = int(config.n_inner) + self.rope_base = float(getattr(config, "rotary_emb_base", 10000.0) or 10000.0) + # Pre-transpose each MoE expert w2 to [hidden, n_experts*ffn] so the second + # expert matmul consumes it as a direct linear weight — transposing a packed + # quantized weight inside the graph is not lowerable. Idempotent by shape so + # repeated adapter construction on the same model is safe. + packed_rows = self.n_experts * self.ffn_dim + for layer in self.encoder.layers: + if getattr(layer, "moe", False): + w2 = layer.mlp.experts.mlp.w2 + if w2.shape[0] == packed_rows: + layer.mlp.experts.mlp.w2 = torch.nn.Parameter( + w2.detach().t().contiguous(), requires_grad=False + ) + + def _rope_cos_sin(self, seq_len: int, device, dtype): + inv_freq = 1.0 / (self.rope_base ** (torch.arange(0, self.head_dim, 2, device=device, dtype=torch.float32) / self.head_dim)) + positions = torch.arange(seq_len, device=device, dtype=torch.float32) + freqs = positions.unsqueeze(1) * inv_freq.unsqueeze(0) + cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).to(dtype) + sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).to(dtype) + return cos, sin + + def _attention(self, layer, hidden_states, additive_mask, cos, sin): + batch, seq, _ = hidden_states.shape + qkv = layer.attn.Wqkv(hidden_states).view(batch, seq, 3, self.n_heads, self.head_dim) + query, key, value = qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2] + cos_b = cos[None, :, None, :] + sin_b = sin[None, :, None, :] + query = query * cos_b + _nomic_rotate_half(query) * sin_b + key = key * cos_b + _nomic_rotate_half(key) * sin_b + query = query.permute(0, 2, 1, 3) + key = key.permute(0, 2, 1, 3) + value = value.permute(0, 2, 1, 3) + scores = torch.matmul(query, key.transpose(-1, -2)) / (self.head_dim ** 0.5) + scores = scores + additive_mask + context = torch.matmul(F.softmax(scores, dim=-1), value) + context = context.permute(0, 2, 1, 3).reshape(batch, seq, self.hidden_dim) + return layer.attn.out_proj(context) + + def _topk_gate(self, weights): + # torch.topk does not lower; build the top-k softmax gate via iterative + # max + masking (top_k is small). gate[n, e] = softmax weight when expert e + # is among the top-k for token n, else 0. + neg = torch.finfo(weights.dtype).min + remaining = weights + gate = weights * 0.0 + for _ in range(self.top_k): + current_max = torch.amax(remaining, dim=-1, keepdim=True) + selected = (remaining >= current_max).to(weights.dtype) + gate = gate + selected * weights + remaining = remaining + selected * neg + return gate + + def _moe_mlp(self, moe, hidden_states): + # Dense MoE: run all experts via two fused matmuls against the packed + # expert weights (no per-expert indexing — slicing a quantized weight is + # not lowerable), then gate by scaling each expert's activations. The + # gated sum is identical to top-k routing because non-selected experts + # have gate 0. + batch, seq, _ = hidden_states.shape + flat = hidden_states.reshape(-1, self.hidden_dim) + weights = moe.router.layer(flat).softmax(dim=-1, dtype=torch.float32) + gate = self._topk_gate(weights).to(hidden_states.dtype) + all_hidden = F.linear(flat, moe.experts.mlp.w1) # [tokens, n_experts * ffn] + activated = F.gelu(all_hidden).reshape(-1, self.n_experts, self.ffn_dim) + activated = activated * gate.unsqueeze(-1) + activated = activated.reshape(-1, self.n_experts * self.ffn_dim) + # w2 is pre-transposed to [hidden, n_experts*ffn] in __init__. + out = F.linear(activated, moe.experts.mlp.w2) + out = out + moe.experts.bias + return out.reshape(batch, seq, self.hidden_dim) + + def forward(self, input_ids: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor: + hidden_states = self.emb_ln(self.embeddings.word_embeddings(input_ids)) + seq_len = input_ids.shape[1] + cos, sin = self._rope_cos_sin(seq_len, input_ids.device, hidden_states.dtype) + mask_float = attention_mask[:, None, None, :].to(hidden_states.dtype) + additive_mask = (mask_float - 1.0) * (-torch.finfo(hidden_states.dtype).min) + for layer in self.encoder.layers: + hidden_states = layer.norm1(self._attention(layer, hidden_states, additive_mask, cos, sin) + hidden_states) + mlp_out = self._moe_mlp(layer.mlp, hidden_states) if layer.moe else layer.mlp(hidden_states) + hidden_states = layer.norm2(mlp_out + hidden_states) + return hidden_states + + class WhisperEncoderComponentAdapter(torch.nn.Module): def __init__(self, encoder: torch.nn.Module): super().__init__() @@ -4659,6 +4773,9 @@ def _family_key(model: torch.nn.Module) -> str: return "lfm2_moe" if module_name.startswith("transformers.models.lfm2."): return "lfm2" + model_type = str(getattr(getattr(model, "config", None), "model_type", "") or "").lower() + if "nomic" in module_name.lower() or "nomic" in model_type: + return "nomic" return "generic" @@ -4806,6 +4923,47 @@ def _build_whisper_seq2seq_component_specs( return specs +def _build_nomic_text_embedding_component_specs( + model: torch.nn.Module, + *, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None = None, +) -> list[ComponentModuleSpec]: + input_ids = named_tensors.get("input_ids") + if not isinstance(input_ids, torch.Tensor): + raise RuntimeError("nomic text_embedding transpile requires input_ids") + attention_mask = named_tensors.get("attention_mask") + if not isinstance(attention_mask, torch.Tensor): + attention_mask = torch.ones_like(input_ids) + + adapter = NomicTextEmbeddingAdapter(model).eval() + + common_graph_meta = { + **_transpile_graph_meta( + model, + adapter_family="nomic", + adapter_type="component_pipeline", + input_names=("input_ids", "attention_mask"), + ), + "task": "text_embedding", + "adapter_family": "nomic", + } + if weights_dir: + common_graph_meta["weights_dir"] = weights_dir + + return [ + ComponentModuleSpec( + component="text_embedding", + module=adapter, + example_inputs=(input_ids, attention_mask), + input_keys=("input_ids", "attention_mask"), + output_keys=("last_hidden_state",), + graph_meta={**common_graph_meta, "component": "text_embedding"}, + metadata={"family": "nomic", "task": "text_embedding"}, + ) + ] + + def _build_lfm2_causal_lm_component_specs( model: torch.nn.Module, *, @@ -5163,6 +5321,12 @@ def build_component_module_specs( inputs_metadata=inputs_metadata, weights_dir=weights_dir, ) + if family == "nomic" and task == "text_embedding": + return _build_nomic_text_embedding_component_specs( + model, + named_tensors=named_tensors, + weights_dir=weights_dir, + ) return None @@ -5289,6 +5453,10 @@ def canonicalize_model_interface( input_names=resolved_input_names, family=family, ) + elif task == "text_embedding": + if not resolved_input_names: + resolved_input_names = ("input_ids", "attention_mask") + adapter_factory = NomicTextEmbeddingAdapter elif task == "audio_classification_logits": if not resolved_input_names: resolved_input_names = _infer_input_names( diff --git a/python/tests/test_model.py b/python/tests/test_model.py index d4107535d..94af3f5b6 100644 --- a/python/tests/test_model.py +++ b/python/tests/test_model.py @@ -33,12 +33,6 @@ def _find_asset(name): _TEST_IMAGE = _find_asset("test_monkey.png") _TEST_AUDIO = _find_asset("test.wav") -_EMBED_NOT_IMPLEMENTED = ( - "Embeddings are not wired up for transpiled bundles in the v2 engine " - "(Model::get_embeddings stub + image/audio embed paths gated)." -) - - class TestVLMModel(unittest.TestCase): @classmethod @@ -58,13 +52,7 @@ def test_text_completion(self): self.assertTrue(result.get("success", False)) self.assertGreater(len(result.get("response", "")), 0) - @unittest.skip(_EMBED_NOT_IMPLEMENTED) - def test_text_embedding(self): - embedding = cactus_embed(self.model, "Hello world", True) - self.assertIsInstance(embedding, list) - self.assertGreater(len(embedding), 0) - - @unittest.skip(_EMBED_NOT_IMPLEMENTED) + @unittest.skipUnless(_TEST_IMAGE is not None, "test_monkey.png not found") def test_image_embedding(self): embedding = cactus_image_embed(self.model, str(_TEST_IMAGE)) self.assertIsInstance(embedding, list) @@ -111,12 +99,79 @@ def test_transcription(self): self.assertTrue(result.get("success", False)) self.assertGreater(len(result.get("response", "")), 0) - @unittest.skip(_EMBED_NOT_IMPLEMENTED) + @unittest.skipUnless(_TEST_AUDIO is not None, "test.wav not found") def test_audio_embedding(self): embedding = cactus_audio_embed(self.model, str(_TEST_AUDIO)) self.assertIsInstance(embedding, list) self.assertGreater(len(embedding), 0) +def _cosine(a, b): + import math + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb + 1e-12) + + +class TestNomicEmbedding(unittest.TestCase): + MODEL_ID = "nomic-ai/nomic-embed-text-v2-moe" + + @classmethod + def setUpClass(cls): + cls.weights_dir = ensure_bundle(cls.MODEL_ID) + cls.model = cactus_init(str(cls.weights_dir), None, False) + + @classmethod + def tearDownClass(cls): + cactus_destroy(cls.model) + + def test_text_embedding_shape_and_determinism(self): + v1 = cactus_embed(self.model, "Paris is the capital of France.", True) + v2 = cactus_embed(self.model, "Paris is the capital of France.", True) + self.assertIsInstance(v1, list) + self.assertGreater(len(v1), 0) + self.assertEqual(len(v1), len(v2)) + self.assertLess(max(abs(a - b) for a, b in zip(v1, v2)), 1e-5) + + def test_retrieval_discrimination(self): + query = cactus_embed(self.model, "search_query: What is the capital of France?", True) + relevant = cactus_embed(self.model, "search_document: Paris is the capital of France.", True) + unrelated = cactus_embed(self.model, "search_document: The Great Barrier Reef is off Australia.", True) + self.assertGreater(_cosine(query, relevant), _cosine(query, unrelated)) + + def test_hf_parity(self): + try: + import torch + import torch.nn.functional as F + from transformers import AutoModel, AutoTokenizer + except ImportError: + self.skipTest("transformers/torch not available for HF parity check") + + texts = [ + "search_query: What is the capital of France?", + "search_document: Paris is the capital and largest city of France.", + "search_document: The Great Barrier Reef is located off Australia.", + ] + cactus_vecs = [cactus_embed(self.model, t, True) for t in texts] + + tok = AutoTokenizer.from_pretrained(self.MODEL_ID) + hf = AutoModel.from_pretrained(self.MODEL_ID, trust_remote_code=True, torch_dtype=torch.float32).eval() + + def hf_embed(text): + enc = tok(text, return_tensors="pt", truncation=True, max_length=256) + with torch.no_grad(): + hs = hf(**enc).last_hidden_state + mask = enc["attention_mask"][..., None].float() + pooled = (hs * mask).sum(1) / mask.sum(1).clamp(min=1e-9) + return F.normalize(pooled, p=2, dim=1)[0].tolist() + + for text, cactus_vec in zip(texts, cactus_vecs): + cos = _cosine(cactus_vec, hf_embed(text)) + print(f"\n HF parity cos ({text[:30]}...): {cos:.4f}") + # CQ4-quantized weights vs FP reference; FP parity is ~1.0 (see plan notes). + self.assertGreater(cos, 0.85) + + if __name__ == "__main__": unittest.main() diff --git a/python/tests/test_nomic_text_embedding.py b/python/tests/test_nomic_text_embedding.py new file mode 100644 index 000000000..716c89286 --- /dev/null +++ b/python/tests/test_nomic_text_embedding.py @@ -0,0 +1,37 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from cactus.transpile.component_plan import infer_component_plan_from_config + + +def test_nomic_config_routes_to_text_embedding(): + config = {"model_type": "nomic_bert", "architectures": ["NomicBertModel"]} + plan = infer_component_plan_from_config(config, model_id="nomic-ai/nomic-embed-text-v2-moe") + assert plan is not None + assert plan.task == "text_embedding" + assert plan.components == ("text_embedding",) + assert plan.force_component_pipeline + + +def test_nomic_model_id_routes_to_text_embedding(): + plan = infer_component_plan_from_config({}, model_id="nomic-ai/nomic-embed-text-v2-moe") + assert plan is not None + assert plan.task == "text_embedding" + + +def test_family_key_detects_nomic(): + import torch + + from cactus.transpile.model_adapters import _family_key + + class _Cfg: + model_type = "nomic_bert" + + class _StubNomic(torch.nn.Module): + def __init__(self): + super().__init__() + self.config = _Cfg() + + assert _family_key(_StubNomic()) == "nomic" diff --git a/python/tests/test_server_live.py b/python/tests/test_server_live.py index 724444625..a2655f057 100644 --- a/python/tests/test_server_live.py +++ b/python/tests/test_server_live.py @@ -432,3 +432,74 @@ def test_live_transcription_rejects_non_wav(live_server) -> None: ) assert res.status_code == 400 assert "Only .wav" in json.dumps(res.json()) + + +EMBED_TYPES = {"bert", "nomic"} + + +def _find_embed_bundle() -> Path | None: + for candidate in sorted(WEIGHTS.iterdir()) if WEIGHTS.exists() else []: + if candidate.is_dir() and _valid_bundle(candidate) and _read_model_type(candidate) in EMBED_TYPES: + return candidate + return None + + +@pytest.fixture(scope="module") +def embed_server(): + bundle = _find_embed_bundle() + if bundle is None: + pytest.skip("No embedding (nomic/bert) bundle under weights/; run `cactus convert nomic-ai/nomic-embed-text-v2-moe`") + port = _free_port() + proc = _start_server(bundle, port) + base_url = f"http://127.0.0.1:{port}" + try: + _wait_ready(proc, base_url) + yield base_url, bundle.name + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +def test_live_embeddings_string(embed_server) -> None: + base_url, model = embed_server + res = httpx.post( + f"{base_url}/v1/embeddings", + json={"model": model, "input": "Paris is the capital of France."}, + timeout=60, + ) + assert res.status_code == 200, res.text + body = res.json() + assert body["object"] == "list" + assert len(body["data"]) == 1 + assert body["data"][0]["object"] == "embedding" + assert body["data"][0]["index"] == 0 + assert len(body["data"][0]["embedding"]) > 0 + assert all(isinstance(x, float) for x in body["data"][0]["embedding"][:8]) + + +def test_live_embeddings_list(embed_server) -> None: + base_url, model = embed_server + res = httpx.post( + f"{base_url}/v1/embeddings", + json={"model": model, "input": ["hello world", "a second sentence"]}, + timeout=60, + ) + assert res.status_code == 200, res.text + data = res.json()["data"] + assert [d["index"] for d in data] == [0, 1] + assert len(data[0]["embedding"]) == len(data[1]["embedding"]) > 0 + + +def test_live_embeddings_rejects_llm_model(live_server) -> None: + base_url, model = live_server + res = httpx.post( + f"{base_url}/v1/embeddings", + json={"model": model, "input": "hello"}, + timeout=30, + ) + assert res.status_code == 400 + assert "not an embedding model" in json.dumps(res.json())