Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cactus-engine/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ set(ENGINE_SOURCES
src/sp.cpp
src/constraints.cpp
src/model.cpp
src/model_npu.cpp
src/engine_image.cpp
src/index.cpp
src/index_ffi.cpp
Expand Down
46 changes: 14 additions & 32 deletions cactus-engine/src/engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,36 +50,6 @@ class NPUEncoder {
std::unique_ptr<NPUEncoder> create_encoder();
bool is_npu_available();

struct NPUBufferRef {
const __fp16* data;
size_t count;
};

struct NPUPrefillDirectResult {
NPUBufferRef hidden;
std::vector<NPUBufferRef> k_caches;
std::vector<NPUBufferRef> v_caches;
bool valid;
};

class NPUPrefill {
public:
virtual ~NPUPrefill() = default;
virtual bool load(const std::string& model_path) = 0;
virtual bool is_available() const = 0;
virtual int get_chunk_size() const = 0;
virtual int get_hidden_dim() const = 0;
virtual int get_num_layers() const = 0;
virtual int get_num_kv_heads() const = 0;
virtual int get_head_dim() const = 0;
virtual NPUPrefillDirectResult prefill_chunk_direct(
const std::vector<__fp16>& embeddings,
int position_offset = 0,
const std::string& input_name = "x") = 0;
};

std::unique_ptr<NPUPrefill> create_prefill();

} // namespace npu
namespace engine {

Expand Down Expand Up @@ -638,13 +608,17 @@ class Model {
void set_cache_window(size_t window_size, size_t sink_size = 4);
size_t get_cache_size() const { return cache_total_seq_len_; }

bool load_npu_prefill(const std::string& model_path);
bool has_npu_prefill() const { return false; }
size_t get_prefill_chunk_size() const { return 128; }
double last_prefill_cache_copy_ms() const { return last_prefill_cache_copy_ms_; }
size_t last_prefill_padding_tokens() const { return last_prefill_padding_tokens_; }
size_t last_prefill_scalar_tail_tokens() const { return last_prefill_scalar_tail_tokens_; }

bool load_npu_audio_encoder(const std::string& model_path);
bool has_npu_audio_encoder() const { return npu_audio_encoder_ != nullptr; }

bool load_npu_vision_encoder(const std::string& model_path);
bool has_npu_vision_encoder() const { return npu_vision_encoder_ != nullptr; }

void remove_thinking_tokens(const std::vector<std::pair<size_t, size_t>>& ranges);
void compact_kv_cache() {}

Expand Down Expand Up @@ -750,6 +724,14 @@ class Model {
Component* lm_encoder_media_chunk_ = nullptr;

std::string family_;
std::string npu_audio_encoder_mlpackage_;
std::string npu_vision_encoder_mlpackage_;

std::unique_ptr<npu::NPUEncoder> npu_audio_encoder_;
std::unique_ptr<npu::NPUEncoder> npu_vision_encoder_;

bool audio_encode_via_npu(const std::vector<float>& audio_features);
bool vision_encode_via_npu(const std::vector<float>& pixel_values);

std::map<std::string, std::vector<uint8_t>> media_features_;
std::map<std::string, std::vector<size_t>> media_feature_shapes_;
Expand Down
104 changes: 96 additions & 8 deletions cactus-engine/src/model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,8 @@ bool Model::init(const std::string& bundle_dir, size_t context_size,
audio_encoder_ = components_.count("audio_encoder") ? &components_.at("audio_encoder") : nullptr;
lm_encoder_media_step_ = components_.count("lm_encoder_media_step") ? &components_.at("lm_encoder_media_step") : nullptr;
lm_encoder_ = components_.count("lm_encoder") ? &components_.at("lm_encoder") : nullptr;
lm_encoder_text_chunk_ = components_.count("lm_encoder_text_chunk") ? &components_.at("lm_encoder_text_chunk") : nullptr;
lm_encoder_media_chunk_ = components_.count("lm_encoder_media_chunk") ? &components_.at("lm_encoder_media_chunk") : nullptr;
std::vector<Component*> to_bind = {
encoder_,
prefill_encoder_,
Expand All @@ -296,6 +298,8 @@ bool Model::init(const std::string& bundle_dir, size_t context_size,
audio_encoder_,
lm_encoder_media_step_,
lm_encoder_,
lm_encoder_text_chunk_,
lm_encoder_media_chunk_,
};
if (components_.count("decoder_cross_kv")) {
to_bind.push_back(&components_.at("decoder_cross_kv"));
Expand All @@ -317,6 +321,20 @@ bool Model::init(const std::string& bundle_dir, size_t context_size,
}

cache_max_seq_len_ = context_size;

if (!npu_audio_encoder_mlpackage_.empty()) {
std::string full_path = bundle_dir + "/" + npu_audio_encoder_mlpackage_;
if (!load_npu_audio_encoder(full_path)) {
CACTUS_LOG_WARN("model", "NPU audio encoder load failed for " << full_path << "; falling back to CPU");
}
}
if (!npu_vision_encoder_mlpackage_.empty()) {
std::string full_path = bundle_dir + "/" + npu_vision_encoder_mlpackage_;
if (!load_npu_vision_encoder(full_path)) {
CACTUS_LOG_WARN("model", "NPU vision encoder load failed for " << full_path << "; falling back to CPU");
}
}

initialized_ = true;
return true;
}
Expand All @@ -334,6 +352,12 @@ bool Model::load_manifest() {
if (obj.count("family") && obj.at("family").is<std::string>()) {
family_ = obj.at("family").get<std::string>();
}
if (obj.count("npu_audio_encoder") && obj.at("npu_audio_encoder").is<std::string>()) {
npu_audio_encoder_mlpackage_ = obj.at("npu_audio_encoder").get<std::string>();
}
if (obj.count("npu_vision_encoder") && obj.at("npu_vision_encoder").is<std::string>()) {
npu_vision_encoder_mlpackage_ = obj.at("npu_vision_encoder").get<std::string>();
}
if (!obj.count("components")) return false;
for (const auto& cv : obj.at("components").get<picojson::array>()) {
const auto& c = cv.get<picojson::object>();
Expand Down Expand Up @@ -975,6 +999,12 @@ void Model::run_vision_encoder(const std::string& image_path) {
throw std::runtime_error("failed to load vision_encoder");
}
Gemma4ImagePreprocessed prep = preprocess_gemma4_image(image_path, config_);
if (has_npu_vision_encoder() && vision_encode_via_npu(prep.pixel_values)) {
return;
}
if (!load_component_graph(*vision_encoder_)) {
throw std::runtime_error("failed to load vision_encoder");
}
write_bytes_input(*vision_encoder_, "pixel_values", prep.pixel_values.data(),
prep.pixel_values.size() * sizeof(float));
write_bytes_input(*vision_encoder_, "pixel_position_ids", prep.pixel_position_ids.data(),
Expand Down Expand Up @@ -1018,6 +1048,9 @@ void Model::run_audio_encoder_messages(const std::vector<std::vector<float>>& au

void Model::run_audio_encoder(const std::vector<float>& audio_features) {
if (!audio_encoder_) return;
if (has_npu_audio_encoder() && audio_encode_via_npu(audio_features)) {
return;
}
const std::vector<std::string> candidate_input_names = {"input_features", "audio_features"};
int feature_idx = -1;
for (const auto& name : candidate_input_names) {
Expand Down Expand Up @@ -1548,6 +1581,9 @@ bool Model::run_chunk_prefill_path(const std::vector<uint32_t>& tokens,
for (const auto& path : image_paths) {
if (family_ == "lfm2_vl") {
Lfm2VlImagePreprocessed prep = preprocess_lfm2_vl_image(path, config_);
if (has_npu_vision_encoder() && vision_encode_via_npu(prep.pixel_values)) {
continue;
}
int pv_idx = input_index(*vision_encoder_, "pixel_values");
if (pv_idx >= 0) {
auto& pv_buf = vision_encoder_->input_buffers[pv_idx];
Expand Down Expand Up @@ -2141,7 +2177,56 @@ std::vector<uint32_t> Model::transcribe_parakeet_tdt(const std::vector<float>& a
write_typed_buffer(feat_buf, feat_desc.precision, transposed.data(),
transposed.size() * sizeof(float), Precision::FP32);

audio_enc->graph->execute();
std::vector<__fp16> npu_hidden_storage;
bool used_npu = false;
size_t npu_hidden_T = 0;
if (has_npu_audio_encoder()) {
const std::vector<int> in_shape = npu_audio_encoder_->get_input_shape();
const std::vector<int> out_shape = npu_audio_encoder_->get_output_shape();
if (in_shape.size() >= 3 && out_shape.size() >= 3 &&
in_shape[1] > 0 && in_shape[2] > 0 && out_shape[1] > 0 && out_shape[2] > 0 &&
static_cast<size_t>(in_shape[2]) == expected_mels) {
const size_t window_frames = static_cast<size_t>(in_shape[1]);
const size_t window_hidden = static_cast<size_t>(out_shape[1]);
const size_t hidden_dim_npu = static_cast<size_t>(out_shape[2]);
const size_t chunk_input_elems = window_frames * expected_mels;
const size_t chunk_output_elems = window_hidden * hidden_dim_npu;
const size_t num_chunks = (copy_frames + window_frames - 1) / window_frames;
const size_t total_hidden_T = num_chunks * window_hidden;
npu_hidden_storage.assign(total_hidden_T * hidden_dim_npu, __fp16(0));
std::vector<__fp16> input_fp16(chunk_input_elems);
bool all_ok = num_chunks > 0;
for (size_t c = 0; c < num_chunks && all_ok; ++c) {
const size_t frame_start = c * window_frames;
const size_t frame_end = std::min(frame_start + window_frames, copy_frames);
std::fill(input_fp16.begin(), input_fp16.end(), __fp16(0));
for (size_t t = frame_start; t < frame_end; ++t) {
const size_t local = (t - frame_start) * expected_mels;
const size_t src = t * expected_mels;
for (size_t m = 0; m < expected_mels; ++m) {
input_fp16[local + m] = static_cast<__fp16>(transposed[src + m]);
}
}
__fp16* out_ptr = npu_hidden_storage.data() + c * chunk_output_elems;
size_t written = npu_audio_encoder_->encode(
input_fp16.data(), out_ptr, in_shape, "x", "encoded");
if (written == 0) { all_ok = false; break; }
}
if (all_ok) {
used_npu = true;
const size_t valid_input = copy_frames;
npu_hidden_T = (valid_input * window_hidden + window_frames - 1) / window_frames;
if (npu_hidden_T > total_hidden_T) npu_hidden_T = total_hidden_T;
CACTUS_LOG_INFO("model", "Parakeet audio encoder ran on NPU ("
<< num_chunks << " chunks, " << npu_hidden_T << " valid hidden frames)");
} else {
CACTUS_LOG_WARN("model", "NPU audio encoder chunk failed; falling back to CPU graph");
}
}
}
if (!used_npu) {
audio_enc->graph->execute();
}

int hidden_idx = output_index(*audio_enc, "encoder_hidden_states");
if (hidden_idx < 0) {
Expand All @@ -2150,14 +2235,20 @@ std::vector<uint32_t> Model::transcribe_parakeet_tdt(const std::vector<float>& a
}
size_t hidden_node = static_cast<size_t>(audio_enc->output_node_ids[hidden_idx]);
const auto& hidden_desc = audio_enc->graph->get_output_buffer(hidden_node);
const uint8_t* hidden_ptr = static_cast<const uint8_t*>(audio_enc->graph->get_output(hidden_node));
const uint8_t* hidden_ptr;
if (used_npu) {
hidden_ptr = reinterpret_cast<const uint8_t*>(npu_hidden_storage.data());
} else {
hidden_ptr = static_cast<const uint8_t*>(audio_enc->graph->get_output(hidden_node));
}
if (hidden_desc.shape.size() < 3 || hidden_ptr == nullptr) {
CACTUS_LOG_ERROR("model", "encoder_hidden_states must be 3D [B, T, D]");
return emitted;
}
const size_t T = hidden_desc.shape[1];
const size_t T = used_npu ? npu_hidden_T : hidden_desc.shape[1];
const size_t D = hidden_desc.shape[2];
const size_t hidden_elem = PrecisionTraits::size_of(hidden_desc.precision);
const Precision hidden_precision = used_npu ? Precision::FP16 : hidden_desc.precision;
const size_t hidden_elem = PrecisionTraits::size_of(hidden_precision);
const size_t frame_bytes = D * hidden_elem;

auto zero_state = [&](const std::string& name) {
Expand Down Expand Up @@ -2223,7 +2314,7 @@ std::vector<uint32_t> Model::transcribe_parakeet_tdt(const std::vector<float>& a

while (time_index < T) {
const uint8_t* frame_ptr = hidden_ptr + time_index * frame_bytes;
write_typed_buffer(ef_buf, ef_desc.precision, frame_ptr, frame_bytes, hidden_desc.precision);
write_typed_buffer(ef_buf, ef_desc.precision, frame_ptr, frame_bytes, hidden_precision);

size_t symbols_added = 0;
bool advanced = false;
Expand Down Expand Up @@ -2701,9 +2792,6 @@ const std::vector<Model::DebugNode>& Model::get_debug_nodes() const {
return debug_nodes_;
}

bool Model::load_npu_prefill(const std::string& /*model_path*/) {
return false;
}

double Model::score_tokens_window_logprob(const std::vector<uint32_t>& /*tokens*/, size_t /*start*/,
size_t /*end*/, size_t /*context*/, size_t* tokens_scored) {
Expand Down
Loading
Loading