diff --git a/.github/actions/stamp-version/action.yml b/.github/actions/stamp-version/action.yml new file mode 100644 index 000000000..2a40301c2 --- /dev/null +++ b/.github/actions/stamp-version/action.yml @@ -0,0 +1,24 @@ +name: Stamp version +description: Resolve the release version and write python/cactus/_version.py +runs: + using: composite + steps: + - shell: bash + env: + TAG: ${{ github.event.release.tag_name }} + INPUT_VERSION: ${{ github.event.inputs.version }} + run: | + if [ -n "$TAG" ]; then + raw="${TAG#v}" + elif [ -n "$INPUT_VERSION" ]; then + raw="${INPUT_VERSION#v}" + else + raw=$(sed 's/^v//' CACTUS_VERSION) + fi + version=$(RAW="$raw" python3 -c 'import os; r=os.environ["RAW"]; print(r + ".0" if r.count(".") == 1 else r)') + if [ -z "$version" ]; then + echo "::error::No version source available (TAG, INPUT_VERSION, CACTUS_VERSION all empty)" + exit 1 + fi + printf '__version__ = "%s"\n' "$version" > python/cactus/_version.py + echo "Stamped version: $version" diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 0a58c7a66..e7c3c0c51 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -3,6 +3,11 @@ name: PyPI Publish on: workflow_call: workflow_dispatch: + inputs: + version: + description: "Version to publish (e.g. 2.0.0). Leave empty to read from CACTUS_VERSION." + required: false + type: string jobs: wheels-macos: @@ -16,6 +21,8 @@ jobs: with: python-version: "3.12" + - uses: ./.github/actions/stamp-version + - name: Build native library run: | pip install -e python/ @@ -47,6 +54,8 @@ jobs: dnf install -y libcurl-devel openssl-devel echo "/opt/python/cp312-cp312/bin" >> "$GITHUB_PATH" + - uses: ./.github/actions/stamp-version + - name: Build native library run: | pip install -e python/ @@ -75,6 +84,8 @@ jobs: with: python-version: "3.12" + - uses: ./.github/actions/stamp-version + - name: Build sdist working-directory: python run: | diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index 29d6e4346..7f2422b2a 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -347,6 +347,13 @@ CACTUS_FFI_EXPORT int cactus_graph_conv_cache_state( cactus_graph_t graph, size_t window_size, size_t hidden_dim, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_conv_cache_append( cactus_graph_t graph, cactus_node_t new_data, cactus_node_t cache_state, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv_cache_initialize( + cactus_graph_t graph, cactus_node_t rows, cactus_node_t cache_state, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_recurrent_cache_state( + cactus_graph_t graph, const size_t* shape, size_t shape_len, int precision, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_recurrent_cache_write( + cactus_graph_t graph, cactus_node_t new_value, cactus_node_t cache_input, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_rfft( cactus_graph_t graph, cactus_node_t input, cactus_node_t* out); diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 3ba72c9e5..874c2175a 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -382,8 +382,8 @@ class Tokenizer { void detect_model_type(const std::string& config_path); void load_chat_template(const std::string& template_file); std::string format_gemma4_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; - std::string format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; - std::string format_lfm2_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; + std::string format_lfm2_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; }; class BPETokenizer : public Tokenizer { diff --git a/cactus-engine/src/graph_ffi.cpp b/cactus-engine/src/graph_ffi.cpp index c06c00d71..4d9b78c47 100644 --- a/cactus-engine/src/graph_ffi.cpp +++ b/cactus-engine/src/graph_ffi.cpp @@ -993,6 +993,42 @@ int cactus_graph_conv_cache_append(cactus_graph_t graph, cactus_node_t new_data, } } +int cactus_graph_conv_cache_initialize(cactus_graph_t graph, cactus_node_t rows, cactus_node_t cache_state, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv_cache_initialize"); + try { + *out = static_cast(as_graph(graph)->graph.conv_cache_initialize(static_cast(rows), static_cast(cache_state))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_recurrent_cache_state(cactus_graph_t graph, const size_t* shape, size_t shape_len, int precision, cactus_node_t* out) { + if (!graph || !out || (!shape && shape_len > 0)) { + return fail_invalid("Invalid args to cactus_graph_recurrent_cache_state"); + } + try { + std::vector shape_vec(shape, shape + shape_len); + *out = static_cast(as_graph(graph)->graph.recurrent_cache_state(shape_vec, static_cast(precision))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_recurrent_cache_write(cactus_graph_t graph, cactus_node_t new_value, cactus_node_t cache_input, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_recurrent_cache_write"); + try { + *out = static_cast(as_graph(graph)->graph.recurrent_cache_write(static_cast(new_value), static_cast(cache_input))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + int cactus_graph_rfft(cactus_graph_t graph, cactus_node_t input, cactus_node_t* out) { if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rfft"); try { diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 41b94cb5a..aae2b5f7d 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -663,6 +663,32 @@ void Model::copy_cache_states(const Component& source, Component& target, size_t } void* src_ptr = source.graph->get_output(static_cast(src_node)); void* dst_ptr = target.graph->get_output(static_cast(dst_node)); + + const OpType src_op = source.graph->get_node_op_type(static_cast(src_node)); + const OpType dst_op = target.graph->get_node_op_type(static_cast(dst_node)); + if (src_op != dst_op) { + throw std::runtime_error( + "cache state op_type mismatch between prefill and step at layer " + + src.layer_key); + } + if (src_op == OpType::RECURRENT_CACHE_STATE) { + if (src_desc.byte_size != dst_desc.byte_size) { + throw std::runtime_error( + "recurrent cache buffer shape mismatch between prefill and step at layer " + + src.layer_key); + } + std::memcpy(dst_ptr, src_ptr, src_desc.byte_size); + continue; + } + if (src_op == OpType::CONV_CACHE_STATE) { + if (src_desc.byte_size != dst_desc.byte_size) { + throw std::runtime_error( + "conv cache buffer shape mismatch between prefill and step at layer " + + src.layer_key); + } + std::memcpy(dst_ptr, src_ptr, src_desc.byte_size); + continue; + } if (src_desc.byte_size == dst_desc.byte_size && logical_current == std::numeric_limits::max()) { std::memcpy(dst_ptr, src_ptr, src_desc.byte_size); continue; @@ -778,11 +804,29 @@ void Model::reset_component_cache_states(Component& comp) { for (int node_id : {state.key_node_id, state.value_node_id}) { if (node_id < 0) continue; const auto& desc = comp.graph->get_output_buffer(static_cast(node_id)); - if (desc.byte_size < sizeof(uint64_t) || !desc.get_data()) continue; + if (desc.byte_size == 0 || !desc.get_data()) continue; void* ptr = comp.graph->get_output(static_cast(node_id)); if (!ptr) continue; - auto* metadata = static_cast(ptr); - metadata[0] = 0; + const OpType op_type = comp.graph->get_node_op_type(static_cast(node_id)); + switch (op_type) { + case OpType::KV_CACHE_STATE: + if (desc.byte_size >= sizeof(uint64_t)) { + static_cast(ptr)[0] = 0; + } + break; + case OpType::CONV_CACHE_STATE: + if (desc.byte_size >= 2 * sizeof(uint64_t)) { + auto* meta = static_cast(ptr); + meta[0] = 0; // head + meta[1] = 0; // count + } + break; + case OpType::RECURRENT_CACHE_STATE: + std::memset(ptr, 0, desc.byte_size); + break; + default: + break; + } } } } @@ -819,10 +863,28 @@ Model::ChunkedPrefillResult Model::run_chunked_prefill(const std::vector 0 ? std::min(chunk_size, component_tokens) : component_tokens; if (effective_chunk != component_tokens) effective_chunk = component_tokens; - const size_t whole_chunks_end = (tokens.size() / effective_chunk) * effective_chunk; + size_t whole_chunks_end = (tokens.size() / effective_chunk) * effective_chunk; + const bool has_recurrent_state = [&]() { + if (!decoder_prefill_->graph) return false; + for (const auto& state : decoder_prefill_->cache_states) { + for (int node_id : {state.key_node_id, state.value_node_id}) { + if (node_id < 0) continue; + if (decoder_prefill_->graph->get_node_op_type(static_cast(node_id)) + == OpType::RECURRENT_CACHE_STATE) { + return true; + } + } + } + return false; + }(); + if (has_recurrent_state && whole_chunks_end > effective_chunk) { + whole_chunks_end = effective_chunk; + } const size_t tail_tokens = tokens.size() - whole_chunks_end; const size_t padding_cutoff = std::max(1, effective_chunk / 16); - const bool pad_tail = family_ != "lfm2_vl" && tail_tokens >= padding_cutoff; + const bool pad_tail = family_ != "lfm2_vl" + && !has_recurrent_state + && tail_tokens >= padding_cutoff; const size_t executable_tokens = whole_chunks_end + (pad_tail ? effective_chunk : 0); if (executable_tokens == 0) { result.scalar_tail_tokens = tail_tokens; diff --git a/cactus-engine/src/tokenizer.cpp b/cactus-engine/src/tokenizer.cpp index 9cd05edaa..8f5930d8e 100644 --- a/cactus-engine/src/tokenizer.cpp +++ b/cactus-engine/src/tokenizer.cpp @@ -383,16 +383,16 @@ std::vector Tokenizer::apply_chat_template(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported) const { if (model_type_ == ModelType::QWEN) { - return format_qwen_style(messages, add_generation_prompt, tools_json); + return format_qwen_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported); } if (model_type_ == ModelType::LFM2) { - return format_lfm2_style(messages, add_generation_prompt, tools_json); + return format_lfm2_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported); } return format_gemma4_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported); } std::string Tokenizer::format_qwen_style(const std::vector& messages, bool add_generation_prompt, - const std::string& tools_json) const { + const std::string& tools_json, bool enable_thinking_if_supported) const { std::string result; if (!tools_json.empty()) { @@ -437,14 +437,17 @@ std::string Tokenizer::format_qwen_style(const std::vector& message if (add_generation_prompt) { result += "<|im_start|>assistant\n"; + if (chat_template_.find("") != std::string::npos) { + result += enable_thinking_if_supported ? "\n" : "\n\n\n\n"; + } } return result; } std::string Tokenizer::format_lfm2_style(const std::vector& messages, bool add_generation_prompt, - const std::string& tools_json) const { - return "<|startoftext|>" + format_qwen_style(messages, add_generation_prompt, tools_json); + const std::string& tools_json, bool enable_thinking_if_supported) const { + return "<|startoftext|>" + format_qwen_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported); } std::string Tokenizer::format_gemma4_style(const std::vector& messages, bool add_generation_prompt, diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index 02a6eadda..7c77adf30 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -115,7 +115,10 @@ enum class OpType { CLAMP, DENSE_MLP_TQ_FUSED, NOT_EQUAL, - SCALAR_NOT_EQUAL + SCALAR_NOT_EQUAL, + RECURRENT_CACHE_STATE, + RECURRENT_CACHE_WRITE, + CONV_CACHE_INITIALIZE }; struct PrecisionTraits { @@ -578,6 +581,10 @@ class CactusGraph { size_t conv_cache_state(size_t window_size, size_t hidden_dim); size_t conv_cache_append(size_t new_data, size_t cache_state_node); + size_t conv_cache_initialize(size_t rows, size_t cache_state_node); + + size_t recurrent_cache_state(const std::vector& shape, Precision precision); + size_t recurrent_cache_write(size_t new_value, size_t cache_state); size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1); size_t conv1d_k3(size_t input, size_t weight, size_t stride); @@ -699,6 +706,7 @@ class CactusGraph { size_t add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params = {}); const BufferDesc& get_output_buffer(size_t node_id) const; + OpType get_node_op_type(size_t node_id) const; void allocate_buffers(); size_t get_node_count() const; diff --git a/cactus-graph/src/builder.cpp b/cactus-graph/src/builder.cpp index fb90adc58..3f513d0d5 100644 --- a/cactus-graph/src/builder.cpp +++ b/cactus-graph/src/builder.cpp @@ -1253,6 +1253,10 @@ const BufferDesc& CactusGraph::get_output_buffer(size_t node_id) const { return nodes_[node_index_map_.at(node_id)]->output_buffer; } +OpType CactusGraph::get_node_op_type(size_t node_id) const { + return nodes_[node_index_map_.at(node_id)]->op_type; +} + size_t CactusGraph::persistent(size_t source_node) { const auto& source_buffer = get_output_buffer(source_node); OpParams params; @@ -1464,6 +1468,60 @@ size_t CactusGraph::conv_cache_append(size_t new_data, size_t cache_state_node) return add_node(OpType::CONV_CACHE_APPEND, {new_data, cache_state_node}, {ws, hd}, params); } +size_t CactusGraph::conv_cache_initialize(size_t rows, size_t cache_state_node) { + if (get_node_op_type(cache_state_node) != OpType::CONV_CACHE_STATE) { + throw std::invalid_argument( + "conv_cache_initialize target must be a CONV_CACHE_STATE node"); + } + const auto& rows_buf = get_output_buffer(rows); + if (rows_buf.precision != Precision::FP16 && rows_buf.precision != Precision::FP32) { + throw std::invalid_argument( + "conv_cache_initialize requires FP16/FP32 input rows"); + } + const size_t hidden_dim = nodes_[node_index_map_.at(cache_state_node)]->params.head_dim; + if (hidden_dim == 0) { + throw std::invalid_argument( + "conv_cache_initialize: cache hidden_dim is zero"); + } + if (rows_buf.shape.empty() || rows_buf.shape.back() != hidden_dim) { + throw std::invalid_argument( + "conv_cache_initialize: rows last dim must equal cache hidden_dim"); + } + if (rows_buf.total_size % hidden_dim != 0) { + throw std::invalid_argument( + "conv_cache_initialize: rows total size must be a multiple of cache hidden_dim"); + } + OpParams params{}; + params.output_precision = Precision::FP16; + return add_node(OpType::CONV_CACHE_INITIALIZE, {rows, cache_state_node}, {0}, params); +} + +size_t CactusGraph::recurrent_cache_state(const std::vector& shape, Precision precision) { + OpParams params{}; + params.output_precision = precision; + size_t node_id = add_node(OpType::RECURRENT_CACHE_STATE, {}, shape, params); + persistent_node_ids_.insert(node_id); + return node_id; +} + +size_t CactusGraph::recurrent_cache_write(size_t new_value, size_t cache_state) { + const auto& src_buf = get_output_buffer(new_value); + const auto& dst_buf = get_output_buffer(cache_state); + if (get_node_op_type(cache_state) != OpType::RECURRENT_CACHE_STATE) { + throw std::invalid_argument( + "recurrent_cache_write target must be a RECURRENT_CACHE_STATE node"); + } + if (src_buf.shape != dst_buf.shape) { + throw std::invalid_argument("recurrent_cache_write requires matching shapes"); + } + if (src_buf.precision != dst_buf.precision) { + throw std::invalid_argument("recurrent_cache_write requires matching precision"); + } + OpParams params{}; + params.output_precision = dst_buf.precision; + return add_node(OpType::RECURRENT_CACHE_WRITE, {new_value, cache_state}, {0}, params); +} + size_t CactusGraph::image_preprocess( size_t pixel_input, int src_width, int src_height, diff --git a/cactus-graph/src/core.cpp b/cactus-graph/src/core.cpp index 78a0a6523..7aff9cf49 100644 --- a/cactus-graph/src/core.cpp +++ b/cactus-graph/src/core.cpp @@ -253,16 +253,20 @@ void CactusGraph::clear_debug_nodes() { void CactusGraph::allocate_buffers() { for (auto& node : nodes_) { - if (node->op_type != OpType::INPUT) { - node->output_buffer.allocate(); - } + if (node->op_type == OpType::INPUT) continue; + if (node->op_type == OpType::KV_CACHE_STATE + || node->op_type == OpType::CONV_CACHE_STATE + || node->op_type == OpType::RECURRENT_CACHE_STATE) continue; + node->output_buffer.allocate(); } } void CactusGraph::release_runtime_buffers() { for (auto& node : nodes_) { if (node->op_type == OpType::INPUT) continue; - if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) continue; + if (node->op_type == OpType::KV_CACHE_STATE + || node->op_type == OpType::CONV_CACHE_STATE + || node->op_type == OpType::RECURRENT_CACHE_STATE) continue; if (persistent_node_ids_.count(node->id)) continue; node->output_buffer.release_memory(buffer_pool_); } diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index a7dc86e1f..7a24bbbc3 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -76,6 +76,9 @@ DECLARE_COMPUTE(compute_kv_cache_append_node); DECLARE_COMPUTE(compute_attention_cached_node); DECLARE_COMPUTE(compute_conv_cache_state_node); DECLARE_COMPUTE(compute_conv_cache_append_node); +DECLARE_COMPUTE(compute_recurrent_cache_state_node); +DECLARE_COMPUTE(compute_recurrent_cache_write_node); +DECLARE_COMPUTE(compute_conv_cache_initialize_node); DECLARE_COMPUTE(compute_image_preprocess_node); DECLARE_COMPUTE(compute_rfft_node); DECLARE_COMPUTE(compute_irfft_node); @@ -84,7 +87,7 @@ DECLARE_COMPUTE(compute_spectrogram_node); extern void shrink_thread_local_buffers(); #undef DECLARE_COMPUTE -static constexpr int OP_TYPE_COUNT = static_cast(OpType::SCALAR_NOT_EQUAL) + 1; +static constexpr int OP_TYPE_COUNT = static_cast(OpType::CONV_CACHE_INITIALIZE) + 1; static_assert(OP_TYPE_COUNT <= 256, "OpType dispatch table overflow"); static ComputeFn dispatch_flat[OP_TYPE_COUNT] = {}; @@ -177,6 +180,9 @@ static bool init_dispatch() { dispatch_flat[static_cast(OpType::ATTENTION_CACHED)] = compute_attention_cached_node; dispatch_flat[static_cast(OpType::CONV_CACHE_STATE)] = compute_conv_cache_state_node; dispatch_flat[static_cast(OpType::CONV_CACHE_APPEND)] = compute_conv_cache_append_node; + dispatch_flat[static_cast(OpType::RECURRENT_CACHE_STATE)] = compute_recurrent_cache_state_node; + dispatch_flat[static_cast(OpType::RECURRENT_CACHE_WRITE)] = compute_recurrent_cache_write_node; + dispatch_flat[static_cast(OpType::CONV_CACHE_INITIALIZE)] = compute_conv_cache_initialize_node; dispatch_flat[static_cast(OpType::IMAGE_PREPROCESS)] = compute_image_preprocess_node; dispatch_flat[static_cast(OpType::RFFT)] = compute_rfft_node; dispatch_flat[static_cast(OpType::IRFFT)] = compute_irfft_node; @@ -224,7 +230,10 @@ static const char* op_type_names[] = { "CONV_CACHE_STATE", "CONV_CACHE_APPEND", "RFFT", "IRFFT", "MEL_FILTER_BANK", "SPECTROGRAM", "IMAGE_PREPROCESS", "CLAMP", "DENSE_MLP_TQ_FUSED", - "NOT_EQUAL", "SCALAR_NOT_EQUAL" + "NOT_EQUAL", "SCALAR_NOT_EQUAL", + "RECURRENT_CACHE_STATE", + "RECURRENT_CACHE_WRITE", + "CONV_CACHE_INITIALIZE" }; static const char* get_op_name(OpType op) { @@ -362,7 +371,9 @@ void CactusGraph::execute(const std::string& profile_file) { auto can_release_node = [&](size_t node_idx) { const auto& node = nodes_[node_idx]; if (node->op_type == OpType::INPUT) return false; - if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) return false; + if (node->op_type == OpType::KV_CACHE_STATE + || node->op_type == OpType::CONV_CACHE_STATE + || node->op_type == OpType::RECURRENT_CACHE_STATE) return false; if (persistent_node_ids_.count(node->id)) return false; if (retained_output_node_ids_.count(node->id)) return false; return true; @@ -414,7 +425,9 @@ void CactusGraph::execute(const std::string& profile_file) { for (size_t i = 0; i < n; ++i) { auto& node = nodes_[i]; if (node->op_type == OpType::INPUT) continue; - if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) { + if (node->op_type == OpType::KV_CACHE_STATE + || node->op_type == OpType::CONV_CACHE_STATE + || node->op_type == OpType::RECURRENT_CACHE_STATE) { dispatch_node(*node, nodes_, node_index_map_); populated_node_ids_.insert(node->id); for (size_t release_idx : release_after[i]) { @@ -495,6 +508,21 @@ void CactusGraph::execute(const std::string& profile_file) { continue; } + if (node->op_type == OpType::KV_CACHE_STATE + || node->op_type == OpType::CONV_CACHE_STATE + || node->op_type == OpType::RECURRENT_CACHE_STATE) { + dispatch_node(*node, nodes_, node_index_map_); + if (trace_execution) { + std::cerr << "[cactus:execute] cache-state idx=" << node_idx + << " id=" << node->id + << " op=" << get_op_name(node->op_type) + << std::endl; + } + trace_nonfinite(node_idx, *node); + populated_node_ids_.insert(node->id); + continue; + } + node->output_buffer.allocate_from_pool(pool); if (trace_execution) { @@ -509,11 +537,7 @@ void CactusGraph::execute(const std::string& profile_file) { std::cerr << "]" << std::endl; } - if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) { - dispatch_node(*node, nodes_, node_index_map_); - trace_nonfinite(node_idx, *node); - populated_node_ids_.insert(node->id); - } else if (enable_profiling) { + if (enable_profiling) { auto start = std::chrono::high_resolution_clock::now(); dispatch_node(*node, nodes_, node_index_map_); trace_nonfinite(node_idx, *node); diff --git a/cactus-graph/src/io.cpp b/cactus-graph/src/io.cpp index 7165b59f0..b7fff474d 100644 --- a/cactus-graph/src/io.cpp +++ b/cactus-graph/src/io.cpp @@ -230,7 +230,7 @@ namespace { GraphFile::NodeEntry node; node.index = read_u32(in); uint32_t op_type_val = read_u32(in); - if (op_type_val > static_cast(OpType::SCALAR_NOT_EQUAL)) { + if (op_type_val > static_cast(OpType::CONV_CACHE_INITIALIZE)) { throw std::runtime_error("Graph file corrupted: invalid op type"); } node.op_type = static_cast(op_type_val); @@ -352,7 +352,10 @@ CactusGraph CactusGraph::from_serialized(const GraphFile::SerializedGraph& sg) { populate_derived_params(graph, node_entry, runtime_inputs, params); new_node_id = graph.add_node(node_entry.op_type, runtime_inputs, node_entry.output_shape, params); - if (node_entry.op_type == OpType::PERSISTENT) { + if (node_entry.op_type == OpType::PERSISTENT + || node_entry.op_type == OpType::KV_CACHE_STATE + || node_entry.op_type == OpType::CONV_CACHE_STATE + || node_entry.op_type == OpType::RECURRENT_CACHE_STATE) { graph.persistent_node_ids_.insert(new_node_id); } } diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index ad722f7ca..7df2a588a 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -406,3 +406,64 @@ void compute_conv_cache_append_node( } } } + +void compute_conv_cache_initialize_node( + GraphNode& node, + const nodes_vector& nodes, + const node_index_map_t& node_index_map) { + + const auto& new_data = get_input(node, 0, nodes, node_index_map); + auto& cache_buf = nodes[node_index_map.at(node.input_ids[1])]->output_buffer; + auto* meta = get_conv_meta(cache_buf); + + const size_t ws = meta->window_size; + const size_t hd = meta->hidden_dim; + + __fp16* cache_data = get_conv_data(cache_buf); + std::memset(cache_data, 0, ws * hd * sizeof(__fp16)); + meta->head = 0; + meta->count = 0; + + const size_t num_rows = new_data.total_size / hd; + if (num_rows == 0) return; + + const __fp16* src; + std::vector<__fp16> converted; + if (new_data.precision == Precision::FP16) { + src = new_data.data_as<__fp16>(); + } else if (new_data.precision == Precision::FP32) { + converted.resize(new_data.total_size); + Quantization::fp32_to_fp16(new_data.data_as(), converted.data(), new_data.total_size); + src = converted.data(); + } else { + converted.resize(new_data.total_size); + Quantization::int8_to_fp16(new_data.data_as(), converted.data(), new_data.total_size); + src = converted.data(); + } + + const size_t copy_rows = std::min(num_rows, ws); + const size_t start_row = num_rows - copy_rows; + std::memcpy(cache_data, src + start_row * hd, copy_rows * hd * sizeof(__fp16)); + meta->head = copy_rows % ws; + meta->count = copy_rows; +} + +void compute_recurrent_cache_state_node( + GraphNode& node, + const nodes_vector&, + const node_index_map_t&) { + + if (node.output_buffer.get_data()) return; + node.output_buffer.allocate(); + std::memset(node.output_buffer.get_data(), 0, node.output_buffer.byte_size); +} + +void compute_recurrent_cache_write_node( + GraphNode& node, + const nodes_vector& nodes, + const node_index_map_t& node_index_map) { + + const auto& src = get_input(node, 0, nodes, node_index_map); + auto& cache_buf = nodes[node_index_map.at(node.input_ids[1])]->output_buffer; + std::memcpy(cache_buf.get_data(), src.get_data(), src.byte_size); +} diff --git a/cactus-graph/tests/test_cache.cpp b/cactus-graph/tests/test_cache.cpp index 21cbdca7d..07429b8c2 100644 --- a/cactus-graph/tests/test_cache.cpp +++ b/cactus-graph/tests/test_cache.cpp @@ -430,6 +430,334 @@ bool test_conv_cache_persistent() { return true; } +bool test_conv_cache_initialize_populates_trailing_rows() { + CactusGraph g; + + constexpr size_t ws = 4; + constexpr size_t hd = 8; + constexpr size_t num_rows = 6; + + size_t cache = g.conv_cache_state(ws, hd); + + size_t rows_input = g.input({num_rows, hd}, Precision::FP16); + std::vector<__fp16> data(num_rows * hd); + for (size_t r = 0; r < num_rows; ++r) { + for (size_t c = 0; c < hd; ++c) { + data[r * hd + c] = static_cast<__fp16>(static_cast(r + 1)); + } + } + g.set_input(rows_input, data.data(), Precision::FP16); + + g.conv_cache_initialize(rows_input, cache); + g.execute(); + + g.soft_reset_keep_pool(); + + size_t append_input = g.input({1, hd}, Precision::FP16); + std::vector<__fp16> append_data(hd, static_cast<__fp16>(99.0f)); + g.set_input(append_input, append_data.data(), Precision::FP16); + + size_t window = g.conv_cache_append(append_input, cache); + g.execute(); + + const __fp16* w = static_cast(g.get_output(window)); + if (!w) return false; + const float expected[ws] = {4.0f, 5.0f, 6.0f, 99.0f}; + for (size_t r = 0; r < ws; ++r) { + for (size_t c = 0; c < hd; ++c) { + if (static_cast(w[r * hd + c]) != expected[r]) return false; + } + } + return true; +} + +bool test_conv_cache_initialize_rejects_non_state_input() { + CactusGraph g; + size_t plain_input = g.input({4, 8}, Precision::FP16); + std::vector<__fp16> zeros(32, static_cast<__fp16>(0.0f)); + g.set_input(plain_input, zeros.data(), Precision::FP16); + size_t rows_input = g.input({2, 8}, Precision::FP16); + g.set_input(rows_input, zeros.data(), Precision::FP16); + try { + g.conv_cache_initialize(rows_input, plain_input); + } catch (const std::invalid_argument&) { + return true; + } + return false; +} + +bool test_conv_cache_initialize_output_is_zero_byte() { + CactusGraph g; + size_t cache = g.conv_cache_state(/*window=*/4, /*hidden_dim=*/8); + size_t rows = g.input({2, 8}, Precision::FP16); + std::vector<__fp16> data(16, static_cast<__fp16>(1.0f)); + g.set_input(rows, data.data(), Precision::FP16); + size_t out_node = g.conv_cache_initialize(rows, cache); + const auto& desc = g.get_output_buffer(out_node); + return desc.byte_size == 0; +} + +bool test_conv_cache_initialize_resets_dirty_cache() { + CactusGraph g; + constexpr size_t ws = 4; + constexpr size_t hd = 8; + size_t cache = g.conv_cache_state(ws, hd); + + size_t dirty_input = g.input({1, hd}, Precision::FP16); + std::vector<__fp16> dirty(hd, static_cast<__fp16>(77.0f)); + g.set_input(dirty_input, dirty.data(), Precision::FP16); + g.conv_cache_append(dirty_input, cache); + g.execute(); + + g.soft_reset_keep_pool(); + size_t rows_input = g.input({ws, hd}, Precision::FP16); + std::vector<__fp16> data(ws * hd); + for (size_t r = 0; r < ws; ++r) { + for (size_t c = 0; c < hd; ++c) { + data[r * hd + c] = static_cast<__fp16>(static_cast(r + 1)); + } + } + g.set_input(rows_input, data.data(), Precision::FP16); + g.conv_cache_initialize(rows_input, cache); + g.execute(); + + g.soft_reset_keep_pool(); + size_t append_input = g.input({1, hd}, Precision::FP16); + std::vector<__fp16> append_data(hd, static_cast<__fp16>(99.0f)); + g.set_input(append_input, append_data.data(), Precision::FP16); + size_t window = g.conv_cache_append(append_input, cache); + g.execute(); + + const __fp16* w = static_cast(g.get_output(window)); + if (!w) return false; + const float expected[ws] = {2.0f, 3.0f, 4.0f, 99.0f}; + for (size_t r = 0; r < ws; ++r) { + for (size_t c = 0; c < hd; ++c) { + if (static_cast(w[r * hd + c]) != expected[r]) return false; + } + } + return true; +} + +bool test_recurrent_cache_write_carries_state_across_executions() { + CactusGraph g; + + const std::vector shape{2, 4}; + const size_t elements = 2 * 4; + + size_t cache_state = g.recurrent_cache_state(shape, Precision::FP16); + + size_t new_value = g.input(shape, Precision::FP16); + std::vector<__fp16> first(elements); + for (size_t i = 0; i < elements; ++i) first[i] = static_cast<__fp16>(static_cast(i + 1)); + g.set_input(new_value, first.data(), Precision::FP16); + + g.recurrent_cache_write(new_value, cache_state); + g.execute(); + + const __fp16* cache_after = static_cast(g.get_output(cache_state)); + if (!cache_after) return false; + for (size_t i = 0; i < elements; ++i) { + if (static_cast(cache_after[i]) != static_cast(first[i])) return false; + } + + g.soft_reset_keep_pool(); + size_t second_input = g.input(shape, Precision::FP16); + std::vector<__fp16> second(elements); + for (size_t i = 0; i < elements; ++i) second[i] = static_cast<__fp16>(static_cast(i + 1) * 10.0f); + g.set_input(second_input, second.data(), Precision::FP16); + g.recurrent_cache_write(second_input, cache_state); + g.execute(); + + const __fp16* cache_after2 = static_cast(g.get_output(cache_state)); + if (!cache_after2) return false; + for (size_t i = 0; i < elements; ++i) { + if (static_cast(cache_after2[i]) != static_cast(second[i])) return false; + } + + return true; +} + +bool test_recurrent_cache_state_initializes_to_zero() { + CactusGraph g; + const std::vector shape{3, 5}; + const size_t elements = 3 * 5; + size_t cache_state = g.recurrent_cache_state(shape, Precision::FP16); + g.execute(); + const __fp16* data = static_cast(g.get_output(cache_state)); + if (!data) return false; + for (size_t i = 0; i < elements; ++i) { + if (static_cast(data[i]) != 0.0f) return false; + } + return true; +} + +bool test_recurrent_cache_write_rejects_non_state_input() { + CactusGraph g; + size_t plain_input = g.input({2, 4}, Precision::FP16); + std::vector<__fp16> zeros(8, static_cast<__fp16>(0.0f)); + g.set_input(plain_input, zeros.data(), Precision::FP16); + size_t new_value = g.input({2, 4}, Precision::FP16); + g.set_input(new_value, zeros.data(), Precision::FP16); + try { + g.recurrent_cache_write(new_value, plain_input); + } catch (const std::invalid_argument&) { + return true; + } + return false; +} + +bool test_recurrent_cache_write_rejects_shape_mismatch() { + CactusGraph g; + + size_t cache_state = g.recurrent_cache_state({2, 4}, Precision::FP16); + + size_t new_value = g.input({2, 8}, Precision::FP16); + std::vector<__fp16> data(16, static_cast<__fp16>(0.0f)); + g.set_input(new_value, data.data(), Precision::FP16); + + try { + g.recurrent_cache_write(new_value, cache_state); + } catch (const std::invalid_argument&) { + return true; + } + return false; +} + +bool test_allocate_buffers_preserves_recurrent_state_init() { + CactusGraph g; + const std::vector shape{3, 5}; + const size_t elements = 3 * 5; + size_t cache_state = g.recurrent_cache_state(shape, Precision::FP16); + g.allocate_buffers(); + g.execute(); + const __fp16* data = static_cast(g.get_output(cache_state)); + if (!data) return false; + for (size_t i = 0; i < elements; ++i) { + if (static_cast(data[i]) != 0.0f) return false; + } + return true; +} + +bool test_conv_cache_initialize_rejects_hidden_dim_mismatch() { + CactusGraph g; + size_t cache = g.conv_cache_state(/*window=*/4, /*hidden_dim=*/8); + size_t rows = g.input({2, 7}, Precision::FP16); + std::vector<__fp16> data(14, static_cast<__fp16>(0.0f)); + g.set_input(rows, data.data(), Precision::FP16); + try { + g.conv_cache_initialize(rows, cache); + } catch (const std::invalid_argument&) { + return true; + } + return false; +} + +bool test_conv_cache_initialize_exact_window() { + CactusGraph g; + constexpr size_t ws = 4, hd = 8; + size_t cache = g.conv_cache_state(ws, hd); + + size_t rows = g.input({ws, hd}, Precision::FP16); + std::vector<__fp16> data(ws * hd); + for (size_t r = 0; r < ws; ++r) { + for (size_t c = 0; c < hd; ++c) { + data[r * hd + c] = static_cast<__fp16>(static_cast(r + 1)); + } + } + g.set_input(rows, data.data(), Precision::FP16); + g.conv_cache_initialize(rows, cache); + g.execute(); + + g.soft_reset_keep_pool(); + size_t append_input = g.input({1, hd}, Precision::FP16); + std::vector<__fp16> append_data(hd, static_cast<__fp16>(99.0f)); + g.set_input(append_input, append_data.data(), Precision::FP16); + size_t window = g.conv_cache_append(append_input, cache); + g.execute(); + + const __fp16* w = static_cast(g.get_output(window)); + if (!w) return false; + const float expected[ws] = {2.0f, 3.0f, 4.0f, 99.0f}; + for (size_t r = 0; r < ws; ++r) { + for (size_t c = 0; c < hd; ++c) { + if (static_cast(w[r * hd + c]) != expected[r]) return false; + } + } + return true; +} + +bool test_conv_cache_initialize_single_row() { + CactusGraph g; + constexpr size_t ws = 4, hd = 8; + size_t cache = g.conv_cache_state(ws, hd); + + size_t rows = g.input({1, hd}, Precision::FP16); + std::vector<__fp16> data(hd, static_cast<__fp16>(42.0f)); + g.set_input(rows, data.data(), Precision::FP16); + g.conv_cache_initialize(rows, cache); + g.execute(); + + g.soft_reset_keep_pool(); + size_t append_input = g.input({1, hd}, Precision::FP16); + std::vector<__fp16> append_data(hd, static_cast<__fp16>(7.0f)); + g.set_input(append_input, append_data.data(), Precision::FP16); + size_t window = g.conv_cache_append(append_input, cache); + g.execute(); + + const __fp16* w = static_cast(g.get_output(window)); + if (!w) return false; + const float expected[ws] = {0.0f, 0.0f, 42.0f, 7.0f}; + for (size_t r = 0; r < ws; ++r) { + for (size_t c = 0; c < hd; ++c) { + if (static_cast(w[r * hd + c]) != expected[r]) return false; + } + } + return true; +} + +bool test_allocate_buffers_idempotent() { + CactusGraph g; + const std::vector shape{2, 4}; + size_t cache_state = g.recurrent_cache_state(shape, Precision::FP16); + g.allocate_buffers(); + g.allocate_buffers(); + g.execute(); + const __fp16* data = static_cast(g.get_output(cache_state)); + if (!data) return false; + for (size_t i = 0; i < 8; ++i) { + if (static_cast(data[i]) != 0.0f) return false; + } + return true; +} + +bool test_recurrent_cache_write_idempotent() { + CactusGraph g; + const std::vector shape{2, 4}; + size_t cache_state = g.recurrent_cache_state(shape, Precision::FP16); + + size_t new_value = g.input(shape, Precision::FP16); + std::vector<__fp16> data(8); + for (size_t i = 0; i < 8; ++i) data[i] = static_cast<__fp16>(static_cast(i + 1)); + g.set_input(new_value, data.data(), Precision::FP16); + + g.recurrent_cache_write(new_value, cache_state); + g.execute(); + + g.soft_reset_keep_pool(); + size_t new_value2 = g.input(shape, Precision::FP16); + g.set_input(new_value2, data.data(), Precision::FP16); + g.recurrent_cache_write(new_value2, cache_state); + g.execute(); + + const __fp16* read = static_cast(g.get_output(cache_state)); + if (!read) return false; + for (size_t i = 0; i < 8; ++i) { + if (static_cast(read[i]) != static_cast(data[i])) return false; + } + return true; +} + bool run_benchmarks() { auto bench = [](const char* label, auto setup, auto run) { setup(); @@ -527,6 +855,20 @@ int main() { runner.run_test("Conv Cache Append Basic", test_conv_cache_append_basic()); runner.run_test("Conv Cache Circular", test_conv_cache_append_circular()); runner.run_test("Conv Cache Persistent", test_conv_cache_persistent()); + runner.run_test("Conv Cache Initialize Populates Trailing", test_conv_cache_initialize_populates_trailing_rows()); + runner.run_test("Conv Cache Initialize Rejects Non-State Input", test_conv_cache_initialize_rejects_non_state_input()); + runner.run_test("Conv Cache Initialize Resets Dirty Cache", test_conv_cache_initialize_resets_dirty_cache()); + runner.run_test("Conv Cache Initialize Output Zero Byte", test_conv_cache_initialize_output_is_zero_byte()); + runner.run_test("Recurrent Cache State Initializes to Zero", test_recurrent_cache_state_initializes_to_zero()); + runner.run_test("Recurrent Cache Write Carries State", test_recurrent_cache_write_carries_state_across_executions()); + runner.run_test("Recurrent Cache Write Rejects Non-State Input", test_recurrent_cache_write_rejects_non_state_input()); + runner.run_test("Recurrent Cache Write Rejects Mismatch", test_recurrent_cache_write_rejects_shape_mismatch()); + runner.run_test("Allocate Buffers Preserves Recurrent Init", test_allocate_buffers_preserves_recurrent_state_init()); + runner.run_test("Conv Cache Initialize Rejects Hidden Dim Mismatch", test_conv_cache_initialize_rejects_hidden_dim_mismatch()); + runner.run_test("Conv Cache Initialize Exact Window", test_conv_cache_initialize_exact_window()); + runner.run_test("Conv Cache Initialize Single Row", test_conv_cache_initialize_single_row()); + runner.run_test("Allocate Buffers Idempotent", test_allocate_buffers_idempotent()); + runner.run_test("Recurrent Cache Write Idempotent", test_recurrent_cache_write_idempotent()); runner.print_benchmarks_header(); runner.run_bench("benchmarks", run_benchmarks()); runner.print_summary(); diff --git a/cactus-graph/tests/test_io.cpp b/cactus-graph/tests/test_io.cpp index 298e6c440..aa84c4224 100644 --- a/cactus-graph/tests/test_io.cpp +++ b/cactus-graph/tests/test_io.cpp @@ -372,6 +372,58 @@ bool test_graph_save_for_inspection() { } } +bool test_save_load_preserves_recurrent_cache_persistence() { + try { + const std::string filename = "test_recurrent_save_load.cg"; + + CactusGraph graph; + const std::vector shape{2, 4}; + size_t cache = graph.recurrent_cache_state(shape, Precision::FP16); + (void)cache; + graph.save(filename); + + CactusGraph loaded = CactusGraph::load(filename); + loaded.execute(); + + size_t loaded_cache_id = 0; + bool found = false; + for (size_t i = 0; i < loaded.get_node_count(); ++i) { + const auto& node = loaded.nodes_[i]; + if (node->op_type == OpType::RECURRENT_CACHE_STATE) { + loaded_cache_id = node->id; + found = true; + break; + } + } + if (!found) { + std::remove(filename.c_str()); + return false; + } + + const __fp16* data = static_cast(loaded.get_output(loaded_cache_id)); + if (!data) { + std::remove(filename.c_str()); + return false; + } + for (size_t i = 0; i < 8; ++i) { + if (static_cast(data[i]) != 0.0f) { + std::remove(filename.c_str()); + return false; + } + } + + loaded.release_runtime_buffers(); + const __fp16* still_there = static_cast(loaded.get_output(loaded_cache_id)); + bool persisted = still_there != nullptr; + + std::remove(filename.c_str()); + return persisted; + } catch (const std::exception& e) { + std::cout << "[recurrent_save_load] exception: " << e.what() << std::endl; + return false; + } +} + bool run_benchmarks() { const int ITERS = 100; const std::string temp_file = "bench_io_50nodes.cg"; @@ -432,6 +484,8 @@ int main() { runner.run_test("Graph Save/Load Roundtrip Execution", test_graph_save_load_roundtrip_execution()); runner.run_test("Graph Save/Load Supported Ops Roundtrip", test_graph_save_load_supported_ops_roundtrip()); runner.run_test("Graph Save For Inspection", test_graph_save_for_inspection()); + runner.run_test("Save/Load Preserves Recurrent Cache Persistence", + test_save_load_preserves_recurrent_cache_persistence()); runner.print_benchmarks_header(); runner.run_bench("benchmarks", run_benchmarks()); runner.print_summary(); diff --git a/cactus-kernels/src/fused.cpp b/cactus-kernels/src/fused.cpp index 69f60d162..fc187a2a0 100644 --- a/cactus-kernels/src/fused.cpp +++ b/cactus-kernels/src/fused.cpp @@ -279,6 +279,7 @@ void gated_deltanet_step( } const float prev_scale = state_scale; + state_scale = prev_scale * safe_exp_gate(gate_log_safe); std::fill(proj.begin(), proj.end(), 0.0f); for (size_t kd = 0; kd < k_dim; ++kd) { @@ -299,14 +300,14 @@ void gated_deltanet_step( } { size_t vd = 0; - const float32x4_t scale4 = vdupq_n_f32(prev_scale); + const float32x4_t scale4 = vdupq_n_f32(state_scale); for (; vd + 4 <= v_dim; vd += 4) { float32x4_t p = vld1q_f32(proj.data() + vd); p = vmulq_f32(p, scale4); vst1q_f32(proj.data() + vd, p); } for (; vd < v_dim; ++vd) { - proj[vd] *= prev_scale; + proj[vd] *= state_scale; } } @@ -315,7 +316,6 @@ void gated_deltanet_step( delta[vd] = (v_val - proj[vd]) * beta_safe; } - state_scale = prev_scale * safe_exp_gate(gate_log_safe); if (!std::isfinite(state_scale) || std::fabs(state_scale) < 1e-8f || std::fabs(state_scale) > 1e8f) { @@ -612,7 +612,7 @@ void gated_deltanet_prefill_chunked_f16( } for (size_t t = 0; t < C; ++t) { - const float pp = p_prev[t]; + const float pp = p_curr[t]; float* coeff_row = coeff + t * C; for (size_t j = 0; j < t; ++j) { const float inv_pc = inv_p_curr[j]; @@ -621,7 +621,7 @@ void gated_deltanet_prefill_chunked_f16( } for (size_t t = 0; t < C; ++t) { - const float pp = p_prev[t]; + const float pp = p_curr[t]; const float bt = beta_chunk[t]; const float* n0_row = n0_proj + t * V; const float* v_row = v_chunk + t * V; diff --git a/python/cactus/__init__.py b/python/cactus/__init__.py index 887bd5a7d..f5d0c383d 100644 --- a/python/cactus/__init__.py +++ b/python/cactus/__init__.py @@ -1,22 +1,6 @@ """Cactus — on-device AI inference.""" -from importlib.metadata import version as _version, PackageNotFoundError -from pathlib import Path as _Path - - -def _read_version(): - try: - return _version("cactus-compute") - except PackageNotFoundError: - pass - vfile = _Path(__file__).resolve().parent.parent.parent / "CACTUS_VERSION" - if vfile.exists(): - raw = vfile.read_text().strip().lstrip("v") - return raw + ".0" if raw.count(".") == 1 else raw - return "0.0.0" - - -__version__ = _read_version() +from ._version import __version__ from .cli.download import ensure_model, get_weights_dir, get_model_dir_name from .cli import main diff --git a/python/cactus/_version.py b/python/cactus/_version.py new file mode 100644 index 000000000..f627338ef --- /dev/null +++ b/python/cactus/_version.py @@ -0,0 +1,13 @@ +"""Stamped by CI before building; dev fallback reads CACTUS_VERSION.""" +from pathlib import Path as _Path + + +def _dev_version(): + vfile = _Path(__file__).resolve().parent.parent.parent / "CACTUS_VERSION" + if vfile.exists(): + raw = vfile.read_text().strip().lstrip("v") + return raw + ".0" if raw.count(".") == 1 else raw + return "0.0.0+dev" + + +__version__ = _dev_version() diff --git a/python/cactus/bindings/cactus.py b/python/cactus/bindings/cactus.py index 15197d470..50b555b2a 100644 --- a/python/cactus/bindings/cactus.py +++ b/python/cactus/bindings/cactus.py @@ -430,6 +430,28 @@ class cactus_tensor_info_t(ctypes.Structure): ctypes.POINTER(cactus_node_t), ] _lib.cactus_graph_conv_cache_append.restype = ctypes.c_int +_lib.cactus_graph_conv_cache_initialize.argtypes = [ + cactus_graph_t, + cactus_node_t, + cactus_node_t, + ctypes.POINTER(cactus_node_t), +] +_lib.cactus_graph_conv_cache_initialize.restype = ctypes.c_int +_lib.cactus_graph_recurrent_cache_state.argtypes = [ + cactus_graph_t, + ctypes.POINTER(ctypes.c_size_t), + ctypes.c_size_t, + ctypes.c_int, + ctypes.POINTER(cactus_node_t), +] +_lib.cactus_graph_recurrent_cache_state.restype = ctypes.c_int +_lib.cactus_graph_recurrent_cache_write.argtypes = [ + cactus_graph_t, + cactus_node_t, + cactus_node_t, + ctypes.POINTER(cactus_node_t), +] +_lib.cactus_graph_recurrent_cache_write.restype = ctypes.c_int _lib.cactus_graph_rel_pos_bias.argtypes = [ cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) ] diff --git a/python/cactus/bindings/graph.py b/python/cactus/bindings/graph.py index 915d54cd8..a5b2f70b2 100644 --- a/python/cactus/bindings/graph.py +++ b/python/cactus/bindings/graph.py @@ -780,6 +780,49 @@ def conv_cache_append(self, new_data, cache_state): raise RuntimeError(_err("graph_conv_cache_append failed")) return self._tensor_from_node(out.value) + def conv_cache_initialize(self, rows, cache_state): + rows = self._ensure_tensor(rows) + cache_state = self._ensure_tensor(cache_state) + out = cactus_node_t() + rc = _lib.cactus_graph_conv_cache_initialize( + self.h, + cactus_node_t(rows.id), + cactus_node_t(cache_state.id), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_conv_cache_initialize failed")) + return self._tensor_from_node(out.value) + + def recurrent_cache_state(self, shape, dtype=FP16): + shape = tuple(int(x) for x in shape) + arr = (ctypes.c_size_t * len(shape))(*shape) + out = cactus_node_t() + rc = _lib.cactus_graph_recurrent_cache_state( + self.h, + arr, + ctypes.c_size_t(len(shape)), + ctypes.c_int(int(dtype)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_recurrent_cache_state failed")) + return self._tensor_from_node(out.value) + + def recurrent_cache_write(self, new_value, cache_input): + new_value = self._ensure_tensor(new_value) + cache_input = self._ensure_tensor(cache_input) + out = cactus_node_t() + rc = _lib.cactus_graph_recurrent_cache_write( + self.h, + cactus_node_t(new_value.id), + cactus_node_t(cache_input.id), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_recurrent_cache_write failed")) + return self._tensor_from_node(out.value) + def rel_pos_bias(self, query, relative_key, scale): query = self._ensure_tensor(query) relative_key = self._ensure_tensor(relative_key) diff --git a/python/cactus/cli/test.py b/python/cactus/cli/test.py index cef3b2c22..adba63b07 100644 --- a/python/cactus/cli/test.py +++ b/python/cactus/cli/test.py @@ -28,18 +28,20 @@ def discover_suites(): def cmd_test(args): - from .model import ensure_weights + from .model import ensure_bundle, resolve_bundle_dir print_color(BLUE, "Running test suite...") if args.ios and not args.reconvert: print_color(YELLOW, "Warning: iOS tests without --reconvert may use stale weights.") - try: - weights_dir = ensure_weights(args.model_id, token=args.token, reconvert=args.reconvert) - except RuntimeError as e: - print_color(RED, f"Failed to download model weights: {e}") - return 1 + bundle_dir = resolve_bundle_dir(args.model_id) + if bundle_dir is None: + try: + bundle_dir = ensure_bundle(args.model_id, token=args.token, reconvert=args.reconvert) + except RuntimeError as e: + print_color(RED, f"Model setup failed: {e}") + return 1 suite = args.suite test_cwd = LAYERS.get(suite, ENGINE_DIR) @@ -49,7 +51,7 @@ def cmd_test(args): print_color(RED, f"Test script not found: {test_script}") return 1 - cmd = [str(test_script), "--model", str(weights_dir)] + cmd = [str(test_script), "--model", str(bundle_dir)] if args.android: cmd.append("--android") if args.ios: diff --git a/python/cactus/convert/model_adapters/adapters.py b/python/cactus/convert/model_adapters/adapters.py index 6e6da78f9..24d74715f 100644 --- a/python/cactus/convert/model_adapters/adapters.py +++ b/python/cactus/convert/model_adapters/adapters.py @@ -433,8 +433,13 @@ def load_processor(self, model_id_or_path: str): if processor is not None and hasattr(processor, "tokenizer"): return processor root = Path(model_id_or_path) - if not root.exists() or not (root / "tokenizer.json").exists(): - return None + has_vl_processor_config = ( + root.exists() + and (root / "tokenizer.json").exists() + and (root / "preprocessor_config.json").exists() + ) + if not has_vl_processor_config: + return processor from transformers import Lfm2VlImageProcessorFast, Lfm2VlProcessor, PreTrainedTokenizerFast cfg_path = root / "tokenizer_config.json" diff --git a/python/cactus/convert/model_adapters/policy.py b/python/cactus/convert/model_adapters/policy.py index 380ac3f24..578218aef 100644 --- a/python/cactus/convert/model_adapters/policy.py +++ b/python/cactus/convert/model_adapters/policy.py @@ -61,7 +61,13 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("fallback", "FP16", None, component, False, "none", "gemma4 media tower accuracy") output_head_or_tied_embedding = out in {"token_embeddings.weights", "decoder_token_embeddings.weights", "output_weight.weights"} if component == "embedding" or output_head_or_tied_embedding: - layout = "interleaved_4row" if output_head_or_tied_embedding else "row_major" + use_interleaved = ( + output_head_or_tied_embedding + and len(shape) == 2 + and int(shape[0]) % 4 == 0 + and int(shape[1]) % 32 == 0 + ) + layout = "interleaved_4row" if use_interleaved else "row_major" return TensorPolicy("convert", "CQ4", 4, component, False, "orthogonal", layout=layout) if component == "audio" or component == "transcription": return TensorPolicy("convert", f"CQ{user_bits}", user_bits, component, False, "hadamard") diff --git a/python/cactus/convert/tests/test_policy.py b/python/cactus/convert/tests/test_policy.py index 1de4101b0..03c568a3e 100644 --- a/python/cactus/convert/tests/test_policy.py +++ b/python/cactus/convert/tests/test_policy.py @@ -43,14 +43,17 @@ def test_policy_lfm_qwen_tied_embeddings_interleaved_cq4(): _validate_cq_layout(lm_head_policy, (65536, 2048), lm_head.source_name, lm_head.output_name) -def test_interleaved_lm_head_policy_rejects_unsupported_shape(): +def test_interleaved_lm_head_policy_falls_back_to_row_major_for_unsupported_shape(): match = cactus_name_for_tensor("lm_head.weight", "generic", 1) p = policy_for_tensor(match, (101, 64), 2, "generic") - try: - _validate_cq_layout(p, (101, 64), "lm_head.weight", "output_weight.weights") - except RuntimeError: - return - raise AssertionError("expected unsupported interleaved LM-head shape to fail") + assert p.layout == "row_major" + _validate_cq_layout(p, (101, 64), "lm_head.weight", "output_weight.weights") + + +def test_interleaved_lm_head_policy_uses_interleaved_for_supported_shape(): + match = cactus_name_for_tensor("lm_head.weight", "generic", 1) + p = policy_for_tensor(match, (128, 256), 2, "generic") + assert p.layout == "interleaved_4row" def test_adapter_registry_preserves_family_names(): diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py index b2f33db41..f1cb93f31 100644 --- a/python/cactus/transpile/lower.py +++ b/python/cactus/transpile/lower.py @@ -439,7 +439,7 @@ def _should_lower_gated_deltanet_with_internal_cache( if op not in {"gated_deltanet_prefill", "gated_deltanet_decode"}: return False component = str(ir.meta.get("component", "") or "").strip().lower() - return component in {"decoder_step", "decoder_media_step"} and int(seq_len) == 1 + return component in {"decoder_prefill_chunk", "decoder_step", "decoder_media_step"} def _gated_deltanet_layer_key(ir: IRGraph, node: IRNode) -> str: @@ -472,7 +472,7 @@ def _lower_gated_deltanet_initial_state( layer_key = f"gdn:{_gated_deltanet_layer_key(ir, node)}" gdn_states: dict[str, Tensor] = env.setdefault("__internal_gated_deltanet_cache_states", {}) # type: ignore[assignment] if layer_key not in gdn_states: - gdn_states[layer_key] = _materialize_constant_tensor(g, torch.zeros(state_shape, dtype=torch.float16)) + gdn_states[layer_key] = g.recurrent_cache_state(state_shape, dtype=g.FP16) return layer_key, gdn_states[layer_key] @@ -502,11 +502,21 @@ def _lower_gated_deltanet_conv1d( cache_entries.append((layer_key, cache_state, cache_state)) cache_state = conv_states[layer_key] + if batch_size != 1: + raise NotImplementedError( + f"gated_deltanet conv cache lowering currently supports batch_size=1 only, got {batch_size}" + ) x_rows = g.reshape(mixed_qkv, (batch_size * seq_len, mixed_qkv_dim)) - window = g.conv_cache_append(x_rows, cache_state) - window_nlc = g.reshape(window, (batch_size, kernel_size, mixed_qkv_dim)) - conv_out = g.conv1d_causal(window_nlc, conv_weight, kernel_size=kernel_size, dilation=1) - return g.slice(conv_out, axis=1, start=kernel_size - 1, length=1) + + if seq_len == 1: + window = g.conv_cache_append(x_rows, cache_state) + window_nlc = g.reshape(window, (batch_size, kernel_size, mixed_qkv_dim)) + conv_out = g.conv1d_causal(window_nlc, conv_weight, kernel_size=kernel_size, dilation=1) + return g.slice(conv_out, axis=1, start=kernel_size - 1, length=1) + + conv_out = g.conv1d_causal(mixed_qkv, conv_weight, kernel_size=kernel_size, dilation=1) + g.conv_cache_initialize(x_rows, cache_state) + return conv_out def _lower_constant_value( @@ -2138,14 +2148,15 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> ) deltanet_scale = 1.0 / math.sqrt(float(key_dim)) - if op == "gated_deltanet_decode": + if seq_len == 1: deltanet_out = g.gated_deltanet_decode(q_4d, k_4d, v_4d, gate_log, beta, initial_state, deltanet_scale) else: deltanet_out = g.gated_deltanet_prefill(q_4d, k_4d, v_4d, gate_log, beta, initial_state, chunk_size, deltanet_scale) if cache_layer_key is not None: final_state = g.slice(deltanet_out, axis=1, start=seq_len, length=key_dim) + g.recurrent_cache_write(final_state, initial_state) cache_entries: list[tuple[str, Tensor, Tensor]] = env.setdefault("__internal_kv_cache_state_entries", []) # type: ignore[assignment] - cache_entries.append((cache_layer_key, initial_state, final_state)) + cache_entries.append((cache_layer_key, initial_state, initial_state)) y_4d = g.slice(deltanet_out, axis=1, start=0, length=seq_len) y_2d = g.reshape(y_4d, (seq_len * num_v_heads, value_dim)) diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index a31973b0e..47f8f73fb 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -1476,6 +1476,62 @@ def get_transpile_metadata(self): } +class Lfm2CausalLMStepAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): + super().__init__() + self.model = model + model_root = getattr(model, "model", None) + language_model = getattr(model_root, "language_model", None) + self.backbone = language_model if isinstance(language_model, torch.nn.Module) else model_root + self.lm_head = getattr(model, "lm_head", None) + self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model) + + def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor: + backbone = self.backbone + lm_head = self.lm_head + if not isinstance(backbone, torch.nn.Module) or not isinstance(lm_head, torch.nn.Module): + raise TypeError("LFM2 causal step adapter requires backbone and lm_head modules") + + inputs_embeds = backbone.embed_tokens(input_ids) + text_position_ids = position_ids.to(dtype=torch.int64) + seq_len = int(inputs_embeds.shape[1]) + causal_mask = None + + hidden_states = inputs_embeds + position_embeddings = backbone.rotary_emb(hidden_states, position_ids=text_position_ids) + layer_types = tuple(getattr(backbone.config, "layer_types", ())) + + for layer_index, decoder_layer in enumerate(backbone.layers[: backbone.config.num_hidden_layers]): + layer_type = layer_types[layer_index] if layer_index < len(layer_types) else "full_attention" + layer_mask = causal_mask if layer_type == "full_attention" else None + hidden_states = decoder_layer( + hidden_states, + attention_mask=layer_mask, + position_embeddings=position_embeddings, + position_ids=text_position_ids, + past_key_values=None, + use_cache=False, + ) + + hidden_states = backbone.embedding_norm(hidden_states) + return lm_head(hidden_states[:, -1:, :]) + + def get_transpile_metadata(self): + layer_types = tuple(str(value) for value in (getattr(self.backbone.config, "layer_types", ()) or ())) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="lfm2", + adapter_type=type(self).__name__, + input_names=("input_ids", "position_ids"), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": layer_types, + } + } + + def _lfm2_vl_model_root(model: torch.nn.Module) -> torch.nn.Module: root = getattr(model, "model", None) if not isinstance(root, torch.nn.Module): @@ -4750,6 +4806,62 @@ def _build_whisper_seq2seq_component_specs( return specs +def _build_lfm2_causal_lm_component_specs( + model: torch.nn.Module, + *, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None = None, + components: tuple[str, ...] | None = None, +) -> list[ComponentModuleSpec] | None: + input_ids = named_tensors.get("input_ids") + if input_ids is None: + return None + pad_token_id = _resolve_model_pad_token_id(model) + decoder = Lfm2CausalLMLogitsAdapter(model, pad_token_id=pad_token_id).eval() + decoder_step = Lfm2CausalLMStepAdapter(model, pad_token_id=pad_token_id).eval() + + requested = tuple(components or ("decoder", "decoder_step")) + requested_set = set(requested) + common_graph_meta = { + "weights_dir": weights_dir, + "task": "causal_lm_logits", + "adapter_family": "lfm2", + } + specs: list[ComponentModuleSpec] = [] + if "decoder" in requested_set: + specs.append(ComponentModuleSpec( + component="decoder", + module=decoder, + example_inputs=(input_ids,), + input_keys=("input_ids",), + output_keys=("logits",), + graph_meta={**common_graph_meta, "component": "decoder"}, + metadata={"family": "lfm2", "task": "causal_lm_logits"}, + )) + if "decoder_step" in requested_set: + step_input_ids = input_ids[:, :1] + step_position_ids = torch.zeros_like(step_input_ids, dtype=torch.int64) + max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) + specs.append(ComponentModuleSpec( + component="decoder_step", + module=decoder_step, + example_inputs=(step_input_ids, step_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=("logits",), + graph_meta={ + **common_graph_meta, + "component": "decoder_step", + "use_internal_kv_cache": True, + "use_internal_conv_cache": True, + "use_internal_gated_deltanet_cache": True, + "max_cache_seq_len": max_cache_seq_len, + "cache_sink_size": 4, + }, + metadata={"family": "lfm2", "task": "causal_lm_logits"}, + )) + return specs + + def _build_lfm2_vl_multimodal_component_specs( model: torch.nn.Module, *, @@ -5029,6 +5141,13 @@ def build_component_module_specs( weights_dir=weights_dir, components=components, ) + if family == "lfm2" and task == "causal_lm_logits": + return _build_lfm2_causal_lm_component_specs( + model, + named_tensors=named_tensors, + weights_dir=weights_dir, + components=components, + ) if family == "parakeet_tdt" and task == "tdt_transcription": from cactus.transpile.tdt_runtime import build_parakeet_tdt_component_specs diff --git a/python/cactus/transpile/model_profiles.py b/python/cactus/transpile/model_profiles.py index 6a07d9c7c..4f875d69b 100644 --- a/python/cactus/transpile/model_profiles.py +++ b/python/cactus/transpile/model_profiles.py @@ -70,14 +70,14 @@ class ModelProfile: ) -LFM2_VL_PROFILE = ModelProfile( - family="lfm2_vl", - model_types=("lfm2_vl",), +LFM2_PROFILE = ModelProfile( + family="lfm2", + model_types=("lfm2_vl", "lfm2"), multimodal_context_tokens=256, input_combinations=((), ("image",)), model_id_aliases=(("lfm", "LiquidAI/LFM2-VL-450M"),), model_id_markers=("lfm2-vl", "lfm-vl"), - family_aliases=("lfm2", "lfm"), + family_aliases=("lfm2_vl", "lfm"), stop_tokens=("<|im_end|>",), avoid_native_loader=True, cached_step_components=("lm_encoder_step", "decoder_step"), @@ -203,7 +203,7 @@ class ModelProfile: PROFILES: tuple[ModelProfile, ...] = ( GEMMA4_PROFILE, - LFM2_VL_PROFILE, + LFM2_PROFILE, PARAKEET_TDT_PROFILE, WHISPER_PROFILE, QWEN_PROFILE, diff --git a/python/cactus/transpile/weight_binding.py b/python/cactus/transpile/weight_binding.py index 66e493cef..6e1e7d416 100644 --- a/python/cactus/transpile/weight_binding.py +++ b/python/cactus/transpile/weight_binding.py @@ -193,13 +193,22 @@ def _add_multimodal_backbone_aliases(candidate: str) -> None: tail = raw[len("model.") :] _add_with_wrappers(tail) - if raw.startswith("backbone."): - tail = raw[len("backbone.") :] - _add_with_wrappers(tail) - _add_with_wrappers(f"model.{tail}") - _add_with_wrappers(f"model.language_model.{tail}") + def _add_text_backbone_aliases(candidate: str) -> None: + for prefix in ( + "module.backbone.", + "module.model.backbone.", + "adapter.backbone.", + "adapter.model.backbone.", + "backbone.", + ): + if candidate.startswith(prefix): + tail = candidate[len(prefix) :] + _add_with_wrappers(tail) + _add_with_wrappers(f"model.{tail}") + _add_with_wrappers(f"model.language_model.{tail}") for candidate in tuple(aliases): + _add_text_backbone_aliases(candidate) _add_multimodal_backbone_aliases(candidate) return tuple(aliases) diff --git a/python/pyproject.toml b/python/pyproject.toml index 0425c1b2a..8d0473bb0 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -55,7 +55,7 @@ requires = ["setuptools>=61.0", "wheel"] build-backend = "setuptools.build_meta" [tool.setuptools.dynamic] -version = {attr = "cactus.__version__"} +version = {attr = "cactus._version.__version__"} [tool.setuptools.packages.find] where = ["."] diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py index 110becb7c..62ea0c99a 100644 --- a/python/tests/test_cli_transpile_defaults.py +++ b/python/tests/test_cli_transpile_defaults.py @@ -21,8 +21,7 @@ def _write_gemma4_multimodal_config(output_dir: Path) -> None: def _gemma4_multimodal_extra_args(model_dir: Path, artifact_dir: Path) -> list[str]: - from cactus.cli.common import PROJECT_ROOT - assets_dir = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" + assets_dir = Path(model_mod.__file__).resolve().parent.parent / "assets" return [ "--weights-dir", str(model_dir), @@ -164,7 +163,7 @@ def _fake_cq_main(command): extra_args = transpile_calls[0]["extra_args"] assert extra_args[extra_args.index("--task") + 1] == "tdt_transcription" assert "--audio-file" in extra_args - assert extra_args[extra_args.index("--audio-file") + 1].endswith("cactus-engine/tests/assets/test.wav") + assert extra_args[extra_args.index("--audio-file") + 1].endswith("python/cactus/assets/test.wav") def test_cmd_convert_supplies_default_audio_for_whisper(monkeypatch, tmp_path: Path) -> None: diff --git a/python/tests/test_transpile_weight_compat.py b/python/tests/test_transpile_weight_compat.py index be52f54e5..29fc9c8f1 100644 --- a/python/tests/test_transpile_weight_compat.py +++ b/python/tests/test_transpile_weight_compat.py @@ -259,6 +259,37 @@ def test_resolve_weight_binding_expands_language_model_backbone_aliases(tmp_path assert binding.path.endswith("layer_0_scalar.weights") +def test_resolve_weight_binding_maps_module_backbone_to_model_manifest(tmp_path: Path) -> None: + (tmp_path / "layer_0_q.weights").write_bytes(b"") + (tmp_path / "weights_manifest.json").write_text( + json.dumps( + { + "weights": [ + { + "source_name": "model.layers.0.self_attn.q_proj.weight", + "hf_name": "model.layers.0.self_attn.q_proj.weight", + "adapter_name": "model.layers.0.self_attn.q_proj.weight", + "output_name": "layer_0_q.weights", + "component": "language", + } + ] + } + ) + ) + + for source_name in ( + "module.backbone.layers.0.self_attn.q_proj.weight", + "adapter.backbone.layers.0.self_attn.q_proj.weight", + ): + binding = resolve_weight_binding( + weights_dir=str(tmp_path), + source_name=source_name, + ) + + assert binding is not None, source_name + assert binding.path.endswith("layer_0_q.weights"), source_name + + def test_resolve_weight_binding_maps_tied_lm_head_to_token_embedding_manifest(tmp_path: Path) -> None: (tmp_path / "token_embeddings.weights").write_bytes(b"") (tmp_path / "weights_manifest.json").write_text(