Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions .github/actions/stamp-version/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
name: Stamp version
description: Resolve the release version and write python/cactus/_version.py
runs:
using: composite
steps:
- shell: bash
env:
TAG: ${{ github.event.release.tag_name }}
INPUT_VERSION: ${{ github.event.inputs.version }}
run: |
if [ -n "$TAG" ]; then
raw="${TAG#v}"
elif [ -n "$INPUT_VERSION" ]; then
raw="${INPUT_VERSION#v}"
else
raw=$(sed 's/^v//' CACTUS_VERSION)
fi
version=$(RAW="$raw" python3 -c 'import os; r=os.environ["RAW"]; print(r + ".0" if r.count(".") == 1 else r)')
if [ -z "$version" ]; then
echo "::error::No version source available (TAG, INPUT_VERSION, CACTUS_VERSION all empty)"
exit 1
fi
printf '__version__ = "%s"\n' "$version" > python/cactus/_version.py
echo "Stamped version: $version"
11 changes: 11 additions & 0 deletions .github/workflows/pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ name: PyPI Publish
on:
workflow_call:
workflow_dispatch:
inputs:
version:
description: "Version to publish (e.g. 2.0.0). Leave empty to read from CACTUS_VERSION."
required: false
type: string

jobs:
wheels-macos:
Expand All @@ -16,6 +21,8 @@ jobs:
with:
python-version: "3.12"

- uses: ./.github/actions/stamp-version

- name: Build native library
run: |
pip install -e python/
Expand Down Expand Up @@ -47,6 +54,8 @@ jobs:
dnf install -y libcurl-devel openssl-devel
echo "/opt/python/cp312-cp312/bin" >> "$GITHUB_PATH"

- uses: ./.github/actions/stamp-version

- name: Build native library
run: |
pip install -e python/
Expand Down Expand Up @@ -75,6 +84,8 @@ jobs:
with:
python-version: "3.12"

- uses: ./.github/actions/stamp-version

- name: Build sdist
working-directory: python
run: |
Expand Down
7 changes: 7 additions & 0 deletions cactus-engine/cactus_engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,13 @@ CACTUS_FFI_EXPORT int cactus_graph_conv_cache_state(
cactus_graph_t graph, size_t window_size, size_t hidden_dim, cactus_node_t* out);
CACTUS_FFI_EXPORT int cactus_graph_conv_cache_append(
cactus_graph_t graph, cactus_node_t new_data, cactus_node_t cache_state, cactus_node_t* out);
CACTUS_FFI_EXPORT int cactus_graph_conv_cache_initialize(
cactus_graph_t graph, cactus_node_t rows, cactus_node_t cache_state, cactus_node_t* out);

CACTUS_FFI_EXPORT int cactus_graph_recurrent_cache_state(
cactus_graph_t graph, const size_t* shape, size_t shape_len, int precision, cactus_node_t* out);
CACTUS_FFI_EXPORT int cactus_graph_recurrent_cache_write(
cactus_graph_t graph, cactus_node_t new_value, cactus_node_t cache_input, cactus_node_t* out);

CACTUS_FFI_EXPORT int cactus_graph_rfft(
cactus_graph_t graph, cactus_node_t input, cactus_node_t* out);
Expand Down
4 changes: 2 additions & 2 deletions cactus-engine/src/engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -382,8 +382,8 @@ class Tokenizer {
void detect_model_type(const std::string& config_path);
void load_chat_template(const std::string& template_file);
std::string format_gemma4_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const;
std::string format_qwen_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
std::string format_lfm2_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
std::string format_qwen_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const;
std::string format_lfm2_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const;
};

class BPETokenizer : public Tokenizer {
Expand Down
36 changes: 36 additions & 0 deletions cactus-engine/src/graph_ffi.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -993,6 +993,42 @@ int cactus_graph_conv_cache_append(cactus_graph_t graph, cactus_node_t new_data,
}
}

int cactus_graph_conv_cache_initialize(cactus_graph_t graph, cactus_node_t rows, cactus_node_t cache_state, cactus_node_t* out) {
if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv_cache_initialize");
try {
*out = static_cast<cactus_node_t>(as_graph(graph)->graph.conv_cache_initialize(static_cast<size_t>(rows), static_cast<size_t>(cache_state)));
return 0;
} catch (const std::exception& e) {
last_error_message = e.what();
return -1;
}
}

int cactus_graph_recurrent_cache_state(cactus_graph_t graph, const size_t* shape, size_t shape_len, int precision, cactus_node_t* out) {
if (!graph || !out || (!shape && shape_len > 0)) {
return fail_invalid("Invalid args to cactus_graph_recurrent_cache_state");
}
try {
std::vector<size_t> shape_vec(shape, shape + shape_len);
*out = static_cast<cactus_node_t>(as_graph(graph)->graph.recurrent_cache_state(shape_vec, static_cast<Precision>(precision)));
return 0;
} catch (const std::exception& e) {
last_error_message = e.what();
return -1;
}
}

int cactus_graph_recurrent_cache_write(cactus_graph_t graph, cactus_node_t new_value, cactus_node_t cache_input, cactus_node_t* out) {
if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_recurrent_cache_write");
try {
*out = static_cast<cactus_node_t>(as_graph(graph)->graph.recurrent_cache_write(static_cast<size_t>(new_value), static_cast<size_t>(cache_input)));
return 0;
} catch (const std::exception& e) {
last_error_message = e.what();
return -1;
}
}

int cactus_graph_rfft(cactus_graph_t graph, cactus_node_t input, cactus_node_t* out) {
if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rfft");
try {
Expand Down
72 changes: 67 additions & 5 deletions cactus-engine/src/model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -663,6 +663,32 @@ void Model::copy_cache_states(const Component& source, Component& target, size_t
}
void* src_ptr = source.graph->get_output(static_cast<size_t>(src_node));
void* dst_ptr = target.graph->get_output(static_cast<size_t>(dst_node));

const OpType src_op = source.graph->get_node_op_type(static_cast<size_t>(src_node));
const OpType dst_op = target.graph->get_node_op_type(static_cast<size_t>(dst_node));
if (src_op != dst_op) {
throw std::runtime_error(
"cache state op_type mismatch between prefill and step at layer "
+ src.layer_key);
}
if (src_op == OpType::RECURRENT_CACHE_STATE) {
if (src_desc.byte_size != dst_desc.byte_size) {
throw std::runtime_error(
"recurrent cache buffer shape mismatch between prefill and step at layer "
+ src.layer_key);
}
std::memcpy(dst_ptr, src_ptr, src_desc.byte_size);
continue;
}
if (src_op == OpType::CONV_CACHE_STATE) {
if (src_desc.byte_size != dst_desc.byte_size) {
throw std::runtime_error(
"conv cache buffer shape mismatch between prefill and step at layer "
+ src.layer_key);
}
std::memcpy(dst_ptr, src_ptr, src_desc.byte_size);
continue;
}
if (src_desc.byte_size == dst_desc.byte_size && logical_current == std::numeric_limits<size_t>::max()) {
std::memcpy(dst_ptr, src_ptr, src_desc.byte_size);
continue;
Expand Down Expand Up @@ -778,11 +804,29 @@ void Model::reset_component_cache_states(Component& comp) {
for (int node_id : {state.key_node_id, state.value_node_id}) {
if (node_id < 0) continue;
const auto& desc = comp.graph->get_output_buffer(static_cast<size_t>(node_id));
if (desc.byte_size < sizeof(uint64_t) || !desc.get_data()) continue;
if (desc.byte_size == 0 || !desc.get_data()) continue;
void* ptr = comp.graph->get_output(static_cast<size_t>(node_id));
if (!ptr) continue;
auto* metadata = static_cast<uint64_t*>(ptr);
metadata[0] = 0;
const OpType op_type = comp.graph->get_node_op_type(static_cast<size_t>(node_id));
switch (op_type) {
case OpType::KV_CACHE_STATE:
if (desc.byte_size >= sizeof(uint64_t)) {
static_cast<uint64_t*>(ptr)[0] = 0;
}
break;
case OpType::CONV_CACHE_STATE:
if (desc.byte_size >= 2 * sizeof(uint64_t)) {
auto* meta = static_cast<uint64_t*>(ptr);
meta[0] = 0; // head
meta[1] = 0; // count
}
break;
case OpType::RECURRENT_CACHE_STATE:
std::memset(ptr, 0, desc.byte_size);
break;
default:
break;
}
}
}
}
Expand Down Expand Up @@ -819,10 +863,28 @@ Model::ChunkedPrefillResult Model::run_chunked_prefill(const std::vector<uint32_
if (component_tokens <= 1) return result;
size_t effective_chunk = chunk_size > 0 ? std::min(chunk_size, component_tokens) : component_tokens;
if (effective_chunk != component_tokens) effective_chunk = component_tokens;
const size_t whole_chunks_end = (tokens.size() / effective_chunk) * effective_chunk;
size_t whole_chunks_end = (tokens.size() / effective_chunk) * effective_chunk;
const bool has_recurrent_state = [&]() {
if (!decoder_prefill_->graph) return false;
for (const auto& state : decoder_prefill_->cache_states) {
for (int node_id : {state.key_node_id, state.value_node_id}) {
if (node_id < 0) continue;
if (decoder_prefill_->graph->get_node_op_type(static_cast<size_t>(node_id))
== OpType::RECURRENT_CACHE_STATE) {
return true;
}
}
}
return false;
}();
if (has_recurrent_state && whole_chunks_end > effective_chunk) {
whole_chunks_end = effective_chunk;
}
const size_t tail_tokens = tokens.size() - whole_chunks_end;
const size_t padding_cutoff = std::max<size_t>(1, effective_chunk / 16);
const bool pad_tail = family_ != "lfm2_vl" && tail_tokens >= padding_cutoff;
const bool pad_tail = family_ != "lfm2_vl"
&& !has_recurrent_state
&& tail_tokens >= padding_cutoff;
const size_t executable_tokens = whole_chunks_end + (pad_tail ? effective_chunk : 0);
if (executable_tokens == 0) {
result.scalar_tail_tokens = tail_tokens;
Expand Down
13 changes: 8 additions & 5 deletions cactus-engine/src/tokenizer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -383,16 +383,16 @@ std::vector<uint32_t> Tokenizer::apply_chat_template(const std::vector<ChatMessa
std::string Tokenizer::format_chat_prompt(const std::vector<ChatMessage>& messages, bool add_generation_prompt,
const std::string& tools_json, bool enable_thinking_if_supported) const {
if (model_type_ == ModelType::QWEN) {
return format_qwen_style(messages, add_generation_prompt, tools_json);
return format_qwen_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported);
}
if (model_type_ == ModelType::LFM2) {
return format_lfm2_style(messages, add_generation_prompt, tools_json);
return format_lfm2_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported);
}
return format_gemma4_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported);
}

std::string Tokenizer::format_qwen_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt,
const std::string& tools_json) const {
const std::string& tools_json, bool enable_thinking_if_supported) const {
std::string result;

if (!tools_json.empty()) {
Expand Down Expand Up @@ -437,14 +437,17 @@ std::string Tokenizer::format_qwen_style(const std::vector<ChatMessage>& message

if (add_generation_prompt) {
result += "<|im_start|>assistant\n";
if (chat_template_.find("<think>") != std::string::npos) {
result += enable_thinking_if_supported ? "<think>\n" : "<think>\n\n</think>\n\n";
}
}

return result;
}

std::string Tokenizer::format_lfm2_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt,
const std::string& tools_json) const {
return "<|startoftext|>" + format_qwen_style(messages, add_generation_prompt, tools_json);
const std::string& tools_json, bool enable_thinking_if_supported) const {
return "<|startoftext|>" + format_qwen_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported);
}

std::string Tokenizer::format_gemma4_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt,
Expand Down
10 changes: 9 additions & 1 deletion cactus-graph/cactus_graph.h
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,10 @@ enum class OpType {
CLAMP,
DENSE_MLP_TQ_FUSED,
NOT_EQUAL,
SCALAR_NOT_EQUAL
SCALAR_NOT_EQUAL,
RECURRENT_CACHE_STATE,
RECURRENT_CACHE_WRITE,
CONV_CACHE_INITIALIZE
};

struct PrecisionTraits {
Expand Down Expand Up @@ -578,6 +581,10 @@ class CactusGraph {

size_t conv_cache_state(size_t window_size, size_t hidden_dim);
size_t conv_cache_append(size_t new_data, size_t cache_state_node);
size_t conv_cache_initialize(size_t rows, size_t cache_state_node);

size_t recurrent_cache_state(const std::vector<size_t>& shape, Precision precision);
size_t recurrent_cache_write(size_t new_value, size_t cache_state);

size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1);
size_t conv1d_k3(size_t input, size_t weight, size_t stride);
Expand Down Expand Up @@ -699,6 +706,7 @@ class CactusGraph {
size_t add_node(OpType op_type, const std::vector<size_t>& inputs,
const std::vector<size_t>& output_shape, const OpParams& params = {});
const BufferDesc& get_output_buffer(size_t node_id) const;
OpType get_node_op_type(size_t node_id) const;
void allocate_buffers();
size_t get_node_count() const;

Expand Down
58 changes: 58 additions & 0 deletions cactus-graph/src/builder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1253,6 +1253,10 @@ const BufferDesc& CactusGraph::get_output_buffer(size_t node_id) const {
return nodes_[node_index_map_.at(node_id)]->output_buffer;
}

OpType CactusGraph::get_node_op_type(size_t node_id) const {
return nodes_[node_index_map_.at(node_id)]->op_type;
}

size_t CactusGraph::persistent(size_t source_node) {
const auto& source_buffer = get_output_buffer(source_node);
OpParams params;
Expand Down Expand Up @@ -1464,6 +1468,60 @@ size_t CactusGraph::conv_cache_append(size_t new_data, size_t cache_state_node)
return add_node(OpType::CONV_CACHE_APPEND, {new_data, cache_state_node}, {ws, hd}, params);
}

size_t CactusGraph::conv_cache_initialize(size_t rows, size_t cache_state_node) {
if (get_node_op_type(cache_state_node) != OpType::CONV_CACHE_STATE) {
throw std::invalid_argument(
"conv_cache_initialize target must be a CONV_CACHE_STATE node");
}
const auto& rows_buf = get_output_buffer(rows);
if (rows_buf.precision != Precision::FP16 && rows_buf.precision != Precision::FP32) {
throw std::invalid_argument(
"conv_cache_initialize requires FP16/FP32 input rows");
}
const size_t hidden_dim = nodes_[node_index_map_.at(cache_state_node)]->params.head_dim;
if (hidden_dim == 0) {
throw std::invalid_argument(
"conv_cache_initialize: cache hidden_dim is zero");
}
if (rows_buf.shape.empty() || rows_buf.shape.back() != hidden_dim) {
throw std::invalid_argument(
"conv_cache_initialize: rows last dim must equal cache hidden_dim");
}
if (rows_buf.total_size % hidden_dim != 0) {
throw std::invalid_argument(
"conv_cache_initialize: rows total size must be a multiple of cache hidden_dim");
}
OpParams params{};
params.output_precision = Precision::FP16;
return add_node(OpType::CONV_CACHE_INITIALIZE, {rows, cache_state_node}, {0}, params);
}

size_t CactusGraph::recurrent_cache_state(const std::vector<size_t>& shape, Precision precision) {
OpParams params{};
params.output_precision = precision;
size_t node_id = add_node(OpType::RECURRENT_CACHE_STATE, {}, shape, params);
persistent_node_ids_.insert(node_id);
return node_id;
}

size_t CactusGraph::recurrent_cache_write(size_t new_value, size_t cache_state) {
const auto& src_buf = get_output_buffer(new_value);
const auto& dst_buf = get_output_buffer(cache_state);
if (get_node_op_type(cache_state) != OpType::RECURRENT_CACHE_STATE) {
throw std::invalid_argument(
"recurrent_cache_write target must be a RECURRENT_CACHE_STATE node");
}
if (src_buf.shape != dst_buf.shape) {
throw std::invalid_argument("recurrent_cache_write requires matching shapes");
}
if (src_buf.precision != dst_buf.precision) {
throw std::invalid_argument("recurrent_cache_write requires matching precision");
}
OpParams params{};
params.output_precision = dst_buf.precision;
return add_node(OpType::RECURRENT_CACHE_WRITE, {new_value, cache_state}, {0}, params);
}

size_t CactusGraph::image_preprocess(
size_t pixel_input,
int src_width, int src_height,
Expand Down
12 changes: 8 additions & 4 deletions cactus-graph/src/core.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -253,16 +253,20 @@ void CactusGraph::clear_debug_nodes() {

void CactusGraph::allocate_buffers() {
for (auto& node : nodes_) {
if (node->op_type != OpType::INPUT) {
node->output_buffer.allocate();
}
if (node->op_type == OpType::INPUT) continue;
if (node->op_type == OpType::KV_CACHE_STATE
|| node->op_type == OpType::CONV_CACHE_STATE
|| node->op_type == OpType::RECURRENT_CACHE_STATE) continue;
node->output_buffer.allocate();
}
}

void CactusGraph::release_runtime_buffers() {
for (auto& node : nodes_) {
if (node->op_type == OpType::INPUT) continue;
if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) continue;
if (node->op_type == OpType::KV_CACHE_STATE
|| node->op_type == OpType::CONV_CACHE_STATE
|| node->op_type == OpType::RECURRENT_CACHE_STATE) continue;
if (persistent_node_ids_.count(node->id)) continue;
node->output_buffer.release_memory(buffer_pool_);
}
Expand Down
Loading
Loading