Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions cactus-graph/src/builder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,15 @@ size_t CactusGraph::reduction_op(OpType op, size_t input, int axis) {
if (axis == -1) {
out = {1};
} else {
if (buf.shape.empty()) {
throw std::runtime_error("Cannot reduce a scalar tensor along an axis");
}
if (axis < 0) {
axis += static_cast<int>(buf.shape.size());
}
if (axis < 0 || static_cast<size_t>(axis) >= buf.shape.size()) {
throw std::runtime_error("Invalid axis for reduction operation");
}
out = buf.shape;
out.erase(out.begin() + axis);
if (out.empty()) out = {1};
Expand Down Expand Up @@ -1150,7 +1159,15 @@ size_t CactusGraph::slice(size_t input, int axis, size_t start, size_t length) {
throw std::runtime_error("Cannot slice a scalar tensor");
}

size_t axis_index = static_cast<size_t>(axis);
int actual_axis = axis;
if (actual_axis < 0) {
actual_axis += static_cast<int>(input_buffer.shape.size());
}
if (actual_axis < 0 || static_cast<size_t>(actual_axis) >= input_buffer.shape.size()) {
throw std::invalid_argument("Slice axis out of bounds");
}

size_t axis_index = static_cast<size_t>(actual_axis);
size_t axis_size = input_buffer.shape[axis_index];

if (start + length > axis_size) {
Expand Down Expand Up @@ -1220,7 +1237,8 @@ size_t CactusGraph::add_node(OpType op_type, const std::vector<size_t>& inputs,
}

Precision result_precision = params.output_precision;
if (op_type == OpType::PRECISION_CAST || op_type == OpType::EMBEDDING) {
if (op_type == OpType::PRECISION_CAST || op_type == OpType::EMBEDDING ||
op_type == OpType::TOPK || op_type == OpType::SAMPLE) {
result_precision = params.output_precision;
} else if (!inputs.empty()) {
result_precision = nodes_[node_index_map_[inputs[0]]]->output_buffer.precision;
Expand Down
17 changes: 13 additions & 4 deletions cactus-graph/src/execute.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -401,17 +401,26 @@ void CactusGraph::execute(const std::string& profile_file) {
}
}

auto resolve_alias_root = [&](size_t idx) -> size_t {
while (may_alias_input(*nodes_[idx]) && !nodes_[idx]->input_ids.empty()) {
auto it = node_index_map_.find(nodes_[idx]->input_ids[0]);
if (it == node_index_map_.end()) break;
idx = it->second;
}
return idx;
};

std::vector<bool> keep_until_graph_cleanup(n, false);
for (size_t i = 0; i < n; ++i) {
const auto& node = *nodes_[i];
if (!may_alias_input(node) || node.input_ids.empty()) continue;
auto it = node_index_map_.find(node.input_ids[0]);
if (it == node_index_map_.end()) continue;
size_t base_idx = it->second;
if (use_count[i] == 0) {
keep_until_graph_cleanup[base_idx] = true;
size_t root_idx = resolve_alias_root(it->second);
if (use_count[i] == 0 || retained_output_node_ids_.count(node.id)) {
keep_until_graph_cleanup[root_idx] = true;
} else {
last_use[base_idx] = std::max(last_use[base_idx], last_use[i]);
last_use[root_idx] = std::max(last_use[root_idx], last_use[i]);
}
}

Expand Down
3 changes: 3 additions & 0 deletions cactus-graph/src/graph_ffi.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -920,6 +920,9 @@ int cactus_graph_attention_int8_hybrid(cactus_graph_t graph, cactus_node_t query
const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales,
size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out) {
if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_attention_int8_hybrid");
if (cache_len > 0 && (!cached_keys || !cached_values || !k_scales || !v_scales)) {
return fail_invalid("Null cache pointer with cache_len > 0 to cactus_graph_attention_int8_hybrid");
}
try {
*out = static_cast<cactus_node_t>(as_graph(graph)->graph.attention_int8_hybrid(static_cast<size_t>(query), static_cast<size_t>(key_new), static_cast<size_t>(value_new), scale, position_offset, cached_keys, cached_values, k_scales, v_scales, cache_len, num_kv_heads, head_dim, window_size));
return 0;
Expand Down
47 changes: 45 additions & 2 deletions cactus-graph/src/io.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,11 @@ namespace {
inline size_t align_offset(size_t offset, size_t alignment) {
size_t remainder = offset % alignment;
if (remainder == 0) return offset;
return offset + (alignment - remainder);
size_t padding = alignment - remainder;
if (offset > SIZE_MAX - padding) {
throw std::runtime_error("File corrupted: offset alignment overflow");
}
return offset + padding;
}

std::string resolve_quantized_weight_file(const std::string& filename) {
Expand Down Expand Up @@ -980,6 +984,9 @@ void MappedFile::parse_header() {
}

uint32_t prec_val = *reinterpret_cast<const uint32_t*>(ptr + offset);
if (prec_val > static_cast<uint32_t>(Precision::CQ4)) {
throw std::runtime_error("Invalid tensor file: unknown precision");
}
precision_ = static_cast<Precision>(prec_val);
offset += sizeof(uint32_t);

Expand Down Expand Up @@ -1025,14 +1032,50 @@ void MappedFile::parse_header() {

if (scales_bytes_ > 0) {
scales_offset_ = aligned_header;
if (scales_bytes_ > file_size_ || scales_offset_ > file_size_ - scales_bytes_) {
throw std::runtime_error("File corrupted: scales extend beyond file size");
}
if (PrecisionTraits::is_cq(precision_) && group_size_ > 0) {
auto add_checked = [](size_t a, size_t b) -> size_t {
if (a > SIZE_MAX - b) {
throw std::runtime_error("File corrupted: CQ scales layout overflow");
}
return a + b;
};
auto mul_checked = [](size_t a, size_t b) -> size_t {
if (a != 0 && b > SIZE_MAX / a) {
throw std::runtime_error("File corrupted: CQ scales layout overflow");
}
return a * b;
};
const size_t fp16 = sizeof(__fp16);
const size_t gs = group_size_;
const size_t K = mul_checked(gs, num_groups_);
const size_t N = shape_.size() >= 2 ? shape_[0] : 1;
const size_t cb_size = static_cast<size_t>(1) << PrecisionTraits::cq_bits(precision_);

size_t footprint = mul_checked(cb_size, fp16);
footprint = add_checked(footprint, mul_checked(K, fp16));
footprint = add_checked(footprint, mul_checked(K, fp16));
footprint = add_checked(footprint, mul_checked(mul_checked(N, num_groups_), fp16));
if (is_orthogonal_rotation_) {
footprint = add_checked(footprint, mul_checked(mul_checked(gs, gs), fp16));
} else {
footprint = add_checked(footprint, add_checked(gs, gs));
footprint = add_checked(footprint, mul_checked(gs, sizeof(uint32_t)));
}
if (footprint > scales_bytes_) {
throw std::runtime_error("File corrupted: CQ scales smaller than declared layout");
}
}
size_t scales_end = scales_offset_ + scales_bytes_;
data_offset_ = align_offset(scales_end, alignment_);
} else {
scales_offset_ = 0;
data_offset_ = aligned_header;
}

if (data_offset_ + byte_size_ > file_size_) {
if (byte_size_ > file_size_ || data_offset_ > file_size_ - byte_size_) {
throw std::runtime_error("File corrupted: data extends beyond file size");
}

Expand Down
10 changes: 10 additions & 0 deletions cactus-graph/src/ops_image.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,16 @@ void compute_image_preprocess_node(
int ch = node.params.image_channels;
float rescale = node.params.rescale_factor;

if (src_w <= 0 || src_h <= 0 || ch <= 0) {
throw std::runtime_error("IMAGE_PREPROCESS requires positive source dimensions and channels");
}
if (ch > 3) {
throw std::runtime_error("IMAGE_PREPROCESS supports at most 3 channels");
}
if (input.total_size != static_cast<size_t>(src_w) * src_h * ch) {
throw std::runtime_error("IMAGE_PREPROCESS input size does not match src_width*src_height*channels");
}

std::vector<float> src_float(static_cast<size_t>(src_w) * src_h * ch);
if (input.precision == Precision::FP32) {
std::memcpy(src_float.data(), input.get_data(), src_float.size() * sizeof(float));
Expand Down
12 changes: 11 additions & 1 deletion cactus-graph/src/ops_math.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,9 @@ void dispatch_binary_op_f16(OpType op, const __fp16* lhs, const __fp16* rhs, __f
static float apply_binary_op_f32(OpType op, float lhs, float rhs) {
switch (op) {
case OpType::ADD:
case OpType::ADD_CLIPPED:
return lhs + rhs;
case OpType::ADD_CLIPPED:
return std::fmin(std::fmax(lhs + rhs, -65500.0f), 65500.0f);
case OpType::SUBTRACT:
return lhs - rhs;
case OpType::MULTIPLY:
Expand Down Expand Up @@ -226,6 +227,11 @@ void compute_binary_op_node(GraphNode& node, const std::vector<std::unique_ptr<G
lhs_strides.data(), rhs_strides.data(),
node.params.broadcast_info.output_shape.data(),
node.params.broadcast_info.output_shape.size());
if (node.op_type == OpType::ADD_CLIPPED) {
cactus_clamp_f16(node.output_buffer.data_as<__fp16>(),
node.output_buffer.data_as<__fp16>(),
total_elements, -65500.0f, 65500.0f);
}
break;
case OpType::SUBTRACT:
cactus_subtract_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(),
Expand Down Expand Up @@ -428,6 +434,10 @@ void compute_reduce_node(GraphNode& node, const std::vector<std::unique_ptr<Grap
return;
}

if (axis != -1 && (axis < 0 || static_cast<size_t>(axis) >= input_buffer.shape.size())) {
throw std::runtime_error("Reduction axis out of range");
}

if (input_buffer.precision == Precision::FP32) {
const float* input = input_buffer.data_as<float>();
float* output = node.output_buffer.data_as<float>();
Expand Down
14 changes: 11 additions & 3 deletions cactus-graph/src/ops_nn.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,14 @@ namespace {

void ensure_moe_buffers(size_t max_tokens, size_t hidden_dim, size_t intermediate_dim,
size_t num_experts, size_t top_k) {
size_t hidden_size = max_tokens * hidden_dim;
size_t inter_size = max_tokens * intermediate_dim;
size_t max_rows = max_tokens * top_k;
size_t hidden_size = max_rows * hidden_dim;
size_t inter_size = max_rows * intermediate_dim;
if (moe_compact_hidden_buf.size() < hidden_size) moe_compact_hidden_buf.resize(hidden_size);
if (moe_gate_buf.size() < inter_size) moe_gate_buf.resize(inter_size);
if (moe_up_buf.size() < inter_size) moe_up_buf.resize(inter_size);
if (moe_expert_out_buf.size() < hidden_size) moe_expert_out_buf.resize(hidden_size);
size_t total_assignments = max_tokens * top_k;
size_t total_assignments = max_rows;
if (moe_expert_offsets_buf.size() < num_experts + 1) moe_expert_offsets_buf.resize(num_experts + 1);
if (moe_expert_tokens_buf.size() < total_assignments) moe_expert_tokens_buf.resize(total_assignments);
if (moe_routing_denom_buf.size() < max_tokens) moe_routing_denom_buf.resize(max_tokens);
Expand Down Expand Up @@ -165,6 +166,13 @@ void compute_moe_layer_node(GraphNode& node, const std::vector<std::unique_ptr<G
const size_t hidden_dim = hidden_buffer.shape[1];
const size_t total_num_experts = routing_buffer.shape[1];

if (topk_idx_buffer.shape.size() != 2 || topk_idx_buffer.shape[0] != token_count || topk_idx_buffer.shape[1] != top_k) {
throw std::runtime_error("moe_layer topk indices must be [token_count, num_experts_per_tok]");
}
if (num_experts > total_num_experts) {
throw std::runtime_error("moe_layer routing width < num_experts");
}

const auto& w1_0_buffer = get_input(node, 3, nodes, node_index_map);
const size_t expert_intermediate_dim = w1_0_buffer.shape[0];

Expand Down
6 changes: 6 additions & 0 deletions cactus-graph/src/ops_sample.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ void compute_sample_node(GraphNode& node, const std::vector<std::unique_ptr<Grap

size_t seq_len = logits_buffer.shape[0];
size_t vocab_size = logits_buffer.shape[1];
if (seq_len == 0) {
throw std::runtime_error("Sample requires seq_len >= 1");
}
size_t last_token_offset = (seq_len - 1) * vocab_size;

if (logits_buffer.precision == Precision::FP16) {
Expand All @@ -49,6 +52,9 @@ void compute_topk_node(GraphNode& node, const std::vector<std::unique_ptr<GraphN
size_t k = node.params.top_k;
size_t batch_size = input_buffer.shape[0];
size_t feature_size = input_buffer.shape[1];
if (k > feature_size) {
throw std::runtime_error("TopK k exceeds feature dimension");
}
size_t block_size = batch_size * k;

std::vector<float> input_float(input_buffer.total_size);
Expand Down
4 changes: 2 additions & 2 deletions cactus-graph/src/ops_tensor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,10 @@ void compute_slice_node(GraphNode& node, const std::vector<std::unique_ptr<Graph

const size_t axis_size = input_buffer.shape[axis_index];
const size_t slice_start = node.params.slice_start;
size_t slice_length = node.params.slice_length;
const size_t slice_length = node.output_buffer.shape[axis_index];

if (slice_length == 0) {
slice_length = axis_size - slice_start;
return;
}

if (axis_index == 0) {
Expand Down
Loading