Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions cactus-graph/src/io.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,59 @@ namespace {
return filename;
}

void validate_interleaved_4row_tensor(
Precision precision,
const std::vector<size_t>& shape,
size_t group_size,
size_t num_groups,
size_t byte_size,
size_t scales_bytes,
bool orthogonal) {
if (!PrecisionTraits::is_cq(precision) || group_size == 0 || num_groups == 0) {
throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires grouped CQ precision");
}
if (shape.size() != 2) {
throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires a rank-2 tensor");
}
const uint32_t bits = PrecisionTraits::cq_bits(precision);
const size_t n = shape[0];
const size_t k = shape[1];
if (bits < 1 || bits > 4) {
throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW only supports CQ1..CQ4");
}
if ((n % 4) != 0) {
throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires N % 4 == 0");
}
if (group_size * num_groups != k) {
throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW group metadata does not match tensor shape");
}
if (orthogonal) {
if (bits != 4 || num_groups != 1 || group_size != k || (k % 32) != 0) {
throw std::runtime_error("Invalid tensor file: orthogonal INTERLEAVED_4ROW requires full-width CQ4 with K % 32 == 0");
}
} else if ((group_size % 32) != 0 || group_size > 256) {
throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires group_size % 32 == 0 and group_size <= 256");
}

const size_t packed_group_bytes = (group_size * bits + 7) / 8;
const size_t expected_bytes = n * num_groups * packed_group_bytes;
if (byte_size != expected_bytes) {
throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW packed byte size mismatch");
}

const size_t codebook_bytes = (static_cast<size_t>(1) << bits) * sizeof(__fp16);
const size_t input_scale_bytes = k * sizeof(__fp16);
const size_t norms_bytes = n * num_groups * sizeof(__fp16);
const size_t rotation_bytes = orthogonal
? k * k * sizeof(__fp16)
: 2 * group_size + group_size * sizeof(uint32_t);
const size_t expected_scales_bytes =
codebook_bytes + 2 * input_scale_bytes + norms_bytes + rotation_bytes;
if (scales_bytes != expected_scales_bytes) {
throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW metadata byte size mismatch");
}
}

inline void write_u32(std::ostream& out, uint32_t v) {
out.write(reinterpret_cast<const char*>(&v), sizeof(v));
}
Expand Down Expand Up @@ -1018,6 +1071,11 @@ void MappedFile::parse_header() {
}
}

if (is_interleaved_4row_) {
validate_interleaved_4row_tensor(
precision_, shape_, group_size_, num_groups_, byte_size_, scales_bytes_, is_orthogonal_rotation_);
}

size_t aligned_header = align_offset(header_size, alignment_);

if (scales_bytes_ > 0) {
Expand Down
1 change: 1 addition & 0 deletions cactus-graph/src/ops_tensor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ void compute_embedding_node(GraphNode& node, const std::vector<std::unique_ptr<G
embeddings_buffer.cq_norms,
embeddings_buffer.cq_input_scale_recip,
embeddings_buffer.cq_rotation,
embeddings_buffer.cq_flags,
output + i * hidden_dim);
} else {
cactus_quant_dequantize_hadamard_embedding_row(
Expand Down
64 changes: 64 additions & 0 deletions cactus-graph/tests/test_io.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,69 @@ bool test_graph_save_for_inspection() {
}
}

bool test_rejects_malformed_interleaved_cq() {
const std::string filename = "test_bad_interleaved_cq.weights";
const uint32_t flags = (1u << 1) | (1u << 2);
const uint32_t alignment = 32;
const uint64_t k = 32;
const uint32_t precision = static_cast<uint32_t>(Precision::CQ4);
auto pad_to = [](std::ofstream& out, size_t offset, size_t alignment) {
size_t aligned = ((offset + alignment - 1) / alignment) * alignment;
for (size_t i = offset; i < aligned; ++i) out.put('\0');
return aligned;
};
auto write_file = [&](uint64_t n, uint64_t scales_bytes) {
const uint64_t data_bytes = n * k / 2;
std::ofstream out(filename, std::ios::binary);
out.write("CACT", 4);
out.write(reinterpret_cast<const char*>(&flags), sizeof(flags));
out.write(reinterpret_cast<const char*>(&alignment), sizeof(alignment));
uint32_t ndim = 2;
out.write(reinterpret_cast<const char*>(&ndim), sizeof(ndim));
uint64_t dims[4] = {n, k, 0, 0};
out.write(reinterpret_cast<const char*>(dims), sizeof(dims));
out.write(reinterpret_cast<const char*>(&precision), sizeof(precision));
out.write(reinterpret_cast<const char*>(&data_bytes), sizeof(data_bytes));
out.write(reinterpret_cast<const char*>(&scales_bytes), sizeof(scales_bytes));
uint32_t group_size = static_cast<uint32_t>(k);
uint32_t num_groups = 1;
out.write(reinterpret_cast<const char*>(&group_size), sizeof(group_size));
out.write(reinterpret_cast<const char*>(&num_groups), sizeof(num_groups));
out.write(reinterpret_cast<const char*>(&n), sizeof(n));
size_t offset = pad_to(out, 84, alignment);
for (uint64_t i = 0; i < scales_bytes; ++i) out.put('\0');
offset = pad_to(out, offset + static_cast<size_t>(scales_bytes), alignment);
for (uint64_t i = 0; i < data_bytes; ++i) out.put('\0');
};

const uint64_t valid_n = 8;
const uint64_t valid_scales_bytes =
16 * sizeof(__fp16) + 2 * k * sizeof(__fp16) + valid_n * sizeof(__fp16) + k * k * sizeof(__fp16);
write_file(5, 16 * sizeof(__fp16) + 2 * k * sizeof(__fp16) + 5 * sizeof(__fp16) + k * k * sizeof(__fp16));
bool rejected = false;
try {
CactusGraph graph;
graph.mmap_weights(filename);
} catch (const std::exception&) {
rejected = true;
}
if (!rejected) {
std::remove(filename.c_str());
return false;
}

write_file(valid_n, valid_scales_bytes - sizeof(__fp16));
rejected = false;
try {
CactusGraph graph;
graph.mmap_weights(filename);
} catch (const std::exception&) {
rejected = true;
}
std::remove(filename.c_str());
return rejected;
}

bool run_benchmarks() {
const int ITERS = 100;
const std::string temp_file = "bench_io_50nodes.cg";
Expand Down Expand Up @@ -432,6 +495,7 @@ int main() {
runner.run_test("Graph Save/Load Roundtrip Execution", test_graph_save_load_roundtrip_execution());
runner.run_test("Graph Save/Load Supported Ops Roundtrip", test_graph_save_load_supported_ops_roundtrip());
runner.run_test("Graph Save For Inspection", test_graph_save_for_inspection());
runner.run_test("Reject Malformed Interleaved CQ", test_rejects_malformed_interleaved_cq());
runner.print_benchmarks_header();
runner.run_bench("benchmarks", run_benchmarks());
runner.print_summary();
Expand Down
123 changes: 123 additions & 0 deletions cactus-graph/tests/test_ops.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <fstream>
#include <cstring>
#include <cmath>
#include <algorithm>
#include <iostream>
#include <iomanip>
#include <cstdio>
Expand Down Expand Up @@ -552,6 +553,126 @@ bool test_cq_embedding_operation() {
return true;
}

static void set_cq4_index(std::vector<uint8_t>& packed, uint32_t K, uint32_t row, uint32_t col, uint8_t value) {
const size_t byte_index = static_cast<size_t>(row) * (K / 2) + col / 2;
if ((col & 1u) == 0) {
packed[byte_index] = static_cast<uint8_t>((packed[byte_index] & 0xF0) | (value & 0x0F));
} else {
packed[byte_index] = static_cast<uint8_t>((packed[byte_index] & 0x0F) | ((value & 0x0F) << 4));
}
}

static std::vector<uint8_t> make_interleaved_cq4(const std::vector<uint8_t>& row_major, uint32_t N, uint32_t K) {
const uint32_t pgb = cactus_quant_packed_group_bytes(4, K);
std::vector<uint8_t> interleaved(static_cast<size_t>(N / 4) * 4 * pgb, 0);
for (uint32_t row = 0; row < N; ++row) {
const uint8_t* src = row_major.data() + static_cast<size_t>(row) * pgb;
uint8_t* panel = interleaved.data() + static_cast<size_t>(row / 4) * 4 * pgb;
for (uint32_t k = 0; k < K; ++k) {
const uint8_t idx = static_cast<uint8_t>((src[k / 2] >> ((k & 1u) * 4)) & 0x0F);
const uint32_t chunk = k / 8;
const uint32_t sub = k & 7u;
uint8_t& dst = panel[chunk * 16 + (row & 3u) * 4 + (sub & 3u)];
if (sub & 4u) dst = static_cast<uint8_t>((dst & 0x0F) | (idx << 4));
else dst = static_cast<uint8_t>((dst & 0xF0) | idx);
}
}
return interleaved;
}

bool test_orthogonal_interleaved_embedding_row() {
const uint32_t N = 8;
const uint32_t K = 32;
std::vector<__fp16> codebook(16);
for (uint32_t i = 0; i < 16; ++i) codebook[i] = static_cast<__fp16>((static_cast<int>(i) - 8) * 0.125f);
std::vector<__fp16> input_scale_recip(K, static_cast<__fp16>(1.0f));
std::vector<__fp16> norms(N);
for (uint32_t row = 0; row < N; ++row) norms[row] = static_cast<__fp16>(0.5f + 0.03125f * row);
std::vector<__fp16> rotation(static_cast<size_t>(K) * K, static_cast<__fp16>(0.0f));
for (uint32_t i = 0; i < K; ++i) rotation[static_cast<size_t>(i) * K + i] = static_cast<__fp16>(1.0f);

std::vector<uint8_t> row_major(static_cast<size_t>(N) * K / 2, 0);
for (uint32_t row = 0; row < N; ++row)
for (uint32_t col = 0; col < K; ++col)
set_cq4_index(row_major, K, row, col, static_cast<uint8_t>((row * 3 + col) & 0x0F));
std::vector<uint8_t> interleaved = make_interleaved_cq4(row_major, N, K);

for (uint32_t row : {0u, 1u, 3u, 4u, 7u}) {
std::vector<__fp16> ref(K);
std::vector<__fp16> got(K);
cactus_quant_dequantize_orthogonal_embedding_row(
4, K, row, row_major.data(), codebook.data(), norms.data(), input_scale_recip.data(),
rotation.data(), CACTUS_QUANT_FLAG_ORTHOGONAL, ref.data());
cactus_quant_dequantize_orthogonal_embedding_row(
4, K, row, interleaved.data(), codebook.data(), norms.data(), input_scale_recip.data(),
rotation.data(), CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW, got.data());
for (uint32_t i = 0; i < K; ++i) {
if (std::abs(static_cast<float>(ref[i]) - static_cast<float>(got[i])) > 0.001f) return false;
}
}
return true;
}

bool test_orthogonal_interleaved_lmhead_matmul() {
const uint32_t N = 8;
const uint32_t K = 32;
std::vector<__fp16> codebook(16);
for (uint32_t i = 0; i < 16; ++i) codebook[i] = static_cast<__fp16>((static_cast<int>(i) - 8) * 0.125f);
std::vector<__fp16> input_scale_recip(K, static_cast<__fp16>(1.0f));
std::vector<__fp16> norms(N);
for (uint32_t row = 0; row < N; ++row) norms[row] = static_cast<__fp16>(0.05f + 0.003f * row);
std::vector<__fp16> rotation(static_cast<size_t>(K) * K, static_cast<__fp16>(0.0f));
for (uint32_t i = 0; i < K; ++i) {
uint32_t j = (i * 5) % K;
rotation[static_cast<size_t>(i) * K + j] = static_cast<__fp16>((i & 1u) ? -1.0f : 1.0f);
}
std::vector<uint8_t> row_major(static_cast<size_t>(N) * K / 2, 0);
for (uint32_t row = 0; row < N; ++row)
for (uint32_t col = 0; col < K; ++col)
set_cq4_index(row_major, K, row, col, static_cast<uint8_t>((row + col * 5) & 0x0F));
std::vector<uint8_t> interleaved = make_interleaved_cq4(row_major, N, K);
std::vector<__fp16> activation(static_cast<size_t>(2) * K);
for (uint32_t i = 0; i < K; ++i) {
activation[i] = static_cast<__fp16>((static_cast<int>(i % 9) - 4) * 0.1f);
activation[K + i] = static_cast<__fp16>((static_cast<int>((i * 3) % 11) - 5) * 0.07f);
}

CactusQuantMatrix row_mat{
.bits = 4, .K = K, .N = N, .group_size = K, .num_groups = 1,
.flags = CACTUS_QUANT_FLAG_ORTHOGONAL, .codebook = codebook.data(),
.input_scale = nullptr, .input_scale_recip = input_scale_recip.data(), .norms = norms.data(),
.packed_indices = row_major.data(), .left_signs = nullptr, .right_signs = nullptr,
.permutation = nullptr, .rotation = rotation.data(), .expanded = nullptr, .norm_f32 = nullptr,
};
CactusQuantMatrix inter_mat = row_mat;
inter_mat.flags = CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW;
inter_mat.packed_indices = interleaved.data();
std::vector<__fp16> ref(N);
std::vector<__fp16> got(N);
cactus_quant_orthogonal_matmul(&row_mat, activation.data(), 1, ref.data());
cactus_quant_orthogonal_matmul(&inter_mat, activation.data(), 1, got.data());

float diff2 = 0.0f;
float ref2 = 0.0f;
for (uint32_t i = 0; i < N; ++i) {
const float d = static_cast<float>(got[i]) - static_cast<float>(ref[i]);
diff2 += d * d;
ref2 += static_cast<float>(ref[i]) * static_cast<float>(ref[i]);
}
if (std::sqrt(diff2 / std::max(ref2, 1.0e-12f)) >= 0.08f) return false;

std::vector<__fp16> ref_fallback(static_cast<size_t>(2) * N);
std::vector<__fp16> got_fallback(static_cast<size_t>(2) * N);
cactus_quant_orthogonal_matmul(&row_mat, activation.data(), 2, ref_fallback.data());
cactus_quant_orthogonal_matmul(&inter_mat, activation.data(), 2, got_fallback.data());
for (uint32_t i = 0; i < 2 * N; ++i) {
if (std::abs(static_cast<float>(got_fallback[i]) - static_cast<float>(ref_fallback[i])) > 0.001f) {
return false;
}
}
return true;
}

bool test_embedding_from_file() {
CactusGraph graph;

Expand Down Expand Up @@ -657,6 +778,8 @@ int main() {
runner.run_test("Memory-Mapped Gather", test_mmap_gather());
runner.run_test("Embedding Operation", test_embedding_operation());
runner.run_test("CQ Embedding Operation", test_cq_embedding_operation());
runner.run_test("Orthogonal Interleaved Embedding Row", test_orthogonal_interleaved_embedding_row());
runner.run_test("Orthogonal Interleaved LMHead MatMul", test_orthogonal_interleaved_lmhead_matmul());
runner.run_test("Embedding from File", test_embedding_from_file());
runner.print_benchmarks_header();
runner.run_bench("benchmarks", run_benchmarks());
Expand Down
1 change: 1 addition & 0 deletions cactus-kernels/cactus_kernels.h
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,7 @@ void cactus_quant_dequantize_orthogonal_embedding_row(
const __fp16* norms,
const __fp16* input_scale_recip,
const __fp16* rotation,
uint32_t flags,
__fp16* out_row);

void cactus_rms_norm_f16(
Expand Down
Loading
Loading