Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cactus-graph/src/ops_tensor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ void compute_embedding_node(GraphNode& node, const std::vector<std::unique_ptr<G
embeddings_buffer.cq_norms,
embeddings_buffer.cq_input_scale_recip,
embeddings_buffer.cq_rotation,
embeddings_buffer.cq_flags,
output + i * hidden_dim);
} else {
cactus_quant_dequantize_hadamard_embedding_row(
Expand Down
123 changes: 123 additions & 0 deletions cactus-graph/tests/test_ops.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <fstream>
#include <cstring>
#include <cmath>
#include <algorithm>
#include <iostream>
#include <iomanip>
#include <cstdio>
Expand Down Expand Up @@ -552,6 +553,126 @@ bool test_cq_embedding_operation() {
return true;
}

static void set_cq4_index(std::vector<uint8_t>& packed, uint32_t K, uint32_t row, uint32_t col, uint8_t value) {
const size_t byte_index = static_cast<size_t>(row) * (K / 2) + col / 2;
if ((col & 1u) == 0) {
packed[byte_index] = static_cast<uint8_t>((packed[byte_index] & 0xF0) | (value & 0x0F));
} else {
packed[byte_index] = static_cast<uint8_t>((packed[byte_index] & 0x0F) | ((value & 0x0F) << 4));
}
}

static std::vector<uint8_t> make_interleaved_cq4(const std::vector<uint8_t>& row_major, uint32_t N, uint32_t K) {
const uint32_t pgb = cactus_quant_packed_group_bytes(4, K);
std::vector<uint8_t> interleaved(static_cast<size_t>(N / 4) * 4 * pgb, 0);
for (uint32_t row = 0; row < N; ++row) {
const uint8_t* src = row_major.data() + static_cast<size_t>(row) * pgb;
uint8_t* panel = interleaved.data() + static_cast<size_t>(row / 4) * 4 * pgb;
for (uint32_t k = 0; k < K; ++k) {
const uint8_t idx = static_cast<uint8_t>((src[k / 2] >> ((k & 1u) * 4)) & 0x0F);
const uint32_t chunk = k / 8;
const uint32_t sub = k & 7u;
uint8_t& dst = panel[chunk * 16 + (row & 3u) * 4 + (sub & 3u)];
if (sub & 4u) dst = static_cast<uint8_t>((dst & 0x0F) | (idx << 4));
else dst = static_cast<uint8_t>((dst & 0xF0) | idx);
}
}
return interleaved;
}

bool test_orthogonal_interleaved_embedding_row() {
const uint32_t N = 8;
const uint32_t K = 32;
std::vector<__fp16> codebook(16);
for (uint32_t i = 0; i < 16; ++i) codebook[i] = static_cast<__fp16>((static_cast<int>(i) - 8) * 0.125f);
std::vector<__fp16> input_scale_recip(K, static_cast<__fp16>(1.0f));
std::vector<__fp16> norms(N);
for (uint32_t row = 0; row < N; ++row) norms[row] = static_cast<__fp16>(0.5f + 0.03125f * row);
std::vector<__fp16> rotation(static_cast<size_t>(K) * K, static_cast<__fp16>(0.0f));
for (uint32_t i = 0; i < K; ++i) rotation[static_cast<size_t>(i) * K + i] = static_cast<__fp16>(1.0f);

std::vector<uint8_t> row_major(static_cast<size_t>(N) * K / 2, 0);
for (uint32_t row = 0; row < N; ++row)
for (uint32_t col = 0; col < K; ++col)
set_cq4_index(row_major, K, row, col, static_cast<uint8_t>((row * 3 + col) & 0x0F));
std::vector<uint8_t> interleaved = make_interleaved_cq4(row_major, N, K);

for (uint32_t row : {0u, 1u, 3u, 4u, 7u}) {
std::vector<__fp16> ref(K);
std::vector<__fp16> got(K);
cactus_quant_dequantize_orthogonal_embedding_row(
4, K, row, row_major.data(), codebook.data(), norms.data(), input_scale_recip.data(),
rotation.data(), CACTUS_QUANT_FLAG_ORTHOGONAL, ref.data());
cactus_quant_dequantize_orthogonal_embedding_row(
4, K, row, interleaved.data(), codebook.data(), norms.data(), input_scale_recip.data(),
rotation.data(), CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW, got.data());
for (uint32_t i = 0; i < K; ++i) {
if (std::abs(static_cast<float>(ref[i]) - static_cast<float>(got[i])) > 0.001f) return false;
}
}
return true;
}

bool test_orthogonal_interleaved_lmhead_matmul() {
const uint32_t N = 8;
const uint32_t K = 32;
std::vector<__fp16> codebook(16);
for (uint32_t i = 0; i < 16; ++i) codebook[i] = static_cast<__fp16>((static_cast<int>(i) - 8) * 0.125f);
std::vector<__fp16> input_scale_recip(K, static_cast<__fp16>(1.0f));
std::vector<__fp16> norms(N);
for (uint32_t row = 0; row < N; ++row) norms[row] = static_cast<__fp16>(0.05f + 0.003f * row);
std::vector<__fp16> rotation(static_cast<size_t>(K) * K, static_cast<__fp16>(0.0f));
for (uint32_t i = 0; i < K; ++i) {
uint32_t j = (i * 5) % K;
rotation[static_cast<size_t>(i) * K + j] = static_cast<__fp16>((i & 1u) ? -1.0f : 1.0f);
}
std::vector<uint8_t> row_major(static_cast<size_t>(N) * K / 2, 0);
for (uint32_t row = 0; row < N; ++row)
for (uint32_t col = 0; col < K; ++col)
set_cq4_index(row_major, K, row, col, static_cast<uint8_t>((row + col * 5) & 0x0F));
std::vector<uint8_t> interleaved = make_interleaved_cq4(row_major, N, K);
std::vector<__fp16> activation(static_cast<size_t>(2) * K);
for (uint32_t i = 0; i < K; ++i) {
activation[i] = static_cast<__fp16>((static_cast<int>(i % 9) - 4) * 0.1f);
activation[K + i] = static_cast<__fp16>((static_cast<int>((i * 3) % 11) - 5) * 0.07f);
}

CactusQuantMatrix row_mat{
.bits = 4, .K = K, .N = N, .group_size = K, .num_groups = 1,
.flags = CACTUS_QUANT_FLAG_ORTHOGONAL, .codebook = codebook.data(),
.input_scale = nullptr, .input_scale_recip = input_scale_recip.data(), .norms = norms.data(),
.packed_indices = row_major.data(), .left_signs = nullptr, .right_signs = nullptr,
.permutation = nullptr, .rotation = rotation.data(), .expanded = nullptr, .norm_f32 = nullptr,
};
CactusQuantMatrix inter_mat = row_mat;
inter_mat.flags = CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW;
inter_mat.packed_indices = interleaved.data();
std::vector<__fp16> ref(N);
std::vector<__fp16> got(N);
cactus_quant_orthogonal_matmul(&row_mat, activation.data(), 1, ref.data());
cactus_quant_orthogonal_matmul(&inter_mat, activation.data(), 1, got.data());

float diff2 = 0.0f;
float ref2 = 0.0f;
for (uint32_t i = 0; i < N; ++i) {
const float d = static_cast<float>(got[i]) - static_cast<float>(ref[i]);
diff2 += d * d;
ref2 += static_cast<float>(ref[i]) * static_cast<float>(ref[i]);
}
if (std::sqrt(diff2 / std::max(ref2, 1.0e-12f)) >= 0.08f) return false;

std::vector<__fp16> ref_fallback(static_cast<size_t>(2) * N);
std::vector<__fp16> got_fallback(static_cast<size_t>(2) * N);
cactus_quant_orthogonal_matmul(&row_mat, activation.data(), 2, ref_fallback.data());
cactus_quant_orthogonal_matmul(&inter_mat, activation.data(), 2, got_fallback.data());
for (uint32_t i = 0; i < 2 * N; ++i) {
if (std::abs(static_cast<float>(got_fallback[i]) - static_cast<float>(ref_fallback[i])) > 0.001f) {
return false;
}
}
return true;
}

bool test_embedding_from_file() {
CactusGraph graph;

Expand Down Expand Up @@ -657,6 +778,8 @@ int main() {
runner.run_test("Memory-Mapped Gather", test_mmap_gather());
runner.run_test("Embedding Operation", test_embedding_operation());
runner.run_test("CQ Embedding Operation", test_cq_embedding_operation());
runner.run_test("Orthogonal Interleaved Embedding Row", test_orthogonal_interleaved_embedding_row());
runner.run_test("Orthogonal Interleaved LMHead MatMul", test_orthogonal_interleaved_lmhead_matmul());
runner.run_test("Embedding from File", test_embedding_from_file());
runner.print_benchmarks_header();
runner.run_bench("benchmarks", run_benchmarks());
Expand Down
1 change: 1 addition & 0 deletions cactus-kernels/cactus_kernels.h
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,7 @@ void cactus_quant_dequantize_orthogonal_embedding_row(
const __fp16* norms,
const __fp16* input_scale_recip,
const __fp16* rotation,
uint32_t flags,
__fp16* out_row);

void cactus_rms_norm_f16(
Expand Down
138 changes: 133 additions & 5 deletions cactus-kernels/src/matmul.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include <cstring>
#include <algorithm>
#include <cmath>
#include <stdexcept>
#include <vector>

#ifdef __APPLE__
Expand Down Expand Up @@ -1906,17 +1907,34 @@ void cactus_quant_dequantize_orthogonal_embedding_row(
const __fp16* norms,
const __fp16* input_scale_recip,
const __fp16* rotation,
uint32_t flags,
__fp16* out_row) {
if (!packed_base || !codebook || !norms || !rotation || !out_row || bits == 0 || bits > 4) return;
if (K == 0) return;

const uint32_t packed_group_bytes = cactus_quant_packed_group_bytes(bits, K);
const uint8_t* packed = packed_base + static_cast<size_t>(row) * packed_group_bytes;
const float norm = static_cast<float>(norms[row]);
const bool interleaved = (flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) != 0;
if (interleaved && (bits != 4 || (K % 8) != 0)) return;
const uint8_t* packed = interleaved ? nullptr : packed_base + static_cast<size_t>(row) * packed_group_bytes;
const float norm = interleaved
? static_cast<float>(norms[(row / 4) * 4 + (row & 3u)])
: static_cast<float>(norms[row]);

std::vector<float> dq(K);
for (uint32_t i = 0; i < K; ++i) {
dq[i] = static_cast<float>(codebook[tq_extract_idx(packed, i, bits)]);
uint32_t idx = 0;
if (interleaved) {
const size_t panel_bytes = static_cast<size_t>(4) * packed_group_bytes;
const uint8_t* panel = packed_base + (row / 4) * panel_bytes;
const uint32_t chunk = i / 8;
const uint32_t sub = i & 7u;
const uint8_t packed_byte = panel[chunk * 16 + (row & 3u) * 4 + (sub & 3u)];
idx = (sub & 4u) ? static_cast<uint32_t>(packed_byte >> 4)
: static_cast<uint32_t>(packed_byte & 0x0F);
} else {
idx = tq_extract_idx(packed, i, bits);
}
dq[i] = static_cast<float>(codebook[idx]);
}

for (uint32_t j = 0; j < K; ++j) {
Expand All @@ -1929,6 +1947,103 @@ void cactus_quant_dequantize_orthogonal_embedding_row(
}
}

static inline uint32_t tq_extract_interleaved_4row_4bit(
const uint8_t* packed_base,
uint32_t K,
size_t row,
uint32_t k) {
const uint32_t packed_group_bytes = cactus_quant_packed_group_bytes(4, K);
const size_t panel_bytes = static_cast<size_t>(4) * packed_group_bytes;
const uint8_t* panel = packed_base + (row / 4) * panel_bytes;
const uint32_t chunk = k / 8;
const uint32_t sub = k & 7u;
const uint8_t packed_byte = panel[chunk * 16 + (row & 3u) * 4 + (sub & 3u)];
return (sub & 4u) ? static_cast<uint32_t>(packed_byte >> 4)
: static_cast<uint32_t>(packed_byte & 0x0F);
}

static bool cactus_quant_orthogonal_interleaved_lmhead_matmul(
const CactusQuantMatrix* W,
const __fp16* A,
uint32_t M,
__fp16* C) {
if (!W || !A || !C || M != 1) return false;
if (W->bits != 4 || W->num_groups != 1 || W->group_size != W->K) return false;
if ((W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) == 0) return false;
if ((W->K % 32) != 0 || (W->N % 4) != 0) return false;

const uint32_t K = W->K;
const uint32_t N = W->N;
const uint32_t packed_group_bytes = cactus_quant_packed_group_bytes(4, K);
const size_t panel_bytes = static_cast<size_t>(4) * packed_group_bytes;
const size_t N_blocks = N / 4;

thread_local std::vector<float> ar32;
thread_local std::vector<int8_t> act_i8;
if (ar32.size() < K) ar32.resize(K);
if (act_i8.size() < K) act_i8.resize(K);
std::fill(ar32.begin(), ar32.begin() + K, 0.0f);

for (uint32_t k = 0; k < K; ++k) {
float a_val = static_cast<float>(A[k]);
if (W->input_scale_recip) a_val *= static_cast<float>(W->input_scale_recip[k]);
if (a_val == 0.0f) continue;
const __fp16* r = W->rotation + static_cast<size_t>(k) * K;
const float32x4_t av = vdupq_n_f32(a_val);
uint32_t i = 0;
for (; i + 8 <= K; i += 8) {
const float16x8_t rv = vld1q_f16(r + i);
vst1q_f32(ar32.data() + i,
vfmaq_f32(vld1q_f32(ar32.data() + i), av, vcvt_f32_f16(vget_low_f16(rv))));
vst1q_f32(ar32.data() + i + 4,
vfmaq_f32(vld1q_f32(ar32.data() + i + 4), av, vcvt_f32_f16(vget_high_f16(rv))));
}
for (; i < K; ++i) ar32[i] += a_val * static_cast<float>(r[i]);
}

float max_abs = 0.0f;
for (uint32_t i = 0; i < K; ++i) max_abs = std::max(max_abs, std::abs(ar32[i]));
const float act_scale = max_abs > 0.0f ? max_abs / 127.0f : 1.0f;
const float inv_act_scale = 1.0f / act_scale;
for (uint32_t i = 0; i < K; ++i) {
int q = static_cast<int>(std::lrintf(ar32[i] * inv_act_scale));
q = std::max(-127, std::min(127, q));
act_i8[i] = static_cast<int8_t>(q);
}

int8_t cb_i8[16] = {};
const float cb_scale = tq_quantize_codebook_i8(W->codebook, cb_i8, 16);
const int8x16_t cb_lut = vld1q_s8(cb_i8);
const uint8x16_t lo_mask = vdupq_n_u8(0x0F);
const int8_t* act_i8_data = act_i8.data();

cactus_quant_parallel_ranges(N_blocks, 64, [&](size_t block_start, size_t block_end) {
for (size_t nb = block_start; nb < block_end; ++nb) {
const uint8_t* panel = W->packed_indices + nb * panel_bytes;
int32x4_t dot_a = vdupq_n_s32(0);
int32x4_t dot_b = vdupq_n_s32(0);
for (uint32_t kb = 0; kb < K; kb += 16) {
const int8x16_t a_v = vld1q_s8(act_i8_data + kb);
const uint8x16_t p0 = vld1q_u8(panel + (kb / 8 + 0) * 16);
const int8x16_t w0 = vqtbl1q_s8(cb_lut, vandq_u8(p0, lo_mask));
const int8x16_t w1 = vqtbl1q_s8(cb_lut, vshrq_n_u8(p0, 4));
dot_a = CACTUS_DOTQ_LANE(dot_a, w0, a_v, 0);
dot_b = CACTUS_DOTQ_LANE(dot_b, w1, a_v, 1);
const uint8x16_t p1 = vld1q_u8(panel + (kb / 8 + 1) * 16);
const int8x16_t w2 = vqtbl1q_s8(cb_lut, vandq_u8(p1, lo_mask));
const int8x16_t w3 = vqtbl1q_s8(cb_lut, vshrq_n_u8(p1, 4));
dot_a = CACTUS_DOTQ_LANE(dot_a, w2, a_v, 2);
dot_b = CACTUS_DOTQ_LANE(dot_b, w3, a_v, 3);
}
const int32x4_t dot = vaddq_s32(dot_a, dot_b);
float32x4_t scale = vcvt_f32_f16(vld1_f16(W->norms + nb * 4));
scale = vmulq_n_f32(scale, cb_scale * act_scale);
vst1_f16(C + nb * 4, vcvt_f16_f32(vmulq_f32(vcvtq_f32_s32(dot), scale)));
}
});
return true;
}

void cactus_quant_orthogonal_matmul(
const CactusQuantMatrix* W,
const __fp16* A,
Expand All @@ -1941,6 +2056,17 @@ void cactus_quant_orthogonal_matmul(
const uint32_t N = W->N;
const uint32_t bits = W->bits;
const uint32_t pgb = cactus_quant_packed_group_bytes(bits, K);
const bool interleaved = (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) != 0;

if (cactus_quant_orthogonal_interleaved_lmhead_matmul(W, A, M, C)) {
return;
}
const bool invalid_interleaved =
interleaved && (bits != 4 || W->num_groups != 1 || W->group_size != K ||
(K % 8) != 0 || (N % 4) != 0);
if (invalid_interleaved) {
throw std::runtime_error("Invalid orthogonal INTERLEAVED_4ROW matmul layout");
}

const uint32_t n_cb = 1u << bits;

Expand Down Expand Up @@ -1971,7 +2097,7 @@ void cactus_quant_orthogonal_matmul(
}
}

if (bits == 4 && (K % 16) == 0) {
if (!interleaved && bits == 4 && (K % 16) == 0) {
__fp16 cb_f16[16];
uint8_t cb_lo_arr[16], cb_hi_arr[16];
for (uint32_t c = 0; c < 16; ++c) {
Expand Down Expand Up @@ -2057,7 +2183,9 @@ void cactus_quant_orthogonal_matmul(
const float* ar = A_rot.data() + static_cast<size_t>(m) * K;
float acc = 0.0f;
for (uint32_t i = 0; i < K; ++i) {
uint8_t idx = static_cast<uint8_t>(tq_extract_idx(packed, i, bits));
uint8_t idx = interleaved
? static_cast<uint8_t>(tq_extract_interleaved_4row_4bit(W->packed_indices, K, n, i))
: static_cast<uint8_t>(tq_extract_idx(packed, i, bits));
acc += cb_f32[idx] * ar[i];
}
C[static_cast<size_t>(m) * N + n] = static_cast<__fp16>(acc * norm_n);
Expand Down
19 changes: 18 additions & 1 deletion python/cactus/convert/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,20 @@ def _scale_cq_norms(cq, factor: float):
return replace(cq, norms=(cq.norms.astype(np.float32) * float(factor)).astype(np.float16))


def _validate_cq_layout(policy, shape: tuple[int, ...], source_name: str, output_name: str) -> None:
if getattr(policy, "layout", "row_major") != "interleaved_4row":
return
if policy.rotation != "orthogonal" or int(policy.bits or 0) != 4:
raise RuntimeError(f"{source_name}: INTERLEAVED_4ROW output {output_name} requires orthogonal CQ4")
if len(shape) != 2:
raise RuntimeError(f"{source_name}: INTERLEAVED_4ROW output {output_name} requires rank-2 tensor, got shape={shape}")
n, k = int(shape[0]), int(shape[1])
if n % 4 != 0 or k % 32 != 0:
raise RuntimeError(
f"{source_name}: INTERLEAVED_4ROW output {output_name} requires N % 4 == 0 and K % 32 == 0, got shape={shape}"
)


def _adapt_tensor_for_cactus(tensor, output_name: str | None, family: str):
match = type("_CompatMatch", (), {"output_name": output_name})()
transformed, _name = adapter_for_family(family).transform_tensor(match, tensor)
Expand Down Expand Up @@ -481,6 +495,7 @@ def convert(args: argparse.Namespace) -> None:
if not match.recognized and args.strict:
raise RuntimeError(f"unrecognized tensor in strict mode: {name}")
if emit_policy.action == "convert" and out_path is not None:
_validate_cq_layout(emit_policy, _tensor_shape(emit_tensor), name, out_path.name)
if emit_policy.use_gptq and int(hessian_samples.get(module_name, 0)) <= 0:
hessian_missing_reason = "expected GPTQ target had zero samples"
if args.strict:
Expand All @@ -502,6 +517,8 @@ def convert(args: argparse.Namespace) -> None:
input_scale=input_scale,
)
cq = _scale_cq_norms(cq, adapter.scale_factor(out_path.name))
if getattr(emit_policy, "layout", "row_major") == "interleaved_4row":
cq = replace(cq, interleaved_4row=True)
write_cq_tensor(out_path, cq)
gptq_used = cq.gptq_used
status = "converted" if match.recognized else "unrecognized"
Expand All @@ -511,7 +528,7 @@ def convert(args: argparse.Namespace) -> None:
elif emit_policy.action == "ignored":
status = "ignored"
except Exception as exc:
if args.strict:
if args.strict or getattr(emit_policy, "layout", "row_major") == "interleaved_4row":
raise
if out_path is not None:
_save_fallback_tensor(emit_tensor, out_path, "FP16", family)
Expand Down
Loading
Loading