Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 67 additions & 54 deletions cactus-kernels/src/matmul.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2492,43 +2492,40 @@ void cactus_quant_3bit_gemv_interleaved(
if (W->bits != 3 || W->N % 4 != 0 || (W->group_size % 32) != 0 || W->group_size > 256) return;
const uint32_t gs = W->group_size;
const uint32_t num_groups = W->num_groups;
const size_t panel_bytes = static_cast<size_t>(gs) * 3 / 2;
const size_t N_blocks = W->N / 4;
const size_t n_chunks = (N_blocks + 15) / 16;
auto& pool = CactusThreading::get_thread_pool();
const size_t sb_per_thread = cactus_quant_gemv_sb_per_thread();
const size_t nt_budget = std::max<size_t>(1, (n_chunks + sb_per_thread - 1) / sb_per_thread);
const size_t nt = std::min(pool.num_workers(), std::min(nt_budget, n_chunks));

thread_local std::vector<__fp16> code_basis_buf;
if (code_basis_buf.size() < W->K) code_basis_buf.resize(W->K);
cactus_quant_transform_hadamard_activations(*W, x, 1, code_basis_buf.data());
const __fp16* code_basis = code_basis_buf.data();

thread_local std::vector<int8_t> act_i8_buf;
thread_local std::vector<float> act_scales_buf;
if (act_i8_buf.size() < W->K) act_i8_buf.resize(W->K);
if (act_scales_buf.size() < num_groups) act_scales_buf.resize(num_groups);
for (uint32_t g = 0; g < num_groups; ++g) {
act_scales_buf[g] = tq_quantize_group_i8(
code_basis + static_cast<size_t>(g) * gs,
act_i8_buf.data() + static_cast<size_t>(g) * gs, gs);
}
const int8_t* act_i8 = act_i8_buf.data();
const float* act_scales = act_scales_buf.data();
static thread_local std::vector<int8_t> tl_act_i8;
static thread_local std::vector<float> tl_act_scales;
if (tl_act_i8.size() < W->K) tl_act_i8.resize(W->K);
if (tl_act_scales.size() < num_groups) tl_act_scales.resize(num_groups);
int8_t* act_i8 = tl_act_i8.data();
float* act_scales = tl_act_scales.data();

int8_t cb_i8[16] = {};
const float cb_scale = tq_quantize_codebook_i8(W->codebook, cb_i8, 8); // 8 codebook entries
const float cb_scale = tq_quantize_codebook_i8(W->codebook, cb_i8, 8);
const int8x16_t cb_lut = vld1q_s8(cb_i8);

const size_t panel_bytes = static_cast<size_t>(gs) * 3 / 2;
const size_t chunks_per_panel = gs / 4;
const size_t N_blocks = W->N / 4;
auto phase_a_group = [&](uint32_t g) {
__fp16 basis[256];
cactus_quant_transform_hadamard_group(*W, x + static_cast<size_t>(g) * gs, g, basis);
act_scales[g] = tq_quantize_group_i8(basis, act_i8 + static_cast<size_t>(g) * gs, gs);
};

cactus_quant_parallel_ranges(N_blocks, 64, [&](size_t block_start, size_t block_end) {
for (size_t nb = block_start; nb < block_end; ++nb) {
auto phase_b = [&](size_t b0, size_t b1) {
for (size_t nb = b0; nb < b1; ++nb) {
float32x4_t acc = vdupq_n_f32(0.f);
for (uint32_t g = 0; g < num_groups; ++g) {
const uint8_t* p_base = packed_interleaved + (nb * num_groups + g) * panel_bytes;
const int8_t* a_grp = act_i8 + static_cast<size_t>(g) * gs;
const float a_sc = act_scales[g];

int32x4_t dot_a = vdupq_n_s32(0);
int32x4_t dot_b = vdupq_n_s32(0);

for (uint32_t kb = 0; kb < gs; kb += 16) {
int8x16_t a_v = vld1q_s8(a_grp + kb);
const uint8_t* band_base = p_base + (kb / 16) * (4 * 6);
Expand All @@ -2541,16 +2538,26 @@ void cactus_quant_3bit_gemv_interleaved(
dot_a = vdotq_laneq_s32(dot_a, w2, a_v, 2);
dot_b = vdotq_laneq_s32(dot_b, w3, a_v, 3);
}

int32x4_t dot = vaddq_s32(dot_a, dot_b);
float32x4_t norm = vcvt_f32_f16(vld1_f16(norms_interleaved + (nb * num_groups + g) * 4));
norm = vmulq_n_f32(norm, cb_scale * a_sc);
acc = vfmaq_f32(acc, vcvtq_f32_s32(dot), norm);
}
vst1_f16(y + nb * 4, vcvt_f16_f32(acc));
}
});
(void)chunks_per_panel;
};

if (nt <= 1) {
for (uint32_t g = 0; g < num_groups; ++g) phase_a_group(g);
phase_b(0, N_blocks);
return;
}
cactus_quant_two_phase_run(nt, num_groups, static_cast<uint32_t>(n_chunks), phase_a_group,
[&](size_t, uint32_t ck, uint32_t cnt) {
const size_t b0 = static_cast<size_t>(ck) * 16;
const size_t b1 = std::min(N_blocks, b0 + static_cast<size_t>(cnt) * 16);
phase_b(b0, b1);
});
}

void cactus_quant_2bit_gemv_interleaved(
Expand All @@ -2564,23 +2571,20 @@ void cactus_quant_2bit_gemv_interleaved(
const uint32_t gs = W->group_size;
const uint32_t pgb = cactus_quant_packed_group_bytes(2, gs);
const uint32_t num_groups = W->num_groups;
const size_t panel_bytes = 4 * pgb;
const size_t N_blocks = W->N / 4;
const size_t n_chunks = (N_blocks + 15) / 16;
auto& pool = CactusThreading::get_thread_pool();
const size_t sb_per_thread = cactus_quant_gemv_sb_per_thread();
const size_t nt_budget = std::max<size_t>(1, (n_chunks + sb_per_thread - 1) / sb_per_thread);
const size_t nt = std::min(pool.num_workers(), std::min(nt_budget, n_chunks));

thread_local std::vector<__fp16> code_basis_buf;
if (code_basis_buf.size() < W->K) code_basis_buf.resize(W->K);
cactus_quant_transform_hadamard_activations(*W, x, 1, code_basis_buf.data());
const __fp16* code_basis = code_basis_buf.data();

thread_local std::vector<int8_t> act_i8_buf;
thread_local std::vector<float> act_scales_buf;
if (act_i8_buf.size() < W->K) act_i8_buf.resize(W->K);
if (act_scales_buf.size() < num_groups) act_scales_buf.resize(num_groups);
for (uint32_t g = 0; g < num_groups; ++g) {
act_scales_buf[g] = tq_quantize_group_i8(
code_basis + static_cast<size_t>(g) * gs,
act_i8_buf.data() + static_cast<size_t>(g) * gs, gs);
}
const int8_t* act_i8 = act_i8_buf.data();
const float* act_scales = act_scales_buf.data();
static thread_local std::vector<int8_t> tl_act_i8;
static thread_local std::vector<float> tl_act_scales;
if (tl_act_i8.size() < W->K) tl_act_i8.resize(W->K);
if (tl_act_scales.size() < num_groups) tl_act_scales.resize(num_groups);
int8_t* act_i8 = tl_act_i8.data();
float* act_scales = tl_act_scales.data();

int8_t cb_i8[16] = {};
const float cb_scale = tq_quantize_codebook_i8(W->codebook, cb_i8, 4);
Expand All @@ -2595,50 +2599,59 @@ void cactus_quant_2bit_gemv_interleaved(
const uint8x16_t lookup_s3 = (uint8x16_t){12,12,12,12, 13,13,13,13, 14,14,14,14, 15,15,15,15};
const uint8x16_t idx_mask = vdupq_n_u8(0x03);

const size_t panel_bytes = 4 * pgb;
const size_t N_blocks = W->N / 4;
auto phase_a_group = [&](uint32_t g) {
__fp16 basis[256];
cactus_quant_transform_hadamard_group(*W, x + static_cast<size_t>(g) * gs, g, basis);
act_scales[g] = tq_quantize_group_i8(basis, act_i8 + static_cast<size_t>(g) * gs, gs);
};

cactus_quant_parallel_ranges(N_blocks, 64, [&](size_t block_start, size_t block_end) {
for (size_t nb = block_start; nb < block_end; ++nb) {
auto phase_b = [&](size_t b0, size_t b1) {
for (size_t nb = b0; nb < b1; ++nb) {
float32x4_t acc = vdupq_n_f32(0.f);
for (uint32_t g = 0; g < num_groups; ++g) {
const uint8_t* p_base = packed_interleaved + (nb * num_groups + g) * panel_bytes;
const int8_t* a_grp = act_i8 + static_cast<size_t>(g) * gs;
const float a_sc = act_scales[g];

int32x4_t dot_a = vdupq_n_s32(0);
int32x4_t dot_b = vdupq_n_s32(0);

for (uint32_t chunk = 0; chunk < gs / 16; ++chunk) {
int8x16_t a_v = vld1q_s8(a_grp + chunk * 16);
uint8x16_t bytes = vld1q_u8(p_base + chunk * 16);

auto unpack_set = [&](uint8x16_t lookup) -> int8x16_t {
uint8x16_t spread = vqtbl1q_u8(bytes, lookup);
uint8x16_t shifted = vshlq_u8(spread, shifts);
uint8x16_t idx = vandq_u8(shifted, idx_mask);
return vqtbl1q_s8(cb_lut, idx);
};

int8x16_t w0 = unpack_set(lookup_s0);
int8x16_t w1 = unpack_set(lookup_s1);
int8x16_t w2 = unpack_set(lookup_s2);
int8x16_t w3 = unpack_set(lookup_s3);

dot_a = vdotq_laneq_s32(dot_a, w0, a_v, 0);
dot_b = vdotq_laneq_s32(dot_b, w1, a_v, 1);
dot_a = vdotq_laneq_s32(dot_a, w2, a_v, 2);
dot_b = vdotq_laneq_s32(dot_b, w3, a_v, 3);
}

int32x4_t dot = vaddq_s32(dot_a, dot_b);
float32x4_t norm = vcvt_f32_f16(vld1_f16(norms_interleaved + (nb * num_groups + g) * 4));
norm = vmulq_n_f32(norm, cb_scale * a_sc);
acc = vfmaq_f32(acc, vcvtq_f32_s32(dot), norm);
}
vst1_f16(y + nb * 4, vcvt_f16_f32(acc));
}
});
};

if (nt <= 1) {
for (uint32_t g = 0; g < num_groups; ++g) phase_a_group(g);
phase_b(0, N_blocks);
return;
}
cactus_quant_two_phase_run(nt, num_groups, static_cast<uint32_t>(n_chunks), phase_a_group,
[&](size_t, uint32_t ck, uint32_t cnt) {
const size_t b0 = static_cast<size_t>(ck) * 16;
const size_t b1 = std::min(N_blocks, b0 + static_cast<size_t>(cnt) * 16);
phase_b(b0, b1);
});
}

void cactus_quant_1bit_gemv_interleaved(
Expand Down
Loading