From 4b0ccb3a276a662a3cddb7eccb201138c298d3a6 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Mon, 25 May 2026 16:54:08 -0700 Subject: [PATCH] Optimize orthogonal CQ4 LM head layout Add support for using the existing 4-row interleaved CQ4 layout with full-width orthogonal output heads. The LM-head path now dispatches to an optimized interleaved decode kernel while preserving the scalar fallback for unsupported shapes. Teach CQ weight export, QDQ, and runtime loading about explicit INTERLEAVED_4ROW storage for orthogonal CQ4 tensors. The loader validates malformed interleaved tensors strictly, including metadata byte counts, embedding lookup can read tied interleaved weights, and the converter policy selects this layout for large output heads and tied token embeddings. Add a temporary artifact patching tool for rewriting existing full-width orthogonal CQ4 weight files to the interleaved layout without running a full reconversion. This keeps current artifacts testable while the converter pipeline is updated. Validation: pytest python/cactus/convert/tests/test_cq.py python/cactus/convert/tests/test_policy.py python/tests/test_transpile_weight_compat.py -q; cactus-graph/build/test_ops; cactus-graph/build/test_io; cactus-kernels/build/test_matmul; cactus build --android; local e2e generation checks for Gemma, LFM, and Qwen temp interleaved bundles; Pixel Gemma shared-KV generation and token benchmark with interleaved and row-major tied weights. Signed-off-by: Noah Cylich --- cactus-graph/src/io.cpp | 58 ++++++++ cactus-graph/src/ops_tensor.cpp | 1 + cactus-graph/tests/test_io.cpp | 64 +++++++++ cactus-graph/tests/test_ops.cpp | 123 ++++++++++++++++ cactus-kernels/cactus_kernels.h | 1 + cactus-kernels/src/matmul.cpp | 135 +++++++++++++++++- python/cactus/convert/cli.py | 19 ++- python/cactus/convert/export/qdq.py | 25 +++- .../convert/interleave_orthogonal_cq4.py | 95 ++++++++++++ .../cactus/convert/model_adapters/policy.py | 7 +- python/cactus/convert/quantization/cq.py | 31 +++- python/cactus/convert/tests/test_cq.py | 33 +++++ python/cactus/convert/tests/test_policy.py | 40 +++++- python/cactus/transpile/weight_compat.py | 19 ++- python/tests/test_transpile_weight_compat.py | 30 ++++ 15 files changed, 661 insertions(+), 20 deletions(-) create mode 100644 python/cactus/convert/interleave_orthogonal_cq4.py diff --git a/cactus-graph/src/io.cpp b/cactus-graph/src/io.cpp index 7165b59f0..13908a99e 100644 --- a/cactus-graph/src/io.cpp +++ b/cactus-graph/src/io.cpp @@ -52,6 +52,59 @@ namespace { return filename; } + void validate_interleaved_4row_tensor( + Precision precision, + const std::vector& shape, + size_t group_size, + size_t num_groups, + size_t byte_size, + size_t scales_bytes, + bool orthogonal) { + if (!PrecisionTraits::is_cq(precision) || group_size == 0 || num_groups == 0) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires grouped CQ precision"); + } + if (shape.size() != 2) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires a rank-2 tensor"); + } + const uint32_t bits = PrecisionTraits::cq_bits(precision); + const size_t n = shape[0]; + const size_t k = shape[1]; + if (bits < 1 || bits > 4) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW only supports CQ1..CQ4"); + } + if ((n % 4) != 0) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires N % 4 == 0"); + } + if (group_size * num_groups != k) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW group metadata does not match tensor shape"); + } + if (orthogonal) { + if (bits != 4 || num_groups != 1 || group_size != k || (k % 32) != 0) { + throw std::runtime_error("Invalid tensor file: orthogonal INTERLEAVED_4ROW requires full-width CQ4 with K % 32 == 0"); + } + } else if ((group_size % 32) != 0 || group_size > 256) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires group_size % 32 == 0 and group_size <= 256"); + } + + const size_t packed_group_bytes = (group_size * bits + 7) / 8; + const size_t expected_bytes = n * num_groups * packed_group_bytes; + if (byte_size != expected_bytes) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW packed byte size mismatch"); + } + + const size_t codebook_bytes = (static_cast(1) << bits) * sizeof(__fp16); + const size_t input_scale_bytes = k * sizeof(__fp16); + const size_t norms_bytes = n * num_groups * sizeof(__fp16); + const size_t rotation_bytes = orthogonal + ? k * k * sizeof(__fp16) + : 2 * group_size + group_size * sizeof(uint32_t); + const size_t expected_scales_bytes = + codebook_bytes + 2 * input_scale_bytes + norms_bytes + rotation_bytes; + if (scales_bytes != expected_scales_bytes) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW metadata byte size mismatch"); + } + } + inline void write_u32(std::ostream& out, uint32_t v) { out.write(reinterpret_cast(&v), sizeof(v)); } @@ -1018,6 +1071,11 @@ void MappedFile::parse_header() { } } + if (is_interleaved_4row_) { + validate_interleaved_4row_tensor( + precision_, shape_, group_size_, num_groups_, byte_size_, scales_bytes_, is_orthogonal_rotation_); + } + size_t aligned_header = align_offset(header_size, alignment_); if (scales_bytes_ > 0) { diff --git a/cactus-graph/src/ops_tensor.cpp b/cactus-graph/src/ops_tensor.cpp index 42f967ce5..857c677b9 100644 --- a/cactus-graph/src/ops_tensor.cpp +++ b/cactus-graph/src/ops_tensor.cpp @@ -186,6 +186,7 @@ void compute_embedding_node(GraphNode& node, const std::vector(Precision::CQ4); + auto pad_to = [](std::ofstream& out, size_t offset, size_t alignment) { + size_t aligned = ((offset + alignment - 1) / alignment) * alignment; + for (size_t i = offset; i < aligned; ++i) out.put('\0'); + return aligned; + }; + auto write_file = [&](uint64_t n, uint64_t scales_bytes) { + const uint64_t data_bytes = n * k / 2; + std::ofstream out(filename, std::ios::binary); + out.write("CACT", 4); + out.write(reinterpret_cast(&flags), sizeof(flags)); + out.write(reinterpret_cast(&alignment), sizeof(alignment)); + uint32_t ndim = 2; + out.write(reinterpret_cast(&ndim), sizeof(ndim)); + uint64_t dims[4] = {n, k, 0, 0}; + out.write(reinterpret_cast(dims), sizeof(dims)); + out.write(reinterpret_cast(&precision), sizeof(precision)); + out.write(reinterpret_cast(&data_bytes), sizeof(data_bytes)); + out.write(reinterpret_cast(&scales_bytes), sizeof(scales_bytes)); + uint32_t group_size = static_cast(k); + uint32_t num_groups = 1; + out.write(reinterpret_cast(&group_size), sizeof(group_size)); + out.write(reinterpret_cast(&num_groups), sizeof(num_groups)); + out.write(reinterpret_cast(&n), sizeof(n)); + size_t offset = pad_to(out, 84, alignment); + for (uint64_t i = 0; i < scales_bytes; ++i) out.put('\0'); + offset = pad_to(out, offset + static_cast(scales_bytes), alignment); + for (uint64_t i = 0; i < data_bytes; ++i) out.put('\0'); + }; + + const uint64_t valid_n = 8; + const uint64_t valid_scales_bytes = + 16 * sizeof(__fp16) + 2 * k * sizeof(__fp16) + valid_n * sizeof(__fp16) + k * k * sizeof(__fp16); + write_file(5, 16 * sizeof(__fp16) + 2 * k * sizeof(__fp16) + 5 * sizeof(__fp16) + k * k * sizeof(__fp16)); + bool rejected = false; + try { + CactusGraph graph; + graph.mmap_weights(filename); + } catch (const std::exception&) { + rejected = true; + } + if (!rejected) { + std::remove(filename.c_str()); + return false; + } + + write_file(valid_n, valid_scales_bytes - sizeof(__fp16)); + rejected = false; + try { + CactusGraph graph; + graph.mmap_weights(filename); + } catch (const std::exception&) { + rejected = true; + } + std::remove(filename.c_str()); + return rejected; +} + bool run_benchmarks() { const int ITERS = 100; const std::string temp_file = "bench_io_50nodes.cg"; @@ -432,6 +495,7 @@ int main() { runner.run_test("Graph Save/Load Roundtrip Execution", test_graph_save_load_roundtrip_execution()); runner.run_test("Graph Save/Load Supported Ops Roundtrip", test_graph_save_load_supported_ops_roundtrip()); runner.run_test("Graph Save For Inspection", test_graph_save_for_inspection()); + runner.run_test("Reject Malformed Interleaved CQ", test_rejects_malformed_interleaved_cq()); runner.print_benchmarks_header(); runner.run_bench("benchmarks", run_benchmarks()); runner.print_summary(); diff --git a/cactus-graph/tests/test_ops.cpp b/cactus-graph/tests/test_ops.cpp index 2b015d4ac..061f45a79 100644 --- a/cactus-graph/tests/test_ops.cpp +++ b/cactus-graph/tests/test_ops.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -552,6 +553,126 @@ bool test_cq_embedding_operation() { return true; } +static void set_cq4_index(std::vector& packed, uint32_t K, uint32_t row, uint32_t col, uint8_t value) { + const size_t byte_index = static_cast(row) * (K / 2) + col / 2; + if ((col & 1u) == 0) { + packed[byte_index] = static_cast((packed[byte_index] & 0xF0) | (value & 0x0F)); + } else { + packed[byte_index] = static_cast((packed[byte_index] & 0x0F) | ((value & 0x0F) << 4)); + } +} + +static std::vector make_interleaved_cq4(const std::vector& row_major, uint32_t N, uint32_t K) { + const uint32_t pgb = cactus_quant_packed_group_bytes(4, K); + std::vector interleaved(static_cast(N / 4) * 4 * pgb, 0); + for (uint32_t row = 0; row < N; ++row) { + const uint8_t* src = row_major.data() + static_cast(row) * pgb; + uint8_t* panel = interleaved.data() + static_cast(row / 4) * 4 * pgb; + for (uint32_t k = 0; k < K; ++k) { + const uint8_t idx = static_cast((src[k / 2] >> ((k & 1u) * 4)) & 0x0F); + const uint32_t chunk = k / 8; + const uint32_t sub = k & 7u; + uint8_t& dst = panel[chunk * 16 + (row & 3u) * 4 + (sub & 3u)]; + if (sub & 4u) dst = static_cast((dst & 0x0F) | (idx << 4)); + else dst = static_cast((dst & 0xF0) | idx); + } + } + return interleaved; +} + +bool test_orthogonal_interleaved_embedding_row() { + const uint32_t N = 8; + const uint32_t K = 32; + std::vector<__fp16> codebook(16); + for (uint32_t i = 0; i < 16; ++i) codebook[i] = static_cast<__fp16>((static_cast(i) - 8) * 0.125f); + std::vector<__fp16> input_scale_recip(K, static_cast<__fp16>(1.0f)); + std::vector<__fp16> norms(N); + for (uint32_t row = 0; row < N; ++row) norms[row] = static_cast<__fp16>(0.5f + 0.03125f * row); + std::vector<__fp16> rotation(static_cast(K) * K, static_cast<__fp16>(0.0f)); + for (uint32_t i = 0; i < K; ++i) rotation[static_cast(i) * K + i] = static_cast<__fp16>(1.0f); + + std::vector row_major(static_cast(N) * K / 2, 0); + for (uint32_t row = 0; row < N; ++row) + for (uint32_t col = 0; col < K; ++col) + set_cq4_index(row_major, K, row, col, static_cast((row * 3 + col) & 0x0F)); + std::vector interleaved = make_interleaved_cq4(row_major, N, K); + + for (uint32_t row : {0u, 1u, 3u, 4u, 7u}) { + std::vector<__fp16> ref(K); + std::vector<__fp16> got(K); + cactus_quant_dequantize_orthogonal_embedding_row( + 4, K, row, row_major.data(), codebook.data(), norms.data(), input_scale_recip.data(), + rotation.data(), CACTUS_QUANT_FLAG_ORTHOGONAL, ref.data()); + cactus_quant_dequantize_orthogonal_embedding_row( + 4, K, row, interleaved.data(), codebook.data(), norms.data(), input_scale_recip.data(), + rotation.data(), CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW, got.data()); + for (uint32_t i = 0; i < K; ++i) { + if (std::abs(static_cast(ref[i]) - static_cast(got[i])) > 0.001f) return false; + } + } + return true; +} + +bool test_orthogonal_interleaved_lmhead_matmul() { + const uint32_t N = 8; + const uint32_t K = 32; + std::vector<__fp16> codebook(16); + for (uint32_t i = 0; i < 16; ++i) codebook[i] = static_cast<__fp16>((static_cast(i) - 8) * 0.125f); + std::vector<__fp16> input_scale_recip(K, static_cast<__fp16>(1.0f)); + std::vector<__fp16> norms(N); + for (uint32_t row = 0; row < N; ++row) norms[row] = static_cast<__fp16>(0.05f + 0.003f * row); + std::vector<__fp16> rotation(static_cast(K) * K, static_cast<__fp16>(0.0f)); + for (uint32_t i = 0; i < K; ++i) { + uint32_t j = (i * 5) % K; + rotation[static_cast(i) * K + j] = static_cast<__fp16>((i & 1u) ? -1.0f : 1.0f); + } + std::vector row_major(static_cast(N) * K / 2, 0); + for (uint32_t row = 0; row < N; ++row) + for (uint32_t col = 0; col < K; ++col) + set_cq4_index(row_major, K, row, col, static_cast((row + col * 5) & 0x0F)); + std::vector interleaved = make_interleaved_cq4(row_major, N, K); + std::vector<__fp16> activation(static_cast(2) * K); + for (uint32_t i = 0; i < K; ++i) { + activation[i] = static_cast<__fp16>((static_cast(i % 9) - 4) * 0.1f); + activation[K + i] = static_cast<__fp16>((static_cast((i * 3) % 11) - 5) * 0.07f); + } + + CactusQuantMatrix row_mat{ + .bits = 4, .K = K, .N = N, .group_size = K, .num_groups = 1, + .flags = CACTUS_QUANT_FLAG_ORTHOGONAL, .codebook = codebook.data(), + .input_scale = nullptr, .input_scale_recip = input_scale_recip.data(), .norms = norms.data(), + .packed_indices = row_major.data(), .left_signs = nullptr, .right_signs = nullptr, + .permutation = nullptr, .rotation = rotation.data(), .expanded = nullptr, .norm_f32 = nullptr, + }; + CactusQuantMatrix inter_mat = row_mat; + inter_mat.flags = CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW; + inter_mat.packed_indices = interleaved.data(); + std::vector<__fp16> ref(N); + std::vector<__fp16> got(N); + cactus_quant_orthogonal_matmul(&row_mat, activation.data(), 1, ref.data()); + cactus_quant_orthogonal_matmul(&inter_mat, activation.data(), 1, got.data()); + + float diff2 = 0.0f; + float ref2 = 0.0f; + for (uint32_t i = 0; i < N; ++i) { + const float d = static_cast(got[i]) - static_cast(ref[i]); + diff2 += d * d; + ref2 += static_cast(ref[i]) * static_cast(ref[i]); + } + if (std::sqrt(diff2 / std::max(ref2, 1.0e-12f)) >= 0.08f) return false; + + std::vector<__fp16> ref_fallback(static_cast(2) * N); + std::vector<__fp16> got_fallback(static_cast(2) * N); + cactus_quant_orthogonal_matmul(&row_mat, activation.data(), 2, ref_fallback.data()); + cactus_quant_orthogonal_matmul(&inter_mat, activation.data(), 2, got_fallback.data()); + for (uint32_t i = 0; i < 2 * N; ++i) { + if (std::abs(static_cast(got_fallback[i]) - static_cast(ref_fallback[i])) > 0.001f) { + return false; + } + } + return true; +} + bool test_embedding_from_file() { CactusGraph graph; @@ -657,6 +778,8 @@ int main() { runner.run_test("Memory-Mapped Gather", test_mmap_gather()); runner.run_test("Embedding Operation", test_embedding_operation()); runner.run_test("CQ Embedding Operation", test_cq_embedding_operation()); + runner.run_test("Orthogonal Interleaved Embedding Row", test_orthogonal_interleaved_embedding_row()); + runner.run_test("Orthogonal Interleaved LMHead MatMul", test_orthogonal_interleaved_lmhead_matmul()); runner.run_test("Embedding from File", test_embedding_from_file()); runner.print_benchmarks_header(); runner.run_bench("benchmarks", run_benchmarks()); diff --git a/cactus-kernels/cactus_kernels.h b/cactus-kernels/cactus_kernels.h index f79ef7169..f371cffbe 100644 --- a/cactus-kernels/cactus_kernels.h +++ b/cactus-kernels/cactus_kernels.h @@ -333,6 +333,7 @@ void cactus_quant_dequantize_orthogonal_embedding_row( const __fp16* norms, const __fp16* input_scale_recip, const __fp16* rotation, + uint32_t flags, __fp16* out_row); void cactus_rms_norm_f16( diff --git a/cactus-kernels/src/matmul.cpp b/cactus-kernels/src/matmul.cpp index b244388b3..151e11711 100644 --- a/cactus-kernels/src/matmul.cpp +++ b/cactus-kernels/src/matmul.cpp @@ -1906,17 +1906,34 @@ void cactus_quant_dequantize_orthogonal_embedding_row( const __fp16* norms, const __fp16* input_scale_recip, const __fp16* rotation, + uint32_t flags, __fp16* out_row) { if (!packed_base || !codebook || !norms || !rotation || !out_row || bits == 0 || bits > 4) return; if (K == 0) return; const uint32_t packed_group_bytes = cactus_quant_packed_group_bytes(bits, K); - const uint8_t* packed = packed_base + static_cast(row) * packed_group_bytes; - const float norm = static_cast(norms[row]); + const bool interleaved = (flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) != 0; + if (interleaved && (bits != 4 || (K % 8) != 0)) return; + const uint8_t* packed = interleaved ? nullptr : packed_base + static_cast(row) * packed_group_bytes; + const float norm = interleaved + ? static_cast(norms[(row / 4) * 4 + (row & 3u)]) + : static_cast(norms[row]); std::vector dq(K); for (uint32_t i = 0; i < K; ++i) { - dq[i] = static_cast(codebook[tq_extract_idx(packed, i, bits)]); + uint32_t idx = 0; + if (interleaved) { + const size_t panel_bytes = static_cast(4) * packed_group_bytes; + const uint8_t* panel = packed_base + (row / 4) * panel_bytes; + const uint32_t chunk = i / 8; + const uint32_t sub = i & 7u; + const uint8_t packed_byte = panel[chunk * 16 + (row & 3u) * 4 + (sub & 3u)]; + idx = (sub & 4u) ? static_cast(packed_byte >> 4) + : static_cast(packed_byte & 0x0F); + } else { + idx = tq_extract_idx(packed, i, bits); + } + dq[i] = static_cast(codebook[idx]); } for (uint32_t j = 0; j < K; ++j) { @@ -1929,6 +1946,103 @@ void cactus_quant_dequantize_orthogonal_embedding_row( } } +static inline uint32_t tq_extract_interleaved_4row_4bit( + const uint8_t* packed_base, + uint32_t K, + size_t row, + uint32_t k) { + const uint32_t packed_group_bytes = cactus_quant_packed_group_bytes(4, K); + const size_t panel_bytes = static_cast(4) * packed_group_bytes; + const uint8_t* panel = packed_base + (row / 4) * panel_bytes; + const uint32_t chunk = k / 8; + const uint32_t sub = k & 7u; + const uint8_t packed_byte = panel[chunk * 16 + (row & 3u) * 4 + (sub & 3u)]; + return (sub & 4u) ? static_cast(packed_byte >> 4) + : static_cast(packed_byte & 0x0F); +} + +static bool cactus_quant_orthogonal_interleaved_lmhead_matmul( + const CactusQuantMatrix* W, + const __fp16* A, + uint32_t M, + __fp16* C) { + if (!W || !A || !C || M != 1) return false; + if (W->bits != 4 || W->num_groups != 1 || W->group_size != W->K) return false; + if ((W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) == 0) return false; + if ((W->K % 32) != 0 || (W->N % 4) != 0) return false; + + const uint32_t K = W->K; + const uint32_t N = W->N; + const uint32_t packed_group_bytes = cactus_quant_packed_group_bytes(4, K); + const size_t panel_bytes = static_cast(4) * packed_group_bytes; + const size_t N_blocks = N / 4; + + thread_local std::vector ar32; + thread_local std::vector act_i8; + if (ar32.size() < K) ar32.resize(K); + if (act_i8.size() < K) act_i8.resize(K); + std::fill(ar32.begin(), ar32.begin() + K, 0.0f); + + for (uint32_t k = 0; k < K; ++k) { + float a_val = static_cast(A[k]); + if (W->input_scale_recip) a_val *= static_cast(W->input_scale_recip[k]); + if (a_val == 0.0f) continue; + const __fp16* r = W->rotation + static_cast(k) * K; + const float32x4_t av = vdupq_n_f32(a_val); + uint32_t i = 0; + for (; i + 8 <= K; i += 8) { + const float16x8_t rv = vld1q_f16(r + i); + vst1q_f32(ar32.data() + i, + vfmaq_f32(vld1q_f32(ar32.data() + i), av, vcvt_f32_f16(vget_low_f16(rv)))); + vst1q_f32(ar32.data() + i + 4, + vfmaq_f32(vld1q_f32(ar32.data() + i + 4), av, vcvt_f32_f16(vget_high_f16(rv)))); + } + for (; i < K; ++i) ar32[i] += a_val * static_cast(r[i]); + } + + float max_abs = 0.0f; + for (uint32_t i = 0; i < K; ++i) max_abs = std::max(max_abs, std::abs(ar32[i])); + const float act_scale = max_abs > 0.0f ? max_abs / 127.0f : 1.0f; + const float inv_act_scale = 1.0f / act_scale; + for (uint32_t i = 0; i < K; ++i) { + int q = static_cast(std::lrintf(ar32[i] * inv_act_scale)); + q = std::max(-127, std::min(127, q)); + act_i8[i] = static_cast(q); + } + + int8_t cb_i8[16] = {}; + const float cb_scale = tq_quantize_codebook_i8(W->codebook, cb_i8, 16); + const int8x16_t cb_lut = vld1q_s8(cb_i8); + const uint8x16_t lo_mask = vdupq_n_u8(0x0F); + const int8_t* act_i8_data = act_i8.data(); + + cactus_quant_parallel_ranges(N_blocks, 64, [&](size_t block_start, size_t block_end) { + for (size_t nb = block_start; nb < block_end; ++nb) { + const uint8_t* panel = W->packed_indices + nb * panel_bytes; + int32x4_t dot_a = vdupq_n_s32(0); + int32x4_t dot_b = vdupq_n_s32(0); + for (uint32_t kb = 0; kb < K; kb += 16) { + const int8x16_t a_v = vld1q_s8(act_i8_data + kb); + const uint8x16_t p0 = vld1q_u8(panel + (kb / 8 + 0) * 16); + const int8x16_t w0 = vqtbl1q_s8(cb_lut, vandq_u8(p0, lo_mask)); + const int8x16_t w1 = vqtbl1q_s8(cb_lut, vshrq_n_u8(p0, 4)); + dot_a = CACTUS_DOTQ_LANE(dot_a, w0, a_v, 0); + dot_b = CACTUS_DOTQ_LANE(dot_b, w1, a_v, 1); + const uint8x16_t p1 = vld1q_u8(panel + (kb / 8 + 1) * 16); + const int8x16_t w2 = vqtbl1q_s8(cb_lut, vandq_u8(p1, lo_mask)); + const int8x16_t w3 = vqtbl1q_s8(cb_lut, vshrq_n_u8(p1, 4)); + dot_a = CACTUS_DOTQ_LANE(dot_a, w2, a_v, 2); + dot_b = CACTUS_DOTQ_LANE(dot_b, w3, a_v, 3); + } + const int32x4_t dot = vaddq_s32(dot_a, dot_b); + float32x4_t scale = vcvt_f32_f16(vld1_f16(W->norms + nb * 4)); + scale = vmulq_n_f32(scale, cb_scale * act_scale); + vst1_f16(C + nb * 4, vcvt_f16_f32(vmulq_f32(vcvtq_f32_s32(dot), scale))); + } + }); + return true; +} + void cactus_quant_orthogonal_matmul( const CactusQuantMatrix* W, const __fp16* A, @@ -1941,6 +2055,15 @@ void cactus_quant_orthogonal_matmul( const uint32_t N = W->N; const uint32_t bits = W->bits; const uint32_t pgb = cactus_quant_packed_group_bytes(bits, K); + const bool interleaved = (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) != 0; + + if (cactus_quant_orthogonal_interleaved_lmhead_matmul(W, A, M, C)) { + return; + } + if (interleaved && (bits != 4 || W->num_groups != 1 || W->group_size != K || + (K % 8) != 0 || (N % 4) != 0)) { + return; + } const uint32_t n_cb = 1u << bits; @@ -1971,7 +2094,7 @@ void cactus_quant_orthogonal_matmul( } } - if (bits == 4 && (K % 16) == 0) { + if (!interleaved && bits == 4 && (K % 16) == 0) { __fp16 cb_f16[16]; uint8_t cb_lo_arr[16], cb_hi_arr[16]; for (uint32_t c = 0; c < 16; ++c) { @@ -2057,7 +2180,9 @@ void cactus_quant_orthogonal_matmul( const float* ar = A_rot.data() + static_cast(m) * K; float acc = 0.0f; for (uint32_t i = 0; i < K; ++i) { - uint8_t idx = static_cast(tq_extract_idx(packed, i, bits)); + uint8_t idx = interleaved + ? static_cast(tq_extract_interleaved_4row_4bit(W->packed_indices, K, n, i)) + : static_cast(tq_extract_idx(packed, i, bits)); acc += cb_f32[idx] * ar[i]; } C[static_cast(m) * N + n] = static_cast<__fp16>(acc * norm_n); diff --git a/python/cactus/convert/cli.py b/python/cactus/convert/cli.py index 4c9ac0349..c94d6022c 100644 --- a/python/cactus/convert/cli.py +++ b/python/cactus/convert/cli.py @@ -214,6 +214,20 @@ def _scale_cq_norms(cq, factor: float): return replace(cq, norms=(cq.norms.astype(np.float32) * float(factor)).astype(np.float16)) +def _validate_cq_layout(policy, shape: tuple[int, ...], source_name: str, output_name: str) -> None: + if getattr(policy, "layout", "row_major") != "interleaved_4row": + return + if policy.rotation != "orthogonal" or int(policy.bits or 0) != 4: + raise RuntimeError(f"{source_name}: INTERLEAVED_4ROW output {output_name} requires orthogonal CQ4") + if len(shape) != 2: + raise RuntimeError(f"{source_name}: INTERLEAVED_4ROW output {output_name} requires rank-2 tensor, got shape={shape}") + n, k = int(shape[0]), int(shape[1]) + if n % 4 != 0 or k % 32 != 0: + raise RuntimeError( + f"{source_name}: INTERLEAVED_4ROW output {output_name} requires N % 4 == 0 and K % 32 == 0, got shape={shape}" + ) + + def _adapt_tensor_for_cactus(tensor, output_name: str | None, family: str): match = type("_CompatMatch", (), {"output_name": output_name})() transformed, _name = adapter_for_family(family).transform_tensor(match, tensor) @@ -481,6 +495,7 @@ def convert(args: argparse.Namespace) -> None: if not match.recognized and args.strict: raise RuntimeError(f"unrecognized tensor in strict mode: {name}") if emit_policy.action == "convert" and out_path is not None: + _validate_cq_layout(emit_policy, _tensor_shape(emit_tensor), name, out_path.name) if emit_policy.use_gptq and int(hessian_samples.get(module_name, 0)) <= 0: hessian_missing_reason = "expected GPTQ target had zero samples" if args.strict: @@ -502,6 +517,8 @@ def convert(args: argparse.Namespace) -> None: input_scale=input_scale, ) cq = _scale_cq_norms(cq, adapter.scale_factor(out_path.name)) + if getattr(emit_policy, "layout", "row_major") == "interleaved_4row": + cq = replace(cq, interleaved_4row=True) write_cq_tensor(out_path, cq) gptq_used = cq.gptq_used status = "converted" if match.recognized else "unrecognized" @@ -511,7 +528,7 @@ def convert(args: argparse.Namespace) -> None: elif emit_policy.action == "ignored": status = "ignored" except Exception as exc: - if args.strict: + if args.strict or getattr(emit_policy, "layout", "row_major") == "interleaved_4row": raise if out_path is not None: _save_fallback_tensor(emit_tensor, out_path, "FP16", family) diff --git a/python/cactus/convert/export/qdq.py b/python/cactus/convert/export/qdq.py index dfbaecd8b..87eec6f62 100644 --- a/python/cactus/convert/export/qdq.py +++ b/python/cactus/convert/export/qdq.py @@ -210,6 +210,17 @@ def unpack_lsb_values(packed: np.ndarray, count: int, bits: int) -> np.ndarray: return out +def unpack_interleaved_4row_4bit(packed: np.ndarray, rows: int, cols: int) -> np.ndarray: + if rows % 4 != 0 or cols % 8 != 0: + raise ValueError(f"INTERLEAVED_4ROW CQ4 requires rows % 4 == 0 and cols % 8 == 0, got {(rows, cols)}") + chunks = cols // 8 + view = packed.reshape(rows // 4, chunks, 4, 4) + lo = (view & 0x0F).astype(np.uint8) + hi = ((view >> 4) & 0x0F).astype(np.uint8) + out = np.concatenate([lo, hi], axis=3).transpose(0, 2, 1, 3).copy() + return out.reshape(rows, cols) + + def dequantize_fp_file(path: Path, header: CactusHeader, out_dtype: torch.dtype) -> torch.Tensor: offset = align_offset(HEADER_SIZE, header.alignment) dtype = np.float16 if header.precision == PRECISION_FP16 else np.float32 @@ -319,13 +330,23 @@ def dequantize_cq_file(path: Path, header: CactusHeader, out_dtype: torch.dtype, packed_row_bytes = math.ceil(k * bits / 8) codebook, input_scale, norms, rotation = parse_orthogonal_metadata(scales_blob, header) out = torch.empty(n, k, dtype=out_dtype) - packed_rows = packed.reshape(n, packed_row_bytes) + interleaved = bool(header.flags & FLAG_INTERLEAVED_4ROW) + if interleaved: + if bits != 4 or header.num_groups != 1 or header.group_size != k: + raise ValueError(f"{path}: orthogonal INTERLEAVED_4ROW requires full-width CQ4") + packed_indices = unpack_interleaved_4row_4bit(packed, n, k) + norms = norms.reshape(n // 4, 4).reshape(n) + else: + packed_rows = packed.reshape(n, packed_row_bytes) rt = rotation.float().T.contiguous() scale = input_scale.float().unsqueeze(0) codebook = codebook.float() for start in range(0, n, row_batch_size): end = min(start + row_batch_size, n) - idx_np = np.stack([unpack_lsb_values(row, k, bits) for row in packed_rows[start:end]]) + if interleaved: + idx_np = packed_indices[start:end] + else: + idx_np = np.stack([unpack_lsb_values(row, k, bits) for row in packed_rows[start:end]]) idx = torch.from_numpy(idx_np.astype(np.int64, copy=False)) recon = (codebook[idx] @ rt) * norms[start:end].float().unsqueeze(1) out[start:end] = (recon / scale).to(out_dtype) diff --git a/python/cactus/convert/interleave_orthogonal_cq4.py b/python/cactus/convert/interleave_orthogonal_cq4.py new file mode 100644 index 000000000..cc0c6fdc2 --- /dev/null +++ b/python/cactus/convert/interleave_orthogonal_cq4.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import argparse +import struct +from pathlib import Path + +import numpy as np + +from .export.qdq import ( + ALIGNMENT_DEFAULT, + CACTUS_MAGIC, + FLAG_INTERLEAVED_4ROW, + FLAG_ORTHOGONAL_ROTATION, + HEADER_SIZE, + align_offset, + read_cq_payload, + read_header, + unpack_lsb_values, +) +from .quantization.cq import PRECISION_CQ, pack_indices_interleaved_4row + + +def _padding(offset: int, alignment: int) -> bytes: + return b"\0" * (align_offset(offset, alignment) - offset) + + +def interleave_orthogonal_cq4_file(input_path: Path, output_path: Path, *, force: bool = False) -> None: + if output_path.exists() and not force: + raise FileExistsError(f"{output_path} exists; pass --force to overwrite") + + header = read_header(input_path) + if header.ndim != 2: + raise ValueError(f"{input_path}: expected 2D CQ tensor") + n, k = header.shape + if header.precision != PRECISION_CQ[4] or header.bits != 4: + raise ValueError(f"{input_path}: expected CQ4") + if (header.flags & FLAG_ORTHOGONAL_ROTATION) == 0: + raise ValueError(f"{input_path}: expected orthogonal CQ rotation") + if header.flags & FLAG_INTERLEAVED_4ROW: + raise ValueError(f"{input_path}: already INTERLEAVED_4ROW") + if header.num_groups != 1 or header.group_size != k: + raise ValueError( + f"{input_path}: expected one full-width group, got group_size={header.group_size} num_groups={header.num_groups}" + ) + if n % 4 != 0 or k % 32 != 0: + raise ValueError(f"{input_path}: expected N % 4 == 0 and K % 32 == 0, got shape={header.shape}") + + scales_blob, packed = read_cq_payload(input_path, header) + rows = packed.reshape(n, k // 2) + indices = np.stack([unpack_lsb_values(row, k, 4) for row in rows]) + interleaved = pack_indices_interleaved_4row(indices, k, 4) + + output_path.parent.mkdir(parents=True, exist_ok=True) + alignment = header.alignment or ALIGNMENT_DEFAULT + scales_offset = align_offset(HEADER_SIZE, alignment) + scales_end = scales_offset + len(scales_blob) + data_offset = align_offset(scales_end, alignment) + flags = header.flags | FLAG_INTERLEAVED_4ROW + + with output_path.open("wb") as out: + out.write(CACTUS_MAGIC) + out.write(struct.pack(" None: + parser = argparse.ArgumentParser(description="Rewrite a full-width orthogonal CQ4 file to 4-row interleaved storage.") + parser.add_argument("input", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--force", action="store_true") + args = parser.parse_args() + interleave_orthogonal_cq4_file(args.input, args.output, force=args.force) + print(f"wrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/python/cactus/convert/model_adapters/policy.py b/python/cactus/convert/model_adapters/policy.py index a56d7fe31..380ac3f24 100644 --- a/python/cactus/convert/model_adapters/policy.py +++ b/python/cactus/convert/model_adapters/policy.py @@ -14,6 +14,7 @@ class TensorPolicy: use_gptq: bool rotation: str fallback_reason: str | None = None + layout: str = "row_major" def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, family: str) -> TensorPolicy: @@ -58,8 +59,10 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("convert", f"CQ{user_bits}", user_bits, component, False, "hadamard") if family == "gemma4" and component in {"audio", "vision"}: return TensorPolicy("fallback", "FP16", None, component, False, "none", "gemma4 media tower accuracy") - if component == "embedding" or out in {"token_embeddings.weights", "output_weight.weights"}: - return TensorPolicy("convert", "CQ4", 4, component, False, "orthogonal") + output_head_or_tied_embedding = out in {"token_embeddings.weights", "decoder_token_embeddings.weights", "output_weight.weights"} + if component == "embedding" or output_head_or_tied_embedding: + layout = "interleaved_4row" if output_head_or_tied_embedding else "row_major" + return TensorPolicy("convert", "CQ4", 4, component, False, "orthogonal", layout=layout) if component == "audio" or component == "transcription": return TensorPolicy("convert", f"CQ{user_bits}", user_bits, component, False, "hadamard") return TensorPolicy("convert", f"CQ{user_bits}", user_bits, component, True, "hadamard") diff --git a/python/cactus/convert/quantization/cq.py b/python/cactus/convert/quantization/cq.py index 67c126b06..2d3942389 100644 --- a/python/cactus/convert/quantization/cq.py +++ b/python/cactus/convert/quantization/cq.py @@ -36,6 +36,7 @@ class CQTensor: rotation_family: str = "hadamard" seed: int = 1234 gptq_used: bool = False + interleaved_4row: bool = False _CODEBOOK_CACHE: dict[tuple[int, int], np.ndarray] = {} @@ -325,18 +326,34 @@ def write_cq_tensor(out_path: Path, cq: CQTensor) -> None: is_embedding = out_path.stem in _EMBEDDING_TENSOR_STEMS + if cq.interleaved_4row: + if cq.bits not in (1, 2, 3, 4): + raise ValueError(f"INTERLEAVED_4ROW only supports CQ1..CQ4, got CQ{cq.bits}") + if n % 4 != 0: + raise ValueError(f"INTERLEAVED_4ROW requires N % 4 == 0, got N={n}") + if cq.rotation_family == "orthogonal": + if cq.bits != 4 or group_size != k: + raise ValueError("orthogonal INTERLEAVED_4ROW requires full-width CQ4") + if k % 32 != 0: + raise ValueError(f"orthogonal INTERLEAVED_4ROW requires K % 32 == 0, got K={k}") + elif group_size % 32 != 0 or group_size > 256: + raise ValueError(f"INTERLEAVED_4ROW requires group_size % 32 == 0 and group_size <= 256, got {group_size}") + interleaved = ( - cq.bits in (1, 2, 3, 4) - and cq.rotation_family != "orthogonal" - and n % 4 == 0 - and group_size % 32 == 0 - and group_size <= 256 - and not is_embedding + cq.interleaved_4row + or ( + cq.bits in (1, 2, 3, 4) + and cq.rotation_family != "orthogonal" + and n % 4 == 0 + and group_size % 32 == 0 + and group_size <= 256 + and not is_embedding + ) ) codebook_f32 = make_codebook(group_size, cq.bits).astype(np.float32) norms_f32 = cq.norms.astype(np.float32, copy=True) - if interleaved: + if interleaved and cq.rotation_family != "orthogonal": cb_max = float(np.max(np.abs(codebook_f32))) cb_factor = cb_max / 127.0 if cb_max > 1e-12 else 1.0 / 127.0 codebook_f32 = codebook_f32 / cb_factor diff --git a/python/cactus/convert/tests/test_cq.py b/python/cactus/convert/tests/test_cq.py index 60d7e6ae4..2cd816e25 100644 --- a/python/cactus/convert/tests/test_cq.py +++ b/python/cactus/convert/tests/test_cq.py @@ -1,8 +1,10 @@ from __future__ import annotations +from dataclasses import replace import struct import numpy as np +import torch from cactus.convert.cactus_adapters.tensor_io import ( FLAG_HAS_SCALES, @@ -11,6 +13,8 @@ save_pointwise_conv1d_int8_with_header, save_tensor_with_header, ) +from cactus.convert.export.qdq import FLAG_INTERLEAVED_4ROW, FLAG_ORTHOGONAL_ROTATION, dequantize_cq_file, read_header +from cactus.convert.interleave_orthogonal_cq4 import interleave_orthogonal_cq4_file from cactus.convert.quantization.cq import PRECISION_CQ, pack_indices_lsb, quantize_hadamard, quantize_orthogonal, write_cq_tensor @@ -52,6 +56,35 @@ def test_orthogonal_embedding_is_cq4(tmp_path): assert cq.rotation_family == "orthogonal" +def test_orthogonal_interleaved_cq4_qdq_matches_row_major(tmp_path): + w = np.random.default_rng(2).standard_normal((8, 32), dtype=np.float32) + cq = quantize_orthogonal(w, bits=4) + row_path = tmp_path / "row.weights" + inter_path = tmp_path / "inter.weights" + write_cq_tensor(row_path, cq) + write_cq_tensor(inter_path, replace(cq, interleaved_4row=True)) + + inter_header = read_header(inter_path) + assert inter_header.flags & FLAG_ORTHOGONAL_ROTATION + assert inter_header.flags & FLAG_INTERLEAVED_4ROW + + row = dequantize_cq_file(row_path, read_header(row_path), torch.float32, 4) + inter = dequantize_cq_file(inter_path, inter_header, torch.float32, 4) + assert torch.max(torch.abs(row - inter)).item() <= 1e-6 + + +def test_interleave_orthogonal_cq4_file_preserves_qdq(tmp_path): + w = np.random.default_rng(3).standard_normal((8, 32), dtype=np.float32) + src = tmp_path / "src.weights" + dst = tmp_path / "dst.weights" + write_cq_tensor(src, quantize_orthogonal(w, bits=4)) + interleave_orthogonal_cq4_file(src, dst) + + src_tensor = dequantize_cq_file(src, read_header(src), torch.float32, 4) + dst_tensor = dequantize_cq_file(dst, read_header(dst), torch.float32, 4) + assert torch.max(torch.abs(src_tensor - dst_tensor)).item() <= 1e-6 + + def test_int8_bias_uses_cactus_grouped_layout(tmp_path): bias = np.array([-1.0, 0.0, 2.0], dtype=np.float32) out = tmp_path / "bias.weights" diff --git a/python/cactus/convert/tests/test_policy.py b/python/cactus/convert/tests/test_policy.py index 3a39f9fe8..1de4101b0 100644 --- a/python/cactus/convert/tests/test_policy.py +++ b/python/cactus/convert/tests/test_policy.py @@ -1,6 +1,6 @@ from cactus.convert.model_adapters.naming import cactus_name_for_tensor from cactus.convert.model_adapters.policy import policy_for_tensor -from cactus.convert.cli import _bits_for_component +from cactus.convert.cli import _bits_for_component, _validate_cq_layout from cactus.convert.model_adapters.detection import detect_family from cactus.convert.cactus_adapters.config_utils import extract_parakeet_tdt_config, extract_whisper_config from cactus.convert.cli import _augment_state_dict_for_family @@ -13,6 +13,44 @@ def test_policy_embedding_cq4(): p = policy_for_tensor(match, (100, 64), 2, "generic") assert p.precision == "CQ4" assert p.rotation == "orthogonal" + assert p.layout == "interleaved_4row" + + +def test_policy_lm_head_interleaved_cq4(): + match = cactus_name_for_tensor("lm_head.weight", "generic", 1) + p = policy_for_tensor(match, (100, 64), 2, "generic") + assert p.precision == "CQ4" + assert p.rotation == "orthogonal" + assert p.layout == "interleaved_4row" + + +def test_policy_lfm_qwen_tied_embeddings_interleaved_cq4(): + for family in ("lfm2", "qwen"): + embed = cactus_name_for_tensor("model.language_model.embed_tokens.weight", family, 1) + embed_policy = policy_for_tensor(embed, (65536, 2048), 2, family) + assert embed.output_name == "token_embeddings.weights" + assert embed_policy.precision == "CQ4" + assert embed_policy.rotation == "orthogonal" + assert embed_policy.layout == "interleaved_4row" + _validate_cq_layout(embed_policy, (65536, 2048), embed.source_name, embed.output_name) + + lm_head = cactus_name_for_tensor("lm_head.weight", family, 1) + lm_head_policy = policy_for_tensor(lm_head, (65536, 2048), 2, family) + assert lm_head.output_name == "output_weight.weights" + assert lm_head_policy.precision == "CQ4" + assert lm_head_policy.rotation == "orthogonal" + assert lm_head_policy.layout == "interleaved_4row" + _validate_cq_layout(lm_head_policy, (65536, 2048), lm_head.source_name, lm_head.output_name) + + +def test_interleaved_lm_head_policy_rejects_unsupported_shape(): + match = cactus_name_for_tensor("lm_head.weight", "generic", 1) + p = policy_for_tensor(match, (101, 64), 2, "generic") + try: + _validate_cq_layout(p, (101, 64), "lm_head.weight", "output_weight.weights") + except RuntimeError: + return + raise AssertionError("expected unsupported interleaved LM-head shape to fail") def test_adapter_registry_preserves_family_names(): diff --git a/python/cactus/transpile/weight_compat.py b/python/cactus/transpile/weight_compat.py index b856cd122..7b6c7421b 100644 --- a/python/cactus/transpile/weight_compat.py +++ b/python/cactus/transpile/weight_compat.py @@ -15,12 +15,14 @@ from cactus.transpile.runtime_compat import Graph from cactus.transpile.weight_binding import WeightBinding from cactus.convert.quantization.cq import FLAG_ORTHOGONAL_ROTATION +from cactus.convert.quantization.cq import FLAG_INTERLEAVED_4ROW from cactus.convert.quantization.cq import GROUP_SIZE as CQ_GROUP_SIZE from cactus.convert.quantization.cq import PRECISION_CQ from cactus.convert.quantization.cq import make_codebook from cactus.convert.quantization.cq import make_hadamard_components from cactus.convert.quantization.cq import make_hadamard_matrix from cactus.convert.quantization.cq import make_orthogonal_rotation +from cactus.convert.quantization.cq import pack_indices_interleaved_4row from cactus.convert.quantization.cq import pack_indices_lsb from cactus.convert.quantization.cq import quantize_hadamard from cactus.convert.quantization.cq import quantize_orthogonal @@ -94,6 +96,7 @@ def ensure_embedding_binding_compatible(binding: WeightBinding) -> WeightBinding compat_path, bits=int(config["bits"]), rotation=str(config["rotation"]), + layout=str(config.get("layout", "row_major")), ) _cleanup_legacy_fp16_cache(source_path) return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) @@ -104,7 +107,7 @@ def _embedding_cache_config(filename: str, *, source_name: str) -> dict[str, obj if filename in _PER_LAYER_EMBEDDING_FILENAMES or normalized_source.endswith("embed_tokens_per_layer.weight"): return {"bits": 2, "rotation": "hadamard"} if filename in _TOKEN_EMBEDDING_FILENAMES: - return {"bits": 4, "rotation": "orthogonal"} + return {"bits": 4, "rotation": "orthogonal", "layout": "interleaved_4row"} return {"bits": 4, "rotation": "orthogonal"} @@ -286,6 +289,7 @@ def _materialize_cq_embedding_cache( *, bits: int, rotation: str, + layout: str = "row_major", ) -> None: src_mtime_ns = opened.path.stat().st_mtime_ns if out_path.exists() and out_path.stat().st_mtime_ns >= src_mtime_ns: @@ -296,6 +300,12 @@ def _materialize_cq_embedding_cache( rows = int(opened.original_n or opened.shape[0]) cols = int(opened.shape[1]) + interleaved = layout == "interleaved_4row" + if interleaved: + if rotation != "orthogonal" or bits != 4: + raise RuntimeError(f"INTERLEAVED_4ROW embedding cache requires orthogonal CQ4, got rotation={rotation} bits={bits}") + if rows % 4 != 0 or cols % 32 != 0: + raise RuntimeError(f"INTERLEAVED_4ROW embedding cache requires rows % 4 == 0 and cols % 32 == 0, got {(rows, cols)}") if rotation == "hadamard" and cols % CQ_GROUP_SIZE != 0: raise RuntimeError( f"Hadamard CQ embedding conversion requires hidden dim divisible by {CQ_GROUP_SIZE}, " @@ -319,6 +329,8 @@ def _materialize_cq_embedding_cache( trailer_parts: list[bytes] = [] if rotation == "orthogonal": flags |= FLAG_ORTHOGONAL_ROTATION + if interleaved: + flags |= FLAG_INTERLEAVED_4ROW trailer_parts.append(make_orthogonal_rotation(group_size, seed=1234).astype(np.float16).tobytes()) else: left, right, perm = make_hadamard_components(group_size, seed=1234) @@ -334,7 +346,7 @@ def _materialize_cq_embedding_cache( stop = min(start + chunk_rows, rows) chunk = _dequantize_int8_rows(opened, start, stop) if rotation == "orthogonal": - packed, norms = _quantize_orthogonal_chunk(chunk, bits=bits, input_scale=input_scale) + packed, norms = _quantize_orthogonal_chunk(chunk, bits=bits, input_scale=input_scale, interleaved=interleaved) else: packed, norms = _quantize_hadamard_chunk(chunk, bits=bits, input_scale=input_scale) norms_handle.write(np.ascontiguousarray(norms, dtype=np.float16).tobytes()) @@ -456,6 +468,7 @@ def _quantize_orthogonal_chunk( *, bits: int, input_scale: np.ndarray, + interleaved: bool = False, ) -> tuple[np.ndarray, np.ndarray]: work = chunk.astype(np.float32, copy=False) * input_scale.astype(np.float32, copy=False)[None, :] rows, cols = work.shape @@ -464,6 +477,8 @@ def _quantize_orthogonal_chunk( norms = np.linalg.norm(work, axis=1).clip(min=1e-8).astype(np.float32, copy=False) rotated = (work / norms[:, None]) @ rotation indices = np.abs(rotated[..., None] - codebook[None, None, :]).argmin(axis=-1).astype(np.uint8) + if interleaved: + return pack_indices_interleaved_4row(indices, cols, bits), norms.astype(np.float16).reshape(rows, 1) return pack_indices_lsb(indices, cols, bits), norms.astype(np.float16).reshape(rows, 1) diff --git a/python/tests/test_transpile_weight_compat.py b/python/tests/test_transpile_weight_compat.py index 1643e28c2..be52f54e5 100644 --- a/python/tests/test_transpile_weight_compat.py +++ b/python/tests/test_transpile_weight_compat.py @@ -259,6 +259,36 @@ def test_resolve_weight_binding_expands_language_model_backbone_aliases(tmp_path assert binding.path.endswith("layer_0_scalar.weights") +def test_resolve_weight_binding_maps_tied_lm_head_to_token_embedding_manifest(tmp_path: Path) -> None: + (tmp_path / "token_embeddings.weights").write_bytes(b"") + (tmp_path / "weights_manifest.json").write_text( + json.dumps( + { + "weights": [ + { + "source_name": "model.language_model.embed_tokens.weight", + "hf_name": "model.language_model.embed_tokens.weight", + "adapter_name": "model.language_model.embed_tokens.weight", + "source_names": ["model.language_model.embed_tokens.weight"], + "output_name": "token_embeddings.weights", + "component": "embedding", + } + ] + } + ) + ) + + for source_name in ("lm_head.weight", "model.lm_head.weight"): + binding = resolve_weight_binding( + weights_dir=str(tmp_path), + source_name=source_name, + ) + + assert binding is not None + assert binding.path.endswith("token_embeddings.weights") + assert binding.kind == "embedding" + + def test_resolve_weight_binding_expands_nested_model_tower_aliases(tmp_path: Path) -> None: (tmp_path / "vision_q.weights").write_bytes(b"") (tmp_path / "weights_manifest.json").write_text(