diff --git a/cactus-graph/src/io.cpp b/cactus-graph/src/io.cpp index 7165b59f0..13908a99e 100644 --- a/cactus-graph/src/io.cpp +++ b/cactus-graph/src/io.cpp @@ -52,6 +52,59 @@ namespace { return filename; } + void validate_interleaved_4row_tensor( + Precision precision, + const std::vector& shape, + size_t group_size, + size_t num_groups, + size_t byte_size, + size_t scales_bytes, + bool orthogonal) { + if (!PrecisionTraits::is_cq(precision) || group_size == 0 || num_groups == 0) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires grouped CQ precision"); + } + if (shape.size() != 2) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires a rank-2 tensor"); + } + const uint32_t bits = PrecisionTraits::cq_bits(precision); + const size_t n = shape[0]; + const size_t k = shape[1]; + if (bits < 1 || bits > 4) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW only supports CQ1..CQ4"); + } + if ((n % 4) != 0) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires N % 4 == 0"); + } + if (group_size * num_groups != k) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW group metadata does not match tensor shape"); + } + if (orthogonal) { + if (bits != 4 || num_groups != 1 || group_size != k || (k % 32) != 0) { + throw std::runtime_error("Invalid tensor file: orthogonal INTERLEAVED_4ROW requires full-width CQ4 with K % 32 == 0"); + } + } else if ((group_size % 32) != 0 || group_size > 256) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW requires group_size % 32 == 0 and group_size <= 256"); + } + + const size_t packed_group_bytes = (group_size * bits + 7) / 8; + const size_t expected_bytes = n * num_groups * packed_group_bytes; + if (byte_size != expected_bytes) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW packed byte size mismatch"); + } + + const size_t codebook_bytes = (static_cast(1) << bits) * sizeof(__fp16); + const size_t input_scale_bytes = k * sizeof(__fp16); + const size_t norms_bytes = n * num_groups * sizeof(__fp16); + const size_t rotation_bytes = orthogonal + ? k * k * sizeof(__fp16) + : 2 * group_size + group_size * sizeof(uint32_t); + const size_t expected_scales_bytes = + codebook_bytes + 2 * input_scale_bytes + norms_bytes + rotation_bytes; + if (scales_bytes != expected_scales_bytes) { + throw std::runtime_error("Invalid tensor file: INTERLEAVED_4ROW metadata byte size mismatch"); + } + } + inline void write_u32(std::ostream& out, uint32_t v) { out.write(reinterpret_cast(&v), sizeof(v)); } @@ -1018,6 +1071,11 @@ void MappedFile::parse_header() { } } + if (is_interleaved_4row_) { + validate_interleaved_4row_tensor( + precision_, shape_, group_size_, num_groups_, byte_size_, scales_bytes_, is_orthogonal_rotation_); + } + size_t aligned_header = align_offset(header_size, alignment_); if (scales_bytes_ > 0) { diff --git a/cactus-graph/src/ops_tensor.cpp b/cactus-graph/src/ops_tensor.cpp index 42f967ce5..857c677b9 100644 --- a/cactus-graph/src/ops_tensor.cpp +++ b/cactus-graph/src/ops_tensor.cpp @@ -186,6 +186,7 @@ void compute_embedding_node(GraphNode& node, const std::vector(Precision::CQ4); + auto pad_to = [](std::ofstream& out, size_t offset, size_t alignment) { + size_t aligned = ((offset + alignment - 1) / alignment) * alignment; + for (size_t i = offset; i < aligned; ++i) out.put('\0'); + return aligned; + }; + auto write_file = [&](uint64_t n, uint64_t scales_bytes) { + const uint64_t data_bytes = n * k / 2; + std::ofstream out(filename, std::ios::binary); + out.write("CACT", 4); + out.write(reinterpret_cast(&flags), sizeof(flags)); + out.write(reinterpret_cast(&alignment), sizeof(alignment)); + uint32_t ndim = 2; + out.write(reinterpret_cast(&ndim), sizeof(ndim)); + uint64_t dims[4] = {n, k, 0, 0}; + out.write(reinterpret_cast(dims), sizeof(dims)); + out.write(reinterpret_cast(&precision), sizeof(precision)); + out.write(reinterpret_cast(&data_bytes), sizeof(data_bytes)); + out.write(reinterpret_cast(&scales_bytes), sizeof(scales_bytes)); + uint32_t group_size = static_cast(k); + uint32_t num_groups = 1; + out.write(reinterpret_cast(&group_size), sizeof(group_size)); + out.write(reinterpret_cast(&num_groups), sizeof(num_groups)); + out.write(reinterpret_cast(&n), sizeof(n)); + size_t offset = pad_to(out, 84, alignment); + for (uint64_t i = 0; i < scales_bytes; ++i) out.put('\0'); + offset = pad_to(out, offset + static_cast(scales_bytes), alignment); + for (uint64_t i = 0; i < data_bytes; ++i) out.put('\0'); + }; + + const uint64_t valid_n = 8; + const uint64_t valid_scales_bytes = + 16 * sizeof(__fp16) + 2 * k * sizeof(__fp16) + valid_n * sizeof(__fp16) + k * k * sizeof(__fp16); + write_file(5, 16 * sizeof(__fp16) + 2 * k * sizeof(__fp16) + 5 * sizeof(__fp16) + k * k * sizeof(__fp16)); + bool rejected = false; + try { + CactusGraph graph; + graph.mmap_weights(filename); + } catch (const std::exception&) { + rejected = true; + } + if (!rejected) { + std::remove(filename.c_str()); + return false; + } + + write_file(valid_n, valid_scales_bytes - sizeof(__fp16)); + rejected = false; + try { + CactusGraph graph; + graph.mmap_weights(filename); + } catch (const std::exception&) { + rejected = true; + } + std::remove(filename.c_str()); + return rejected; +} + bool run_benchmarks() { const int ITERS = 100; const std::string temp_file = "bench_io_50nodes.cg"; @@ -432,6 +495,7 @@ int main() { runner.run_test("Graph Save/Load Roundtrip Execution", test_graph_save_load_roundtrip_execution()); runner.run_test("Graph Save/Load Supported Ops Roundtrip", test_graph_save_load_supported_ops_roundtrip()); runner.run_test("Graph Save For Inspection", test_graph_save_for_inspection()); + runner.run_test("Reject Malformed Interleaved CQ", test_rejects_malformed_interleaved_cq()); runner.print_benchmarks_header(); runner.run_bench("benchmarks", run_benchmarks()); runner.print_summary(); diff --git a/cactus-graph/tests/test_ops.cpp b/cactus-graph/tests/test_ops.cpp index 2b015d4ac..061f45a79 100644 --- a/cactus-graph/tests/test_ops.cpp +++ b/cactus-graph/tests/test_ops.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -552,6 +553,126 @@ bool test_cq_embedding_operation() { return true; } +static void set_cq4_index(std::vector& packed, uint32_t K, uint32_t row, uint32_t col, uint8_t value) { + const size_t byte_index = static_cast(row) * (K / 2) + col / 2; + if ((col & 1u) == 0) { + packed[byte_index] = static_cast((packed[byte_index] & 0xF0) | (value & 0x0F)); + } else { + packed[byte_index] = static_cast((packed[byte_index] & 0x0F) | ((value & 0x0F) << 4)); + } +} + +static std::vector make_interleaved_cq4(const std::vector& row_major, uint32_t N, uint32_t K) { + const uint32_t pgb = cactus_quant_packed_group_bytes(4, K); + std::vector interleaved(static_cast(N / 4) * 4 * pgb, 0); + for (uint32_t row = 0; row < N; ++row) { + const uint8_t* src = row_major.data() + static_cast(row) * pgb; + uint8_t* panel = interleaved.data() + static_cast(row / 4) * 4 * pgb; + for (uint32_t k = 0; k < K; ++k) { + const uint8_t idx = static_cast((src[k / 2] >> ((k & 1u) * 4)) & 0x0F); + const uint32_t chunk = k / 8; + const uint32_t sub = k & 7u; + uint8_t& dst = panel[chunk * 16 + (row & 3u) * 4 + (sub & 3u)]; + if (sub & 4u) dst = static_cast((dst & 0x0F) | (idx << 4)); + else dst = static_cast((dst & 0xF0) | idx); + } + } + return interleaved; +} + +bool test_orthogonal_interleaved_embedding_row() { + const uint32_t N = 8; + const uint32_t K = 32; + std::vector<__fp16> codebook(16); + for (uint32_t i = 0; i < 16; ++i) codebook[i] = static_cast<__fp16>((static_cast(i) - 8) * 0.125f); + std::vector<__fp16> input_scale_recip(K, static_cast<__fp16>(1.0f)); + std::vector<__fp16> norms(N); + for (uint32_t row = 0; row < N; ++row) norms[row] = static_cast<__fp16>(0.5f + 0.03125f * row); + std::vector<__fp16> rotation(static_cast(K) * K, static_cast<__fp16>(0.0f)); + for (uint32_t i = 0; i < K; ++i) rotation[static_cast(i) * K + i] = static_cast<__fp16>(1.0f); + + std::vector row_major(static_cast(N) * K / 2, 0); + for (uint32_t row = 0; row < N; ++row) + for (uint32_t col = 0; col < K; ++col) + set_cq4_index(row_major, K, row, col, static_cast((row * 3 + col) & 0x0F)); + std::vector interleaved = make_interleaved_cq4(row_major, N, K); + + for (uint32_t row : {0u, 1u, 3u, 4u, 7u}) { + std::vector<__fp16> ref(K); + std::vector<__fp16> got(K); + cactus_quant_dequantize_orthogonal_embedding_row( + 4, K, row, row_major.data(), codebook.data(), norms.data(), input_scale_recip.data(), + rotation.data(), CACTUS_QUANT_FLAG_ORTHOGONAL, ref.data()); + cactus_quant_dequantize_orthogonal_embedding_row( + 4, K, row, interleaved.data(), codebook.data(), norms.data(), input_scale_recip.data(), + rotation.data(), CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW, got.data()); + for (uint32_t i = 0; i < K; ++i) { + if (std::abs(static_cast(ref[i]) - static_cast(got[i])) > 0.001f) return false; + } + } + return true; +} + +bool test_orthogonal_interleaved_lmhead_matmul() { + const uint32_t N = 8; + const uint32_t K = 32; + std::vector<__fp16> codebook(16); + for (uint32_t i = 0; i < 16; ++i) codebook[i] = static_cast<__fp16>((static_cast(i) - 8) * 0.125f); + std::vector<__fp16> input_scale_recip(K, static_cast<__fp16>(1.0f)); + std::vector<__fp16> norms(N); + for (uint32_t row = 0; row < N; ++row) norms[row] = static_cast<__fp16>(0.05f + 0.003f * row); + std::vector<__fp16> rotation(static_cast(K) * K, static_cast<__fp16>(0.0f)); + for (uint32_t i = 0; i < K; ++i) { + uint32_t j = (i * 5) % K; + rotation[static_cast(i) * K + j] = static_cast<__fp16>((i & 1u) ? -1.0f : 1.0f); + } + std::vector row_major(static_cast(N) * K / 2, 0); + for (uint32_t row = 0; row < N; ++row) + for (uint32_t col = 0; col < K; ++col) + set_cq4_index(row_major, K, row, col, static_cast((row + col * 5) & 0x0F)); + std::vector interleaved = make_interleaved_cq4(row_major, N, K); + std::vector<__fp16> activation(static_cast(2) * K); + for (uint32_t i = 0; i < K; ++i) { + activation[i] = static_cast<__fp16>((static_cast(i % 9) - 4) * 0.1f); + activation[K + i] = static_cast<__fp16>((static_cast((i * 3) % 11) - 5) * 0.07f); + } + + CactusQuantMatrix row_mat{ + .bits = 4, .K = K, .N = N, .group_size = K, .num_groups = 1, + .flags = CACTUS_QUANT_FLAG_ORTHOGONAL, .codebook = codebook.data(), + .input_scale = nullptr, .input_scale_recip = input_scale_recip.data(), .norms = norms.data(), + .packed_indices = row_major.data(), .left_signs = nullptr, .right_signs = nullptr, + .permutation = nullptr, .rotation = rotation.data(), .expanded = nullptr, .norm_f32 = nullptr, + }; + CactusQuantMatrix inter_mat = row_mat; + inter_mat.flags = CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW; + inter_mat.packed_indices = interleaved.data(); + std::vector<__fp16> ref(N); + std::vector<__fp16> got(N); + cactus_quant_orthogonal_matmul(&row_mat, activation.data(), 1, ref.data()); + cactus_quant_orthogonal_matmul(&inter_mat, activation.data(), 1, got.data()); + + float diff2 = 0.0f; + float ref2 = 0.0f; + for (uint32_t i = 0; i < N; ++i) { + const float d = static_cast(got[i]) - static_cast(ref[i]); + diff2 += d * d; + ref2 += static_cast(ref[i]) * static_cast(ref[i]); + } + if (std::sqrt(diff2 / std::max(ref2, 1.0e-12f)) >= 0.08f) return false; + + std::vector<__fp16> ref_fallback(static_cast(2) * N); + std::vector<__fp16> got_fallback(static_cast(2) * N); + cactus_quant_orthogonal_matmul(&row_mat, activation.data(), 2, ref_fallback.data()); + cactus_quant_orthogonal_matmul(&inter_mat, activation.data(), 2, got_fallback.data()); + for (uint32_t i = 0; i < 2 * N; ++i) { + if (std::abs(static_cast(got_fallback[i]) - static_cast(ref_fallback[i])) > 0.001f) { + return false; + } + } + return true; +} + bool test_embedding_from_file() { CactusGraph graph; @@ -657,6 +778,8 @@ int main() { runner.run_test("Memory-Mapped Gather", test_mmap_gather()); runner.run_test("Embedding Operation", test_embedding_operation()); runner.run_test("CQ Embedding Operation", test_cq_embedding_operation()); + runner.run_test("Orthogonal Interleaved Embedding Row", test_orthogonal_interleaved_embedding_row()); + runner.run_test("Orthogonal Interleaved LMHead MatMul", test_orthogonal_interleaved_lmhead_matmul()); runner.run_test("Embedding from File", test_embedding_from_file()); runner.print_benchmarks_header(); runner.run_bench("benchmarks", run_benchmarks()); diff --git a/cactus-kernels/cactus_kernels.h b/cactus-kernels/cactus_kernels.h index f79ef7169..f371cffbe 100644 --- a/cactus-kernels/cactus_kernels.h +++ b/cactus-kernels/cactus_kernels.h @@ -333,6 +333,7 @@ void cactus_quant_dequantize_orthogonal_embedding_row( const __fp16* norms, const __fp16* input_scale_recip, const __fp16* rotation, + uint32_t flags, __fp16* out_row); void cactus_rms_norm_f16( diff --git a/cactus-kernels/src/matmul.cpp b/cactus-kernels/src/matmul.cpp index b244388b3..151e11711 100644 --- a/cactus-kernels/src/matmul.cpp +++ b/cactus-kernels/src/matmul.cpp @@ -1906,17 +1906,34 @@ void cactus_quant_dequantize_orthogonal_embedding_row( const __fp16* norms, const __fp16* input_scale_recip, const __fp16* rotation, + uint32_t flags, __fp16* out_row) { if (!packed_base || !codebook || !norms || !rotation || !out_row || bits == 0 || bits > 4) return; if (K == 0) return; const uint32_t packed_group_bytes = cactus_quant_packed_group_bytes(bits, K); - const uint8_t* packed = packed_base + static_cast(row) * packed_group_bytes; - const float norm = static_cast(norms[row]); + const bool interleaved = (flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) != 0; + if (interleaved && (bits != 4 || (K % 8) != 0)) return; + const uint8_t* packed = interleaved ? nullptr : packed_base + static_cast(row) * packed_group_bytes; + const float norm = interleaved + ? static_cast(norms[(row / 4) * 4 + (row & 3u)]) + : static_cast(norms[row]); std::vector dq(K); for (uint32_t i = 0; i < K; ++i) { - dq[i] = static_cast(codebook[tq_extract_idx(packed, i, bits)]); + uint32_t idx = 0; + if (interleaved) { + const size_t panel_bytes = static_cast(4) * packed_group_bytes; + const uint8_t* panel = packed_base + (row / 4) * panel_bytes; + const uint32_t chunk = i / 8; + const uint32_t sub = i & 7u; + const uint8_t packed_byte = panel[chunk * 16 + (row & 3u) * 4 + (sub & 3u)]; + idx = (sub & 4u) ? static_cast(packed_byte >> 4) + : static_cast(packed_byte & 0x0F); + } else { + idx = tq_extract_idx(packed, i, bits); + } + dq[i] = static_cast(codebook[idx]); } for (uint32_t j = 0; j < K; ++j) { @@ -1929,6 +1946,103 @@ void cactus_quant_dequantize_orthogonal_embedding_row( } } +static inline uint32_t tq_extract_interleaved_4row_4bit( + const uint8_t* packed_base, + uint32_t K, + size_t row, + uint32_t k) { + const uint32_t packed_group_bytes = cactus_quant_packed_group_bytes(4, K); + const size_t panel_bytes = static_cast(4) * packed_group_bytes; + const uint8_t* panel = packed_base + (row / 4) * panel_bytes; + const uint32_t chunk = k / 8; + const uint32_t sub = k & 7u; + const uint8_t packed_byte = panel[chunk * 16 + (row & 3u) * 4 + (sub & 3u)]; + return (sub & 4u) ? static_cast(packed_byte >> 4) + : static_cast(packed_byte & 0x0F); +} + +static bool cactus_quant_orthogonal_interleaved_lmhead_matmul( + const CactusQuantMatrix* W, + const __fp16* A, + uint32_t M, + __fp16* C) { + if (!W || !A || !C || M != 1) return false; + if (W->bits != 4 || W->num_groups != 1 || W->group_size != W->K) return false; + if ((W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) == 0) return false; + if ((W->K % 32) != 0 || (W->N % 4) != 0) return false; + + const uint32_t K = W->K; + const uint32_t N = W->N; + const uint32_t packed_group_bytes = cactus_quant_packed_group_bytes(4, K); + const size_t panel_bytes = static_cast(4) * packed_group_bytes; + const size_t N_blocks = N / 4; + + thread_local std::vector ar32; + thread_local std::vector act_i8; + if (ar32.size() < K) ar32.resize(K); + if (act_i8.size() < K) act_i8.resize(K); + std::fill(ar32.begin(), ar32.begin() + K, 0.0f); + + for (uint32_t k = 0; k < K; ++k) { + float a_val = static_cast(A[k]); + if (W->input_scale_recip) a_val *= static_cast(W->input_scale_recip[k]); + if (a_val == 0.0f) continue; + const __fp16* r = W->rotation + static_cast(k) * K; + const float32x4_t av = vdupq_n_f32(a_val); + uint32_t i = 0; + for (; i + 8 <= K; i += 8) { + const float16x8_t rv = vld1q_f16(r + i); + vst1q_f32(ar32.data() + i, + vfmaq_f32(vld1q_f32(ar32.data() + i), av, vcvt_f32_f16(vget_low_f16(rv)))); + vst1q_f32(ar32.data() + i + 4, + vfmaq_f32(vld1q_f32(ar32.data() + i + 4), av, vcvt_f32_f16(vget_high_f16(rv)))); + } + for (; i < K; ++i) ar32[i] += a_val * static_cast(r[i]); + } + + float max_abs = 0.0f; + for (uint32_t i = 0; i < K; ++i) max_abs = std::max(max_abs, std::abs(ar32[i])); + const float act_scale = max_abs > 0.0f ? max_abs / 127.0f : 1.0f; + const float inv_act_scale = 1.0f / act_scale; + for (uint32_t i = 0; i < K; ++i) { + int q = static_cast(std::lrintf(ar32[i] * inv_act_scale)); + q = std::max(-127, std::min(127, q)); + act_i8[i] = static_cast(q); + } + + int8_t cb_i8[16] = {}; + const float cb_scale = tq_quantize_codebook_i8(W->codebook, cb_i8, 16); + const int8x16_t cb_lut = vld1q_s8(cb_i8); + const uint8x16_t lo_mask = vdupq_n_u8(0x0F); + const int8_t* act_i8_data = act_i8.data(); + + cactus_quant_parallel_ranges(N_blocks, 64, [&](size_t block_start, size_t block_end) { + for (size_t nb = block_start; nb < block_end; ++nb) { + const uint8_t* panel = W->packed_indices + nb * panel_bytes; + int32x4_t dot_a = vdupq_n_s32(0); + int32x4_t dot_b = vdupq_n_s32(0); + for (uint32_t kb = 0; kb < K; kb += 16) { + const int8x16_t a_v = vld1q_s8(act_i8_data + kb); + const uint8x16_t p0 = vld1q_u8(panel + (kb / 8 + 0) * 16); + const int8x16_t w0 = vqtbl1q_s8(cb_lut, vandq_u8(p0, lo_mask)); + const int8x16_t w1 = vqtbl1q_s8(cb_lut, vshrq_n_u8(p0, 4)); + dot_a = CACTUS_DOTQ_LANE(dot_a, w0, a_v, 0); + dot_b = CACTUS_DOTQ_LANE(dot_b, w1, a_v, 1); + const uint8x16_t p1 = vld1q_u8(panel + (kb / 8 + 1) * 16); + const int8x16_t w2 = vqtbl1q_s8(cb_lut, vandq_u8(p1, lo_mask)); + const int8x16_t w3 = vqtbl1q_s8(cb_lut, vshrq_n_u8(p1, 4)); + dot_a = CACTUS_DOTQ_LANE(dot_a, w2, a_v, 2); + dot_b = CACTUS_DOTQ_LANE(dot_b, w3, a_v, 3); + } + const int32x4_t dot = vaddq_s32(dot_a, dot_b); + float32x4_t scale = vcvt_f32_f16(vld1_f16(W->norms + nb * 4)); + scale = vmulq_n_f32(scale, cb_scale * act_scale); + vst1_f16(C + nb * 4, vcvt_f16_f32(vmulq_f32(vcvtq_f32_s32(dot), scale))); + } + }); + return true; +} + void cactus_quant_orthogonal_matmul( const CactusQuantMatrix* W, const __fp16* A, @@ -1941,6 +2055,15 @@ void cactus_quant_orthogonal_matmul( const uint32_t N = W->N; const uint32_t bits = W->bits; const uint32_t pgb = cactus_quant_packed_group_bytes(bits, K); + const bool interleaved = (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) != 0; + + if (cactus_quant_orthogonal_interleaved_lmhead_matmul(W, A, M, C)) { + return; + } + if (interleaved && (bits != 4 || W->num_groups != 1 || W->group_size != K || + (K % 8) != 0 || (N % 4) != 0)) { + return; + } const uint32_t n_cb = 1u << bits; @@ -1971,7 +2094,7 @@ void cactus_quant_orthogonal_matmul( } } - if (bits == 4 && (K % 16) == 0) { + if (!interleaved && bits == 4 && (K % 16) == 0) { __fp16 cb_f16[16]; uint8_t cb_lo_arr[16], cb_hi_arr[16]; for (uint32_t c = 0; c < 16; ++c) { @@ -2057,7 +2180,9 @@ void cactus_quant_orthogonal_matmul( const float* ar = A_rot.data() + static_cast(m) * K; float acc = 0.0f; for (uint32_t i = 0; i < K; ++i) { - uint8_t idx = static_cast(tq_extract_idx(packed, i, bits)); + uint8_t idx = interleaved + ? static_cast(tq_extract_interleaved_4row_4bit(W->packed_indices, K, n, i)) + : static_cast(tq_extract_idx(packed, i, bits)); acc += cb_f32[idx] * ar[i]; } C[static_cast(m) * N + n] = static_cast<__fp16>(acc * norm_n); diff --git a/python/cactus/convert/cli.py b/python/cactus/convert/cli.py index 4c9ac0349..c94d6022c 100644 --- a/python/cactus/convert/cli.py +++ b/python/cactus/convert/cli.py @@ -214,6 +214,20 @@ def _scale_cq_norms(cq, factor: float): return replace(cq, norms=(cq.norms.astype(np.float32) * float(factor)).astype(np.float16)) +def _validate_cq_layout(policy, shape: tuple[int, ...], source_name: str, output_name: str) -> None: + if getattr(policy, "layout", "row_major") != "interleaved_4row": + return + if policy.rotation != "orthogonal" or int(policy.bits or 0) != 4: + raise RuntimeError(f"{source_name}: INTERLEAVED_4ROW output {output_name} requires orthogonal CQ4") + if len(shape) != 2: + raise RuntimeError(f"{source_name}: INTERLEAVED_4ROW output {output_name} requires rank-2 tensor, got shape={shape}") + n, k = int(shape[0]), int(shape[1]) + if n % 4 != 0 or k % 32 != 0: + raise RuntimeError( + f"{source_name}: INTERLEAVED_4ROW output {output_name} requires N % 4 == 0 and K % 32 == 0, got shape={shape}" + ) + + def _adapt_tensor_for_cactus(tensor, output_name: str | None, family: str): match = type("_CompatMatch", (), {"output_name": output_name})() transformed, _name = adapter_for_family(family).transform_tensor(match, tensor) @@ -481,6 +495,7 @@ def convert(args: argparse.Namespace) -> None: if not match.recognized and args.strict: raise RuntimeError(f"unrecognized tensor in strict mode: {name}") if emit_policy.action == "convert" and out_path is not None: + _validate_cq_layout(emit_policy, _tensor_shape(emit_tensor), name, out_path.name) if emit_policy.use_gptq and int(hessian_samples.get(module_name, 0)) <= 0: hessian_missing_reason = "expected GPTQ target had zero samples" if args.strict: @@ -502,6 +517,8 @@ def convert(args: argparse.Namespace) -> None: input_scale=input_scale, ) cq = _scale_cq_norms(cq, adapter.scale_factor(out_path.name)) + if getattr(emit_policy, "layout", "row_major") == "interleaved_4row": + cq = replace(cq, interleaved_4row=True) write_cq_tensor(out_path, cq) gptq_used = cq.gptq_used status = "converted" if match.recognized else "unrecognized" @@ -511,7 +528,7 @@ def convert(args: argparse.Namespace) -> None: elif emit_policy.action == "ignored": status = "ignored" except Exception as exc: - if args.strict: + if args.strict or getattr(emit_policy, "layout", "row_major") == "interleaved_4row": raise if out_path is not None: _save_fallback_tensor(emit_tensor, out_path, "FP16", family) diff --git a/python/cactus/convert/export/qdq.py b/python/cactus/convert/export/qdq.py index dfbaecd8b..87eec6f62 100644 --- a/python/cactus/convert/export/qdq.py +++ b/python/cactus/convert/export/qdq.py @@ -210,6 +210,17 @@ def unpack_lsb_values(packed: np.ndarray, count: int, bits: int) -> np.ndarray: return out +def unpack_interleaved_4row_4bit(packed: np.ndarray, rows: int, cols: int) -> np.ndarray: + if rows % 4 != 0 or cols % 8 != 0: + raise ValueError(f"INTERLEAVED_4ROW CQ4 requires rows % 4 == 0 and cols % 8 == 0, got {(rows, cols)}") + chunks = cols // 8 + view = packed.reshape(rows // 4, chunks, 4, 4) + lo = (view & 0x0F).astype(np.uint8) + hi = ((view >> 4) & 0x0F).astype(np.uint8) + out = np.concatenate([lo, hi], axis=3).transpose(0, 2, 1, 3).copy() + return out.reshape(rows, cols) + + def dequantize_fp_file(path: Path, header: CactusHeader, out_dtype: torch.dtype) -> torch.Tensor: offset = align_offset(HEADER_SIZE, header.alignment) dtype = np.float16 if header.precision == PRECISION_FP16 else np.float32 @@ -319,13 +330,23 @@ def dequantize_cq_file(path: Path, header: CactusHeader, out_dtype: torch.dtype, packed_row_bytes = math.ceil(k * bits / 8) codebook, input_scale, norms, rotation = parse_orthogonal_metadata(scales_blob, header) out = torch.empty(n, k, dtype=out_dtype) - packed_rows = packed.reshape(n, packed_row_bytes) + interleaved = bool(header.flags & FLAG_INTERLEAVED_4ROW) + if interleaved: + if bits != 4 or header.num_groups != 1 or header.group_size != k: + raise ValueError(f"{path}: orthogonal INTERLEAVED_4ROW requires full-width CQ4") + packed_indices = unpack_interleaved_4row_4bit(packed, n, k) + norms = norms.reshape(n // 4, 4).reshape(n) + else: + packed_rows = packed.reshape(n, packed_row_bytes) rt = rotation.float().T.contiguous() scale = input_scale.float().unsqueeze(0) codebook = codebook.float() for start in range(0, n, row_batch_size): end = min(start + row_batch_size, n) - idx_np = np.stack([unpack_lsb_values(row, k, bits) for row in packed_rows[start:end]]) + if interleaved: + idx_np = packed_indices[start:end] + else: + idx_np = np.stack([unpack_lsb_values(row, k, bits) for row in packed_rows[start:end]]) idx = torch.from_numpy(idx_np.astype(np.int64, copy=False)) recon = (codebook[idx] @ rt) * norms[start:end].float().unsqueeze(1) out[start:end] = (recon / scale).to(out_dtype) diff --git a/python/cactus/convert/interleave_orthogonal_cq4.py b/python/cactus/convert/interleave_orthogonal_cq4.py new file mode 100644 index 000000000..cc0c6fdc2 --- /dev/null +++ b/python/cactus/convert/interleave_orthogonal_cq4.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import argparse +import struct +from pathlib import Path + +import numpy as np + +from .export.qdq import ( + ALIGNMENT_DEFAULT, + CACTUS_MAGIC, + FLAG_INTERLEAVED_4ROW, + FLAG_ORTHOGONAL_ROTATION, + HEADER_SIZE, + align_offset, + read_cq_payload, + read_header, + unpack_lsb_values, +) +from .quantization.cq import PRECISION_CQ, pack_indices_interleaved_4row + + +def _padding(offset: int, alignment: int) -> bytes: + return b"\0" * (align_offset(offset, alignment) - offset) + + +def interleave_orthogonal_cq4_file(input_path: Path, output_path: Path, *, force: bool = False) -> None: + if output_path.exists() and not force: + raise FileExistsError(f"{output_path} exists; pass --force to overwrite") + + header = read_header(input_path) + if header.ndim != 2: + raise ValueError(f"{input_path}: expected 2D CQ tensor") + n, k = header.shape + if header.precision != PRECISION_CQ[4] or header.bits != 4: + raise ValueError(f"{input_path}: expected CQ4") + if (header.flags & FLAG_ORTHOGONAL_ROTATION) == 0: + raise ValueError(f"{input_path}: expected orthogonal CQ rotation") + if header.flags & FLAG_INTERLEAVED_4ROW: + raise ValueError(f"{input_path}: already INTERLEAVED_4ROW") + if header.num_groups != 1 or header.group_size != k: + raise ValueError( + f"{input_path}: expected one full-width group, got group_size={header.group_size} num_groups={header.num_groups}" + ) + if n % 4 != 0 or k % 32 != 0: + raise ValueError(f"{input_path}: expected N % 4 == 0 and K % 32 == 0, got shape={header.shape}") + + scales_blob, packed = read_cq_payload(input_path, header) + rows = packed.reshape(n, k // 2) + indices = np.stack([unpack_lsb_values(row, k, 4) for row in rows]) + interleaved = pack_indices_interleaved_4row(indices, k, 4) + + output_path.parent.mkdir(parents=True, exist_ok=True) + alignment = header.alignment or ALIGNMENT_DEFAULT + scales_offset = align_offset(HEADER_SIZE, alignment) + scales_end = scales_offset + len(scales_blob) + data_offset = align_offset(scales_end, alignment) + flags = header.flags | FLAG_INTERLEAVED_4ROW + + with output_path.open("wb") as out: + out.write(CACTUS_MAGIC) + out.write(struct.pack(" None: + parser = argparse.ArgumentParser(description="Rewrite a full-width orthogonal CQ4 file to 4-row interleaved storage.") + parser.add_argument("input", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--force", action="store_true") + args = parser.parse_args() + interleave_orthogonal_cq4_file(args.input, args.output, force=args.force) + print(f"wrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/python/cactus/convert/model_adapters/policy.py b/python/cactus/convert/model_adapters/policy.py index a56d7fe31..380ac3f24 100644 --- a/python/cactus/convert/model_adapters/policy.py +++ b/python/cactus/convert/model_adapters/policy.py @@ -14,6 +14,7 @@ class TensorPolicy: use_gptq: bool rotation: str fallback_reason: str | None = None + layout: str = "row_major" def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, family: str) -> TensorPolicy: @@ -58,8 +59,10 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("convert", f"CQ{user_bits}", user_bits, component, False, "hadamard") if family == "gemma4" and component in {"audio", "vision"}: return TensorPolicy("fallback", "FP16", None, component, False, "none", "gemma4 media tower accuracy") - if component == "embedding" or out in {"token_embeddings.weights", "output_weight.weights"}: - return TensorPolicy("convert", "CQ4", 4, component, False, "orthogonal") + output_head_or_tied_embedding = out in {"token_embeddings.weights", "decoder_token_embeddings.weights", "output_weight.weights"} + if component == "embedding" or output_head_or_tied_embedding: + layout = "interleaved_4row" if output_head_or_tied_embedding else "row_major" + return TensorPolicy("convert", "CQ4", 4, component, False, "orthogonal", layout=layout) if component == "audio" or component == "transcription": return TensorPolicy("convert", f"CQ{user_bits}", user_bits, component, False, "hadamard") return TensorPolicy("convert", f"CQ{user_bits}", user_bits, component, True, "hadamard") diff --git a/python/cactus/convert/quantization/cq.py b/python/cactus/convert/quantization/cq.py index 67c126b06..2d3942389 100644 --- a/python/cactus/convert/quantization/cq.py +++ b/python/cactus/convert/quantization/cq.py @@ -36,6 +36,7 @@ class CQTensor: rotation_family: str = "hadamard" seed: int = 1234 gptq_used: bool = False + interleaved_4row: bool = False _CODEBOOK_CACHE: dict[tuple[int, int], np.ndarray] = {} @@ -325,18 +326,34 @@ def write_cq_tensor(out_path: Path, cq: CQTensor) -> None: is_embedding = out_path.stem in _EMBEDDING_TENSOR_STEMS + if cq.interleaved_4row: + if cq.bits not in (1, 2, 3, 4): + raise ValueError(f"INTERLEAVED_4ROW only supports CQ1..CQ4, got CQ{cq.bits}") + if n % 4 != 0: + raise ValueError(f"INTERLEAVED_4ROW requires N % 4 == 0, got N={n}") + if cq.rotation_family == "orthogonal": + if cq.bits != 4 or group_size != k: + raise ValueError("orthogonal INTERLEAVED_4ROW requires full-width CQ4") + if k % 32 != 0: + raise ValueError(f"orthogonal INTERLEAVED_4ROW requires K % 32 == 0, got K={k}") + elif group_size % 32 != 0 or group_size > 256: + raise ValueError(f"INTERLEAVED_4ROW requires group_size % 32 == 0 and group_size <= 256, got {group_size}") + interleaved = ( - cq.bits in (1, 2, 3, 4) - and cq.rotation_family != "orthogonal" - and n % 4 == 0 - and group_size % 32 == 0 - and group_size <= 256 - and not is_embedding + cq.interleaved_4row + or ( + cq.bits in (1, 2, 3, 4) + and cq.rotation_family != "orthogonal" + and n % 4 == 0 + and group_size % 32 == 0 + and group_size <= 256 + and not is_embedding + ) ) codebook_f32 = make_codebook(group_size, cq.bits).astype(np.float32) norms_f32 = cq.norms.astype(np.float32, copy=True) - if interleaved: + if interleaved and cq.rotation_family != "orthogonal": cb_max = float(np.max(np.abs(codebook_f32))) cb_factor = cb_max / 127.0 if cb_max > 1e-12 else 1.0 / 127.0 codebook_f32 = codebook_f32 / cb_factor diff --git a/python/cactus/convert/tests/test_cq.py b/python/cactus/convert/tests/test_cq.py index 60d7e6ae4..2cd816e25 100644 --- a/python/cactus/convert/tests/test_cq.py +++ b/python/cactus/convert/tests/test_cq.py @@ -1,8 +1,10 @@ from __future__ import annotations +from dataclasses import replace import struct import numpy as np +import torch from cactus.convert.cactus_adapters.tensor_io import ( FLAG_HAS_SCALES, @@ -11,6 +13,8 @@ save_pointwise_conv1d_int8_with_header, save_tensor_with_header, ) +from cactus.convert.export.qdq import FLAG_INTERLEAVED_4ROW, FLAG_ORTHOGONAL_ROTATION, dequantize_cq_file, read_header +from cactus.convert.interleave_orthogonal_cq4 import interleave_orthogonal_cq4_file from cactus.convert.quantization.cq import PRECISION_CQ, pack_indices_lsb, quantize_hadamard, quantize_orthogonal, write_cq_tensor @@ -52,6 +56,35 @@ def test_orthogonal_embedding_is_cq4(tmp_path): assert cq.rotation_family == "orthogonal" +def test_orthogonal_interleaved_cq4_qdq_matches_row_major(tmp_path): + w = np.random.default_rng(2).standard_normal((8, 32), dtype=np.float32) + cq = quantize_orthogonal(w, bits=4) + row_path = tmp_path / "row.weights" + inter_path = tmp_path / "inter.weights" + write_cq_tensor(row_path, cq) + write_cq_tensor(inter_path, replace(cq, interleaved_4row=True)) + + inter_header = read_header(inter_path) + assert inter_header.flags & FLAG_ORTHOGONAL_ROTATION + assert inter_header.flags & FLAG_INTERLEAVED_4ROW + + row = dequantize_cq_file(row_path, read_header(row_path), torch.float32, 4) + inter = dequantize_cq_file(inter_path, inter_header, torch.float32, 4) + assert torch.max(torch.abs(row - inter)).item() <= 1e-6 + + +def test_interleave_orthogonal_cq4_file_preserves_qdq(tmp_path): + w = np.random.default_rng(3).standard_normal((8, 32), dtype=np.float32) + src = tmp_path / "src.weights" + dst = tmp_path / "dst.weights" + write_cq_tensor(src, quantize_orthogonal(w, bits=4)) + interleave_orthogonal_cq4_file(src, dst) + + src_tensor = dequantize_cq_file(src, read_header(src), torch.float32, 4) + dst_tensor = dequantize_cq_file(dst, read_header(dst), torch.float32, 4) + assert torch.max(torch.abs(src_tensor - dst_tensor)).item() <= 1e-6 + + def test_int8_bias_uses_cactus_grouped_layout(tmp_path): bias = np.array([-1.0, 0.0, 2.0], dtype=np.float32) out = tmp_path / "bias.weights" diff --git a/python/cactus/convert/tests/test_policy.py b/python/cactus/convert/tests/test_policy.py index 3a39f9fe8..1de4101b0 100644 --- a/python/cactus/convert/tests/test_policy.py +++ b/python/cactus/convert/tests/test_policy.py @@ -1,6 +1,6 @@ from cactus.convert.model_adapters.naming import cactus_name_for_tensor from cactus.convert.model_adapters.policy import policy_for_tensor -from cactus.convert.cli import _bits_for_component +from cactus.convert.cli import _bits_for_component, _validate_cq_layout from cactus.convert.model_adapters.detection import detect_family from cactus.convert.cactus_adapters.config_utils import extract_parakeet_tdt_config, extract_whisper_config from cactus.convert.cli import _augment_state_dict_for_family @@ -13,6 +13,44 @@ def test_policy_embedding_cq4(): p = policy_for_tensor(match, (100, 64), 2, "generic") assert p.precision == "CQ4" assert p.rotation == "orthogonal" + assert p.layout == "interleaved_4row" + + +def test_policy_lm_head_interleaved_cq4(): + match = cactus_name_for_tensor("lm_head.weight", "generic", 1) + p = policy_for_tensor(match, (100, 64), 2, "generic") + assert p.precision == "CQ4" + assert p.rotation == "orthogonal" + assert p.layout == "interleaved_4row" + + +def test_policy_lfm_qwen_tied_embeddings_interleaved_cq4(): + for family in ("lfm2", "qwen"): + embed = cactus_name_for_tensor("model.language_model.embed_tokens.weight", family, 1) + embed_policy = policy_for_tensor(embed, (65536, 2048), 2, family) + assert embed.output_name == "token_embeddings.weights" + assert embed_policy.precision == "CQ4" + assert embed_policy.rotation == "orthogonal" + assert embed_policy.layout == "interleaved_4row" + _validate_cq_layout(embed_policy, (65536, 2048), embed.source_name, embed.output_name) + + lm_head = cactus_name_for_tensor("lm_head.weight", family, 1) + lm_head_policy = policy_for_tensor(lm_head, (65536, 2048), 2, family) + assert lm_head.output_name == "output_weight.weights" + assert lm_head_policy.precision == "CQ4" + assert lm_head_policy.rotation == "orthogonal" + assert lm_head_policy.layout == "interleaved_4row" + _validate_cq_layout(lm_head_policy, (65536, 2048), lm_head.source_name, lm_head.output_name) + + +def test_interleaved_lm_head_policy_rejects_unsupported_shape(): + match = cactus_name_for_tensor("lm_head.weight", "generic", 1) + p = policy_for_tensor(match, (101, 64), 2, "generic") + try: + _validate_cq_layout(p, (101, 64), "lm_head.weight", "output_weight.weights") + except RuntimeError: + return + raise AssertionError("expected unsupported interleaved LM-head shape to fail") def test_adapter_registry_preserves_family_names(): diff --git a/python/cactus/transpile/weight_compat.py b/python/cactus/transpile/weight_compat.py index b856cd122..7b6c7421b 100644 --- a/python/cactus/transpile/weight_compat.py +++ b/python/cactus/transpile/weight_compat.py @@ -15,12 +15,14 @@ from cactus.transpile.runtime_compat import Graph from cactus.transpile.weight_binding import WeightBinding from cactus.convert.quantization.cq import FLAG_ORTHOGONAL_ROTATION +from cactus.convert.quantization.cq import FLAG_INTERLEAVED_4ROW from cactus.convert.quantization.cq import GROUP_SIZE as CQ_GROUP_SIZE from cactus.convert.quantization.cq import PRECISION_CQ from cactus.convert.quantization.cq import make_codebook from cactus.convert.quantization.cq import make_hadamard_components from cactus.convert.quantization.cq import make_hadamard_matrix from cactus.convert.quantization.cq import make_orthogonal_rotation +from cactus.convert.quantization.cq import pack_indices_interleaved_4row from cactus.convert.quantization.cq import pack_indices_lsb from cactus.convert.quantization.cq import quantize_hadamard from cactus.convert.quantization.cq import quantize_orthogonal @@ -94,6 +96,7 @@ def ensure_embedding_binding_compatible(binding: WeightBinding) -> WeightBinding compat_path, bits=int(config["bits"]), rotation=str(config["rotation"]), + layout=str(config.get("layout", "row_major")), ) _cleanup_legacy_fp16_cache(source_path) return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) @@ -104,7 +107,7 @@ def _embedding_cache_config(filename: str, *, source_name: str) -> dict[str, obj if filename in _PER_LAYER_EMBEDDING_FILENAMES or normalized_source.endswith("embed_tokens_per_layer.weight"): return {"bits": 2, "rotation": "hadamard"} if filename in _TOKEN_EMBEDDING_FILENAMES: - return {"bits": 4, "rotation": "orthogonal"} + return {"bits": 4, "rotation": "orthogonal", "layout": "interleaved_4row"} return {"bits": 4, "rotation": "orthogonal"} @@ -286,6 +289,7 @@ def _materialize_cq_embedding_cache( *, bits: int, rotation: str, + layout: str = "row_major", ) -> None: src_mtime_ns = opened.path.stat().st_mtime_ns if out_path.exists() and out_path.stat().st_mtime_ns >= src_mtime_ns: @@ -296,6 +300,12 @@ def _materialize_cq_embedding_cache( rows = int(opened.original_n or opened.shape[0]) cols = int(opened.shape[1]) + interleaved = layout == "interleaved_4row" + if interleaved: + if rotation != "orthogonal" or bits != 4: + raise RuntimeError(f"INTERLEAVED_4ROW embedding cache requires orthogonal CQ4, got rotation={rotation} bits={bits}") + if rows % 4 != 0 or cols % 32 != 0: + raise RuntimeError(f"INTERLEAVED_4ROW embedding cache requires rows % 4 == 0 and cols % 32 == 0, got {(rows, cols)}") if rotation == "hadamard" and cols % CQ_GROUP_SIZE != 0: raise RuntimeError( f"Hadamard CQ embedding conversion requires hidden dim divisible by {CQ_GROUP_SIZE}, " @@ -319,6 +329,8 @@ def _materialize_cq_embedding_cache( trailer_parts: list[bytes] = [] if rotation == "orthogonal": flags |= FLAG_ORTHOGONAL_ROTATION + if interleaved: + flags |= FLAG_INTERLEAVED_4ROW trailer_parts.append(make_orthogonal_rotation(group_size, seed=1234).astype(np.float16).tobytes()) else: left, right, perm = make_hadamard_components(group_size, seed=1234) @@ -334,7 +346,7 @@ def _materialize_cq_embedding_cache( stop = min(start + chunk_rows, rows) chunk = _dequantize_int8_rows(opened, start, stop) if rotation == "orthogonal": - packed, norms = _quantize_orthogonal_chunk(chunk, bits=bits, input_scale=input_scale) + packed, norms = _quantize_orthogonal_chunk(chunk, bits=bits, input_scale=input_scale, interleaved=interleaved) else: packed, norms = _quantize_hadamard_chunk(chunk, bits=bits, input_scale=input_scale) norms_handle.write(np.ascontiguousarray(norms, dtype=np.float16).tobytes()) @@ -456,6 +468,7 @@ def _quantize_orthogonal_chunk( *, bits: int, input_scale: np.ndarray, + interleaved: bool = False, ) -> tuple[np.ndarray, np.ndarray]: work = chunk.astype(np.float32, copy=False) * input_scale.astype(np.float32, copy=False)[None, :] rows, cols = work.shape @@ -464,6 +477,8 @@ def _quantize_orthogonal_chunk( norms = np.linalg.norm(work, axis=1).clip(min=1e-8).astype(np.float32, copy=False) rotated = (work / norms[:, None]) @ rotation indices = np.abs(rotated[..., None] - codebook[None, None, :]).argmin(axis=-1).astype(np.uint8) + if interleaved: + return pack_indices_interleaved_4row(indices, cols, bits), norms.astype(np.float16).reshape(rows, 1) return pack_indices_lsb(indices, cols, bits), norms.astype(np.float16).reshape(rows, 1) diff --git a/python/tests/test_transpile_weight_compat.py b/python/tests/test_transpile_weight_compat.py index 1643e28c2..be52f54e5 100644 --- a/python/tests/test_transpile_weight_compat.py +++ b/python/tests/test_transpile_weight_compat.py @@ -259,6 +259,36 @@ def test_resolve_weight_binding_expands_language_model_backbone_aliases(tmp_path assert binding.path.endswith("layer_0_scalar.weights") +def test_resolve_weight_binding_maps_tied_lm_head_to_token_embedding_manifest(tmp_path: Path) -> None: + (tmp_path / "token_embeddings.weights").write_bytes(b"") + (tmp_path / "weights_manifest.json").write_text( + json.dumps( + { + "weights": [ + { + "source_name": "model.language_model.embed_tokens.weight", + "hf_name": "model.language_model.embed_tokens.weight", + "adapter_name": "model.language_model.embed_tokens.weight", + "source_names": ["model.language_model.embed_tokens.weight"], + "output_name": "token_embeddings.weights", + "component": "embedding", + } + ] + } + ) + ) + + for source_name in ("lm_head.weight", "model.lm_head.weight"): + binding = resolve_weight_binding( + weights_dir=str(tmp_path), + source_name=source_name, + ) + + assert binding is not None + assert binding.path.endswith("token_embeddings.weights") + assert binding.kind == "embedding" + + def test_resolve_weight_binding_expands_nested_model_tower_aliases(tmp_path: Path) -> None: (tmp_path / "vision_q.weights").write_bytes(b"") (tmp_path / "weights_manifest.json").write_text(