From 7976d362deb9fe7b0d191bb538c7410c549eb718 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 03:01:52 -0700 Subject: [PATCH 01/49] deepseek41: gguf constants, arch enums, tensor/key contract --- TENSORS.md | 96 +++++++++++++++++++++++++++++++++++++ gguf-py/gguf/constants.py | 87 +++++++++++++++++++++++++++++++++ gguf-py/gguf/gguf_writer.py | 69 ++++++++++++++++++++++++++ src/llama-arch.cpp | 41 ++++++++++++++++ src/llama-arch.h | 31 ++++++++++++ src/llama-hparams.h | 7 +++ src/llama-model.h | 8 ++++ 7 files changed, 339 insertions(+) create mode 100644 TENSORS.md diff --git a/TENSORS.md b/TENSORS.md new file mode 100644 index 000000000000..96b495909a72 --- /dev/null +++ b/TENSORS.md @@ -0,0 +1,96 @@ +# DeepSeek-V4.1 (`deepseek41`) GGUF contract + +Owner: builder A (arch, graph). Builder B (converter) writes exactly this. Shapes are GGUF `ne` order (fastest dim first), +i.e. the reverse of the HF `[out, in]` shape. HF names are the official checkpoint names (`layers.N.*`, `mtp.M.*`). + +## Arch +`general.architecture = "deepseek41"` (`gguf.MODEL_ARCH.DEEPSEEK41`, `LLM_ARCH_DEEPSEEK41`). +`block_count` = 40 (backbone only) or 43 with `nextn_predict_layers = 3` when the DSpark blocks `mtp.0..2` are written as +`blk.40..42` (optional, later). + +## Tensors +Unchanged from deepseek4 (same names and shapes): +`token_embd`, `output_norm`, `output`; per block `attn_norm`, `attn_sinks` (F32 `{n_head}`), `attn_q_a`, `attn_q_a_norm`, +`attn_q_b`, `attn_kv`, `attn_kv_a_norm`, `attn_output_a`, `attn_output_b`, `hc_{attn,ffn}_{fn,base,scale}` (F32), +`ffn_norm`, `ffn_gate_inp`, `exp_probs_b` (F32 `.bias`), `exp_probs_b_vl` (F32 `.bias`, optional), +`ffn_{gate,up,down}_exps` (MXFP4), `ffn_{gate,up,down}_shexp`. + +Removed vs deepseek4: `output_hc_{fn,base,scale}` (no hc_head: the final collapse uses the last FFN pre-mix), +`attn_compressor_ape`, `indexer.compressor_*`, `ffn_gate_tid2eid`, `nextn.*`. + +| GGUF name | HF name | ne | dtype | layers | +|---|---|---|---|---| +| `blk.N.attn_compressor_kv.weight` | `layers.N.attn.compressor.wkv.weight` | `{n_embd, head_dim}` | BF16 | kv sources (2,8,14,20) | +| `blk.N.attn_compressor_gate.weight` | `layers.N.attn.compressor.wgate.weight` | `{n_embd, head_dim}` | BF16 | kv sources with ratio > 1 (2,8,14) | +| `blk.N.attn_compressor_norm.weight` | `layers.N.attn.compressor.norm.weight` | `{head_dim}` | F32/BF16 | kv sources | +| `blk.N.indexer.attn_k.weight` | `layers.N.attn.indexer.wk.weight` | `{head_dim, index_head_dim}` | BF16 | kv sources | +| `blk.N.indexer.k_norm.weight` | `layers.N.attn.indexer.k_norm.weight` | `{index_head_dim}` | F32/BF16 | kv sources | +| `blk.N.indexer.attn_q_b.weight` | `layers.N.attn.indexer.wq_b.weight` | `{q_lora_rank, index_n_head*index_head_dim}` | BF16 (FP8 dequant) | index sources (2,8,...,36) | +| `blk.N.indexer.proj.weight` | `layers.N.attn.indexer.weights_proj.weight` | `{n_embd, index_n_head}` | BF16 | index sources | +| `blk.N.engram_embd.weight` | `layers.N.engram.embed.weight` + `.scale` | `{engram_head_dim, num_embeddings[i]}` | MXFP8 (B) / BF16 / F32 | engram layers (1,14) | +| `blk.N.engram_kv.weight` | `layers.N.engram.wkv.weight` | `{n_hash_cols*engram_head_dim, n_embd*(hc+1)}` | BF16 (FP8 dequant) | engram layers | +| `blk.N.engram_gate.weight` | `layers.N.engram.q_weight * layers.N.engram.k_weight` (folded, elementwise) | `{n_embd, hc}` | F32 | engram layers | +| `blk.40.dspark_main_proj.weight` | `mtp.0.main_proj.weight` | `{n_embd*len(target_layers), n_embd}` | BF16 | DSpark stage 0 (optional) | +| `blk.40.dspark_main_norm.weight` | `mtp.0.main_norm.weight` | `{n_embd}` | F32 | DSpark stage 0 | +| `blk.42.dspark_norm.weight` | `mtp.2.norm.weight` | `{n_embd}` | F32 | last DSpark stage | +| `markov_w1.weight` | `mtp.2.markov_head.embed.weight` | `{markov_rank, n_vocab}` | BF16 | DSpark | +| `markov_w2.weight` | `mtp.2.markov_head.head.weight` | `{markov_rank, n_vocab}` | BF16 | DSpark | +| `conf_proj.weight` | `mtp.2.confidence_head.proj.weight` | `{n_embd+markov_rank, 1}` | F32 | DSpark | + +Notes +- `n_hash_cols = (engram.max_ngram_size - 1) * engram.head_count` (24). `engram_kv` output rows keep the HF order: + rows `[0, hc*n_embd)` are the `hc` keys (copy-major), rows `[hc*n_embd, (hc+1)*n_embd)` the shared value. +- `engram_embd` rows are written exactly as stored (no TP padding rows: exactly `engram_num_embeddings[i]` rows). The + row for hash id r is `float(w[r]).view(8,32) * 2^(scale[r]-127)`. Never repack, never offload: the graph gathers it on + the host with `ggml_get_rows`. Any type with get_rows on CPU works for the graph. +- FP8 linear weights use 32x32 `ue8m0` blocks (derive the block from the scale shape, do not hard-code 128). +- Every layer `0..39` has all backbone attention/HC/MoE tensors; DSpark blocks have 128 experts (`dspark.expert_count`). + +## Keys (`deepseek41.` prefix) +Same as deepseek4: `context_length`, `embedding_length`, `block_count`, `feed_forward_length` (unused), +`expert_feed_forward_length`, `expert_count` (384), `expert_used_count` (6), `expert_shared_count`, `expert_weights_scale` +(1.5), `expert_weights_norm` (true), `expert_gating_func` (`gguf.ExpertGatingFuncType.SQRTSOFTPLUS` = 4), +`swiglu_clamp_exp` / `swiglu_clamp_shexp` (f32[block_count], 10.0), `attention.head_count` (64), `attention.head_count_kv` (1), +`attention.key_length` = `attention.value_length` (512), `rope.dimension_count` (64), `attention.q_lora_rank`, +`attention.sliding_window` (128), `attention.layer_norm_rms_epsilon` (1e-20), `attention.output_group_count`, +`attention.output_lora_rank`, `attention.compress_ratios` (u32[>= block_count], all 43 entries ok), +`attention.compress_rope_freq_base` (160000), `rope.freq_base` (10000), `rope.scaling.type` yarn, `rope.scaling.factor` 16, +`rope.scaling.original_context_length` 65536, `rope.scaling.yarn_beta_fast` 32, `rope.scaling.yarn_beta_slow` 1, +`attention.indexer.head_count` (32), `attention.indexer.key_length` (128), `attention.indexer.top_k` (512), +`hyper_connection.count` (4), `hyper_connection.sinkhorn_iterations` (20), `hyper_connection.epsilon` (1e-6), +`embedding_length_out` = n_embd*hc (as deepseek4). +Not written: `hash_layer_count`. + +New: +| key | type | value (V4.1-Flash-1017) | +|---|---|---| +| `attention.kv_source_layers` | u32[] | [2, 8, 14, 20] | +| `attention.indexer.source_layers` | u32[] | [2, 8, 14, 20, 24, 28, 32, 36] | +| `attention.indexer.candidate_source_layer` | i32 | 20 (-1 = off) | +| `attention.indexer.candidate_top_k_blocks` | u32 | 2048 | +| `attention.indexer.candidate_block_size` | u32 | 8 | +| `engram.layers` | u32[] | [1, 14] | +| `engram.num_embeddings` | u32[] | [384006168, 384016682] | +| `engram.max_ngram_size` | u32 | 4 | +| `engram.vocab_size` | u32 | 16000000 (prime search start, informational) | +| `engram.head_count` | u32 | 8 | +| `engram.head_dim` | u32 | 256 | +| `engram.pad_token_id` | u32 | 2 (raw id; the graph maps it through token_map) | +| `engram.compressed_vocab_size` | u32 | 99092 | +| `engram.token_map` | i32[n_vocab] | `build_compressed_token_map(tokenizer)` from inference/engram.py | +| `engram.multipliers` | u64[n_engram*max_ngram] | layer-major, `compute_hash_multipliers` | +| `engram.primes` | u32[n_engram*(max_ngram-1)*head_count] | layer, n-gram order (2..max), head; `EngramLayout.primes` | +| `dspark.block_size` | u32 | 5 | +| `dspark.noise_token_id` | u32 | 128799 | +| `dspark.target_layers` | u32[] | [37, 38, 39] | +| `dspark.markov_rank` | u32 | 256 | +| `dspark.expert_count` | u32 | 128 | +| `dspark.expert_used_count` | u32 | 3 | + +Engram offsets are not stored: per layer they are the exclusive cumulative sum of that layer's 24 primes (order-major, +head-minor), exactly `NgramHashState.offsets`. + +gguf-py writer helpers: `add_attention_kv_source_layers`, `add_indexer_source_layers`, `add_indexer_candidate_source_layer`, +`add_indexer_candidate_top_k_blocks`, `add_indexer_candidate_block_size`, `add_engram_{layers,num_embeddings,max_ngram_size, +vocab_size,head_count,head_dim,pad_token_id,compressed_vocab_size,token_map,multipliers,primes}`, +`add_dspark_{block_size,noise_token_id,target_layers,markov_rank,expert_count,expert_used_count}`. diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 0c9872228a8a..190cc2d63f7a 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -209,6 +209,7 @@ class Attention: VALUE_SCALE = "{arch}.attention.value_scale" COMPRESS_RATIOS = "{arch}.attention.compress_ratios" COMPRESS_ROPE_FREQ_BASE = "{arch}.attention.compress_rope_freq_base" + KV_SOURCE_LAYERS = "{arch}.attention.kv_source_layers" # deepseek41 TEMPERATURE_LENGTH = "{arch}.attention.temperature_length" KEY_LENGTH_MLA = "{arch}.attention.key_length_mla" VALUE_LENGTH_MLA = "{arch}.attention.value_length_mla" @@ -234,6 +235,10 @@ class Indexer: TYPES = "{arch}.attention.indexer.types" KPOOL = "{arch}.attention.indexer.kpool" # GLM5-Next KPOOL_SELECT_TAIL = "{arch}.attention.indexer.kpool_select_tail" # GLM5-Next + SOURCE_LAYERS = "{arch}.attention.indexer.source_layers" # deepseek41 + CANDIDATE_SOURCE_LAYER = "{arch}.attention.indexer.candidate_source_layer" # deepseek41, -1 = off + CANDIDATE_TOP_K_BLOCKS = "{arch}.attention.indexer.candidate_top_k_blocks" # deepseek41 + CANDIDATE_BLOCK_SIZE = "{arch}.attention.indexer.candidate_block_size" # deepseek41 class HyperConnection: COUNT = "{arch}.hyper_connection.count" @@ -244,6 +249,27 @@ class HyperConnection: # absent means the mix projection is full rank (DeepSeek-V4 behaviour) LOW_RANK = "{arch}.hyper_connection.low_rank" + class Engram: # deepseek41 hashed n-gram memory + LAYERS = "{arch}.engram.layers" # u32[n_engram] + NUM_EMBEDDINGS = "{arch}.engram.num_embeddings" # u32[n_engram], table rows per layer + MAX_NGRAM_SIZE = "{arch}.engram.max_ngram_size" # u32, n-gram orders 2..max + VOCAB_SIZE = "{arch}.engram.vocab_size" # u32, prime search start (informational) + HEAD_COUNT = "{arch}.engram.head_count" # u32, heads per n-gram order + HEAD_DIM = "{arch}.engram.head_dim" # u32, row width + PAD_TOKEN_ID = "{arch}.engram.pad_token_id" # u32, raw token id filling missing history + COMPRESSED_VOCAB_SIZE = "{arch}.engram.compressed_vocab_size" # u32 + TOKEN_MAP = "{arch}.engram.token_map" # i32[n_vocab], raw id -> compressed id + MULTIPLIERS = "{arch}.engram.multipliers" # u64[n_engram * max_ngram], layer-major + PRIMES = "{arch}.engram.primes" # u32[n_engram * (max_ngram-1) * head_count], layer, order, head + + class DSpark: # deepseek41 draft head (mtp.* blocks) + BLOCK_SIZE = "{arch}.dspark.block_size" + NOISE_TOKEN_ID = "{arch}.dspark.noise_token_id" + TARGET_LAYERS = "{arch}.dspark.target_layers" + MARKOV_RANK = "{arch}.dspark.markov_rank" + EXPERT_COUNT = "{arch}.dspark.expert_count" + EXPERT_USED_COUNT = "{arch}.dspark.expert_used_count" + class PerLayerEmbedding: LAYERS = "{arch}.ple.layers" NGRAM_SIZE = "{arch}.ple.ngram_size" @@ -618,6 +644,7 @@ class MODEL_ARCH(IntEnum): DEEPSEEK2OCR = auto() DEEPSEEK32 = auto() DEEPSEEK4 = auto() + DEEPSEEK41 = auto() CHATGLM = auto() GLM4 = auto() GLM4_MOE = auto() @@ -1283,6 +1310,13 @@ class MODEL_TENSOR(IntEnum): DSPARK_MARKOV_W1 = auto() # markov head: prev-token embed DSPARK_MARKOV_W2 = auto() # markov head: bias projection DSPARK_CONF_PROJ = auto() # confidence head + DSPARK_MAIN_PROJ = auto() # deepseek41: target hidden taps -> n_embd + DSPARK_MAIN_NORM = auto() # deepseek41 + DSPARK_NORM = auto() # deepseek41: final norm of the last DSpark stage + # engram (deepseek41) + ENGRAM_EMBD = auto() + ENGRAM_KV = auto() + ENGRAM_GATE = auto() DFLASH_ATTN_CONV_BASE = auto() DFLASH_ATTN_CONV_PROJ = auto() DFLASH_FFN_CONV_BASE = auto() @@ -1422,6 +1456,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.DEEPSEEK2OCR: "deepseek2-ocr", MODEL_ARCH.DEEPSEEK32: "deepseek32", MODEL_ARCH.DEEPSEEK4: "deepseek4", + MODEL_ARCH.DEEPSEEK41: "deepseek41", MODEL_ARCH.CHATGLM: "chatglm", MODEL_ARCH.GLM4: "glm4", MODEL_ARCH.GLM4_MOE: "glm4moe", @@ -2111,6 +2146,12 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.DSPARK_MARKOV_W1: "markov_w1", MODEL_TENSOR.DSPARK_MARKOV_W2: "markov_w2", MODEL_TENSOR.DSPARK_CONF_PROJ: "conf_proj", + MODEL_TENSOR.DSPARK_MAIN_PROJ: "blk.{bid}.dspark_main_proj", + MODEL_TENSOR.DSPARK_MAIN_NORM: "blk.{bid}.dspark_main_norm", + MODEL_TENSOR.DSPARK_NORM: "blk.{bid}.dspark_norm", + MODEL_TENSOR.ENGRAM_EMBD: "blk.{bid}.engram_embd", + MODEL_TENSOR.ENGRAM_KV: "blk.{bid}.engram_kv", + MODEL_TENSOR.ENGRAM_GATE: "blk.{bid}.engram_gate", MODEL_TENSOR.DFLASH_ATTN_CONV_BASE: "blk.{bid}.attn_conv_base", MODEL_TENSOR.DFLASH_ATTN_CONV_PROJ: "blk.{bid}.attn_conv_proj", MODEL_TENSOR.DFLASH_FFN_CONV_BASE: "blk.{bid}.ffn_conv_base", @@ -4201,6 +4242,52 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ], + MODEL_ARCH.DEEPSEEK41: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_SINKS, + MODEL_TENSOR.ATTN_Q_A, + MODEL_TENSOR.ATTN_Q_B, + MODEL_TENSOR.ATTN_Q_A_NORM, + MODEL_TENSOR.ATTN_KV, + MODEL_TENSOR.ATTN_KV_NORM, + MODEL_TENSOR.ATTN_OUT_A, + MODEL_TENSOR.ATTN_OUT_B, + MODEL_TENSOR.HC_ATTN_FN, + MODEL_TENSOR.HC_ATTN_BASE, + MODEL_TENSOR.HC_ATTN_SCALE, + MODEL_TENSOR.HC_FFN_FN, + MODEL_TENSOR.HC_FFN_BASE, + MODEL_TENSOR.HC_FFN_SCALE, + MODEL_TENSOR.ATTN_COMPRESSOR_WKV, + MODEL_TENSOR.ATTN_COMPRESSOR_WGATE, + MODEL_TENSOR.ATTN_COMPRESSOR_NORM, + MODEL_TENSOR.INDEXER_ATTN_K, + MODEL_TENSOR.INDEXER_K_NORM, + MODEL_TENSOR.INDEXER_ATTN_Q_B, + MODEL_TENSOR.INDEXER_PROJ, + MODEL_TENSOR.ENGRAM_EMBD, + MODEL_TENSOR.ENGRAM_KV, + MODEL_TENSOR.ENGRAM_GATE, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_EXP_PROBS_B_VL, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.DSPARK_MAIN_PROJ, + MODEL_TENSOR.DSPARK_MAIN_NORM, + MODEL_TENSOR.DSPARK_NORM, + MODEL_TENSOR.DSPARK_MARKOV_W1, + MODEL_TENSOR.DSPARK_MARKOV_W2, + MODEL_TENSOR.DSPARK_CONF_PROJ, + ], MODEL_ARCH.ERNIE4_5_MOE: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index cc2bd8e7b3b8..79b488f4efe6 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1074,6 +1074,75 @@ def add_attention_compress_ratios(self, values: Sequence[int]) -> None: def add_attention_compress_rope_freq_base(self, value: float) -> None: self.add_float32(Keys.Attention.COMPRESS_ROPE_FREQ_BASE.format(arch=self.arch), value) + # deepseek41: cross-layer KV / indexer sharing and two-level candidate selection + def add_attention_kv_source_layers(self, layers: Sequence[int]) -> None: + self.add_key_value(Keys.Attention.KV_SOURCE_LAYERS.format(arch=self.arch), list(layers), GGUFValueType.ARRAY, GGUFValueType.UINT32) + + def add_indexer_source_layers(self, layers: Sequence[int]) -> None: + self.add_key_value(Keys.Attention.Indexer.SOURCE_LAYERS.format(arch=self.arch), list(layers), GGUFValueType.ARRAY, GGUFValueType.UINT32) + + def add_indexer_candidate_source_layer(self, layer: int) -> None: + self.add_int32(Keys.Attention.Indexer.CANDIDATE_SOURCE_LAYER.format(arch=self.arch), layer) + + def add_indexer_candidate_top_k_blocks(self, value: int) -> None: + self.add_uint32(Keys.Attention.Indexer.CANDIDATE_TOP_K_BLOCKS.format(arch=self.arch), value) + + def add_indexer_candidate_block_size(self, value: int) -> None: + self.add_uint32(Keys.Attention.Indexer.CANDIDATE_BLOCK_SIZE.format(arch=self.arch), value) + + # deepseek41 engram + def add_engram_layers(self, layers: Sequence[int]) -> None: + self.add_key_value(Keys.Engram.LAYERS.format(arch=self.arch), list(layers), GGUFValueType.ARRAY, GGUFValueType.UINT32) + + def add_engram_num_embeddings(self, values: Sequence[int]) -> None: + self.add_key_value(Keys.Engram.NUM_EMBEDDINGS.format(arch=self.arch), list(values), GGUFValueType.ARRAY, GGUFValueType.UINT32) + + def add_engram_max_ngram_size(self, value: int) -> None: + self.add_uint32(Keys.Engram.MAX_NGRAM_SIZE.format(arch=self.arch), value) + + def add_engram_vocab_size(self, value: int) -> None: + self.add_uint32(Keys.Engram.VOCAB_SIZE.format(arch=self.arch), value) + + def add_engram_head_count(self, value: int) -> None: + self.add_uint32(Keys.Engram.HEAD_COUNT.format(arch=self.arch), value) + + def add_engram_head_dim(self, value: int) -> None: + self.add_uint32(Keys.Engram.HEAD_DIM.format(arch=self.arch), value) + + def add_engram_pad_token_id(self, value: int) -> None: + self.add_uint32(Keys.Engram.PAD_TOKEN_ID.format(arch=self.arch), value) + + def add_engram_compressed_vocab_size(self, value: int) -> None: + self.add_uint32(Keys.Engram.COMPRESSED_VOCAB_SIZE.format(arch=self.arch), value) + + def add_engram_token_map(self, values: Sequence[int]) -> None: + self.add_key_value(Keys.Engram.TOKEN_MAP.format(arch=self.arch), list(values), GGUFValueType.ARRAY, GGUFValueType.INT32) + + def add_engram_multipliers(self, values: Sequence[int]) -> None: + self.add_key_value(Keys.Engram.MULTIPLIERS.format(arch=self.arch), list(values), GGUFValueType.ARRAY, GGUFValueType.UINT64) + + def add_engram_primes(self, values: Sequence[int]) -> None: + self.add_key_value(Keys.Engram.PRIMES.format(arch=self.arch), list(values), GGUFValueType.ARRAY, GGUFValueType.UINT32) + + # deepseek41 DSpark draft head + def add_dspark_block_size(self, value: int) -> None: + self.add_uint32(Keys.DSpark.BLOCK_SIZE.format(arch=self.arch), value) + + def add_dspark_noise_token_id(self, value: int) -> None: + self.add_uint32(Keys.DSpark.NOISE_TOKEN_ID.format(arch=self.arch), value) + + def add_dspark_target_layers(self, layers: Sequence[int]) -> None: + self.add_key_value(Keys.DSpark.TARGET_LAYERS.format(arch=self.arch), list(layers), GGUFValueType.ARRAY, GGUFValueType.UINT32) + + def add_dspark_markov_rank(self, value: int) -> None: + self.add_uint32(Keys.DSpark.MARKOV_RANK.format(arch=self.arch), value) + + def add_dspark_expert_count(self, value: int) -> None: + self.add_uint32(Keys.DSpark.EXPERT_COUNT.format(arch=self.arch), value) + + def add_dspark_expert_used_count(self, value: int) -> None: + self.add_uint32(Keys.DSpark.EXPERT_USED_COUNT.format(arch=self.arch), value) + def add_hyper_connection_count(self, count: int) -> None: self.add_uint32(Keys.HyperConnection.COUNT.format(arch=self.arch), count) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index d1dd4cbaeb67..e80ac1299c1d 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -84,6 +84,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_DEEPSEEK2OCR, "deepseek2-ocr" }, { LLM_ARCH_DEEPSEEK32, "deepseek32" }, { LLM_ARCH_DEEPSEEK4, "deepseek4" }, + { LLM_ARCH_DEEPSEEK41, "deepseek41" }, { LLM_ARCH_CHATGLM, "chatglm" }, { LLM_ARCH_GLM4, "glm4" }, { LLM_ARCH_GLM4_MOE, "glm4moe" }, @@ -304,6 +305,11 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_INDEXER_TYPES, "%s.attention.indexer.types" }, { LLM_KV_ATTENTION_INDEXER_KPOOL, "%s.attention.indexer.kpool" }, { LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, "%s.attention.indexer.kpool_select_tail" }, + { LLM_KV_ATTENTION_INDEXER_SOURCE_LAYERS, "%s.attention.indexer.source_layers" }, + { LLM_KV_ATTENTION_INDEXER_CANDIDATE_SOURCE_LAYER, "%s.attention.indexer.candidate_source_layer" }, + { LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, "%s.attention.indexer.candidate_top_k_blocks" }, + { LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, "%s.attention.indexer.candidate_block_size" }, + { LLM_KV_ATTENTION_KV_SOURCE_LAYERS, "%s.attention.kv_source_layers" }, { LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, "%s.attention.output_group_count" }, { LLM_KV_ATTENTION_OUTPUT_LORA_RANK, "%s.attention.output_lora_rank" }, { LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, "%s.attention.compress_rope_freq_base" }, @@ -317,6 +323,25 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_HYPER_CONNECTION_MAGNITUDE, "%s.hyper_connection.magnitude" }, { LLM_KV_HYPER_CONNECTION_LOW_RANK, "%s.hyper_connection.low_rank" }, + { LLM_KV_ENGRAM_LAYERS, "%s.engram.layers" }, + { LLM_KV_ENGRAM_NUM_EMBEDDINGS, "%s.engram.num_embeddings" }, + { LLM_KV_ENGRAM_MAX_NGRAM_SIZE, "%s.engram.max_ngram_size" }, + { LLM_KV_ENGRAM_VOCAB_SIZE, "%s.engram.vocab_size" }, + { LLM_KV_ENGRAM_HEAD_COUNT, "%s.engram.head_count" }, + { LLM_KV_ENGRAM_HEAD_DIM, "%s.engram.head_dim" }, + { LLM_KV_ENGRAM_PAD_TOKEN_ID, "%s.engram.pad_token_id" }, + { LLM_KV_ENGRAM_COMPRESSED_VOCAB_SIZE, "%s.engram.compressed_vocab_size" }, + { LLM_KV_ENGRAM_TOKEN_MAP, "%s.engram.token_map" }, + { LLM_KV_ENGRAM_MULTIPLIERS, "%s.engram.multipliers" }, + { LLM_KV_ENGRAM_PRIMES, "%s.engram.primes" }, + + { LLM_KV_DSPARK_BLOCK_SIZE, "%s.dspark.block_size" }, + { LLM_KV_DSPARK_NOISE_TOKEN_ID, "%s.dspark.noise_token_id" }, + { LLM_KV_DSPARK_TARGET_LAYERS, "%s.dspark.target_layers" }, + { LLM_KV_DSPARK_MARKOV_RANK, "%s.dspark.markov_rank" }, + { LLM_KV_DSPARK_EXPERT_COUNT, "%s.dspark.expert_count" }, + { LLM_KV_DSPARK_EXPERT_USED_COUNT, "%s.dspark.expert_used_count" }, + { LLM_KV_PLE_LAYERS, "%s.ple.layers" }, { LLM_KV_PLE_NGRAM_SIZE, "%s.ple.ngram_size" }, { LLM_KV_PLE_HEADS_PER_NGRAM, "%s.ple.heads_per_ngram" }, @@ -762,6 +787,12 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_DSPARK_MARKOV_W1, "markov_w1" }, { LLM_TENSOR_DSPARK_MARKOV_W2, "markov_w2" }, { LLM_TENSOR_DSPARK_CONF_PROJ, "conf_proj" }, + { LLM_TENSOR_DSPARK_MAIN_PROJ, "blk.%d.dspark_main_proj" }, + { LLM_TENSOR_DSPARK_MAIN_NORM, "blk.%d.dspark_main_norm" }, + { LLM_TENSOR_DSPARK_NORM, "blk.%d.dspark_norm" }, + { LLM_TENSOR_ENGRAM_EMBD, "blk.%d.engram_embd" }, + { LLM_TENSOR_ENGRAM_KV, "blk.%d.engram_kv" }, + { LLM_TENSOR_ENGRAM_GATE, "blk.%d.engram_gate" }, { LLM_TENSOR_DFLASH_ATTN_CONV_BASE, "blk.%d.attn_conv_base" }, { LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, "blk.%d.attn_conv_proj" }, { LLM_TENSOR_DFLASH_FFN_CONV_BASE, "blk.%d.ffn_conv_base" }, @@ -1092,6 +1123,13 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_DSPARK_MARKOV_W1, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}}, {LLM_TENSOR_DSPARK_MARKOV_W2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, {LLM_TENSOR_DSPARK_CONF_PROJ, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DSPARK_MAIN_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DSPARK_MAIN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_DSPARK_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + // engram tables are gathered rows, kept host-side (never repacked / offloaded) + {LLM_TENSOR_ENGRAM_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, + {LLM_TENSOR_ENGRAM_KV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ENGRAM_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_DFLASH_ATTN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_DFLASH_FFN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, @@ -1211,6 +1249,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) { case LLM_ARCH_QWEN35MOE: case LLM_ARCH_QWEN4EXP: case LLM_ARCH_DEEPSEEK4: + case LLM_ARCH_DEEPSEEK41: case LLM_ARCH_MINIMAX_01: return true; default: @@ -1237,6 +1276,7 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) { case LLM_ARCH_QWEN35MOE: case LLM_ARCH_QWEN4EXP: case LLM_ARCH_DEEPSEEK4: + case LLM_ARCH_DEEPSEEK41: case LLM_ARCH_GLM5_NEXT: case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: @@ -1254,6 +1294,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch) { switch (arch) { case LLM_ARCH_COGVLM: case LLM_ARCH_DEEPSEEK4: + case LLM_ARCH_DEEPSEEK41: case LLM_ARCH_GRANITE_SWITCH: case LLM_ARCH_EAGLE3: case LLM_ARCH_DFLASH: diff --git a/src/llama-arch.h b/src/llama-arch.h index 4e8e53421af3..e49643df283d 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -89,6 +89,7 @@ enum llm_arch { LLM_ARCH_DEEPSEEK2OCR, LLM_ARCH_DEEPSEEK32, LLM_ARCH_DEEPSEEK4, + LLM_ARCH_DEEPSEEK41, LLM_ARCH_CHATGLM, LLM_ARCH_GLM4, LLM_ARCH_GLM4_MOE, @@ -309,6 +310,11 @@ enum llm_kv { LLM_KV_ATTENTION_INDEXER_TYPES, LLM_KV_ATTENTION_INDEXER_KPOOL, LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, + LLM_KV_ATTENTION_INDEXER_SOURCE_LAYERS, + LLM_KV_ATTENTION_INDEXER_CANDIDATE_SOURCE_LAYER, + LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, + LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, + LLM_KV_ATTENTION_KV_SOURCE_LAYERS, LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, LLM_KV_ATTENTION_OUTPUT_LORA_RANK, LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, @@ -322,6 +328,25 @@ enum llm_kv { LLM_KV_HYPER_CONNECTION_MAGNITUDE, LLM_KV_HYPER_CONNECTION_LOW_RANK, + LLM_KV_ENGRAM_LAYERS, + LLM_KV_ENGRAM_NUM_EMBEDDINGS, + LLM_KV_ENGRAM_MAX_NGRAM_SIZE, + LLM_KV_ENGRAM_VOCAB_SIZE, + LLM_KV_ENGRAM_HEAD_COUNT, + LLM_KV_ENGRAM_HEAD_DIM, + LLM_KV_ENGRAM_PAD_TOKEN_ID, + LLM_KV_ENGRAM_COMPRESSED_VOCAB_SIZE, + LLM_KV_ENGRAM_TOKEN_MAP, + LLM_KV_ENGRAM_MULTIPLIERS, + LLM_KV_ENGRAM_PRIMES, + + LLM_KV_DSPARK_BLOCK_SIZE, + LLM_KV_DSPARK_NOISE_TOKEN_ID, + LLM_KV_DSPARK_TARGET_LAYERS, + LLM_KV_DSPARK_MARKOV_RANK, + LLM_KV_DSPARK_EXPERT_COUNT, + LLM_KV_DSPARK_EXPERT_USED_COUNT, + LLM_KV_PLE_LAYERS, LLM_KV_PLE_NGRAM_SIZE, LLM_KV_PLE_HEADS_PER_NGRAM, @@ -768,6 +793,12 @@ enum llm_tensor { LLM_TENSOR_DSPARK_MARKOV_W1, LLM_TENSOR_DSPARK_MARKOV_W2, LLM_TENSOR_DSPARK_CONF_PROJ, + LLM_TENSOR_DSPARK_MAIN_PROJ, + LLM_TENSOR_DSPARK_MAIN_NORM, + LLM_TENSOR_DSPARK_NORM, + LLM_TENSOR_ENGRAM_EMBD, + LLM_TENSOR_ENGRAM_KV, + LLM_TENSOR_ENGRAM_GATE, LLM_TENSOR_DFLASH_ATTN_CONV_BASE, LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, LLM_TENSOR_DFLASH_FFN_CONV_BASE, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 922164a544b8..91d7940698b3 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -324,6 +324,13 @@ struct llama_hparams { float dsv4_hc_eps = 0.0f; std::array dsv4_compress_ratios; + // deepseek41: cross-layer sharing of the compressed KV / index K and of the indexer top-k + std::array dsv41_kv_src; // layer whose compressed KV (and index K) this layer reads, -1 = none + std::array dsv41_topk_src; // index-source layer whose top-k this layer attends with, -1 = none + int32_t dsv41_cand_src = -1; // candidate_source_layer, -1 = two-level selection off + uint32_t dsv41_cand_topk_blocks = 0; + uint32_t dsv41_cand_block_size = 0; + // 0 = full rank (DeepSeek-V4) uint32_t hc_low_rank = 0; diff --git a/src/llama-model.h b/src/llama-model.h index bcfbf1e2cb39..1e456e98bb49 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -590,6 +590,14 @@ struct llama_layer { struct ggml_tensor * indexer_kpool_gate = nullptr; struct ggml_tensor * indexer_kpool_ape = nullptr; + // deepseek41 + struct ggml_tensor * engram_embd = nullptr; // [head_dim, n_rows], host-resident gather table + struct ggml_tensor * engram_kv = nullptr; // [n_hash_cols*head_dim, n_embd*(hc+1)] + struct ggml_tensor * engram_gate = nullptr; // [n_embd, hc], q_weight*k_weight folded + struct ggml_tensor * dspark_main_proj = nullptr; + struct ggml_tensor * dspark_main_norm = nullptr; + struct ggml_tensor * dspark_norm = nullptr; + // MSA struct ggml_tensor * index_q_proj = nullptr; struct ggml_tensor * index_k_proj = nullptr; From baa94f5ce6ffc8b854eec1a16b1a5e3f5751e631 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 03:02:37 -0700 Subject: [PATCH 02/49] ggml : add MXFP8 (E4M3 + E8M0) block type 32 E4M3 (fn) values sharing one E8M0 scale, 33 bytes per block, byte compatible with OCP MX FP8 checkpoints (e.g. DeepSeek-V4.1 Engram tables). Dequant is exact in fp32. CPU: reference quantize, dequantize, get_rows, scalar vec_dot with q8_0. CUDA: get_rows. gguf-py: quantize/dequantize. test-backend-ops: get_rows and mul_mat cases. --- ggml/include/ggml.h | 3 +- ggml/src/ggml-common.h | 8 +++++ ggml/src/ggml-cpu/ggml-cpu.c | 6 ++++ ggml/src/ggml-cpu/ops.cpp | 7 ++++ ggml/src/ggml-cpu/quants.c | 31 ++++++++++++++++ ggml/src/ggml-cpu/quants.h | 2 ++ ggml/src/ggml-cuda/dequantize.cuh | 18 ++++++++++ ggml/src/ggml-cuda/getrows.cu | 4 +++ ggml/src/ggml-cuda/ggml-cuda.cu | 1 + ggml/src/ggml-impl.h | 45 +++++++++++++++++++++++ ggml/src/ggml-quants.c | 56 +++++++++++++++++++++++++++++ ggml/src/ggml-quants.h | 3 ++ ggml/src/ggml.c | 9 +++++ gguf-py/gguf/constants.py | 2 ++ gguf-py/gguf/quants.py | 60 +++++++++++++++++++++++++++++++ tests/test-backend-ops.cpp | 11 ++++++ 16 files changed, 265 insertions(+), 1 deletion(-) diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index c37ba947140e..d3d484ee7d19 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -430,7 +430,8 @@ extern "C" { GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, GGML_TYPE_Q2_0 = 42, - GGML_TYPE_COUNT = 43, + GGML_TYPE_MXFP8 = 43, // MXFP8 E4M3 (32 elements, E8M0 scale) + GGML_TYPE_COUNT = 44, }; // [TAG_GGML_PREC] diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h index 1dbbe326d0fc..a9c2385d580e 100644 --- a/ggml/src/ggml-common.h +++ b/ggml/src/ggml-common.h @@ -218,6 +218,14 @@ typedef struct { } block_mxfp4; static_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + QK_MXFP4/2, "wrong mxfp4 block size/padding"); +// MXFP8 (OCP MX): 32 E4M3 (fn) values sharing one E8M0 scale, value = e4m3(q[j]) * 2^(e - 127) +#define QK_MXFP8 32 +typedef struct { + uint8_t e; // E8M0 + uint8_t qs[QK_MXFP8]; // E4M3 (fn) codes +} block_mxfp8; +static_assert(sizeof(block_mxfp8) == sizeof(uint8_t) + QK_MXFP8, "wrong mxfp8 block size/padding"); + #define QK_NVFP4 64 #define QK_NVFP4_SUB 16 // sub-block size for per-group scales typedef struct { diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index ab82e3d6c043..ba79d439739d 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -290,6 +290,12 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_0, .nrows = 1, }, + [GGML_TYPE_MXFP8] = { + .from_float = quantize_row_mxfp8, + .vec_dot = ggml_vec_dot_mxfp8_q8_0, + .vec_dot_type = GGML_TYPE_Q8_0, + .nrows = 1, + }, [GGML_TYPE_NVFP4] = { .from_float = quantize_row_nvfp4, .vec_dot = ggml_vec_dot_nvfp4_q8_0, diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 7a73e138ef1f..4e7e5abf9856 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -673,6 +673,7 @@ void ggml_compute_forward_add( case GGML_TYPE_Q8_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -1125,6 +1126,7 @@ void ggml_compute_forward_add1( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -1256,6 +1258,7 @@ void ggml_compute_forward_acc( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -4912,6 +4915,7 @@ void ggml_compute_forward_out_prod( case GGML_TYPE_Q8_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5231,6 +5235,7 @@ void ggml_compute_forward_set( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5456,6 +5461,7 @@ void ggml_compute_forward_get_rows( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -6216,6 +6222,7 @@ void ggml_compute_forward_clamp( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index 5e36459f8cbc..b8f2a8a12656 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -58,6 +58,10 @@ void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, i quantize_row_mxfp4_ref(x, y, k); } +void quantize_row_mxfp8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + quantize_row_mxfp8_ref(x, y, k); +} + void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { quantize_row_nvfp4_ref(x, y, k); } @@ -326,6 +330,33 @@ void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } +// MXFP8: scalar reference only (the format is used for host-side row gathers) +void ggml_vec_dot_mxfp8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK_MXFP8 == 0); + static_assert(QK_MXFP8 == QK8_0, "QK_MXFP8 and QK8_0 must be the same"); + + const block_mxfp8 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + const int nb = n / QK_MXFP8; + + float sumf = 0; + + for (int ib = 0; ib < nb; ++ib) { + float sumq = 0; + for (int j = 0; j < QK_MXFP8; ++j) { + sumq += ggml_e4m3_to_fp32(x[ib].qs[j]) * y[ib].qs[j]; + } + sumf += GGML_CPU_FP16_TO_FP32(y[ib].d) * GGML_E8M0_TO_FP32(x[ib].e) * sumq; + } + *s = sumf; +} + // NVFP4: super-block of 64 elements = 4 sub-blocks of 16 = 2 q8_0 blocks void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(nrc == 1); diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index 93ea7eeffe5b..cc8b65fbe24a 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -22,6 +22,7 @@ void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +void quantize_row_mxfp8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); @@ -47,6 +48,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_mxfp8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); diff --git a/ggml/src/ggml-cuda/dequantize.cuh b/ggml/src/ggml-cuda/dequantize.cuh index 651524be7b3e..dc2cbe3d75ec 100644 --- a/ggml/src/ggml-cuda/dequantize.cuh +++ b/ggml/src/ggml-cuda/dequantize.cuh @@ -119,6 +119,24 @@ static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const in v.y *= d; } +// E4M3 (fn) decode, exact in fp32; 0x7F/0xFF are NaN +static __device__ __forceinline__ float ggml_cuda_e4m3_to_fp32(const uint8_t x) { + const int exp = (x >> 3) & 0xF; + const int man = x & 0x7; + float v = exp == 0 ? ldexpf((float) man, -9) : ldexpf((float) (8 + man), exp - 10); + v = (exp == 0xF && man == 0x7) ? __int_as_float(0x7FC00000) : v; + return (x & 0x80) ? -v : v; +} + +static __device__ __forceinline__ void dequantize_mxfp8(const void * vx, const int64_t ib, const int iqs, float2 & v){ + const block_mxfp8 * x = (const block_mxfp8 *) vx; + + const float d = x[ib].e == 0xFF ? __int_as_float(0x7FC00000) : ggml_cuda_e8m0_to_fp32(x[ib].e); + + v.x = ggml_cuda_e4m3_to_fp32(x[ib].qs[iqs + 0]) * d; + v.y = ggml_cuda_e4m3_to_fp32(x[ib].qs[iqs + 1]) * d; +} + //================================== k-quants // Each call dequantizes one super-block of QK_K values into y using the diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cu index d97e8fdb0f40..95f6d2383a78 100644 --- a/ggml/src/ggml-cuda/getrows.cu +++ b/ggml/src/ggml-cuda/getrows.cu @@ -344,6 +344,10 @@ static void ggml_cuda_get_rows_switch_src0_type( get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); break; + case GGML_TYPE_MXFP8: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; case GGML_TYPE_Q2_K: get_rows_cuda_kq<64, dst_t, dequantize_q2_K>(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 7c45d0d1fb23..2feea3840fd9 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -5570,6 +5570,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ1_M: case GGML_TYPE_IQ4_XS: + case GGML_TYPE_MXFP8: return true; case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h index ae26e0c23b46..96ab08ae49f0 100644 --- a/ggml/src/ggml-impl.h +++ b/ggml/src/ggml-impl.h @@ -564,6 +564,51 @@ static inline uint8_t ggml_fp32_to_ue4m3(float x) { return (uint8_t) ((ue4m3_exp << 3) | ue4m3_man); } +// E4M3 (fn): signed, 4 exp bits (bias=7), 3 mantissa bits, no inf, 0x7F/0xFF = NaN (used by MXFP8) +// every E4M3 value is exactly representable in fp32 +static inline float ggml_e4m3_to_fp32(uint8_t x) { + const int sign = x & 0x80; + const int exp = (x >> 3) & 0xF; + const int man = x & 0x7; + float v; + if (exp == 0xF && man == 0x7) { + v = NAN; + } else if (exp == 0) { + v = ldexpf((float) man, -9); + } else { + v = ldexpf((float) (8 + man), exp - 10); + } + return sign ? -v : v; +} + +// fp32 -> E4M3 (fn), round to nearest even, saturating to +-448, NaN -> 0x7F +static inline uint8_t ggml_fp32_to_e4m3(float x) { + if (x != x) { + return 0x7F; + } + const uint8_t sign = signbit(x) ? 0x80 : 0x00; + float a = fabsf(x); + if (a >= 448.0f) { + return sign | 0x7E; + } + if (a < 0.015625f) { // < 2^-6: subnormal grid 2^-9 (m == 8 is the smallest normal, code 0x08) + const int m = (int) nearbyintf(a * 512.0f); + return sign | (uint8_t) m; + } + int e; + const float f = frexpf(a, &e); // a = f * 2^e, f in [0.5, 1) + int m = (int) nearbyintf((f * 2.0f - 1.0f) * 8.0f); + int E = e - 1 + 7; + if (m == 8) { + m = 0; + E++; + } + if (E > 15 || (E == 15 && m == 7)) { + return sign | 0x7E; + } + return sign | (uint8_t) ((E << 3) | m); +} + /** * Converts brain16 to float32. * diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 7750a72ce61b..ab4f5b4ae571 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -347,6 +347,35 @@ static inline int best_index_mxfp4(float x, float e) { return best_index; } +void quantize_row_mxfp8_ref(const float * GGML_RESTRICT x, block_mxfp8 * GGML_RESTRICT y, int64_t k) { + static const int qk = QK_MXFP8; + + assert(k % qk == 0); + + const int nb = k / qk; + + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + for (int j = 0; j < qk; j++) { + amax = MAX(amax, fabsf(x[i*qk + j])); + } + + // smallest power of two scale with amax/scale <= 448 (largest e4m3 value) + int e = 0; + if (amax > 0.0f) { + e = (int) ceilf(log2f(amax / 448.0f)) + 127; + e = MAX(0, MIN(254, e)); + } + const float d = GGML_E8M0_TO_FP32((uint8_t) e); + const float id = 1.0f/d; + + y[i].e = (uint8_t) e; + for (int j = 0; j < qk; ++j) { + y[i].qs[j] = ggml_fp32_to_e4m3(x[i*qk + j]*id); + } + } +} + void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k) { static const int qk = QK_MXFP4; @@ -586,6 +615,23 @@ void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_REST } } +void dequantize_row_mxfp8(const block_mxfp8 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + static const int qk = QK_MXFP8; + + assert(k % qk == 0); + + const int nb = k / qk; + + for (int i = 0; i < nb; i++) { + // exact: e4m3 * 2^(e-127) is representable in fp32 for every code and scale + const float d = x[i].e == 0xFF ? NAN : GGML_E8M0_TO_FP32(x[i].e); + + for (int j = 0; j < qk; ++j) { + y[i*qk + j] = ggml_e4m3_to_fp32(x[i].qs[j])*d; + } + } +} + void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { static const int qk = QK_NVFP4; static const int qk_sub = QK_NVFP4_SUB; @@ -2306,6 +2352,12 @@ size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * ggml_row_size(GGML_TYPE_MXFP4, n_per_row); } +size_t quantize_mxfp8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + GGML_UNUSED(quant_weights); + quantize_row_mxfp8_ref(src, dst, (int64_t)nrow*n_per_row); + return nrow * ggml_row_size(GGML_TYPE_MXFP8, n_per_row); +} + size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { GGML_UNUSED(quant_weights); quantize_row_nvfp4_ref(src, dst, (int64_t)nrow*n_per_row); @@ -5533,6 +5585,10 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte { VALIDATE_ROW_DATA_E_E8M0_IMPL(block_mxfp4, data, nb); } break; + case GGML_TYPE_MXFP8: + { + VALIDATE_ROW_DATA_E_E8M0_IMPL(block_mxfp8, data, nb); + } break; case GGML_TYPE_NVFP4: { // UE4M3 scales are uint8_t — all byte values are valid diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index 75188f1af180..06b1fc5daa5d 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -24,6 +24,7 @@ GGML_API void quantize_row_q8_0_ref(const float * GGML_RESTRICT x, block_q8_0 * GGML_API void quantize_row_q8_1_ref(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k); +GGML_API void quantize_row_mxfp8_ref(const float * GGML_RESTRICT x, block_mxfp8 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k); @@ -53,6 +54,7 @@ GGML_API void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GG //GGML_API void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_mxfp8(const block_mxfp8 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); @@ -103,6 +105,7 @@ GGML_API size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTR GGML_API size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_mxfp8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API void iq2xs_init_impl(enum ggml_type type); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 01dc8d05c316..cf25c283e763 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -768,6 +768,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) dequantize_row_mxfp4, .from_float_ref = (ggml_from_float_t)quantize_row_mxfp4_ref, }, + [GGML_TYPE_MXFP8] = { + .type_name = "mxfp8", + .blck_size = QK_MXFP8, + .type_size = sizeof(block_mxfp8), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_mxfp8, + .from_float_ref = (ggml_from_float_t)quantize_row_mxfp8_ref, + }, [GGML_TYPE_NVFP4] = { .type_name = "nvfp4", .blck_size = QK_NVFP4, @@ -8110,6 +8118,7 @@ size_t ggml_quantize_chunk( case GGML_TYPE_Q5_1: result = quantize_q5_1 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q8_0: result = quantize_q8_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_MXFP4: result = quantize_mxfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_MXFP8: result = quantize_mxfp8 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_NVFP4: result = quantize_nvfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q2_K: result = quantize_q2_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q3_K: result = quantize_q3_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 190cc2d63f7a..5a3d7bc23310 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -6196,6 +6196,7 @@ class GGMLQuantizationType(IntEnum): NVFP4 = 40 Q1_0 = 41 Q2_0 = 42 + MXFP8 = 43 class ExpertGatingFuncType(IntEnum): @@ -6409,6 +6410,7 @@ class VisionProjectorType: GGMLQuantizationType.NVFP4: (64, 4 + 32), GGMLQuantizationType.Q1_0: (128, 2 + 16), GGMLQuantizationType.Q2_0: (64, 2 + 16), + GGMLQuantizationType.MXFP8: (32, 1 + 32), } diff --git a/gguf-py/gguf/quants.py b/gguf-py/gguf/quants.py index 80966b6ef151..2d8eb2021692 100644 --- a/gguf-py/gguf/quants.py +++ b/gguf-py/gguf/quants.py @@ -705,6 +705,66 @@ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray: return (d * qs.astype(np.float32)) +class MXFP8(__Quant, qtype=GGMLQuantizationType.MXFP8): + # 32 E4M3 (fn) codes sharing one E8M0 scale: value = e4m3(q) * 2^(e - 127) + # ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf + + @staticmethod + def e4m3_table() -> np.ndarray: + x = np.arange(256, dtype=np.uint32) + exp = ((x >> 3) & 0xF).astype(np.int32) + man = (x & 0x7).astype(np.float64) + v = np.where(exp == 0, man * 2.0**-9, (8.0 + man) * np.exp2(exp - 10.0)) + v = np.where((x & 0x7F) == 0x7F, np.nan, v) + return np.where(x & 0x80, -v, v).astype(np.float32) + + @staticmethod + def e8m0_to_fp32(x: np.ndarray) -> np.ndarray: + # see ggml_e8m0_to_fp32 in ggml-impl.h (0xFF is NaN) + x = x.astype(np.uint32) + bits = np.where(x == 0, np.uint32(0x00400000), x << np.uint32(23)) + return np.where(x == 0xFF, np.float32(np.nan), bits.view(np.float32)) + + @staticmethod + def fp32_to_e4m3(x: np.ndarray) -> np.ndarray: + # see ggml_fp32_to_e4m3 in ggml-impl.h (round to nearest even, saturating) + sign = np.where(np.signbit(x), np.uint8(0x80), np.uint8(0)) + a = np.abs(x).astype(np.float32) + sub = np.rint(a * np.float32(512)).astype(np.int32) + f, e = np.frexp(a) + m = np.rint((f * 2 - 1) * 8).astype(np.int32) + E = e - 1 + 7 + E = np.where(m == 8, E + 1, E) + m = np.where(m == 8, 0, m) + norm = (E << 3) | m + norm = np.where((E > 15) | ((E == 15) & (m == 7)), 0x7E, norm) + code = np.where(a < 2.0**-6, sub, norm) + code = np.where(a >= 448.0, 0x7E, code) + code = np.where(np.isnan(x), 0x7F, code | sign) + return code.astype(np.uint8) + + @classmethod + def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray: + amax = abs(blocks).max(axis=-1, keepdims=True) + + with np.errstate(divide="ignore"): + e = np.where(amax > 0, np.ceil(np.log2(amax / np.float32(448))) + 127, 0) + e = np.clip(e, 0, 254).astype(np.uint8) + + d = cls.e8m0_to_fp32(e) + qs = cls.fp32_to_e4m3(blocks * (np.float32(1) / d)) + + return np.concatenate([e, qs], axis=-1) + + @classmethod + def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray: + e, qs = np.hsplit(blocks, [1]) + + d = cls.e8m0_to_fp32(e) + + return d * cls.e4m3_table()[qs] + + class NVFP4(__Quant, qtype=GGMLQuantizationType.NVFP4): # E2M1 values doubled (kvalues_mxfp4 convention) kvalues = (0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index ba7f7eb4e56c..121aca1fb1eb 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9802,6 +9802,17 @@ static std::vector> make_test_cases_eval() { } test_cases.emplace_back(new test_get_rows(GGML_TYPE_F32, 256, 8, 2, 1, 1, false, true, 3)); + // MXFP8 (E4M3 + E8M0, 33 B / 32): gather-only type for large host-resident tables + for (int b : {1, 7}) { + for (bool v : {false, true}) { + for (bool vs0 : {false, true}) { + test_cases.emplace_back(new test_get_rows(GGML_TYPE_MXFP8, 256, 5, 4, b, 1, v, vs0)); + } + } + } + test_cases.emplace_back(new test_get_rows(GGML_TYPE_MXFP8, 256, 1000, 24, 1, 1, false)); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP8, GGML_TYPE_F32, 16, 1, 256, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 8, 2, 1, false)); test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 70000, 4, 1, false)); // row count > CUDA grid-y limit (65535) for (ggml_type type : all_types) { From ae08e81831d809eb89723c29446bc05424d56dcb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 03:10:51 -0700 Subject: [PATCH 03/49] convert : add DeepSeek-V4.1 (deepseek41) converter FP8 32x32 ue8m0 linears dequantised exactly to BF16 (block derived from the scale shape and checked against weight_block_size, scale exponent range asserted), routed experts copied bit-exact as MXFP4, Engram tables streamed bit-exact as MXFP8 (BF16/F32 fallback via DSV41_ENGRAM_TYPE), Engram gate folded from q_weight*k_weight, Engram hash constants (token map, multipliers, primes) written as GGUF keys, V4.1 chat template. DSpark blocks mtp.0..2 go to a separate GGUF with --mtp. Vision skipped. --- conversion/__init__.py | 1 + conversion/deepseek41.py | 487 ++++++++++++++++++ .../templates/deepseek-ai-DeepSeek-V4.1.jinja | 275 ++++++++++ 3 files changed, 763 insertions(+) create mode 100644 conversion/deepseek41.py create mode 100644 models/templates/deepseek-ai-DeepSeek-V4.1.jinja diff --git a/conversion/__init__.py b/conversion/__init__.py index d5b480a9bf41..cc7d6f66013b 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -67,6 +67,7 @@ "LingDSparkModel": "qwen", "DeepseekV4ForCausalLM": "deepseek", "DeepseekV4DSparkModel": "deepseek", + "DeepseekV41ForCausalLM": "deepseek41", "DistilBertForMaskedLM": "bert", "DistilBertForSequenceClassification": "bert", "DistilBertModel": "bert", diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py new file mode 100644 index 000000000000..88f80ebc761a --- /dev/null +++ b/conversion/deepseek41.py @@ -0,0 +1,487 @@ +from __future__ import annotations + +import os +import re +from pathlib import Path + +from typing import Any, Callable, Iterable, TYPE_CHECKING + +import numpy as np +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import LazyTorchTensor, ModelBase, TextModel, gguf, logger +from .deepseek import DeepseekV4Model + + +# --------------------------------------------------------------------------------------------------------------------- +# Engram hash constants, ported from inference/engram.py of deepseek-ai/DeepSeek-V4.1-Flash-1017. +# They are written to GGUF so that the runtime never recomputes them (the compressed token map needs the HF +# tokenizers normalizers, the multipliers numpy's PCG64 stream). +# --------------------------------------------------------------------------------------------------------------------- + +def _is_prime(n: int) -> bool: + # deterministic Miller-Rabin for n < 3.3e24 + if n < 2: + return False + small = (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41) + for p in small: + if n % p == 0: + return n == p + d, s = n - 1, 0 + while d % 2 == 0: + d //= 2 + s += 1 + for a in small: + x = pow(a, d, n) + if x in (1, n - 1): + continue + for _ in range(s - 1): + x = x * x % n + if x == n - 1: + break + else: + return False + return True + + +def engram_primes(n_layers: int, max_ngram_size: int, n_heads: int, vocab_size: int) -> list[list[list[int]]]: + """EngramLayout.from_args: [layer][ngram order 2..max][head] bucket moduli, never reused across layers.""" + seen: set[int] = set() + out: list[list[list[int]]] = [] + for _ in range(n_layers): + per_ngram = [] + for _ in range(max_ngram_size - 1): + sizes, cur = [], vocab_size - 1 + for _ in range(n_heads): + cand = cur + 1 + while not _is_prime(cand) or cand in seen: + cand += 1 + cur = cand + seen.add(cur) + sizes.append(cur) + per_ngram.append(sizes) + out.append(per_ngram) + return out + + +def engram_multipliers(layer_ids: list[int], max_ngram_size: int, compressed_vocab_size: int) -> list[list[int]]: + """compute_hash_multipliers: odd, bounded so that token_id * multiplier cannot overflow int64.""" + max_long = np.iinfo(np.int64).max + bound = max(1, (max_long // compressed_vocab_size) // 2) + rows = [] + for layer_id in layer_ids: + rng = np.random.default_rng(10007 * layer_id) + values = rng.integers(low=0, high=bound, size=(max_ngram_size,), dtype=np.int64) + rows.append([int(v) * 2 + 1 for v in values]) + return rows + + +def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]: + """build_compressed_token_map: token id -> compressed id (tokens that normalize alike collapse).""" + from tokenizers import Regex, normalizers + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir) + sentinel = "\ue000" + normalizer = normalizers.Sequence([ + normalizers.NFKC(), + normalizers.NFD(), + normalizers.StripAccents(), + normalizers.Lowercase(), + normalizers.Replace(Regex(r"[ \t\r\n]+"), " "), + normalizers.Replace(Regex(r"^ $"), sentinel), + normalizers.Strip(), + normalizers.Replace(sentinel, " "), + ]) + backend = tokenizer.backend_tokenizer + key_to_new: dict[str, int] = {} + lookup = [0] * len(tokenizer) + for token_id in range(len(tokenizer)): + text = backend.decode([token_id], skip_special_tokens=False) + if "\ufffd" in text: + key = backend.id_to_token(token_id) + else: + normalized = normalizer.normalize_str(text) + key = normalized if normalized else text + new_id = key_to_new.get(key) + if new_id is None: + new_id = len(key_to_new) + key_to_new[key] = new_id + lookup[token_id] = new_id + return lookup, len(key_to_new) + + +# --------------------------------------------------------------------------------------------------------------------- +# Streaming raw tensors: gguf_writer keeps every added tensor until the final write and only needs +# shape / dtype / nbytes / tofile, so producing the bytes chunk by chunk at write time keeps memory flat +# (an Engram table is ~101 GB, a stacked MXFP4 expert tensor ~2.4 GB). +# --------------------------------------------------------------------------------------------------------------------- + +class _StreamedBytes: + def __init__(self, shape: tuple[int, ...], chunks: Callable[[], Iterable[np.ndarray]]): + self.shape = shape + self.dtype = np.dtype(np.uint8) + self.nbytes = int(np.prod(shape)) + self._chunks = chunks + + def tofile(self, fout) -> None: + written = 0 + for chunk in self._chunks(): + chunk = np.ascontiguousarray(chunk, dtype=np.uint8) + chunk.tofile(fout) + written += chunk.nbytes + if written != self.nbytes: + raise ValueError(f"streamed {written} bytes, expected {self.nbytes}") + + def byteswap(self, inplace: bool = False): + return self # raw bytes + + +def _e8m0_check(scale_u8: np.ndarray, name: str, lo: int, hi: int) -> None: + if scale_u8.size == 0: + return + smin, smax = int(scale_u8.min()), int(scale_u8.max()) + if smin < lo or smax > hi: + raise ValueError(f"{name}: E8M0 scale exponents [{smin}, {smax}] outside the exact range [{lo}, {hi}] " + f"(0xFF is NaN, very small exponents underflow BF16)") + + +@ModelBase.register("DeepseekV41ForCausalLM") +@ModelBase.example("deepseek-ai/DeepSeek-V4.1-Flash-1017") +class DeepseekV41Model(DeepseekV4Model): + """DeepSeek-V4.1 text model (arch deepseek41), see TENSORS.md in the port branch. + + Lossless reference layout: FP8 32x32-block linears dequantised to BF16 (exact: e4m3 * 2^k fits BF16), routed + experts copied bit-exact as MXFP4, Engram tables copied bit-exact as MXFP8. DSpark blocks (mtp.0..2) go to a + separate GGUF with --mtp. The vision tower is not converted yet. + + Environment overrides: + DSV41_DENSE_TYPE bf16 (default, lossless) | q8_0 for the dequantised FP8 linears + DSV41_ENGRAM_TYPE mxfp8 (default, lossless) | bf16 (lossless, 2x larger) | f32 + """ + model_arch = gguf.MODEL_ARCH.DEEPSEEK41 + supports_mtp_export = True + + # tensors that stay in their source float dtype (bf16 in the checkpoint, never quantised) + _ENGRAM_EMBED_RE = re.compile(r"layers\.(\d+)\.engram\.embed\.(weight|scale)$") + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + hp = self.hparams + self._dense_type = os.environ.get("DSV41_DENSE_TYPE", "bf16").lower() + self._engram_type = os.environ.get("DSV41_ENGRAM_TYPE", "mxfp8").lower() + if self._dense_type not in ("bf16", "q8_0"): + raise ValueError(f"DSV41_DENSE_TYPE={self._dense_type!r}") + if self._engram_type not in ("mxfp8", "bf16", "f32"): + raise ValueError(f"DSV41_ENGRAM_TYPE={self._engram_type!r}") + + qc = hp.get("quantization_config") or {} + self._fp8_block = tuple(qc.get("weight_block_size") or (128, 128)) + if qc.get("scale_fmt", "ue8m0") != "ue8m0": + raise ValueError(f"unsupported FP8 scale format {qc.get('scale_fmt')!r}") + + # the V4 constructor installed the V4 template; V4.1 has its own + template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.1.jinja" + if template_path.is_file(): + self.gguf_writer.add_chat_template(template_path.read_text(encoding="utf-8")) + + # ---- tensor selection ------------------------------------------------------------------------------------------- + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + hp = {**self.hparams, **self.hparams.get("text_config", {})} # text_config is merged only after indexing + type(self)._dsv4_main_layers = hp["num_hidden_layers"] + type(self)._dsv4_nextn_layers = hp.get("num_nextn_predict_layers", 0) + return TextModel.index_tensors(self, remote_hf_model_id=remote_hf_model_id) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if name.startswith(("vision.", "aligner.", "image_")): + return None # vision tower: mmproj not converted yet + if name.startswith("mtp."): + if not cls.mtp_only: + cls._skipped_mtp_tensors += 1 + return None + m = re.match(r"mtp\.(\d+)\.(.+)$", name) + if m is None: + raise ValueError(f"Unexpected DeepSeek-V4.1 DSpark tensor {name!r}") + stage, rest = int(m.group(1)), m.group(2) + assert cls._dsv4_main_layers is not None + if stage >= cls._dsv4_nextn_layers: + raise ValueError(f"Unexpected DSpark stage {stage}") + bid = cls._dsv4_main_layers + stage + if rest.startswith(("markov_head.", "confidence_head.")): + return f"dspark.{rest}", gen + return f"layers.{bid}.{rest}", gen + if cls.mtp_only and name not in ("embed.weight", "norm.weight", "head.weight"): + return None + return TextModel.filter_tensors((name, gen)) + + # ---- hparams ---------------------------------------------------------------------------------------------------- + + def set_gguf_parameters(self): + TextModel.set_gguf_parameters(self) + hp = self.hparams + w = self.gguf_writer + n_main = hp["num_hidden_layers"] + + w.add_rope_dimension_count(hp["qk_rope_head_dim"]) + w.add_q_lora_rank(hp["q_lora_rank"]) + w.add_sliding_window(hp["sliding_window"]) + w.add_expert_feed_forward_length(hp["moe_intermediate_size"]) + w.add_expert_shared_count(hp["n_shared_experts"]) + w.add_expert_weights_scale(hp["routed_scaling_factor"]) + w.add_expert_weights_norm(hp["norm_topk_prob"]) + w.add_swiglu_clamp_exp([hp["swiglu_limit"]] * self.block_count) + w.add_swiglu_clamp_shexp([hp["swiglu_limit"]] * self.block_count) + + w.add_indexer_head_count(hp["index_n_heads"]) + w.add_indexer_key_length(hp["index_head_dim"]) + w.add_indexer_top_k(hp["index_topk"]) + + w.add_attention_output_group_count(hp["o_groups"]) + w.add_attention_output_lora_rank(hp["o_lora_rank"]) + w.add_attention_compress_ratios(hp["compress_ratios"]) + w.add_attention_compress_rope_freq_base(hp["compress_rope_theta"]) + w.add_hyper_connection_count(hp["hc_mult"]) + w.add_hyper_connection_sinkhorn_iterations(hp["hc_sinkhorn_iters"]) + w.add_hyper_connection_epsilon(hp["hc_eps"]) + w.add_embedding_length_out(hp["hidden_size"] * hp["hc_mult"]) + + # cross-layer sharing and candidate blocks + w.add_attention_kv_source_layers(hp["kv_source_layer_ids"]) + w.add_indexer_source_layers(hp["index_source_layer_ids"]) + w.add_indexer_candidate_source_layer(int(hp.get("candidate_source_layer_id", -1))) + w.add_indexer_candidate_top_k_blocks(hp["candidate_topk_blocks"]) + w.add_indexer_candidate_block_size(hp["candidate_block_size"]) + + # Engram + layer_ids = list(hp["engram_layer_ids"]) + max_ngram, n_heads = hp["engram_max_ngram_size"], hp["engram_n_heads"] + primes = engram_primes(len(layer_ids), max_ngram, n_heads, hp["engram_vocab_size"]) + for li, layer_primes in enumerate(primes): + total = sum(p for per in layer_primes for p in per) + if total != hp["engram_num_embeddings"][li]: + raise ValueError(f"engram layer {layer_ids[li]}: prime sum {total} != num_embeddings {hp['engram_num_embeddings'][li]}") + token_map, cvocab = engram_compressed_token_map(self.dir_model if self.target_model_dir is None else self.target_model_dir) + if cvocab != hp["engram_compressed_vocab_size"]: + raise ValueError(f"compressed vocab {cvocab} != engram_compressed_vocab_size {hp['engram_compressed_vocab_size']}") + if len(token_map) != hp["vocab_size"]: + raise ValueError(f"token map has {len(token_map)} entries, vocab_size is {hp['vocab_size']}") + mults = engram_multipliers(layer_ids, max_ngram, cvocab) + w.add_engram_layers(layer_ids) + w.add_engram_num_embeddings(hp["engram_num_embeddings"]) + w.add_engram_max_ngram_size(max_ngram) + w.add_engram_vocab_size(hp["engram_vocab_size"]) + w.add_engram_head_count(n_heads) + w.add_engram_head_dim(hp["engram_head_dim"]) + w.add_engram_pad_token_id(hp["engram_pad_token_id"]) + w.add_engram_compressed_vocab_size(cvocab) + w.add_engram_token_map(token_map) + w.add_engram_multipliers([m for row in mults for m in row]) + w.add_engram_primes([p for layer in primes for per in layer for p in per]) + logger.info(f"engram: compressed vocab {cvocab}, pad -> {token_map[hp['engram_pad_token_id']]}, multipliers {mults}") + + # DSpark (block layout only matters to the --mtp file, but the keys are cheap) + if "dspark_block_size" in hp: + w.add_dspark_block_size(hp["dspark_block_size"]) + w.add_dspark_noise_token_id(hp["dspark_noise_token_id"]) + w.add_dspark_target_layers(hp["dspark_target_layer_ids"]) + w.add_dspark_markov_rank(hp["dspark_markov_rank"]) + w.add_dspark_expert_count(hp["dspark_n_routed_experts"]) + w.add_dspark_expert_used_count(hp["dspark_num_experts_per_tok"]) + if self.mtp_only: + w.add_nextn_predict_layers(hp.get("num_nextn_predict_layers", 0)) + assert self.block_count == n_main + (hp.get("num_nextn_predict_layers", 0) if self.mtp_only else 0) + + # ---- weights ---------------------------------------------------------------------------------------------------- + + def dequant_model(self): + """FP8 (e4m3, ue8m0 scales) linears -> f32, exact. The block is derived per tensor from the scale shape and + checked against quantization_config.weight_block_size (V4 used 128, V4.1 uses 32x32).""" + fp8_dtypes = self._float8_dtypes() + bo_cfg, bi_cfg = self._fp8_block + to_remove: list[str] = [] + + def dequant(weight: Tensor, scale: Tensor, bo: int, bi: int) -> Tensor: + out_f, in_f = weight.shape + s = self._e8m0_to_float(scale) + s = s.repeat_interleave(bo, 0)[:out_f].repeat_interleave(bi, 1)[:, :in_f] + return weight.float() * s + + for name in list(self.model_tensors.keys()): + if not name.endswith(".scale") or self._ENGRAM_EMBED_RE.match(name): + continue + weight_name = name.removesuffix(".scale") + ".weight" + if weight_name not in self.model_tensors: + continue + wgen, sgen = self.model_tensors[weight_name], self.model_tensors[name] + w_meta = wgen() + if w_meta.dtype not in fp8_dtypes: + continue # MXFP4 experts (I8 packed) are handled in generate_extra_tensors + out_f, in_f = w_meta.shape + s_meta = sgen() + srows, scols = s_meta.shape + bo, bi = -(-out_f // srows), -(-in_f // scols) + if (bo, bi) != (bo_cfg, bi_cfg) or -(-out_f // bo) != srows or -(-in_f // bi) != scols: + raise ValueError(f"{weight_name}: FP8 block {(bo, bi)} from scale {tuple(s_meta.shape)} " + f"does not match weight_block_size {(bo_cfg, bi_cfg)}") + # exactness: e4m3 (|x| in [2^-9, 448]) * 2^(e-127) must stay inside BF16's range + _e8m0_check(LazyTorchTensor.to_eager(s_meta).view(torch.uint8).numpy(), name, 3, 246) + self.model_tensors[weight_name] = lambda w=wgen, s=sgen, bo=bo, bi=bi: dequant(w(), s(), bo, bi) + self._dsv4_fp8_dequantized.add(weight_name) + to_remove.append(name) + + for name in to_remove: + del self.model_tensors[name] + + def _write_mxfp4_experts(self, bid: int, n_experts: int) -> list[str]: + consumed: list[str] = [] + for proj, key in (("w1", gguf.MODEL_TENSOR.FFN_GATE_EXP), ("w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP), + ("w3", gguf.MODEL_TENSOR.FFN_UP_EXP)): + loaders = [] + for eid in range(n_experts): + wn, sn = f"layers.{bid}.ffn.experts.{eid}.{proj}.weight", f"layers.{bid}.ffn.experts.{eid}.{proj}.scale" + if wn not in self.model_tensors or sn not in self.model_tensors: + raise KeyError(f"Missing routed expert tensors {wn}") + loaders.append((self.model_tensors[wn], self.model_tensors[sn])) + consumed += [wn, sn] + extra = sorted(n for n in self.model_tensors + if re.match(rf"layers\.{bid}\.ffn\.experts\.(\d+)\.{proj}\.weight$", n) + and int(n.split(".")[4]) >= n_experts) + if extra: + raise KeyError(f"layer {bid}: unexpected experts beyond {n_experts}: {extra[:3]}") + data = self._mxfp4_expert_tensor(loaders) + new_name = self.format_tensor_name(key, bid) + self.gguf_writer.add_tensor(new_name, data, raw_dtype=gguf.GGMLQuantizationType.MXFP4) + return consumed + + def _engram_embed_tensor(self, bid: int) -> list[str]: + wn, sn = f"layers.{bid}.engram.embed.weight", f"layers.{bid}.engram.embed.scale" + wgen, sgen = self.model_tensors[wn], self.model_tensors[sn] + rows, dim = wgen().shape + if tuple(sgen().shape) != (rows, dim // 32) or dim % 32 != 0: + raise ValueError(f"{wn}: scale shape {tuple(sgen().shape)} is not 1x32 blocks of {(rows, dim)}") + li = list(self.hparams["engram_layer_ids"]).index(bid) + if rows != self.hparams["engram_num_embeddings"][li]: + raise ValueError(f"{wn}: {rows} rows != engram_num_embeddings {self.hparams['engram_num_embeddings'][li]}") + new_name = self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_EMBD, bid) + part = self._tensor_part(wn) + chunk_rows = 1 << 20 + + def raw_chunks() -> Iterable[tuple[np.ndarray, np.ndarray]]: + from safetensors import safe_open + with safe_open(part, framework="pt") as f: + ws, ss = f.get_slice(wn), f.get_slice(sn) + for r0 in range(0, rows, chunk_rows): + r1 = min(rows, r0 + chunk_rows) + q = ws[r0:r1].view(torch.uint8).numpy().reshape(-1, 32) + e = ss[r0:r1].view(torch.uint8).numpy().reshape(-1, 1) + yield q, e + + if self._engram_type == "mxfp8": + def chunks() -> Iterable[np.ndarray]: + for q, e in raw_chunks(): + if e.max() == 0xFF: + raise ValueError(f"{wn}: E8M0 NaN scale (0xFF)") + if ((q & 0x7F) == 0x7F).any(): + raise ValueError(f"{wn}: E4M3 NaN code") + yield np.concatenate([e, q], axis=1) + data = _StreamedBytes((rows, dim // 32 * 33), chunks) + self.gguf_writer.add_tensor(new_name, data, raw_dtype=gguf.GGMLQuantizationType.MXFP8) # type: ignore[arg-type] + else: + to_bf16 = self._engram_type == "bf16" + + def chunks() -> Iterable[np.ndarray]: + for q, e in raw_chunks(): + _e8m0_check(e, wn, 3, 246) + v = torch.from_numpy(q).view(torch.float8_e4m3fn).float() * torch.exp2(torch.from_numpy(e).float() - 127) + v = v.reshape(-1, dim) + yield (v.bfloat16().view(torch.uint8) if to_bf16 else v.view(torch.uint8)).numpy() + esz = 2 if to_bf16 else 4 + data = _StreamedBytes((rows, dim * esz), chunks) + qt = gguf.GGMLQuantizationType.BF16 if to_bf16 else gguf.GGMLQuantizationType.F32 + self.gguf_writer.add_tensor(new_name, data, raw_shape=(rows, dim), raw_dtype=qt) # type: ignore[arg-type] + logger.info(f"{new_name}: Engram table {rows} x {dim} copied as {self._engram_type.upper()}") + return [wn, sn] + + def _tensor_part(self, name: str) -> Path: + import json + with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: + return self.dir_model / json.load(f)["weight_map"][name] + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + if self._dsv4_mxfp4_generated: + return () + hp = self.hparams + consumed: list[str] = [] + n_main = hp["num_hidden_layers"] + for bid in range(self.block_count): + if self.mtp_only and bid < n_main: + continue + n_exp = hp["n_routed_experts"] if bid < n_main else hp["dspark_n_routed_experts"] + consumed += self._write_mxfp4_experts(bid, n_exp) + if not self.mtp_only: + for bid in hp["engram_layer_ids"]: + consumed += self._engram_embed_tensor(bid) + for name in consumed: + del self.model_tensors[name] + self._dsv4_mxfp4_generated = True + return () + + _LAYER_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { + "attn.indexer.wk.weight": (gguf.MODEL_TENSOR.INDEXER_ATTN_K, ".weight"), + "attn.indexer.k_norm.weight": (gguf.MODEL_TENSOR.INDEXER_K_NORM, ".weight"), + "engram.wkv.weight": (gguf.MODEL_TENSOR.ENGRAM_KV, ".weight"), + "main_proj.weight": (gguf.MODEL_TENSOR.DSPARK_MAIN_PROJ, ".weight"), + "main_norm.weight": (gguf.MODEL_TENSOR.DSPARK_MAIN_NORM, ".weight"), + "norm.weight": (gguf.MODEL_TENSOR.DSPARK_NORM, ".weight"), + } + _ROOT_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { + "dspark.markov_head.embed.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W1, ".weight"), + "dspark.markov_head.head.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W2, ".weight"), + "dspark.confidence_head.proj.weight": (gguf.MODEL_TENSOR.DSPARK_CONF_PROJ, ".weight"), + } + + def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]: + if name in self._ROOT_MAP: + return self._ROOT_MAP[name] + m = re.match(r"layers\.(\d+)\.(.+)$", name) + if m is not None and m.group(2) in self._LAYER_MAP: + return self._LAYER_MAP[m.group(2)] + key, suffix = super()._map_dsv4_tensor_name(name, bid) + if key in (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_APE, gguf.MODEL_TENSOR.FFN_GATE_TID2EID) or \ + key.name.startswith(("INDEXER_COMPRESSOR_", "NEXTN_", "HC_HEAD_")): + raise ValueError(f"{name!r} is a DeepSeek-V4 tensor that V4.1 does not have") + return key, suffix + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name): + return [] + if re.match(r"layers\.\d+\.engram\.k_weight$", name): + return [] # folded into engram_gate together with q_weight + if (m := re.match(r"layers\.(\d+)\.engram\.q_weight$", name)) is not None: + k = self.model_tensors.get(f"layers.{m.group(1)}.engram.k_weight") + if k is None: + raise KeyError(f"missing layers.{m.group(1)}.engram.k_weight") + # bf16 * bf16 is exact in f32 + gate = data_torch.float() * k().float() + return [(self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_GATE, bid), gate)] + key, suffix = self._map_dsv4_tensor_name(name, bid) + new_name = self.format_tensor_name(key, None if name.startswith("dspark.") else bid, suffix) + return [(new_name, data_torch)] + + def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: + if new_name.endswith(".engram_gate.weight") or new_name == "conf_proj.weight": + return gguf.GGMLQuantizationType.F32 + if name in self._dsv4_fp8_dequantized and n_dims >= 2: + return gguf.GGMLQuantizationType.Q8_0 if self._dense_type == "q8_0" else gguf.GGMLQuantizationType.BF16 + if name in self._dsv4_f32_tensors: + return gguf.GGMLQuantizationType.F32 + if name in self._dsv4_bf16_tensors and n_dims >= 2: + return gguf.GGMLQuantizationType.BF16 + return False + diff --git a/models/templates/deepseek-ai-DeepSeek-V4.1.jinja b/models/templates/deepseek-ai-DeepSeek-V4.1.jinja new file mode 100644 index 000000000000..aac8438a37df --- /dev/null +++ b/models/templates/deepseek-ai-DeepSeek-V4.1.jinja @@ -0,0 +1,275 @@ +{#- DeepSeek-V4.1 chat template (for DeepSeek-V4.1-Flash-1017), ported from encoding/encoding.py in the model repo. + Based on the deepseek-ai/DeepSeek-V4.1-Flash template, with fixes so it matches encoding.py: + tool-result-only user turns do not start a new turn, tool_result content lists keep "[Unsupported x]" blocks, + top-level response_format is attached to the leading system message. + Extra kwargs: thinking_mode ("thinking" | "chat", default "thinking"; enable_thinking / thinking = true/false are + accepted as aliases), drop_thinking (default true), reasoning_effort (int in [1, 100] or "low"/"medium"/"high"/"max", + default "high"). Assistant reasoning is read from reasoning_content, or from thinking as an alias. + Tool call arguments must be objects (string arguments are rendered as a single "arguments" parameter). -#} +{%- set IMG = "<｜deepseek_image｜>" -%} +{%- set DSML = "｜DSML｜" -%} + +{%- if thinking_mode is not defined or thinking_mode is none -%} + {%- if enable_thinking is defined and enable_thinking is not none -%} + {%- set thinking_mode = "thinking" if enable_thinking else "chat" -%} + {%- elif thinking is defined and thinking is not none and thinking is not string -%} + {%- set thinking_mode = "thinking" if thinking else "chat" -%} + {%- else -%} + {%- set thinking_mode = "thinking" -%} + {%- endif -%} +{%- endif -%} +{%- if thinking_mode not in ["chat", "thinking"] -%} + {{- raise_exception("Invalid thinking_mode `" ~ thinking_mode ~ "`, should be \"chat\" or \"thinking\"") -}} +{%- endif -%} +{%- if drop_thinking is not defined or drop_thinking is none -%}{%- set drop_thinking = true -%}{%- endif -%} +{%- if add_generation_prompt is not defined -%}{%- set add_generation_prompt = false -%}{%- endif -%} +{%- if tools is not defined -%}{%- set tools = none -%}{%- endif -%} +{%- if response_format is not defined -%}{%- set response_format = none -%}{%- endif -%} + +{%- if reasoning_effort is not defined or reasoning_effort is none -%}{%- set reasoning_effort = "high" -%}{%- endif -%} +{%- set effort_map = {"low": 50, "medium": 62, "high": 75, "max": 100} -%} +{%- if reasoning_effort is string and reasoning_effort in effort_map -%} + {%- set budget = effort_map[reasoning_effort] -%} +{%- elif reasoning_effort is integer and reasoning_effort >= 1 and reasoning_effort <= 100 -%} + {%- set budget = reasoning_effort -%} +{%- else -%} + {{- raise_exception("Invalid reasoning effort: " ~ reasoning_effort ~ ", should be int within [1,100] or one of \"low\", \"medium\", \"high\", \"max\"") -}} +{%- endif -%} + +{#- Splits/validates "ns::name" against an explicit namespace and returns the qualified name. -#} +{%- macro qualified_name(name, ns) -%} + {%- if ns is mapping -%}{%- set ns = ns.name -%}{%- endif -%} + {%- set parts = name.split("::", 1) -%} + {%- if parts | length == 2 -%} + {%- if ns is not none and ns != parts[0] -%}{{- raise_exception("Conflicting tool namespaces: " ~ ns ~ " != " ~ parts[0]) -}}{%- endif -%} + {%- set ns = parts[0] -%} + {%- set name = parts[1] -%} + {%- endif -%} + {%- if "::" in name -%}{{- raise_exception("Tool name must not contain '::': " ~ name) -}}{%- endif -%} + {%- if ns is not none and "::" in ns -%}{{- raise_exception("Tool namespace must not contain '::': " ~ ns) -}}{%- endif -%} + {{- name if ns is none else ns ~ "::" ~ name -}} +{%- endmacro -%} + +{%- macro render_tools(tool_list) -%} + {%- set ts = namespace(schemas=[]) -%} + {%- for tool in tool_list -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set ns = tool.namespace if tool.namespace is defined and tool.namespace is not none else fn.get("namespace") -%} + {#- JSON assembled by hand (key order kept, name/description replaced in place): llama.cpp has no ** unpacking. -#} + {%- set qname = qualified_name(fn.name, ns) -%} + {%- set sch = namespace(parts=[]) -%} + {%- for k, v in fn.items() if k != "namespace" -%} + {%- if k == "name" -%}{%- set v = qname -%} + {%- elif k == "description" and ns is mapping and ns.description -%}{%- set v = ns.description ~ "\n" ~ (fn.description or "") -%} + {%- endif -%} + {%- set sch.parts = sch.parts + [(k | tojson) ~ ": " ~ (v | tojson)] -%} + {%- endfor -%} + {%- if "description" not in fn and ns is mapping and ns.description -%} + {%- set sch.parts = sch.parts + ['"description": ' ~ ((ns.description ~ "\n") | tojson)] -%} + {%- endif -%} + {%- set ts.schemas = ts.schemas + ["{" ~ sch.parts | join(", ") ~ "}"] -%} + {%- endfor -%} + {{- "## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜ calls>\" block like the following:\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"$TOOL_NAME\">\n<｜DSML｜ parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n...\n\n<｜DSML｜ invoke name=\"$TOOL_NAME2\">\n...\n\n\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response.\n\nOtherwise, output directly after with tool calls or final response.\n\n### Available Tool Schemas\n\n" -}} + {{- ts.schemas | join("\n") -}} + {{- "\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n" -}} +{%- endmacro -%} + +{#- 1. Merge tool results (and consecutive user turns) into user messages made of content blocks. -#} +{%- set mns = namespace(list=[]) -%} +{%- for msg in messages -%} + {%- set content = msg.get("content") -%} + {%- set is_blocks = content is iterable and content is not string and content is not mapping -%} + {%- if content is string and IMG in content -%} + {{- raise_exception("Message content contains image special token " ~ IMG ~ ". Images should be provided as image content blocks.") -}} + {%- endif -%} + {%- set rc = msg.reasoning_content if msg.get("reasoning_content") is not none else msg.get("thinking") -%} + {%- if rc is string and IMG in rc -%} + {{- raise_exception("reasoning_content contains image special token " ~ IMG) -}} + {%- endif -%} + {%- set prev = mns.list[-1] if mns.list else none -%} + {%- if msg.role == "tool" -%} + {%- set tblock = {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content if content is not none else ""} -%} + {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined -%} + {%- set mns.list = mns.list[:-1] + [dict(prev, content_blocks=prev.content_blocks + [tblock])] -%} + {%- else -%} + {%- set mns.list = mns.list + [{"role": "user", "content_blocks": [tblock]}] -%} + {%- endif -%} + {%- elif msg.role == "user" -%} + {%- set blocks = content if is_blocks else [{"type": "text", "text": content or ""}] -%} + {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined and prev.get("task") is none -%} + {%- set mns.list = mns.list[:-1] + [dict(prev, content_blocks=prev.content_blocks + blocks)] -%} + {%- else -%} + {%- set mns.list = mns.list + [dict(msg, content_blocks=blocks)] -%} + {%- endif -%} + {%- else -%} + {%- set mns.list = mns.list + [msg] -%} + {%- endif -%} +{%- endfor -%} + +{#- Top-level tools / response_format attach to a leading system message, or to a new empty one. -#} +{%- if tools or response_format -%} + {%- if not (mns.list and mns.list[0].role == "system") -%} + {%- set mns.list = [{"role": "system", "content": ""}] + mns.list -%} + {%- endif -%} + {%- set first = mns.list[0] -%} + {%- if tools and not first.get("tools") -%}{%- set first = dict(first, tools=tools) -%}{%- endif -%} + {%- if response_format and not first.get("response_format") -%}{%- set first = dict(first, response_format=response_format) -%}{%- endif -%} + {%- set mns.list = [first] + mns.list[1:] -%} +{%- endif -%} + +{#- 2. Reorder tool results to match the tool_calls order of the preceding assistant message. -#} +{%- set sns = namespace(list=[], ids=[]) -%} +{%- for m in mns.list -%} + {%- if m.role == "assistant" and m.get("tool_calls") -%} + {%- set sns.ids = [] -%} + {%- for tc in m.tool_calls -%} + {%- set sns.ids = sns.ids + [tc.get("id") or (tc.function.get("id", "") if tc.function is defined else "")] -%} + {%- endfor -%} + {%- set sns.list = sns.list + [m] -%} + {%- elif m.role == "user" and m.content_blocks -%} + {%- set tb = namespace(blocks=[]) -%} + {%- for b in m.content_blocks if b.type == "tool_result" -%} + {%- set order = namespace(idx=0) -%} + {%- for tc_id in sns.ids -%} + {%- if tc_id and tc_id == b.get("tool_use_id", "") -%}{%- set order.idx = loop.index0 -%}{%- endif -%} + {%- endfor -%} + {%- set tb.blocks = tb.blocks + [dict(b, _order=order.idx)] -%} + {%- endfor -%} + {%- if tb.blocks | length > 1 and sns.ids -%} + {%- set sorted_blocks = tb.blocks | sort(attribute="_order") -%} + {%- set rb = namespace(blocks=[], i=0) -%} + {%- for b in m.content_blocks -%} + {%- if b.type == "tool_result" -%} + {%- set rb.blocks = rb.blocks + [sorted_blocks[rb.i]] -%} + {%- set rb.i = rb.i + 1 -%} + {%- else -%} + {%- set rb.blocks = rb.blocks + [b] -%} + {%- endif -%} + {%- endfor -%} + {%- set sns.list = sns.list + [dict(m, content_blocks=rb.blocks)] -%} + {%- else -%} + {%- set sns.list = sns.list + [m] -%} + {%- endif -%} + {%- else -%} + {%- set sns.list = sns.list + [m] -%} + {%- endif -%} +{%- endfor -%} +{%- set msgs = sns.list -%} + +{#- Last user turn: users and mid-conversation systems count, except user turns made only of tool results. + Tools anywhere disable drop_thinking. -#} +{%- set state = namespace(last_user=-1, has_tools=false) -%} +{%- for m in msgs -%} + {%- if m.role == "user" or (m.role == "system" and loop.index0 > 0) -%} + {%- set real = namespace(v=true) -%} + {%- if m.get("content_blocks") -%} + {%- set real.v = false -%} + {%- for b in m.content_blocks if b.type in ["text", "image", "image_url"] -%}{%- set real.v = true -%}{%- endfor -%} + {%- endif -%} + {%- if real.v -%}{%- set state.last_user = loop.index0 -%}{%- endif -%} + {%- endif -%} + {%- if m.get("tools") -%}{%- set state.has_tools = true -%}{%- endif -%} +{%- endfor -%} +{%- set effective_drop = drop_thinking and not state.has_tools -%} +{%- set is_thinking = thinking_mode == "thinking" -%} + +{{- "<｜begin▁of▁sentence｜>" -}} +{#- 3. Render each message. -#} +{%- for m in msgs -%} + {%- set idx = loop.index0 -%} + {%- set content = m.get("content") -%} + {%- if content is iterable and content is not string and content is not mapping -%} + {%- set sp = namespace(parts=[]) -%} + {%- for b in content -%} + {%- if b.type == "text" -%}{%- set sp.parts = sp.parts + [b.text or ""] -%} + {%- elif b.type in ["image", "image_url"] -%}{%- set sp.parts = sp.parts + [IMG] -%} + {%- endif -%} + {%- endfor -%} + {%- set content = sp.parts | join("\n\n") -%} + {%- endif -%} + + {%- if idx == 0 -%} + {%- if is_thinking or m.role == "system" -%}{{- "<｜System｜>" -}}{%- endif -%} + {%- if is_thinking -%}{{- "Reasoning Effort: " ~ budget ~ " (range 1-100, the higher the value, the more thorough the reasoning)\n\n" -}}{%- endif -%} + {%- endif -%} + + {%- if m.role == "system" -%} + {%- if idx > 0 -%}{{- "<｜System｜>" -}}{%- endif -%} + {{- content or "" -}} + {%- if m.get("tools") -%}{{- "\n\n" ~ render_tools(m.tools) -}}{%- endif -%} + {%- if m.get("response_format") -%} + {{- "\n\n## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n" ~ (m.response_format | tojson) -}} + {%- endif -%} + + {%- elif m.role == "user" -%} + {{- "<｜User｜>" -}} + {%- set up = namespace(parts=[]) -%} + {%- for b in m.content_blocks -%} + {%- if b.type == "text" -%} + {%- if IMG in (b.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%} + {%- set up.parts = up.parts + [b.text or ""] -%} + {%- elif b.type in ["image", "image_url"] -%} + {%- set up.parts = up.parts + [IMG] -%} + {%- elif b.type == "tool_result" -%} + {%- set tc = b.get("content", "") -%} + {%- if tc is iterable and tc is not string and tc is not mapping -%} + {%- set sub = namespace(parts=[]) -%} + {%- for s in tc -%} + {%- if s.type == "text" -%} + {%- if IMG in (s.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%} + {%- set sub.parts = sub.parts + [s.get("text", "")] -%} + {%- elif s.type in ["image", "image_url"] -%}{%- set sub.parts = sub.parts + [IMG] -%} + {%- else -%}{%- set sub.parts = sub.parts + ["[Unsupported " ~ s.type ~ "]"] -%} + {%- endif -%} + {%- endfor -%} + {%- set tc = sub.parts | join("\n\n") -%} + {%- endif -%} + {%- set up.parts = up.parts + ["" ~ tc ~ ""] -%} + {%- else -%} + {%- set up.parts = up.parts + ["[Unsupported " ~ b.type ~ "]"] -%} + {%- endif -%} + {%- endfor -%} + {{- up.parts | join("\n\n") -}} + + {%- elif m.role == "latest_reminder" -%} + {{- "<｜latest_reminder｜>" ~ (content or "") -}} + + {%- elif m.role == "assistant" -%} + {%- set prev_has_task = idx > 0 and msgs[idx - 1].get("task") is not none -%} + {%- if is_thinking and not prev_has_task and (not effective_drop or idx > state.last_user) -%} + {{- ((m.reasoning_content if m.get("reasoning_content") is not none else m.get("thinking")) or "") ~ "" -}} + {%- endif -%} + {{- content or "" -}} + {%- if m.get("tool_calls") -%} + {%- set tcs = namespace(lines=[]) -%} + {%- for tc in m.tool_calls -%} + {%- set fn = tc.function if tc.function is defined else tc -%} + {%- set params = namespace(lines=[]) -%} + {%- set args = fn.arguments if fn.arguments is mapping else {"arguments": fn.arguments} -%} + {%- for k, v in args.items() -%} + {%- set params.lines = params.lines + ["<" ~ DSML ~ " parameter name=\"" ~ k ~ "\" string=\"" ~ ("true" if v is string else "false") ~ "\">" ~ (v if v is string else v | tojson) ~ ""] -%} + {%- endfor -%} + {%- set name = qualified_name(fn.name, tc.get("namespace") or fn.get("namespace")) -%} + {%- set tcs.lines = tcs.lines + ["<" ~ DSML ~ " invoke name=\"" ~ name ~ "\">\n" ~ params.lines | join("\n") ~ "\n"] -%} + {%- endfor -%} + {{- "\n\n<" ~ DSML ~ " calls>\n" ~ tcs.lines | join("\n") ~ "\n" -}} + {%- endif -%} + {%- if not m.get("wo_eos", false) -%}{{- "<｜end▁of▁sentence｜>" -}}{%- endif -%} + + {%- else -%} + {{- raise_exception("Unknown role: " ~ m.role) -}} + {%- endif -%} + + {#- Transition tokens, only when the next message is an assistant/latest_reminder turn or this is the last one. -#} + {%- if loop.last or msgs[idx + 1].role in ["assistant", "latest_reminder"] -%} + {%- set task = m.get("task") -%} + {%- if task is not none -%} + {%- set task_tokens = {"action": "<｜action｜>", "query": "<｜query｜>", "authority": "<｜authority｜>", "domain": "<｜domain｜>", "title": "<｜title｜>", "read_url": "<｜read_url｜>"} -%} + {%- if task not in task_tokens -%}{{- raise_exception("Invalid task: '" ~ task ~ "'") -}}{%- endif -%} + {%- if task == "action" -%}{{- "<｜Assistant｜>" ~ ("" if is_thinking else "") -}}{%- endif -%} + {{- task_tokens[task] -}} + {%- elif (m.role == "user" or (m.role == "system" and idx > 0)) and (add_generation_prompt or not loop.last) -%} + {{- "<｜Assistant｜>" ~ ("" if is_thinking and (not effective_drop or idx >= state.last_user) else "") -}} + {%- endif -%} + {%- endif -%} +{%- endfor -%} From 3766bbdfd0f14cf680d9401b01ee2f4f4c8a8616 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 03:21:30 -0700 Subject: [PATCH 04/49] convert : deepseek41 stream MXFP4 experts through a thread pool Vectorised nibble repack (byte-identical to repack_mxfp4_blocks on the real experts) and streamed writes, so no stacked expert tensor is held in memory; DSpark expert names map back to mtp.M for the --mtp file. --- conversion/deepseek41.py | 66 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py index 88f80ebc761a..61130b92acdc 100644 --- a/conversion/deepseek41.py +++ b/conversion/deepseek41.py @@ -342,23 +342,75 @@ def _write_mxfp4_experts(self, bid: int, n_experts: int) -> list[str]: consumed: list[str] = [] for proj, key in (("w1", gguf.MODEL_TENSOR.FFN_GATE_EXP), ("w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP), ("w3", gguf.MODEL_TENSOR.FFN_UP_EXP)): - loaders = [] + loaders, names = [], [] for eid in range(n_experts): wn, sn = f"layers.{bid}.ffn.experts.{eid}.{proj}.weight", f"layers.{bid}.ffn.experts.{eid}.{proj}.scale" if wn not in self.model_tensors or sn not in self.model_tensors: raise KeyError(f"Missing routed expert tensors {wn}") loaders.append((self.model_tensors[wn], self.model_tensors[sn])) + names.append((wn, sn)) consumed += [wn, sn] extra = sorted(n for n in self.model_tensors if re.match(rf"layers\.{bid}\.ffn\.experts\.(\d+)\.{proj}\.weight$", n) and int(n.split(".")[4]) >= n_experts) if extra: raise KeyError(f"layer {bid}: unexpected experts beyond {n_experts}: {extra[:3]}") - data = self._mxfp4_expert_tensor(loaders) + data = self._mxfp4_expert_stream(names, loaders) new_name = self.format_tensor_name(key, bid) self.gguf_writer.add_tensor(new_name, data, raw_dtype=gguf.GGMLQuantizationType.MXFP4) return consumed + @staticmethod + def _repack_mxfp4_np(packed: np.ndarray, scale: np.ndarray) -> np.ndarray: + """Same bytes as TextModel.repack_mxfp4_blocks (checked on the real experts), vectorised: + source byte k of a 32-group holds elements 2k (low) / 2k+1 (high); ggml byte j holds j (low) / j+16 (high).""" + rows, packed_cols = packed.shape + nb = packed_cols * 2 // 32 + if packed_cols * 2 % 32 != 0 or scale.shape != (rows, nb): + raise ValueError(f"MXFP4 shapes {packed.shape} / {scale.shape} do not match") + src = packed.reshape(rows, nb, 16) + lo_src, hi_src = src[:, :, :8], src[:, :, 8:] # elements 0..15 and 16..31 + out = np.empty((rows, nb, 17), dtype=np.uint8) + out[:, :, 0] = scale + q = out[:, :, 1:].reshape(rows, nb, 8, 2) + q[..., 0] = (lo_src & 0x0F) | ((hi_src & 0x0F) << 4) # j = 2k + q[..., 1] = (lo_src >> 4) | (hi_src & 0xF0) # j = 2k + 1 + return out.reshape(rows, nb * 17) + + def _mxfp4_expert_stream(self, names: list[tuple[str, str]], loaders) -> _StreamedBytes: + """Stacked [n_expert, rows, blocks*17] MXFP4 tensor, repacked by a thread pool at write time.""" + import threading + from concurrent.futures import ThreadPoolExecutor + from safetensors import safe_open + + rows, packed_cols = loaders[0][0]().shape + shape = (len(names), rows, packed_cols * 2 // 32 * 17) + n_main = self.hparams["num_hidden_layers"] + + def src_name(n: str) -> str: # undo the mtp.M -> layers.(n_main+M) rekey of filter_tensors + m = re.match(r"layers\.(\d+)\.(.+)$", n) + if m is not None and int(m.group(1)) >= n_main: + return f"mtp.{int(m.group(1)) - n_main}.{m.group(2)}" + return n + names = [(src_name(wn), src_name(sn)) for wn, sn in names] + parts = {wn: self._tensor_part(wn) for wn, _ in names} + local = threading.local() + + def one(ws: tuple[str, str]) -> np.ndarray: + wn, sn = ws + hs = local.__dict__.setdefault("h", {}) + part = parts[wn] + if part not in hs: + hs[part] = safe_open(part, framework="pt") + w = hs[part].get_tensor(wn).view(torch.uint8).numpy() + sc = hs[part].get_tensor(sn).view(torch.uint8).numpy() + return self._repack_mxfp4_np(w, sc) + + def chunks() -> Iterable[np.ndarray]: + with ThreadPoolExecutor(int(os.environ.get("DSV41_THREADS", "16"))) as ex: + yield from ex.map(one, names) + return _StreamedBytes(shape, chunks) + def _engram_embed_tensor(self, bid: int) -> list[str]: wn, sn = f"layers.{bid}.engram.embed.weight", f"layers.{bid}.engram.embed.scale" wgen, sgen = self.model_tensors[wn], self.model_tensors[sn] @@ -408,10 +460,14 @@ def chunks() -> Iterable[np.ndarray]: logger.info(f"{new_name}: Engram table {rows} x {dim} copied as {self._engram_type.upper()}") return [wn, sn] + _weight_map: dict[str, str] | None = None + def _tensor_part(self, name: str) -> Path: - import json - with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: - return self.dir_model / json.load(f)["weight_map"][name] + if self._weight_map is None: + import json + with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: + self._weight_map = json.load(f)["weight_map"] + return self.dir_model / self._weight_map[name] def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: if self._dsv4_mxfp4_generated: From bae963fb27b2a5242ce37f9ba0e58b299cef06fb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 03:24:13 -0700 Subject: [PATCH 05/49] deepseek41: model graph (Engram, single-pass mHC, ratio 1/2 compressors, shared KV/index K/top-k, candidate blocks) and DSV4 KV cache ratio generalisation --- src/llama-context.cpp | 3 +- src/llama-graph.cpp | 4 +- src/llama-kv-cache-dsv4.cpp | 105 ++-- src/llama-kv-cache-dsv4.h | 4 + src/llama-model-loader.cpp | 5 +- src/llama-model.cpp | 6 +- src/models/deepseek41.cpp | 996 ++++++++++++++++++++++++++++++++++++ src/models/models.h | 67 +++ 8 files changed, 1155 insertions(+), 35 deletions(-) create mode 100644 src/models/deepseek41.cpp diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 9ee64b566779..4629ce08f1d1 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2520,6 +2520,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { model.arch == LLM_ARCH_CLEF || model.arch == LLM_ARCH_QWEN4EXP || model.arch == LLM_ARCH_DEEPSEEK4 || + model.arch == LLM_ARCH_DEEPSEEK41 || (model.arch == LLM_ARCH_DFLASH && model.hparams.dsv4_hc_mult > 0) || model.arch == LLM_ARCH_NANBEIGE || model.arch == LLM_ARCH_MINIMAX_01 || @@ -4055,7 +4056,7 @@ llama_context * llama_init_from_model( } } - if ((model->hparams.is_mla() || model->arch == LLM_ARCH_DEEPSEEK4) && params.type_k != params.type_v) { + if ((model->hparams.is_mla() || model->arch == LLM_ARCH_DEEPSEEK4 || model->arch == LLM_ARCH_DEEPSEEK41) && params.type_k != params.type_v) { LLAMA_LOG_ERROR("%s: model does not support different K (%s) and V (%s) cache types\n", __func__, ggml_type_name(params.type_k), ggml_type_name(params.type_v)); return nullptr; } diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 6cadda9716ff..902b0269e3ab 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1954,7 +1954,7 @@ ggml_tensor * llm_graph_context::build_ffn( const float limit = hparams.swiglu_clamp_shexp[il]; constexpr float eps = 1e-6f; if (limit > eps) { - if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) { + if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) { cur = ggml_swiglu_clamp(ctx0, cur, tmp, limit); } else { tmp = ggml_clamp(ctx0, tmp, -limit, limit); @@ -2356,7 +2356,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( const float limit = hparams.swiglu_clamp_exp[il]; constexpr float eps = 1e-6f; if (limit > eps) { - if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) { + if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) { cur = ggml_swiglu_clamp(ctx0, cur, up, limit); } else { up = ggml_clamp(ctx0, up, -limit, limit); diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp index f8caae9d6be5..9046ddde41b6 100644 --- a/src/llama-kv-cache-dsv4.cpp +++ b/src/llama-kv-cache-dsv4.cpp @@ -432,6 +432,7 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan( const llama_ubatch & ubatch, uint32_t ratio, bool overlap, + bool pad_blocks, uint32_t state_size, uint32_t kv_size, uint32_t n_stream, @@ -548,7 +549,7 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan( } } - if (ratio == DSV4_CSA_RATIO && !plan.state_pos.empty()) { + if (pad_blocks && !plan.state_pos.empty()) { assert(kv_size > 0); // Pad each stream to the reserve plan's block count. @@ -603,7 +604,7 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan( } } - if (ratio == DSV4_HCA_RATIO && !plan.state_pos.empty() && plan.state_write_idxs.empty()) { + if (!pad_blocks && !plan.state_pos.empty() && plan.state_write_idxs.empty()) { assert(kv_size > 0); // the last slot must not be live, or the dummy write would corrupt it; // a full stream implies a completed block, which implies real writes @@ -744,6 +745,7 @@ static std::vector dsv4_build_comp_plans const std::vector & ubatches, uint32_t ratio, bool overlap, + bool pad_blocks, uint32_t state_size, uint32_t kv_size, uint32_t n_stream, @@ -755,7 +757,7 @@ static std::vector dsv4_build_comp_plans // the first ubatch touching a seq consumes its rollback restore std::vector rs(rs_idx); for (const llama_ubatch & ubatch : ubatches) { - plans.push_back(dsv4_build_comp_plan(ubatch, ratio, overlap, state_size, kv_size, n_stream, n_rs_seq, rs)); + plans.push_back(dsv4_build_comp_plan(ubatch, ratio, overlap, pad_blocks, state_size, kv_size, n_stream, n_rs_seq, rs)); for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) { const llama_seq_id seq_id = ubatch.seq_id_unq[s]; @@ -1056,6 +1058,10 @@ uint32_t llama_dsv4_comp_state::get_ratio() const { return ratio; } +bool llama_dsv4_comp_state::is_overlap() const { + return state_size == 2*ratio; +} + uint32_t llama_dsv4_comp_state::get_state_size() const { return state_size; } @@ -1261,6 +1267,25 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( v_trans, offload, swa_full, unified_raw, kv_size, n_seq_max, n_ubatch, n_pad, nullptr, filter_raw, reuse, nullptr); + // V4: CSA = overlapped ratio-4 pooling (+ its own indexer compressor, "lid"), HCA = ratio-128 pooling. + // V4.1: CSA slot = ratio-2 pooling, HCA slot = ratio-1 projection, no overlap, no "lid": the index K is derived from + // the compressed latent and stored next to it in the same row ([kv | index K]), and only the KV source layers own + // rows (consumers read the source layer's cache). + const bool is_v41 = model.arch == LLM_ARCH_DEEPSEEK41; + const uint32_t ratio_csa = is_v41 ? 2u : DSV4_CSA_RATIO; + const uint32_t ratio_hca = is_v41 ? 1u : DSV4_HCA_RATIO; + + if (is_v41) { + const uint32_t n_embd_comp = model.hparams.n_embd_head_k() + model.hparams.indexer_head_size; + for (llama_hparams * hp : { &hparams_csa, &hparams_hca }) { + std::fill(hp->n_head_kv_arr.begin(), hp->n_head_kv_arr.end(), 1); + hp->n_embd_head_k_full = n_embd_comp; + hp->n_embd_head_v_full = n_embd_comp; + hp->n_embd_head_k_swa = n_embd_comp; + hp->n_embd_head_v_swa = n_embd_comp; + } + } + dsv4_make_k_only(hparams_csa); dsv4_make_k_only(hparams_hca); @@ -1272,12 +1297,19 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( hparams_lid.rope_type = LLAMA_ROPE_TYPE_NEOX; dsv4_make_k_only(hparams_lid); + const auto is_comp_owner = [&](int32_t il, uint32_t ratio) { + if (model.hparams.dsv4_compress_ratios[il] != ratio) { + return false; + } + return !is_v41 || model.hparams.dsv41_kv_src[il] == il; + }; + const layer_filter_cb filter_csa = [&](int32_t il) { if (filter && !filter(il)) { return false; } - return model.hparams.dsv4_compress_ratios[il] == DSV4_CSA_RATIO; + return is_comp_owner(il, ratio_csa); }; const layer_filter_cb filter_hca = [&](int32_t il) { @@ -1285,52 +1317,60 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( return false; } - return model.hparams.dsv4_compress_ratios[il] == DSV4_HCA_RATIO; + return is_comp_owner(il, ratio_hca); + }; + + const layer_filter_cb filter_lid = [&](int32_t il) { + return !is_v41 && filter_csa(il); }; const bool unified_compressed = false; - LLAMA_LOG_INFO("%s: creating DSV4 CSA compressed KV cache, size = %u cells\n", - __func__, dsv4_comp_size(kv_size, DSV4_CSA_RATIO)); + LLAMA_LOG_INFO("%s: creating DSV4 CSA compressed KV cache, ratio = %u, size = %u cells\n", + __func__, ratio_csa, dsv4_comp_size(kv_size, ratio_csa)); kv_csa = std::make_unique( model, hparams_csa, type_k, type_v, - v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, DSV4_CSA_RATIO), 256u), n_seq_max, n_pad, + v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_csa, nullptr, nullptr); - LLAMA_LOG_INFO("%s: creating DSV4 HCA compressed KV cache, size = %u cells\n", - __func__, dsv4_comp_size(kv_size, DSV4_HCA_RATIO)); + LLAMA_LOG_INFO("%s: creating DSV4 HCA compressed KV cache, ratio = %u, size = %u cells\n", + __func__, ratio_hca, dsv4_comp_size(kv_size, ratio_hca)); kv_hca = std::make_unique( model, hparams_hca, type_k, type_v, - v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, DSV4_HCA_RATIO), 256u), n_seq_max, n_pad, + v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_hca), 256u), n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_hca, nullptr, nullptr); LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer KV cache, size = %u cells\n", - __func__, dsv4_comp_size(kv_size, DSV4_CSA_RATIO)); + __func__, dsv4_comp_size(kv_size, ratio_csa)); kv_lid = std::make_unique( model, hparams_lid, type_k, type_v, - v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, DSV4_CSA_RATIO), 256u), n_seq_max, n_pad, - 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_csa, nullptr, nullptr); + v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad, + 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_lid, nullptr, nullptr); LLAMA_LOG_INFO("%s: creating DSV4 CSA compressor state\n", __func__); + // overlapped pooling keeps 2*ratio rows of 2x width; plain pooling keeps ratio rows of head width + const uint32_t csa_state_size = is_v41 ? ratio_csa : 2*ratio_csa; + const uint32_t csa_state_embd = is_v41 ? model.hparams.n_embd_head_k() : 2*model.hparams.n_embd_head_k(); + csa_state = std::make_unique( - model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO, - 2*model.hparams.n_embd_head_k(), n_rs_seq, "csa", filter_csa); + model, offload, unified_compressed, n_seq_max, ratio_csa, csa_state_size, + csa_state_embd, n_rs_seq, "csa", filter_csa); LLAMA_LOG_INFO("%s: creating DSV4 HCA compressor state\n", __func__); hca_state = std::make_unique( - model, offload, unified_compressed, n_seq_max, DSV4_HCA_RATIO, DSV4_HCA_RATIO, + model, offload, unified_compressed, n_seq_max, ratio_hca, ratio_hca, model.hparams.n_embd_head_k(), n_rs_seq, "hca", filter_hca); LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer compressor state\n", __func__); lid_state = std::make_unique( model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO, - 2*model.hparams.indexer_head_size, n_rs_seq, "lid", filter_csa); + 2*model.hparams.indexer_head_size, n_rs_seq, "lid", filter_lid); // DSV4 attention reads compressed-K / compressor-state rows that the current // graph does not necessarily overwrite; uninitialized buffer contents would @@ -1475,9 +1515,9 @@ bool llama_kv_cache_dsv4::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1 bool res = true; res = res & kv_raw->seq_rm(seq_id, p0, -1); - res = res & kv_csa->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1); - res = res & kv_hca->seq_rm(seq_id, p0/DSV4_HCA_RATIO, -1); - res = res & kv_lid->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1); + res = res & kv_csa->seq_rm(seq_id, p0/csa_state->get_ratio(), -1); + res = res & kv_hca->seq_rm(seq_id, p0/hca_state->get_ratio(), -1); + res = res & kv_lid->seq_rm(seq_id, p0/lid_state->get_ratio(), -1); return res; } @@ -1613,11 +1653,11 @@ void llama_kv_cache_dsv4::state_write(llama_io_write_i & io, llama_seq_id seq_id //FIXME : note that we conflate token positions with rows, which is not true for multi-modal case. const uint32_t n_rows_csa = seq_id >= 0 ? - dsv4_state_n_used_k_rows(pos_max, DSV4_CSA_RATIO, kv_csa->get_size()) : kv_csa->get_size(); + dsv4_state_n_used_k_rows(pos_max, csa_state->get_ratio(), kv_csa->get_size()) : kv_csa->get_size(); const uint32_t n_rows_hca = seq_id >= 0 ? - dsv4_state_n_used_k_rows(pos_max, DSV4_HCA_RATIO, kv_hca->get_size()) : kv_hca->get_size(); + dsv4_state_n_used_k_rows(pos_max, hca_state->get_ratio(), kv_hca->get_size()) : kv_hca->get_size(); const uint32_t n_rows_lid = seq_id >= 0 ? - dsv4_state_n_used_k_rows(pos_max, DSV4_CSA_RATIO, kv_lid->get_size()) : kv_lid->get_size(); + dsv4_state_n_used_k_rows(pos_max, lid_state->get_ratio(), kv_lid->get_size()) : kv_lid->get_size(); dsv4_state_write_k_cache(io, kv_csa.get(), seq_id, flags, n_rows_csa); dsv4_state_write_k_cache(io, kv_hca.get(), seq_id, flags, n_rows_hca); @@ -1932,6 +1972,11 @@ void llama_kv_cache_dsv4_raw_context::set_input_kq_mask(ggml_tensor * dst, const kv_swa->set_input_kq_mask(dst, ubatch, causal_attn); } +void llama_kv_cache_dsv4_raw_context::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector & res) const { + GGML_ASSERT(kv_swa != nullptr); + kv_swa->get_prev_tokens(ubatch, n, res); +} + void llama_kv_cache_dsv4_raw_context::set_input_k_rot(ggml_tensor * dst) const { kv_swa->set_input_k_rot(dst); } @@ -2052,13 +2097,13 @@ llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context( std::vector ubatches, std::vector ubatches_raw) : ubatches(std::move(ubatches)), - plans_csa(dsv4_build_comp_plans(this->ubatches, DSV4_CSA_RATIO, true, + plans_csa(dsv4_build_comp_plans(this->ubatches, kv->get_csa_state()->get_ratio(), kv->get_csa_state()->is_overlap(), true, kv->get_csa_state()->get_state_size(), kv->get_csa()->get_size(), kv->get_csa_state()->get_n_stream(), kv->get_n_rs_seq(), kv->get_rs_idx())), - plans_hca(dsv4_build_comp_plans(this->ubatches, DSV4_HCA_RATIO, false, + plans_hca(dsv4_build_comp_plans(this->ubatches, kv->get_hca_state()->get_ratio(), kv->get_hca_state()->is_overlap(), false, kv->get_hca_state()->get_state_size(), kv->get_hca()->get_size(), kv->get_hca_state()->get_n_stream(), kv->get_n_rs_seq(), kv->get_rs_idx())), - plans_lid(dsv4_build_comp_plans(this->ubatches, DSV4_CSA_RATIO, true, + plans_lid(dsv4_build_comp_plans(this->ubatches, kv->get_lid_state()->get_ratio(), kv->get_lid_state()->is_overlap(), true, kv->get_lid_state()->get_state_size(), kv->get_lid()->get_size(), kv->get_lid_state()->get_n_stream(), kv->get_n_rs_seq(), kv->get_rs_idx())), ctx_raw(std::make_unique( @@ -2222,7 +2267,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_ } reserve_plan_csa = dsv4_build_reserve_comp_plan( - ubatch, DSV4_CSA_RATIO, true, + ubatch, csa_state->get_ratio(), csa_state->is_overlap(), csa_state->get_state_size(), get_csa()->get_n_kv(), csa_state->get_n_stream(), csa_state->get_n_rs_seq()); return reserve_plan_csa; @@ -2236,7 +2281,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_ } reserve_plan_hca = dsv4_build_reserve_comp_plan( - ubatch, DSV4_HCA_RATIO, false, + ubatch, hca_state->get_ratio(), hca_state->is_overlap(), hca_state->get_state_size(), get_hca()->get_n_kv(), hca_state->get_n_stream(), hca_state->get_n_rs_seq()); return reserve_plan_hca; @@ -2250,7 +2295,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_ } reserve_plan_lid = dsv4_build_reserve_comp_plan( - ubatch, DSV4_CSA_RATIO, true, + ubatch, lid_state->get_ratio(), lid_state->is_overlap(), lid_state->get_state_size(), get_lid()->get_n_kv(), lid_state->get_n_stream(), lid_state->get_n_rs_seq()); return reserve_plan_lid; diff --git a/src/llama-kv-cache-dsv4.h b/src/llama-kv-cache-dsv4.h index ce39867c0342..2a6e14a0da8e 100644 --- a/src/llama-kv-cache-dsv4.h +++ b/src/llama-kv-cache-dsv4.h @@ -31,6 +31,7 @@ class llama_dsv4_comp_state { void apply_copies(const stream_copy_info & sc_info) const; uint32_t get_ratio() const; + bool is_overlap() const; // overlapped (V4 CSA) pooling: state holds 2*ratio rows of 2x width uint32_t get_state_size() const; uint32_t get_n_stream() const; uint32_t get_n_rs_seq() const; @@ -214,6 +215,9 @@ class llama_kv_cache_dsv4_raw_context : public llama_memory_context_i { void set_input_kq_mask(ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const; void set_input_k_rot(ggml_tensor * dst) const; + // token history of the raw (window) cells, for host-side n-gram hashing (deepseek41 Engram) + void get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector & res) const; + private: size_t i_next = 0; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index bc93c6f9e62d..c5cab2d01279 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -414,6 +414,7 @@ namespace GGUFMeta { template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::array & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); + template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::array & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::array & result, bool required); @@ -441,6 +442,7 @@ namespace GGUFMeta { template bool llama_model_loader::get_key (enum llm_kv kid, bool & result, bool required); template bool llama_model_loader::get_key (enum llm_kv kid, float & result, bool required); template bool llama_model_loader::get_key (enum llm_kv kid, uint32_t & result, bool required); + template bool llama_model_loader::get_key (enum llm_kv kid, int32_t & result, bool required); template bool llama_model_loader::get_key(enum llm_kv kid, std::string & result, bool required); template<> @@ -1200,7 +1202,8 @@ struct ggml_tensor * llama_model_loader::create_tensor( // sanity checks if (info.layer == LLM_TENSOR_LAYER_INPUT || info.layer == LLM_TENSOR_LAYER_OUTPUT) { - if (tn.bid != -1) { + // deepseek41 Engram tables are per-layer but host-resident gather tables, placed like token_embd + if (tn.bid != -1 && tn.tensor != LLM_TENSOR_ENGRAM_EMBD) { GGML_ABORT("input/output layer tensor %s used with a layer number", tn.str().c_str()); } } else { diff --git a/src/llama-model.cpp b/src/llama-model.cpp index ded790b2b86d..2d876b198ffa 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -206,6 +206,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_dots3note(params); case LLM_ARCH_DEEPSEEK4: return new llama_model_deepseek4(params); + case LLM_ARCH_DEEPSEEK41: + return new llama_model_deepseek41(params); case LLM_ARCH_GLM_DSA: return new llama_model_glm_dsa(params); case LLM_ARCH_MISTRAL4: @@ -2995,6 +2997,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, nullptr); } } break; + case LLM_ARCH_DEEPSEEK41: case LLM_ARCH_DEEPSEEK4: { GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); @@ -3404,7 +3407,7 @@ int32_t llama_model_n_head_kv(const llama_model * model) { int32_t llama_model_n_swa(const llama_model * model) { // dsv4 kv-cache has SWA but it cannot be used as a rollback because of // other compression ratios, so we return 0 here - if (model->arch == LLM_ARCH_DEEPSEEK4) { + if (model->arch == LLM_ARCH_DEEPSEEK4 || model->arch == LLM_ARCH_DEEPSEEK41) { return 0; } return model->hparams.n_swa; @@ -3491,6 +3494,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_DEEPSEEK2OCR: case LLM_ARCH_DEEPSEEK32: case LLM_ARCH_DEEPSEEK4: + case LLM_ARCH_DEEPSEEK41: case LLM_ARCH_MUSE_GLIMMER: case LLM_ARCH_PLM: case LLM_ARCH_CHATGLM: diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp new file mode 100644 index 000000000000..2db3c4bd2f09 --- /dev/null +++ b/src/models/deepseek41.cpp @@ -0,0 +1,996 @@ +#include "llama-hparams.h" +#include "models.h" + +#include "llama-kv-cache-dsv4.h" + +#include +#include +#include +#include + +// DeepSeek-V4.1 (ref: deepseek-ai/DeepSeek-V4.1-Flash-1017 inference/model.py, engram.py) +// +// Differences to DeepSeek-V4 handled here: +// - Engram on a few layers: hashed 2..N-gram rows gathered from host-resident tables, mixed into every HC copy +// - single-pass mHC: a sublayer collapses the HC copies with the pre-mix computed by the PREVIOUS sublayer +// (layer 0 attention: copy 0), and there is no hc_head (the output collapses with the last FFN pre-mix) +// - compress ratios 1 and 2 (plain softmax pooling, no ape / overlap), compressed KV and index K produced only by +// the kv source layers and read by the layers after them; index K = k_norm(wk(latent)), stored next to the KV row +// - indexer top-k computed only on index source layers, reused by the layers in between +// - two-level candidate selection: the candidate source layer picks blocks, later index layers score inside them +// - no per-head q norm after wq_b, no hash-routed (tid2eid) layers + +static float dsv41_rope_attn_factor(float freq_scale, float ext_factor) { + if (ext_factor == 0.0f) { + return 1.0f; + } + + return 1.0f / (1.0f + 0.1f*logf(1.0f/freq_scale)); +} + +static constexpr int64_t DSV41_ENGRAM_DEAD = -1; + +// +// loading +// + +void llama_model_deepseek41::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm); + ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all); + if (!ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, 0)) { + hparams.swiglu_clamp_shexp = hparams.swiglu_clamp_exp; + } + + ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); + ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); + ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k); + + ml.get_key(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, hparams.dsv4_o_group_count); + ml.get_key(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, hparams.dsv4_o_lora_rank); + ml.get_key(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, hparams.dsv4_compress_rope_base); + ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult); + ml.get_key(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, hparams.dsv4_hc_sinkhorn_iters); + ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps); + hparams.dsv4_hash_layer_count = 0; + + hparams.n_embd_out_impl = hparams.dsv4_hc_mult * hparams.n_embd; + + uint32_t n_compress_ratios = 0; + ml.get_arr_n(LLM_KV_ATTENTION_COMPRESS_RATIOS, n_compress_ratios); + if (n_compress_ratios < hparams.n_layer_all) { + throw std::runtime_error("DeepSeek-V4.1 compress_ratios is shorter than block_count"); + } + GGML_ASSERT(n_compress_ratios <= LLAMA_MAX_LAYERS); + ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios); + + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); + if (hparams.expert_gating_func != LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) { + throw std::runtime_error("DeepSeek-V4.1 loader expects sqrtsoftplus MoE scoring"); + } + + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + hparams.set_swa_pattern(0); + hparams.non_causal_type = LLAMA_NON_CAUSAL_TYPE_SWA_FULL; + for (uint32_t il = hparams.n_layer(); il < hparams.n_layer_all; ++il) { + hparams.is_swa_impl[il] = true; + } + + // cross-layer sharing: every compressed layer reads the latest kv source at or before it, and attends with the + // top-k of the latest index source at or before it (SharedAttentionRuntime in the reference) + std::vector kv_src; + std::vector idx_src; + ml.get_arr(LLM_KV_ATTENTION_KV_SOURCE_LAYERS, kv_src); + ml.get_arr(LLM_KV_ATTENTION_INDEXER_SOURCE_LAYERS, idx_src); + + const uint32_t n_layer = hparams.n_layer(); + int32_t last_kv = -1; + int32_t last_idx = -1; + for (uint32_t il = 0; il < LLAMA_MAX_LAYERS; ++il) { + hparams.dsv41_kv_src[il] = -1; + hparams.dsv41_topk_src[il] = -1; + } + for (uint32_t il = 0; il < n_layer; ++il) { + const uint32_t ratio = hparams.dsv4_compress_ratios[il]; + if (std::find(kv_src.begin(), kv_src.end(), il) != kv_src.end()) { + if (ratio == 0) { + throw std::runtime_error(format("DeepSeek-V4.1 kv source layer %u has compress ratio 0", il)); + } + last_kv = (int32_t) il; + } + if (std::find(idx_src.begin(), idx_src.end(), il) != idx_src.end()) { + if (ratio == 0) { + throw std::runtime_error(format("DeepSeek-V4.1 index source layer %u has compress ratio 0", il)); + } + last_idx = (int32_t) il; + } + if (ratio == 0) { + continue; + } + if (ratio != 1 && ratio != 2) { + throw std::runtime_error(format("DeepSeek-V4.1 layer %u: compress ratio %u (only 0, 1, 2 supported)", il, ratio)); + } + if (last_kv < 0 || last_idx < 0 || + hparams.dsv4_compress_ratios[last_kv] != ratio || hparams.dsv4_compress_ratios[last_idx] != ratio || + last_idx < last_kv) { + throw std::runtime_error(format("DeepSeek-V4.1 layer %u: no matching kv/index source before it", il)); + } + hparams.dsv41_kv_src[il] = last_kv; + hparams.dsv41_topk_src[il] = last_idx; + } + + hparams.dsv41_cand_src = -1; + hparams.dsv41_cand_topk_blocks = 0; + hparams.dsv41_cand_block_size = 0; + ml.get_key(LLM_KV_ATTENTION_INDEXER_CANDIDATE_SOURCE_LAYER, hparams.dsv41_cand_src, false); + if (hparams.dsv41_cand_src >= 0) { + ml.get_key(LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, hparams.dsv41_cand_topk_blocks); + ml.get_key(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, hparams.dsv41_cand_block_size); + const int32_t cs = hparams.dsv41_cand_src; + if ((uint32_t) cs >= n_layer || hparams.dsv41_topk_src[cs] != cs || hparams.dsv41_cand_block_size == 0 || + hparams.dsv41_cand_topk_blocks == 0) { + throw std::runtime_error("DeepSeek-V4.1 candidate source layer must be an index source"); + } + for (uint32_t il = cs + 1; il < n_layer; ++il) { + if (hparams.dsv41_topk_src[il] == (int32_t) il && hparams.dsv4_compress_ratios[il] != hparams.dsv4_compress_ratios[cs]) { + throw std::runtime_error("DeepSeek-V4.1 candidate consumers must share the source's compress ratio"); + } + } + } + + // Engram + std::vector e_layers; + ml.get_arr(LLM_KV_ENGRAM_LAYERS, e_layers, false); + if (!e_layers.empty()) { + uint32_t pad_id = 0; + std::vector primes; + ml.get_arr(LLM_KV_ENGRAM_NUM_EMBEDDINGS, engram_n_rows); + ml.get_key(LLM_KV_ENGRAM_MAX_NGRAM_SIZE, engram_max_ngram); + ml.get_key(LLM_KV_ENGRAM_HEAD_COUNT, engram_n_head); + ml.get_key(LLM_KV_ENGRAM_HEAD_DIM, engram_head_dim); + ml.get_key(LLM_KV_ENGRAM_PAD_TOKEN_ID, pad_id); + ml.get_arr(LLM_KV_ENGRAM_TOKEN_MAP, engram_token_map); + ml.get_arr(LLM_KV_ENGRAM_MULTIPLIERS, engram_mult); + ml.get_arr(LLM_KV_ENGRAM_PRIMES, primes); + + const size_t n_e = e_layers.size(); + const size_t n_cols = (size_t) (engram_max_ngram - 1)*engram_n_head; + if (engram_max_ngram < 2 || engram_n_head == 0 || engram_n_rows.size() != n_e || + engram_mult.size() != n_e*engram_max_ngram || primes.size() != n_e*n_cols || + pad_id >= engram_token_map.size()) { + throw std::runtime_error("DeepSeek-V4.1 engram metadata is inconsistent"); + } + engram_layers.assign(e_layers.begin(), e_layers.end()); + engram_pad_cid = engram_token_map[pad_id]; + engram_primes.assign(primes.begin(), primes.end()); + engram_offsets.resize(primes.size()); + for (size_t e = 0; e < n_e; ++e) { + uint64_t off = 0; + for (size_t c = 0; c < n_cols; ++c) { + engram_offsets[e*n_cols + c] = off; + off += engram_primes[e*n_cols + c]; + } + if (off > engram_n_rows[e] || off > (uint64_t) INT32_MAX) { + throw std::runtime_error(format("DeepSeek-V4.1 engram layer %u: buckets (%llu) exceed the table rows (%u)", + e_layers[e], (unsigned long long) off, engram_n_rows[e])); + } + } + for (uint32_t il : e_layers) { + if (il >= n_layer) { + throw std::runtime_error(format("DeepSeek-V4.1 engram layer %u is out of range", il)); + } + } + } + + type = LLM_TYPE_UNKNOWN; +} + +void llama_model_deepseek41::load_arch_tensors(llama_model_loader & ml) { + LLAMA_LOAD_LOCALS; + + const int64_t q_lora_rank = hparams.n_lora_q; + const int64_t n_ff_exp = hparams.n_ff_exp(); + const int64_t n_expert_shared = hparams.n_expert_shared; + + const int64_t n_embd_head = hparams.n_embd_head_k(); + const int64_t o_groups = hparams.dsv4_o_group_count; + const int64_t o_lora_rank = hparams.dsv4_o_lora_rank; + const int64_t hc_mult = hparams.dsv4_hc_mult; + const int64_t hc_dim = hc_mult * n_embd; + const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult; + const int64_t n_idx_embd = hparams.indexer_head_size; + const int64_t n_idx_head = hparams.indexer_n_head; + + // DSpark stages (blk.n_layer..) are loaded as optional tensors; the graph does not use them yet + uint32_t n_expert_dspark = n_expert; + ml.get_key(LLM_KV_DSPARK_EXPERT_COUNT, n_expert_dspark, false); + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + for (int i = 0; i < n_layer_all; ++i) { + auto & layer = layers[i]; + const bool mtp = i >= n_layer; + const int flags = mtp ? TENSOR_NOT_REQUIRED : 0; + const int64_t n_exp = mtp ? (int64_t) n_expert_dspark : n_expert; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags); + layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, flags); + layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, flags); + layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, flags); + layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, flags); + layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, flags); + layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, flags); + layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank, o_groups}, flags | TENSOR_ALLOW_RESHAPE); + layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, flags); + + layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, flags); + layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {hc_mix_dim}, flags); + layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, flags); + layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc_dim, hc_mix_dim}, flags); + layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {hc_mix_dim}, flags); + layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, flags); + + if (!mtp && hparams.dsv41_kv_src[i] == i) { + const uint32_t ratio = hparams.dsv4_compress_ratios[i]; + layer.attn_comp_wkv = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WKV, "weight", i), {n_embd, n_embd_head}, 0); + layer.attn_comp_norm = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_NORM, "weight", i), {n_embd_head}, 0); + if (ratio > 1) { + layer.attn_comp_wgate = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WGATE, "weight", i), {n_embd, n_embd_head}, 0); + } + layer.indexer_attn_k = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_K, "weight", i), {n_embd_head, n_idx_embd}, 0); + layer.indexer_k_norm = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "weight", i), {n_idx_embd}, 0); + } + if (!mtp && hparams.dsv41_topk_src[i] == i) { + layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, n_idx_head * n_idx_embd}, 0); + layer.indexer_proj = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ, "weight", i), {n_embd, n_idx_head}, 0); + } + + if (!mtp) { + const int e = engram_index(i); + if (e >= 0) { + const int64_t n_cols = engram_n_cols(); + layer.engram_embd = create_tensor(tn(LLM_TENSOR_ENGRAM_EMBD, "weight", i), {(int64_t) engram_head_dim, (int64_t) engram_n_rows[e]}, 0); + layer.engram_kv = create_tensor(tn(LLM_TENSOR_ENGRAM_KV, "weight", i), {n_cols * engram_head_dim, n_embd * (hc_mult + 1)}, 0); + layer.engram_gate = create_tensor(tn(LLM_TENSOR_ENGRAM_GATE, "weight", i), {n_embd, hc_mult}, 0); + } + } + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_exp}, flags); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_exp}, flags); + layer.ffn_exp_probs_b_vl = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B_VL, "bias", i), {n_exp}, TENSOR_NOT_REQUIRED); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags); + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_exp}, flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_exp}, flags); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_exp}, flags); + + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd }, flags); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); + + if (mtp) { + layer.dspark_main_proj = create_tensor(tn(LLM_TENSOR_DSPARK_MAIN_PROJ, "weight", i), {n_embd * 3, n_embd}, TENSOR_NOT_REQUIRED); + layer.dspark_main_norm = create_tensor(tn(LLM_TENSOR_DSPARK_MAIN_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED); + layer.dspark_norm = create_tensor(tn(LLM_TENSOR_DSPARK_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED); + } + } +} + +std::unique_ptr llama_model_deepseek41::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + throw std::runtime_error("DeepSeek-V4.1 DSpark drafting is not implemented yet"); + } + return std::make_unique(*this, params); +} + +// +// Engram n-gram hash (engram.py NgramHashState.forward) +// + +void llama_model_deepseek41::engram_hash(int e, const llama_ubatch & ubatch, const std::vector & prev, std::vector & out) const { + const int64_t n_tokens = ubatch.n_tokens; + const int64_t n_gram = engram_max_ngram; + const int64_t n_prev = n_gram - 1; + const int64_t n_head = engram_n_head; + const int64_t n_cols = engram_n_cols(); + const int64_t n_map = (int64_t) engram_token_map.size(); + + const uint64_t * mult = engram_mult.data() + (size_t) e*n_gram; + const uint64_t * primes = engram_primes.data() + (size_t) e*n_cols; + const uint64_t * offs = engram_offsets.data() + (size_t) e*n_cols; + + out.resize(n_tokens*n_cols); + + // compressed id of a raw token; embeddings (image spans) and unknown history are DEAD + const auto cid = [&](llama_token t) -> int64_t { + if (t < 0 || t >= n_map) { + return DSV41_ENGRAM_DEAD; + } + return engram_token_map[t]; + }; + + std::vector toks(n_gram); + for (int64_t i = 0; i < n_tokens; ++i) { + const bool is_embd = !ubatch.token || (ubatch.is_mixed() && ubatch.type[i]); + const llama_pos pos = ubatch.pos[i]; + + // shift 0 is the token itself; once a shift is blocked (sequence start or dead token) every longer one is too + int64_t c = is_embd ? DSV41_ENGRAM_DEAD : cid(ubatch.token[i]); + bool blocked = c == DSV41_ENGRAM_DEAD; + toks[0] = blocked ? engram_pad_cid : c; + for (int64_t s = 1; s < n_gram; ++s) { + const llama_token t = prev[i*n_prev + (n_prev - s)]; + c = (pos - s < 0) ? DSV41_ENGRAM_DEAD : cid(t); + blocked = blocked || c == DSV41_ENGRAM_DEAD; + toks[s] = blocked ? engram_pad_cid : c; + } + + uint64_t rolling = toks[0]*mult[0]; + for (int64_t o = 1; o < n_gram; ++o) { + rolling ^= toks[o]*mult[o]; + for (int64_t h = 0; h < n_head; ++h) { + const int64_t col = (o - 1)*n_head + h; + out[i*n_cols + col] = (int32_t) (rolling % primes[col] + offs[col]); + } + } + } +} + +// Host-side inputs: Engram row ids per engram layer, and the candidate-block pin (+1e30 on the block holding each +// query's newest visible compressed row: select_candidate_blocks pins it so a partly filled block always survives). +class llm_graph_input_dsv41 : public llm_graph_input_i { +public: + llm_graph_input_dsv41(const llama_model_deepseek41 & model, const llama_kv_cache_dsv4_context * mctx) : + model(model), mctx(mctx) {} + virtual ~llm_graph_input_dsv41() = default; + + void set_input(const llama_ubatch * ubatch) override; + bool can_reuse(const llm_graph_params & params) override; + + std::vector engram_ids; // per engram layer: I32 [n_cols*n_tokens] + ggml_tensor * cand_pin = nullptr; // F32 [n_blk, n_tokens/n_stream, 1, n_stream] + bool cand_csa = false; // candidate source reads the CSA (ratio 2) slot, else HCA (ratio 1) + + const llama_model_deepseek41 & model; + const llama_kv_cache_dsv4_context * mctx; + + std::vector prev; + std::vector ids; + std::vector pin; + +private: + const llama_kv_cache_dsv4_context::comp_plan & cand_plan(const llama_ubatch & ubatch) const { + return cand_csa ? mctx->get_csa_plan(ubatch) : mctx->get_hca_plan(ubatch); + } +}; + +void llm_graph_input_dsv41::set_input(const llama_ubatch * ubatch) { + if (!engram_ids.empty()) { + for (uint32_t i = 0; i < ubatch->n_tokens; ++i) { + GGML_ASSERT(ubatch->n_seq_id[i] == 1 && "DeepSeek-V4.1 Engram n-grams do not support tokens shared by multiple sequences"); + } + // predecessors come from the raw window cells, which already hold this ubatch + mctx->get_raw()->get_prev_tokens(*ubatch, model.engram_max_ngram - 1, prev); + for (size_t e = 0; e < engram_ids.size(); ++e) { + if (!engram_ids[e]->buffer) { + continue; + } + model.engram_hash((int) e, *ubatch, prev, ids); + ggml_backend_tensor_set(engram_ids[e], ids.data(), 0, ids.size()*sizeof(int32_t)); + } + } + + if (cand_pin && cand_pin->buffer) { + const auto & plan = cand_plan(*ubatch); + const int64_t n_blk = cand_pin->ne[0]; + const int64_t bs = model.hparams.dsv41_cand_block_size; + GGML_ASSERT((int64_t) plan.n_visible.size() == (int64_t) ubatch->n_tokens); + pin.assign(n_blk*ubatch->n_tokens, 0.0f); + for (int64_t i = 0; i < (int64_t) ubatch->n_tokens; ++i) { + const int64_t nv = plan.n_visible[i]; + if (nv > 0) { + const int64_t b = (nv - 1)/bs; + GGML_ASSERT(b < n_blk); + pin[i*n_blk + b] = 1e30f; + } + } + ggml_backend_tensor_set(cand_pin, pin.data(), 0, pin.size()*sizeof(float)); + } +} + +bool llm_graph_input_dsv41::can_reuse(const llm_graph_params & params) { + mctx = static_cast(params.mctx); + + bool res = true; + for (auto * t : engram_ids) { + res &= t->ne[0] == (int64_t) model.engram_n_cols()*params.ubatch.n_tokens; + } + if (cand_pin) { + const auto & plan = cand_plan(params.ubatch); + const int64_t bs = model.hparams.dsv41_cand_block_size; + res &= plan.n_kv > 0 && cand_pin->ne[0] == (plan.n_kv + bs - 1)/bs; + res &= cand_pin->ne[1]*cand_pin->ne[3] == (int64_t) params.ubatch.n_tokens; + } + return res; +} + +// +// graph helpers +// + +static ggml_tensor * dsv41_view_2d(ggml_context * ctx, ggml_tensor * t, int64_t ne0, int64_t ne1, int64_t i0) { + return ggml_view_2d(ctx, t, ne0, ne1, t->nb[1], ggml_row_size(t->type, i0)); +} + +static ggml_tensor * dsv41_view_1d(ggml_context * ctx, ggml_tensor * t, int64_t ne0, int64_t i0) { + return ggml_view_1d(ctx, t, ne0, ggml_row_size(t->type, i0)); +} + +ggml_tensor * llama_model_deepseek41::graph::build_hc_mixes( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base, + ggml_tensor ** post, + ggml_tensor ** comb, + int il) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t hc_dim = hc*n_embd; + const int64_t hc_mix_dim = (2 + hc)*hc; + const int64_t nt = x->ne[2]; + + GGML_ASSERT(hc == 4); + GGML_ASSERT(hc_fn->ne[1] == hc_mix_dim); + + // normalized over the whole flattened hc*d stream, one statistic per token + ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc_dim, nt); + ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, norm_rms_eps); + ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm); + cb(mixes, "hc_mixes", il); + + ggml_tensor * pre = dsv41_view_2d(ctx0, mixes, hc, nt, 0); + pre = ggml_mul(ctx0, pre, dsv41_view_1d(ctx0, hc_scale, 1, 0)); + pre = ggml_add(ctx0, pre, dsv41_view_1d(ctx0, hc_base, hc, 0)); + pre = ggml_sigmoid(ctx0, pre); + pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); + cb(pre, "hc_pre", il); + + *post = dsv41_view_2d(ctx0, mixes, hc, nt, hc); + *post = ggml_mul(ctx0, *post, dsv41_view_1d(ctx0, hc_scale, 1, 1)); + *post = ggml_add(ctx0, *post, dsv41_view_1d(ctx0, hc_base, hc, hc)); + *post = ggml_sigmoid(ctx0, *post); + *post = ggml_scale(ctx0, *post, 2.0f); + cb(*post, "hc_post", il); + + if (cparams.fused_dsv4_hc_comb) { + *comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps, + (int32_t) hparams.dsv4_hc_sinkhorn_iters); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, *comb, il}); + } else { + *comb = dsv41_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); + *comb = ggml_mul(ctx0, *comb, dsv41_view_1d(ctx0, hc_scale, 1, 2)); + *comb = ggml_add(ctx0, *comb, dsv41_view_1d(ctx0, hc_base, hc*hc, 2*hc)); + *comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt); + *comb = build_hc_sinkhorn(*comb, il); + } + cb(*comb, "hc_comb", il); + + return pre; +} + +// Engram.forward: x [n_embd, hc, n_tokens], ids I32 [n_cols*n_tokens] +ggml_tensor * llama_model_deepseek41::graph::build_engram( + const llama_model_deepseek41 & model, + ggml_tensor * x, + ggml_tensor * ids, + int il) const { + const auto & layer = model.layers[il]; + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[2]; + const int64_t n_cols = model.engram_n_cols(); + const int64_t hd = model.engram_head_dim; + + ggml_tensor * rows = ggml_get_rows(ctx0, layer.engram_embd, ids); // [hd, n_cols*nt] + if (rows->type != GGML_TYPE_F32) { + rows = ggml_cast(ctx0, rows, GGML_TYPE_F32); + } + rows = ggml_reshape_2d(ctx0, rows, hd*n_cols, nt); + cb(rows, "engram_rows", il); + + ggml_tensor * kv = build_lora_mm(layer.engram_kv, rows); // [n_embd*(hc+1), nt] + cb(kv, "engram_kv", il); + + ggml_tensor * key = ggml_view_3d(ctx0, kv, n_embd, hc, nt, ggml_row_size(kv->type, n_embd), kv->nb[1], 0); + ggml_tensor * value = ggml_view_3d(ctx0, kv, n_embd, 1, nt, ggml_row_size(kv->type, n_embd), kv->nb[1], ggml_row_size(kv->type, hc*n_embd)); + key = ggml_cont(ctx0, key); + value = ggml_cont(ctx0, value); + + // dot_c = sum_d (x * rstd_x) * (key * rstd_k) * w / sqrt(n_embd), normalized per (token, copy) + ggml_tensor * xn = ggml_rms_norm(ctx0, x, norm_rms_eps); + ggml_tensor * kn = ggml_rms_norm(ctx0, key, norm_rms_eps); + ggml_tensor * dot = ggml_mul(ctx0, ggml_mul(ctx0, xn, kn), layer.engram_gate); + dot = ggml_sum_rows(ctx0, dot); // [1, hc, nt] + dot = ggml_scale(ctx0, dot, 1.0f/sqrtf((float) n_embd)); + cb(dot, "engram_dot", il); + + // gate = sigmoid(sign(dot) * sqrt(max(|dot|, 1e-6))) + ggml_tensor * mag = ggml_sqrt(ctx0, ggml_clamp(ctx0, ggml_abs(ctx0, dot), 1e-6f, INFINITY)); + ggml_tensor * gate = ggml_sigmoid(ctx0, ggml_mul(ctx0, mag, ggml_sgn(ctx0, dot))); + cb(gate, "engram_gate", il); + + ggml_tensor * upd = ggml_mul(ctx0, ggml_repeat_4d(ctx0, value, n_embd, hc, nt, 1), gate); + return ggml_add(ctx0, x, upd); +} + +static void dsv41_state_restore( + ggml_context * ctx, + const llm_graph_input_dsv4::comp_input & inp, + const llama_dsv4_comp_state * state, + int32_t il, + ggml_tensor ** kv, + ggml_tensor ** score) { + *kv = state->get_kv_all(ctx, il); + *score = state->get_score_all(ctx, il); + + if (inp.state_restore_src_idxs == nullptr || inp.state_restore_dst_idxs == nullptr) { + return; + } + + *kv = state->cpy_kv (ctx, ggml_get_rows(ctx, *kv, inp.state_restore_src_idxs), inp.state_restore_dst_idxs, il); + *score = state->cpy_score(ctx, ggml_get_rows(ctx, *score, inp.state_restore_src_idxs), inp.state_restore_dst_idxs, il); +} + +ggml_tensor * llama_model_deepseek41::graph::build_attention41( + const llama_model_deepseek41 & model, + llm_graph_input_dsv4 * inp_dsv4, + ggml_tensor * cur, + ggml_tensor * inp_pos, + ggml_tensor * cand_pin, + int il) { + const auto & layer = model.layers[il]; + llm_graph_input_dsv4_raw * inp_attn = inp_dsv4->get_raw(); + + const int64_t n_embd_head = hparams.n_embd_head_k(); + const int64_t n_embd_head_rope = hparams.n_rot(); + const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope; + const int64_t n_groups = hparams.dsv4_o_group_count; + const int64_t n_heads_group = n_head / n_groups; + const int64_t o_lora_rank = hparams.dsv4_o_lora_rank; + const int64_t o_group_dim = n_heads_group*n_embd_head; + const int64_t n_idx_embd = hparams.indexer_head_size; + const int64_t n_idx_head = hparams.indexer_n_head; + const int64_t nt = cur->ne[1]; + const float kq_scale = 1.0f/sqrtf(float(n_embd_head)); + + GGML_ASSERT(n_embd_head == n_embd_head_v); + GGML_ASSERT(n_head % n_groups == 0); + + const int64_t ratio = hparams.dsv4_compress_ratios[il]; + + // layers with compressed KV use the compress theta + YaRN, window-only layers plain rope + const bool use_compress_rope = ratio != 0; + const float freq_base_l = use_compress_rope ? hparams.dsv4_compress_rope_base : freq_base; + const float freq_scale_l = use_compress_rope ? freq_scale : 1.0f; + const float ext_factor_l = use_compress_rope ? ext_factor : 0.0f; + const float attn_factor_l = dsv41_rope_attn_factor(freq_scale_l, ext_factor_l); + const float beta_fast_l = use_compress_rope ? beta_fast : 0.0f; + const float beta_slow_l = use_compress_rope ? beta_slow : 0.0f; + const int32_t n_ctx_orig_l = use_compress_rope ? n_ctx_orig : 0; + + const auto rope = [&](ggml_tensor * t, ggml_tensor * pos, int64_t n_nope) { + t = ggml_rope_ext(ctx0, t, pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l, + freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); + return ggml_rope_set_offset(t, n_nope); + }; + + ggml_tensor * qr = build_lora_mm(layer.wq_a, cur); + qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(qr, "qr_norm", il); + + ggml_tensor * q = build_lora_mm(layer.wq_b, qr); + q = ggml_reshape_3d(ctx0, q, n_embd_head, n_head, nt); + q = rope(q, inp_pos, n_embd_head_nope); + cb(q, "q", il); + + ggml_tensor * kv = build_lora_mm(layer.wkv, cur); + kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il); + kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, nt); + kv = rope(kv, inp_pos, n_embd_head_nope); + cb(kv, "kv", il); + + ggml_tensor * out = nullptr; + + if (ratio == 0) { + out = build_raw_attention(inp_attn, q, kv, layer.attn_sinks, kq_scale, il); + } else { + GGML_ASSERT(ratio == 1 || ratio == 2); + const bool use_csa = ratio == 2; + const auto & ci = use_csa ? inp_dsv4->get_csa() : inp_dsv4->get_hca(); + const auto * comp_ctx = use_csa ? inp_dsv4->mctx->get_csa() : inp_dsv4->mctx->get_hca(); + const auto * comp_state = use_csa ? inp_dsv4->mctx->get_csa_state() : inp_dsv4->mctx->get_hca_state(); + const int32_t src = hparams.dsv41_kv_src[il]; + GGML_ASSERT(src >= 0 && (int64_t) comp_state->get_ratio() == ratio); + + // ---- compressor (kv sources only): pool ratio tokens, norm, index K, rope, write [kv | index K] + if (src == il && ci.state_pos) { + ggml_tensor * state_kv = build_lora_mm(layer.attn_comp_wkv, cur); + cb(state_kv, "comp_state_kv", il); + ggml_tensor * state_score = ratio > 1 ? build_lora_mm(layer.attn_comp_wgate, cur) : ggml_scale(ctx0, state_kv, 0.0f); + cb(state_score, "comp_state_score", il); + + ggml_tensor * restored_kv = nullptr; + ggml_tensor * restored_score = nullptr; + dsv41_state_restore(ctx0, ci, comp_state, il, &restored_kv, &restored_score); + + if (ci.state_write_idxs) { + const int64_t n_blocks = ci.state_write_pos->ne[0]; + GGML_ASSERT(ci.state_read_idxs->ne[0] == ratio*n_blocks); + + ggml_tensor * src_kv = ggml_concat(ctx0, + dsv41_view_2d(ctx0, restored_kv, restored_kv->ne[0], comp_state->get_n_rows(), 0), state_kv, 1); + ggml_tensor * src_score = ggml_concat(ctx0, + dsv41_view_2d(ctx0, restored_score, restored_score->ne[0], comp_state->get_n_rows(), 0), state_score, 1); + + ggml_tensor * v = ggml_get_rows(ctx0, src_kv, ci.state_read_idxs); + ggml_tensor * s = ggml_get_rows(ctx0, src_score, ci.state_read_idxs); + v = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, v, n_embd_head, ratio, n_blocks), 1, 0, 2, 3)); + s = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, s, n_embd_head, ratio, n_blocks), 1, 0, 2, 3)); + + // per-channel softmax over the ratio slots of a group + ggml_tensor * comp = ggml_sum_rows(ctx0, ggml_mul(ctx0, v, ggml_soft_max(ctx0, s))); // [1, hd, nb] + comp = ggml_cont(ctx0, ggml_permute(ctx0, comp, 1, 0, 2, 3)); // [hd, 1, nb] + comp = build_norm(comp, layer.attn_comp_norm, nullptr, LLM_NORM_RMS, il); + cb(comp, "comp_latent", il); + + // index K from the RoPE-free latent; a latent stands for the first token of its group + ggml_tensor * ik = build_lora_mm(layer.indexer_attn_k, comp); // [n_idx_embd, 1, nb] + ik = build_norm(ik, layer.indexer_k_norm, nullptr, LLM_NORM_RMS, il); + ik = rope(ik, ci.state_write_pos, n_idx_embd - n_embd_head_rope); + cb(ik, "comp_index_k", il); + + comp = rope(comp, ci.state_write_pos, n_embd_head_nope); + cb(comp, "comp_kv", il); + + ggml_tensor * row = ggml_concat(ctx0, comp, ik, 0); // [hd + n_idx_embd, 1, nb] + ggml_build_forward_expand(gf, comp_ctx->cpy_k(ctx0, row, ci.state_write_idxs, il)); + } + + // rollback snapshots and the persistent partial-group state (as V4 HCA) + if (ci.state_snapshot_src_idxs && ci.state_snapshot_dst_idxs) { + ggml_tensor * snap_kv = ggml_concat(ctx0, restored_kv, state_kv, 1); + ggml_tensor * snap_score = ggml_concat(ctx0, restored_score, state_score, 1); + ggml_build_forward_expand(gf, comp_state->cpy_kv(ctx0, + ggml_get_rows(ctx0, snap_kv, ci.state_snapshot_src_idxs), ci.state_snapshot_dst_idxs, il)); + ggml_build_forward_expand(gf, comp_state->cpy_score(ctx0, + ggml_get_rows(ctx0, snap_score, ci.state_snapshot_src_idxs), ci.state_snapshot_dst_idxs, il)); + } + + ggml_build_forward_expand(gf, comp_state->cpy_kv(ctx0, + ggml_get_rows(ctx0, state_kv, ci.state_persist_src_idxs), ci.state_persist_dst_idxs, il)); + ggml_build_forward_expand(gf, comp_state->cpy_score(ctx0, + ggml_get_rows(ctx0, state_score, ci.state_persist_src_idxs), ci.state_persist_dst_idxs, il)); + } + + if (!ci.kq_mask) { + // no compressed row visible to any token of this ubatch: window only + out = build_raw_attention(inp_attn, q, kv, layer.attn_sinks, kq_scale, il); + } else { + const int64_t n_comp = ci.kq_mask->ne[0]; + + ggml_tensor * comp_all = comp_ctx->get_k(ctx0, src); // [hd + n_idx_embd, 1, n_cells, n_stream] + GGML_ASSERT(comp_all->ne[0] == n_embd_head + n_idx_embd); + GGML_ASSERT(n_comp <= comp_all->ne[2]); + const int64_t n_stream = comp_all->ne[3]; + + ggml_tensor * comp_mask = ci.kq_mask; + if (comp_mask->type != GGML_TYPE_F32) { + comp_mask = ggml_cast(ctx0, comp_mask, GGML_TYPE_F32); + } + + // ---- top-k: own indexer on index sources, reused otherwise + const int32_t topk_src = hparams.dsv41_topk_src[il]; + ggml_tensor * top_k = nullptr; + if (topk_src == il) { + ggml_tensor * ik = ggml_view_4d(ctx0, comp_all, n_idx_embd, 1, n_comp, n_stream, + comp_all->nb[1], comp_all->nb[2], comp_all->nb[3], ggml_row_size(comp_all->type, n_embd_head)); + ik = ggml_cont(ctx0, ik); + cb(ik, "idx_k", il); + + ggml_tensor * iq = build_lora_mm(layer.indexer_attn_q_b, qr); + iq = ggml_reshape_3d(ctx0, iq, n_idx_embd, n_idx_head, nt); + iq = rope(iq, inp_pos, n_idx_embd - n_embd_head_rope); + cb(iq, "idx_q", il); + + ggml_tensor * iw = build_lora_mm(layer.indexer_proj, cur); + iw = ggml_scale(ctx0, iw, 1.0f/sqrtf(float(n_idx_embd*n_idx_head))); + cb(iw, "idx_w", il); + + iq = ggml_view_4d(ctx0, iq, iq->ne[0], iq->ne[1], nt/n_stream, n_stream, + iq->nb[1], iq->nb[2], iq->nb[2]*(nt/n_stream), 0); + iw = ggml_view_4d(ctx0, iw, iw->ne[0], nt/n_stream, 1, n_stream, + iw->nb[1], iw->nb[1]*(nt/n_stream), iw->nb[1]*(nt/n_stream), 0); + + // score[j, t] = sum_h relu(q_h . k_j) * w_h + ggml_tensor * kq = ggml_mul_mat(ctx0, + ggml_permute(ctx0, ik, 0, 2, 1, 3), // [d, n_comp, 1, ns] + ggml_permute(ctx0, iq, 0, 2, 1, 3)); // [d, nt/ns, n_ih, ns] -> [n_comp, nt/ns, n_ih, ns] + kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3)); // [n_ih, nt/ns, n_comp, ns] + ggml_tensor * score = ggml_sum_rows(ctx0, ggml_mul(ctx0, ggml_relu(ctx0, kq), iw)); // [1, nt/ns, n_comp, ns] + score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); // [n_comp, nt/ns, 1, ns] + score = ggml_add(ctx0, score, comp_mask); + cb(score, "idx_score", il); + + const int32_t cand_src = hparams.dsv41_cand_src; + if (cand_src == il && cand_pin) { + // level one: best position per block, pin the newest block, keep the top blocks + const int64_t bs = hparams.dsv41_cand_block_size; + const int64_t n_blk = (n_comp + bs - 1)/bs; + GGML_ASSERT(cand_pin->ne[0] == n_blk); + + ggml_tensor * s2 = ggml_reshape_2d(ctx0, score, n_comp, nt); + if (n_blk*bs != n_comp) { + ggml_tensor * pad = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_blk*bs - n_comp, nt); + s2 = ggml_concat(ctx0, s2, ggml_fill(ctx0, pad, -INFINITY), 0); + } + ggml_tensor * bmax = ggml_pool_1d(ctx0, s2, GGML_OP_POOL_MAX, bs, bs, 0); // [n_blk, nt] + bmax = ggml_add(ctx0, bmax, ggml_reshape_2d(ctx0, cand_pin, n_blk, nt)); + cb(bmax, "cand_block_score", il); + + const int64_t k_b = std::min(hparams.dsv41_cand_topk_blocks, n_blk); + ggml_tensor * bsel = ggml_top_k(ctx0, bmax, k_b); // [k_b, nt] + + ggml_tensor * bm = ggml_fill(ctx0, ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, n_blk, nt, 1), -INFINITY); + ggml_tensor * zeros = ggml_fill(ctx0, ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, k_b, nt, 1), 0.0f); + bm = ggml_set_rows(ctx0, bm, zeros, ggml_reshape_3d(ctx0, bsel, k_b, nt, 1)); // [1, n_blk, nt, 1] + bm = ggml_repeat_4d(ctx0, bm, bs, n_blk, nt, 1); + bm = ggml_reshape_2d(ctx0, bm, bs*n_blk, nt); + bm = ggml_cont(ctx0, ggml_view_2d(ctx0, bm, n_comp, nt, bm->nb[1], 0)); + cand_mask = ggml_reshape_4d(ctx0, bm, n_comp, nt/n_stream, 1, n_stream); + cb(cand_mask, "cand_mask", il); + } else if (cand_src >= 0 && cand_src < il && cand_mask) { + // level two: own scores, restricted to the source's candidate blocks + GGML_ASSERT(cand_mask->ne[0] == n_comp); + score = ggml_add(ctx0, score, cand_mask); + cb(score, "idx_score_cand", il); + } + + const int64_t k = std::min(n_comp, hparams.indexer_top_k); + top_k = ggml_cont(ctx0, ggml_top_k(ctx0, score, k)); + cb(top_k, "idx_top_k", il); + topk_of[il] = top_k; + } else { + GGML_ASSERT(topk_src >= 0); + top_k = topk_of[topk_src]; + GGML_ASSERT(top_k && "DeepSeek-V4.1: top-k source layer produced no top-k"); + } + + // ---- sparse attention over [window raw KV | selected compressed rows] + ggml_tensor * k_rot = inp_attn->self_k_rot; + if (k_rot) { + q = llama_mul_mat_hadamard(ctx0, q, k_rot); + kv = llama_mul_mat_hadamard(ctx0, kv, k_rot); + } + + ggml_build_forward_expand(gf, q); + ggml_build_forward_expand(gf, kv); + + const llama_kv_cache_dsv4_raw_context * mctx_raw = inp_attn->mctx; + ggml_build_forward_expand(gf, mctx_raw->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il)); + + ggml_tensor * raw_k = mctx_raw->get_k(ctx0, il); + + ggml_tensor * comp_k = ggml_view_4d(ctx0, comp_all, n_embd_head, 1, n_comp, n_stream, + comp_all->nb[1], comp_all->nb[2], comp_all->nb[3], 0); + if (comp_k->type != raw_k->type) { + comp_k = ggml_cast(ctx0, comp_k, raw_k->type); + } else { + comp_k = ggml_cont(ctx0, comp_k); + } + if (k_rot) { + comp_k = llama_mul_mat_hadamard(ctx0, comp_k, k_rot); + } + cb(comp_k, "comp_k", il); + + ggml_tensor * k_all = ggml_concat(ctx0, raw_k, comp_k, 2); + + ggml_tensor * raw_mask = inp_attn->get_kq_mask(); + ggml_tensor * sel_mask = build_top_k_mask(ci.kq_mask, top_k, "comp_top_k_mask", il); + ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, sel_mask, 0); + cb(kq_mask, "comp_kq_mask", il); + + out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, layer.attn_sinks, nullptr, 0, kq_scale, il); + if (k_rot) { + out = llama_mul_mat_hadamard(ctx0, out, k_rot); + } + cb(out, "attn_comp", il); + } + } + + // undo the query rotation on the rope dims, then the grouped low-rank output projection + out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, nt); + out = ggml_rope_ext_back(ctx0, out, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l, + freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); + out = ggml_rope_set_offset(out, n_embd_head_nope); + cb(out, "attn_derope", il); + + out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, nt); + out = ggml_permute(ctx0, out, 0, 2, 1, 3); + ggml_tensor * oa = ggml_mul_mat(ctx0, layer.wo_a, out); + cb(oa, "attn_wo_a", il); + oa = ggml_permute(ctx0, oa, 0, 2, 1, 3); + oa = ggml_cont_2d(ctx0, oa, o_lora_rank*n_groups, nt); + + out = build_lora_mm(layer.wo_b, oa); + cb(out, "attn_out", il); + + return out; +} + +llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_graph_params & params) : + llama_model_deepseek4::graph(params) { + const auto & model = static_cast(model_base); + + topk_of.assign(n_layer, nullptr); + + ggml_tensor * cur; + + ggml_tensor * inp = build_inp_embd(model.tok_embd); + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + llm_graph_input_dsv4 * inp_dsv4 = build_inp_dsv4(); + ggml_build_forward_expand(gf, inp_dsv4->get_raw()->self_kq_mask); + + // host-side inputs + auto inp41 = std::make_unique(model, inp_dsv4->mctx); + std::vector engram_ids(model.engram_layers.size(), nullptr); + for (size_t e = 0; e < engram_ids.size(); ++e) { + engram_ids[e] = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, (int64_t) model.engram_n_cols()*n_tokens); + ggml_set_input(engram_ids[e]); + ggml_set_name(engram_ids[e], ("engram_ids-" + std::to_string(model.engram_layers[e])).c_str()); + } + inp41->engram_ids = engram_ids; + ggml_tensor * cand_pin = nullptr; + if (hparams.dsv41_cand_src >= 0) { + const int32_t cs = hparams.dsv41_cand_src; + inp41->cand_csa = hparams.dsv4_compress_ratios[cs] == 2; + const auto & ci = inp41->cand_csa ? inp_dsv4->get_csa() : inp_dsv4->get_hca(); + if (ci.kq_mask) { + const int64_t bs = hparams.dsv41_cand_block_size; + cand_pin = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, (ci.kq_mask->ne[0] + bs - 1)/bs, + ci.kq_mask->ne[1], 1, ci.kq_mask->ne[3]); + ggml_set_input(cand_pin); + ggml_set_name(cand_pin, "cand_pin"); + inp41->cand_pin = cand_pin; + } + } + res->add_input(std::move(inp41)); + + const int64_t hc = hparams.dsv4_hc_mult; + ggml_tensor * inpL = ggml_reshape_3d(ctx0, inp, n_embd, 1, n_tokens); + inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1); + cb(inpL, "hc_init", -1); + + // single-pass mHC: the pre-mix used to collapse the stream lags one sublayer + ggml_tensor * pre_prev = nullptr; + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + const int e = model.engram_index(il); + if (e >= 0) { + inpL = build_engram(model, inpL, engram_ids[e], il); + cb(inpL, "engram_out", il); + } + + if ((size_t) il < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[il]) { + res->t_layer_inp[il] = build_hc_mean(inpL); + cb(res->t_layer_inp[il], "layer_inp", il); + ggml_build_forward_expand(gf, res->t_layer_inp[il]); + } + + ggml_tensor * residual = inpL; + ggml_tensor * post = nullptr; + ggml_tensor * comb = nullptr; + + ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il); + + if (pre_prev == nullptr) { + // identity pre-mix of the first layer: copy 0 + cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0)); + } else { + cur = build_hc_pre(inpL, pre_prev, il); + } + cb(cur, "hc_attn_pre", il); + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + cur = build_attention41(model, inp_dsv4, cur, inp_pos, cand_pin, il); + + inpL = build_hc_post(cur, residual, post, comb, il); + cb(inpL, "hc_attn_post", il); + + residual = inpL; + ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il); + + cur = build_hc_pre(inpL, attn_pre, il); + cb(cur, "hc_ffn_pre", il); + + ggml_build_forward_expand(gf, residual); + ggml_build_forward_expand(gf, post); + ggml_build_forward_expand(gf, comb); + + cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + ggml_tensor * exp_probs_b = layer.ffn_exp_probs_b; + ASSERT_EMBD_OR_TOKEN(ubatch); + if (ubatch.embd != nullptr && layer.ffn_exp_probs_b_vl) { + exp_probs_b = layer.ffn_exp_probs_b_vl; + } + + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + exp_probs_b, + n_expert, hparams.n_expert_used(), + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il); + cb(moe_out, "ffn_moe_out", il); + + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, nullptr, nullptr, + layer.ffn_gate_shexp, nullptr, nullptr, + layer.ffn_down_shexp, nullptr, nullptr, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + + inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_cvec(inpL, il); + cb(inpL, "l_last", il); + + pre_prev = ffn_pre; + } + + if ((size_t) n_layer < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[n_layer]) { + res->t_layer_inp[n_layer] = build_hc_mean(inpL); + cb(res->t_layer_inp[n_layer], "layer_inp", n_layer); + ggml_build_forward_expand(gf, res->t_layer_inp[n_layer]); + } + + ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens); + if (inp_out_ids) { + flat = ggml_get_rows(ctx0, flat, inp_out_ids); + pre_prev = ggml_get_rows(ctx0, pre_prev, inp_out_ids); + } + res->t_h_nextn = flat; + + // no hc_head: collapse with the last FFN's pre-mix + inpL = ggml_reshape_3d(ctx0, flat, n_embd, hc, flat->ne[1]); + cur = build_hc_pre(inpL, pre_prev, -1); + cb(cur, "hc_out", -1); + + cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = ggml_mul_mat(ctx0, model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; +} diff --git a/src/models/models.h b/src/models/models.h index 88fd07d973cd..d2d37ec8d448 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1406,6 +1406,73 @@ struct llama_model_deepseek4 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; +// DeepSeek-V4.1: V4 graph pieces (HC, MLA window attention, sparse compressed attention, MoE) plus Engram, +// single-pass mHC, ratio-1/2 compressors with cross-layer KV / index-K / top-k sharing and two-level candidate selection. +struct llama_model_deepseek41 : public llama_model_base { + llama_model_deepseek41(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + // Engram n-gram hash constants (host side; ggml has no int64 mul/xor/mod) + std::vector engram_layers; // backbone layer ids + std::vector engram_n_rows; // table rows per engram layer + uint32_t engram_max_ngram = 0; + uint32_t engram_n_head = 0; + uint32_t engram_head_dim = 0; + int64_t engram_pad_cid = 0; // compressed id of the pad token + std::vector engram_token_map; // raw id -> compressed id + std::vector engram_mult; // [n_engram][max_ngram] + std::vector engram_primes; // [n_engram][(max_ngram-1)*n_head] + std::vector engram_offsets; // [n_engram][(max_ngram-1)*n_head] + + int engram_index(int il) const { + for (size_t i = 0; i < engram_layers.size(); ++i) { + if (engram_layers[i] == il) { + return (int) i; + } + } + return -1; + } + uint32_t engram_n_cols() const { return (engram_max_ngram - 1)*engram_n_head; } + + // hash ids of every token of a ubatch for engram layer e: out[i*n_cols + c] + void engram_hash(int e, const llama_ubatch & ubatch, const std::vector & prev, std::vector & out) const; + + struct graph : public llama_model_deepseek4::graph { + graph(const llama_model & model, const llm_graph_params & params); + + // HC coefficients of one sublayer: returns pre [hc, n_tokens], fills post / comb + ggml_tensor * build_hc_mixes( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base, + ggml_tensor ** post, + ggml_tensor ** comb, + int il) const; + + ggml_tensor * build_engram( + const llama_model_deepseek41 & model, + ggml_tensor * x, + ggml_tensor * ids, + int il) const; + + ggml_tensor * build_attention41( + const llama_model_deepseek41 & model, + llm_graph_input_dsv4 * inp_dsv4, + ggml_tensor * cur, + ggml_tensor * inp_pos, + ggml_tensor * cand_pin, + int il); + + // per-graph sharing state (layers run in order, sources before consumers) + std::vector topk_of; // [n_layer] top-k of each index source + ggml_tensor * cand_mask = nullptr; // candidate-block mask from the candidate source layer + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + struct llama_model_deepseek2ocr : public llama_model_base { llama_model_deepseek2ocr(const struct llama_model_params & params) : llama_model_base(params) {} From f79bf193b55182bb4084a6bbcf3389ae1b1f6b73 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 03:28:22 -0700 Subject: [PATCH 06/49] convert : deepseek41 read weight and scale from their own shards --- conversion/deepseek41.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py index 61130b92acdc..027daf121711 100644 --- a/conversion/deepseek41.py +++ b/conversion/deepseek41.py @@ -393,17 +393,17 @@ def src_name(n: str) -> str: # undo the mtp.M -> layers.(n_main+M) rekey of fil return f"mtp.{int(m.group(1)) - n_main}.{m.group(2)}" return n names = [(src_name(wn), src_name(sn)) for wn, sn in names] - parts = {wn: self._tensor_part(wn) for wn, _ in names} + parts = {n: self._tensor_part(n) for pair in names for n in pair} # weight and scale may sit in different shards local = threading.local() def one(ws: tuple[str, str]) -> np.ndarray: wn, sn = ws hs = local.__dict__.setdefault("h", {}) - part = parts[wn] - if part not in hs: - hs[part] = safe_open(part, framework="pt") - w = hs[part].get_tensor(wn).view(torch.uint8).numpy() - sc = hs[part].get_tensor(sn).view(torch.uint8).numpy() + for part in (parts[wn], parts[sn]): + if part not in hs: + hs[part] = safe_open(part, framework="pt") + w = hs[parts[wn]].get_tensor(wn).view(torch.uint8).numpy() + sc = hs[parts[sn]].get_tensor(sn).view(torch.uint8).numpy() return self._repack_mxfp4_np(w, sc) def chunks() -> Iterable[np.ndarray]: @@ -421,13 +421,13 @@ def _engram_embed_tensor(self, bid: int) -> list[str]: if rows != self.hparams["engram_num_embeddings"][li]: raise ValueError(f"{wn}: {rows} rows != engram_num_embeddings {self.hparams['engram_num_embeddings'][li]}") new_name = self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_EMBD, bid) - part = self._tensor_part(wn) + wpart, spart = self._tensor_part(wn), self._tensor_part(sn) chunk_rows = 1 << 20 def raw_chunks() -> Iterable[tuple[np.ndarray, np.ndarray]]: from safetensors import safe_open - with safe_open(part, framework="pt") as f: - ws, ss = f.get_slice(wn), f.get_slice(sn) + with safe_open(wpart, framework="pt") as fw, safe_open(spart, framework="pt") as fs: + ws, ss = fw.get_slice(wn), fs.get_slice(sn) for r0 in range(0, rows, chunk_rows): r1 = min(rows, r0 + chunk_rows) q = ws[r0:r1].view(torch.uint8).numpy().reshape(-1, 32) From d29889d8f5e7ac1cff17141241aa69361dcbefb3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 03:38:51 -0700 Subject: [PATCH 07/49] deepseek41: test-llama-archs config, saver keys, meta split, engram table placement, GGML_CUDA_F32_EXACT debug knob --- ggml/src/ggml-cuda/mmf.cu | 7 ++++++ src/llama-model-saver.cpp | 44 +++++++++++++++++++++++++++++++++ src/llama-model.cpp | 2 +- tests/test-llama-archs.cpp | 50 +++++++++++++++++++++++++++++++++++--- 4 files changed, 98 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-cuda/mmf.cu b/ggml/src/ggml-cuda/mmf.cu index 646a5899c803..e67b46d47ae5 100644 --- a/ggml/src/ggml-cuda/mmf.cu +++ b/ggml/src/ggml-cuda/mmf.cu @@ -136,6 +136,13 @@ bool ggml_cuda_should_use_mmf(enum ggml_type type, int cc, int warp_size, const return false; } + // debug: GGML_CUDA_F32_EXACT=1 keeps F32 weights off the TF32 tensor-core path (together with + // NVIDIA_TF32_OVERRIDE=0 for cuBLAS), for exact-F32 parity tests against a CPU / PyTorch reference + static const bool f32_exact = getenv("GGML_CUDA_F32_EXACT") != nullptr; + if (type == GGML_TYPE_F32 && f32_exact) { + return false; + } + const size_t ts = ggml_type_size(type); if (src0_ne[0] % (warp_size * (4/ts)) != 0) { return false; diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index e011e24ea605..48d361e2fbfa 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -8,6 +8,7 @@ #include "llama-hparams.h" #include "llama-model.h" #include "llama-vocab.h" +#include "models/models.h" #include #include @@ -337,6 +338,49 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_HASH_LAYER_COUNT, hparams.dsv4_hash_layer_count); add_kv(LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank); + if (model->arch == LLM_ARCH_DEEPSEEK41) { + const auto * m41 = dynamic_cast(model); + GGML_ASSERT(m41 != nullptr); + std::vector kv_src; + std::vector idx_src; + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + if (hparams.dsv41_kv_src[il] == (int32_t) il) { + kv_src.push_back(il); + } + if (hparams.dsv41_topk_src[il] == (int32_t) il) { + idx_src.push_back(il); + } + } + add_kv(LLM_KV_ATTENTION_KV_SOURCE_LAYERS, kv_src); + add_kv(LLM_KV_ATTENTION_INDEXER_SOURCE_LAYERS, idx_src); + if (hparams.dsv41_cand_src >= 0) { + add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_SOURCE_LAYER, hparams.dsv41_cand_src); + add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, hparams.dsv41_cand_topk_blocks); + add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, hparams.dsv41_cand_block_size); + } + if (!m41->engram_layers.empty()) { + const size_t n_cols = m41->engram_n_cols(); + int32_t pad_id = 0; // any raw id whose compressed id is the pad's + for (size_t i = 0; i < m41->engram_token_map.size(); ++i) { + if (m41->engram_token_map[i] == m41->engram_pad_cid) { + pad_id = (int32_t) i; + break; + } + } + std::vector primes(m41->engram_primes.begin(), m41->engram_primes.end()); + GGML_ASSERT(primes.size() == n_cols*m41->engram_layers.size()); + add_kv(LLM_KV_ENGRAM_LAYERS, std::vector(m41->engram_layers.begin(), m41->engram_layers.end())); + add_kv(LLM_KV_ENGRAM_NUM_EMBEDDINGS, m41->engram_n_rows); + add_kv(LLM_KV_ENGRAM_MAX_NGRAM_SIZE, m41->engram_max_ngram); + add_kv(LLM_KV_ENGRAM_HEAD_COUNT, m41->engram_n_head); + add_kv(LLM_KV_ENGRAM_HEAD_DIM, m41->engram_head_dim); + add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, (uint32_t) pad_id); + add_kv(LLM_KV_ENGRAM_TOKEN_MAP, m41->engram_token_map); + add_kv(LLM_KV_ENGRAM_MULTIPLIERS, m41->engram_mult); + add_kv(LLM_KV_ENGRAM_PRIMES, primes); + } + } + // the PLE group only means anything whole: write all of it or none if (hparams.ple_n_heads > 0) { std::vector ple_layers; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 2d876b198ffa..84150117b6e4 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -390,7 +390,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str const llama_meta_device_get_split_state_userdata * ud = (const llama_meta_device_get_split_state_userdata *) userdata; const llama_hparams & hparams = ud->model->hparams; const std::string tensor_name = tensor->name; - const bool is_dsv4 = ud->model->arch == LLM_ARCH_DEEPSEEK4 || + const bool is_dsv4 = ud->model->arch == LLM_ARCH_DEEPSEEK4 || ud->model->arch == LLM_ARCH_DEEPSEEK41 || (ud->model->arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0); static const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight"); diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 3dd8f0655ee9..d305431602dc 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -154,6 +154,12 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_head = 8; n_ff = 1024; n_layer = 4; + } else if (arch == LLM_ARCH_DEEPSEEK41) { + // window-only, Engram, ratio-2 and ratio-1 sources, top-k reuse and a candidate-masked index layer + n_embd = 512; + n_head = 8; + n_ff = 1024; + n_layer = 6; } else if (arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_LAGUNA) { n_embd = 160; // exercise per-head tensor split granularity with head size 80 } else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) { @@ -238,11 +244,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer); } else { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head); - ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(1) : n_head_kv); + ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(1) : n_head_kv); } ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f); - if (arch == LLM_ARCH_DEEPSEEK4) { + if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) { ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, n_embd_head); ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, n_embd_head); ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, n_embd_head/2); @@ -287,7 +293,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f); ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, 1e-5f); ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, uint32_t(8)); - ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(64) : uint32_t(512)); + ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(64) : uint32_t(512)); ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, uint32_t(512)); ms.add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, uint32_t(8)); ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, n_ctx/8); @@ -403,6 +409,41 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true); } + if (arch == LLM_ARCH_DEEPSEEK41) { + ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8)); + ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32)); + ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector({0, 0, 2, 2, 1, 1})); + ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f); + ms.add_kv(LLM_KV_ATTENTION_KV_SOURCE_LAYERS, std::vector({2, 4})); + ms.add_kv(LLM_KV_ATTENTION_INDEXER_SOURCE_LAYERS, std::vector({2, 4, 5})); + ms.add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_SOURCE_LAYER, int32_t(4)); + ms.add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, uint32_t(4096)); + ms.add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, uint32_t(4)); + ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4)); + ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2)); + ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f); + ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f); + ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f); + ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true); + // Engram on layers 1 and 3: 2..3-grams, 2 heads, 64-wide rows + std::vector token_map(n_vocab); + for (uint32_t i = 0; i < n_vocab; ++i) { + token_map[i] = (int32_t) (i % 61); + } + ms.add_kv(LLM_KV_ENGRAM_LAYERS, std::vector({1, 3})); + ms.add_kv(LLM_KV_ENGRAM_NUM_EMBEDDINGS, std::vector({420, 508})); + ms.add_kv(LLM_KV_ENGRAM_MAX_NGRAM_SIZE, uint32_t(3)); + ms.add_kv(LLM_KV_ENGRAM_VOCAB_SIZE, uint32_t(100)); + ms.add_kv(LLM_KV_ENGRAM_HEAD_COUNT, uint32_t(2)); + ms.add_kv(LLM_KV_ENGRAM_HEAD_DIM, uint32_t(64)); + ms.add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, uint32_t(2)); + ms.add_kv(LLM_KV_ENGRAM_COMPRESSED_VOCAB_SIZE, uint32_t(61)); + ms.add_kv(LLM_KV_ENGRAM_TOKEN_MAP, token_map); + ms.add_kv(LLM_KV_ENGRAM_MULTIPLIERS, std::vector({ 76632096046245ull, 4839876093313ull, 35959672319349ull, + 67716810739261ull, 51510806800915ull, 30921347202721ull })); + ms.add_kv(LLM_KV_ENGRAM_PRIMES, std::vector({101, 103, 107, 109, 113, 127, 131, 137})); + } + if (arch == LLM_ARCH_HRM_TEXT) { // 6 cache slots alias 2 physical blocks: 1 low-stack layer + 1 high-stack layer ms.add_kv(LLM_KV_HRM_LAYERS_PER_STACK, uint32_t(1)); @@ -439,7 +480,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(host_experts_test(arch) ? 64 : 2)); ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(2)); ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1)); - ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(4) : uint32_t(2)); // sqrtsoftplus : sigmoid + ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(4) : uint32_t(2)); // sqrtsoftplus : sigmoid ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f); ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1)); if (arch == LLM_ARCH_K2_HORIZON) { @@ -683,6 +724,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_DEEPSEEK32: case LLM_ARCH_DOTS3NOTE: case LLM_ARCH_DEEPSEEK4: + case LLM_ARCH_DEEPSEEK41: case LLM_ARCH_GLM4_MOE: case LLM_ARCH_GLM_DSA: case LLM_ARCH_EXAONE_MOE: From 0c1b44d5705b7d38392e416f3cd8f79c38d7d776 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 03:41:20 -0700 Subject: [PATCH 08/49] deepseek41: builder A status and full-model handoff notes --- STATUS_A.md | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 STATUS_A.md diff --git a/STATUS_A.md b/STATUS_A.md new file mode 100644 index 000000000000..413ffbded4e3 --- /dev/null +++ b/STATUS_A.md @@ -0,0 +1,78 @@ +# Builder A status (DeepSeek-V4.1-Flash-1017 llama.cpp port) + +Branch `dsv41` in `$W/wt_dsv41` (off llama.cpp_pinset fd55cb98f). Contract for builder B: `TENSORS.md`. + +## Milestones +| # | state | commit | +|---|---|---| +| 1 gguf-py constants, arch enums, keys, TENSORS.md | done | 7976d362d | +| 2 tiny fixture from official model.py + reference logits + fixture-only converter | done | scripts in `$W/scripts/dsv41/` | +| 3 graph + KV-cache generalisation, CPU + CUDA parity | done | bae963fb2 | +| 4 test-llama-archs (CPU, CUDA, Meta split), test-backend-ops, V4 regression | done | d29889d8f | +| 5 full-model handoff (below) | written, waiting for B's GGUF | | +| 6 DSpark / MTP graph | not started (tensors load as optional, graph throws on the MTP context) | | + +## What the graph implements (`src/models/deepseek41.cpp`) +Engram (host n-gram hash from the raw-cache token history, CPU get_rows on `engram_embd`, gate as model.py), single-pass mHC +(no hc_head), ratio-1/2 compressors (V4 HCA-style state, no ape/overlap), index K = k_norm(wk(latent)) stored next to the +compressed KV in one cache row, compressed KV / index K read from the kv source layer, top-k computed on index sources and +reused in between, two-level candidate blocks (pool_1d max per block, pin newest block, top blocks, mask later index layers), +no per-head q norm. KV cache: `llama-kv-cache-dsv4` slots are ratio-parametrised (V4: CSA 4 overlap + LID, HCA 128; +V4.1: CSA slot = ratio 2, HCA slot = ratio 1, only kv-source layers own rows, no LID). + +## Tiny fixture (`$W/models/dsv41/tiny_fixture`, GGUF `$W/models/dsv41/tiny_gguf/tiny-f32.gguf`) +10 layers, dim 128, real tokenizer + real 99,092-id Engram token map, Engram on L1/L4, ratios [0,0,2,2,2,2,1,1,1,1], +kv sources 2/4/6, index sources 2/4/6/8, candidate source 6 (4 blocks x 4), index_topk 8, window 16, YaRN orig 64. +Checkpoint in official naming and storage formats (FP8 32x32 ue8m0, FP4 experts, Engram FP8 + per-row-32 scales). +Reference = official model.py on CPU fp32 with activation fake-quant off (`scripts/dsv41/shim/kernel.py`). +Regenerate: `python scripts/dsv41/make_fixture.py`; convert: `python scripts/dsv41/fixture_to_gguf.py --out X.gguf [--experts mxfp4]`; +sweep: `bash scripts/dsv41/run_parity.sh "fa:ub ..."`; ablations: `python scripts/dsv41/ablate.py --got `. + +## Fixture parity (KL(ref||llama) mean / max, top-1; prompts wiki330, rep300, short16) +- CPU F32, fa=0, ub 512 / 7 / 1: KL mean <= 4.9e-12, max <= 1.5e-11, top-1 100% on all three (ub 7 differs from ub 512 by 5.7e-6 abs, still passes). +- CUDA (B200) F32, fa=0, `GGML_CUDA_F32_EXACT=1 NVIDIA_TF32_OVERRIDE=0`, ub 512 / 7 / 1: KL mean <= 2.8e-11, max <= 3.0e-10, top-1 100%. +- CUDA default precision fails the 1e-6 gate (F32 GEMMs run on TF32: hc_mixes rel err 9e-4 at layer 0); with the knob the + per-tensor CPU-vs-CUDA error is <= 1.3e-6 everywhere. +- fa=1 (CPU and CUDA): F16 flash attention. short16 KL 1e-6..2e-6; long prompts 4e-3..1e-1, the same scale as the reference + itself with q/k/v rounded to F16 (4.5e-3 wiki, 1.7e-2 rep300): the random fixture's top-k / routing flips amplify F16 noise. +- Mechanisms are live: KL(ref || ref without X) = Engram 3.1, candidate mask 0.17, top-k reuse (dense instead) 1.2, + single-pass mHC (standard instead) 0.6; llama.cpp matches the full reference, not any ablation. +- MXFP4 experts (bit-exact repack verified): short16 KL 2.8e-4 from the Q8 activation quantisation of the MXFP4 matmul. + +## Other checks +- test-llama-archs deepseek41: CPU OK, CUDA NMSE 3.3e-6, Meta (tensor split) OK, roundtrip OK. deepseek4: still OK. +- V4 regression: random V4 model (ratios 4/128, 240 tokens) logits bitwise identical base vs head build, ub 256 and 7. +- test-backend-ops CUDA0: POOL_1D, TOP_K, SET_ROWS, CONCAT, FILL, GET_ROWS, SGN, ROPE, SOFT_MAX, SUM_ROWS all pass; + MUL_MAT type_a=f32 with GGML_CUDA_F32_EXACT=1 275/275. + +## For builder B +- Keys/tensors exactly as `TENSORS.md`. The loader derives sharing from `attention.kv_source_layers` / + `attention.indexer.source_layers` and validates ratios; Engram offsets are derived from `engram.primes`. +- `engram_embd` is placed like token_embd (host buffer); any type works if CPU `get_rows` supports it (MXFP8 must + implement CPU get_rows -> F32). Do not repack it. `engram_gate` must be F32 `q_weight*k_weight`. +- DSpark blocks may be written as blk.40-42 (loaded as optional, unused) or left out. +- Smoke test for a converted fixture: `python scripts/dsv41/fixture_to_gguf.py` is the reference converter; B's output on + `$W/models/dsv41/tiny_fixture` should give the same parity numbers (compare with `run_parity.sh`). + +## Full-model parity run (once B's reference GGUF exists) +Sizes: experts 289 GB (MXFP4), dense ~14 GB BF16 (FP8 dequant), embed+head 2.7 GB, Engram 203 GB (MXFP8, host). +1. Engram tables stay on the host automatically (input-layer placement); host RAM 2 TB is enough. mmap first-touch is + random reads (48 rows per token): pre-warm the file (`vmtouch -t` or one `cat > /dev/null`) before timing. +2. GPU: ~306 GB weights -> 2 B200 with ~50 GB headroom (3 to be safe). Lease all cards at once: + `python $W/gpu_pool_lease.py take --owner $$ --key $$:dsv41 --gpus A,B --gb 178 --est-s 3600`, run with + `CUDA_VISIBLE_DEVICES=A,B`, then `gpu_pool_lease.py drop --key $$:dsv41 --gpus A,B`. +3. Logits: `$W/wt_dsv41/build-cuda/bin/llama_dump_logits -m DSV41.gguf -o OUT -c 8192 -ub 512 -ngl 99 -fa 0 --kv-f32 + prompts/*.tokens.bin` (the Phase-1 oracle writes the same int32 token files); compare with + `python $W/scripts/dsv41/compare.py --ref ORACLE --got OUT --kl-max 1e-3`. Repeat with `-ub 1` and `-ub 7` + (batch-split invariance) and once with fa=1 (expected higher KL, F16 attention). +4. Long-context gate: a prompt > 16,384 + 8 tokens (candidate pool) and > 512 compressed rows per ratio, so the + candidate mask, top-k and reuse all bind. +5. Oracle caveat: model.py on GPU quantises activations (FP8 before every FP8/FP4 GEMM, FP8 window KV, FP4 index/compressed + KV); the GGUF port does not, so expect KL above the fixture's 1e-11 even when correct. Measure the oracle with and without + fake-quant on a few prompts (`DSV41_FAKE_QUANT=1` in the shim emulates it on CPU). + +## Not proven / open +- Image tokens: Engram DEAD / gate-off for embd inputs is hashed as DEAD but the gate is not forced to 0 yet; vision not ported. +- Multi-sequence batches (n_seq_max > 1) and rollback snapshots (n_rs_seq > 0) are wired like V4 but not parity-tested. +- No ubatch shares a token across sequences (asserts, as the PLE path). +- DSpark drafting graph not implemented. From 7e34c4479dad673d93210333235be87b497c02b6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 03:52:34 -0700 Subject: [PATCH 09/49] deepseek41: tag l_out for parity tooling --- src/models/deepseek41.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index 2db3c4bd2f09..cf5365e06993 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -965,6 +965,11 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g inpL = build_cvec(inpL, il); cb(inpL, "l_last", il); + // flat [n_embd*hc, n_tokens] alias for parity tooling (compare_ref.py reads l_out-) + ggml_tensor * l_out = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens); + cb(l_out, "l_out", il); + ggml_build_forward_expand(gf, l_out); + pre_prev = ffn_pre; } From 445d59c34752cda5386235ae066deae8060a56e4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 03:54:38 -0700 Subject: [PATCH 10/49] convert : deepseek41 dense type follows --outtype, router kept F32 --outtype q8_0 now gives the Q8_0 companion (dense 2D tensors Q8_0, MXFP4 experts and MXFP8 Engram still bit-exact); ffn_gate_inp is written F32 (exact from bf16) as the router pin requires. --- conversion/deepseek41.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py index 027daf121711..80ad63306af4 100644 --- a/conversion/deepseek41.py +++ b/conversion/deepseek41.py @@ -159,7 +159,7 @@ class DeepseekV41Model(DeepseekV4Model): separate GGUF with --mtp. The vision tower is not converted yet. Environment overrides: - DSV41_DENSE_TYPE bf16 (default, lossless) | q8_0 for the dequantised FP8 linears + DSV41_DENSE_TYPE bf16 | q8_0 for the 2D dense tensors (default: q8_0 with --outtype q8_0, else bf16) DSV41_ENGRAM_TYPE mxfp8 (default, lossless) | bf16 (lossless, 2x larger) | f32 """ model_arch = gguf.MODEL_ARCH.DEEPSEEK41 @@ -171,7 +171,9 @@ class DeepseekV41Model(DeepseekV4Model): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) hp = self.hparams - self._dense_type = os.environ.get("DSV41_DENSE_TYPE", "bf16").lower() + # dense (non-expert, non-Engram) linears follow --outtype: bf16 is lossless, q8_0 is the B2 Q8_0 companion + default_dense = "q8_0" if self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0 else "bf16" + self._dense_type = os.environ.get("DSV41_DENSE_TYPE", default_dense).lower() self._engram_type = os.environ.get("DSV41_ENGRAM_TYPE", "mxfp8").lower() if self._dense_type not in ("bf16", "q8_0"): raise ValueError(f"DSV41_DENSE_TYPE={self._dense_type!r}") @@ -531,13 +533,14 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter return [(new_name, data_torch)] def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: - if new_name.endswith(".engram_gate.weight") or new_name == "conf_proj.weight": + # router, engram gate and confidence head stay F32 (exact from bf16; verify_types.py pins the router to F32) + if new_name.endswith((".engram_gate.weight", ".ffn_gate_inp.weight")) or new_name == "conf_proj.weight": return gguf.GGMLQuantizationType.F32 if name in self._dsv4_fp8_dequantized and n_dims >= 2: return gguf.GGMLQuantizationType.Q8_0 if self._dense_type == "q8_0" else gguf.GGMLQuantizationType.BF16 if name in self._dsv4_f32_tensors: return gguf.GGMLQuantizationType.F32 if name in self._dsv4_bf16_tensors and n_dims >= 2: - return gguf.GGMLQuantizationType.BF16 + return gguf.GGMLQuantizationType.Q8_0 if self._dense_type == "q8_0" else gguf.GGMLQuantizationType.BF16 return False From 6438476963920efd3c8938c461688e220ad5a872 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 03:55:39 -0700 Subject: [PATCH 11/49] deepseek41: TENSORS.md F32 pins --- TENSORS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TENSORS.md b/TENSORS.md index 96b495909a72..598eb3db93ec 100644 --- a/TENSORS.md +++ b/TENSORS.md @@ -44,6 +44,8 @@ Notes row for hash id r is `float(w[r]).view(8,32) * 2^(scale[r]-127)`. Never repack, never offload: the graph gathers it on the host with `ggml_get_rows`. Any type with get_rows on CPU works for the graph. - FP8 linear weights use 32x32 `ue8m0` blocks (derive the block from the scale shape, do not hard-code 128). +- Keep F32 in every output file (checked by gguf_build `eval/verify_types.py`): `ffn_gate_inp` (router), `exp_probs_b`, + `exp_probs_b_vl`, every `*norm.weight` (incl. `attn_compressor_norm`, `indexer.k_norm`), `hc_*`, `attn_sinks`, `engram_gate`. - Every layer `0..39` has all backbone attention/HC/MoE tensors; DSpark blocks have 128 experts (`dspark.expert_count`). ## Keys (`deepseek41.` prefix) From 70e2170d83a5df8f73cbcf9b48b833598ce0eafb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 04:49:51 -0700 Subject: [PATCH 12/49] ggml : MXFP8 is gather-only; convert : deepseek41 writes engram.offsets (u64) CPU: MXFP8 has no vec_dot / from_float and supports_op refuses every op on it except GET_ROWS (source 0), so a misuse fails at scheduling instead of computing garbage; CUDA already refused all but GET_ROWS. 64-bit block loops in the MXFP8 row (de)quantizers. The converter also writes the Engram bucket offsets as an explicit u64 array. --- conversion/deepseek41.py | 7 +++++++ ggml/src/ggml-cpu/ggml-cpu.c | 6 ------ ggml/src/ggml-cpu/ggml-cpu.cpp | 10 ++++++++++ ggml/src/ggml-cpu/ops.cpp | 6 ------ ggml/src/ggml-cpu/quants.c | 31 ------------------------------- ggml/src/ggml-cpu/quants.h | 2 -- ggml/src/ggml-quants.c | 8 ++++---- tests/test-backend-ops.cpp | 1 - 8 files changed, 21 insertions(+), 50 deletions(-) diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py index 80ad63306af4..e2f8d9d2d5db 100644 --- a/conversion/deepseek41.py +++ b/conversion/deepseek41.py @@ -285,6 +285,13 @@ def set_gguf_parameters(self): w.add_engram_token_map(token_map) w.add_engram_multipliers([m for row in mults for m in row]) w.add_engram_primes([p for layer in primes for per in layer for p in per]) + # per-layer row offsets of each (order, head) bucket range: exclusive cumsum of that layer's primes (order-major, + # head-minor) = NgramHashState.offsets; u64 so table addressing never narrows (tables are > 100 GB) + offsets = [] + for layer in primes: + flat = [p for per in layer for p in per] + offsets += [sum(flat[:i]) for i in range(len(flat))] + w.add_key_value(f"{self.gguf_writer.arch}.engram.offsets", offsets, gguf.GGUFValueType.ARRAY, gguf.GGUFValueType.UINT64) logger.info(f"engram: compressed vocab {cvocab}, pad -> {token_map[hp['engram_pad_token_id']]}, multipliers {mults}") # DSpark (block layout only matters to the --mtp file, but the keys are cheap) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index ba79d439739d..ab82e3d6c043 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -290,12 +290,6 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_0, .nrows = 1, }, - [GGML_TYPE_MXFP8] = { - .from_float = quantize_row_mxfp8, - .vec_dot = ggml_vec_dot_mxfp8_q8_0, - .vec_dot_type = GGML_TYPE_Q8_0, - .nrows = 1, - }, [GGML_TYPE_NVFP4] = { .from_float = quantize_row_nvfp4, .vec_dot = ggml_vec_dot_nvfp4_q8_0, diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 81ff5d79fa98..5e1a10671b67 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -429,6 +429,16 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st return true; } + // MXFP8 is a gather-only type: GET_ROWS reads it, nothing else does (and nothing writes it) + if (op->type == GGML_TYPE_MXFP8) { + return false; + } + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (op->src[i] && op->src[i]->type == GGML_TYPE_MXFP8 && !(op->op == GGML_OP_GET_ROWS && i == 0)) { + return false; + } + } + // check extra buffer types // note: only the first sources are checked for extra buffer types to reduce overhead, increase if necessary for (int i = 0; i < 4; i++) { diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 4e7e5abf9856..6726e9e1a46f 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -673,7 +673,6 @@ void ggml_compute_forward_add( case GGML_TYPE_Q8_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: - case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -1126,7 +1125,6 @@ void ggml_compute_forward_add1( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: - case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -1258,7 +1256,6 @@ void ggml_compute_forward_acc( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: - case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -4915,7 +4912,6 @@ void ggml_compute_forward_out_prod( case GGML_TYPE_Q8_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: - case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5235,7 +5231,6 @@ void ggml_compute_forward_set( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: - case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -6222,7 +6217,6 @@ void ggml_compute_forward_clamp( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: - case GGML_TYPE_MXFP8: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index b8f2a8a12656..5e36459f8cbc 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -58,10 +58,6 @@ void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, i quantize_row_mxfp4_ref(x, y, k); } -void quantize_row_mxfp8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { - quantize_row_mxfp8_ref(x, y, k); -} - void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { quantize_row_nvfp4_ref(x, y, k); } @@ -330,33 +326,6 @@ void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } -// MXFP8: scalar reference only (the format is used for host-side row gathers) -void ggml_vec_dot_mxfp8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { - assert(nrc == 1); - UNUSED(nrc); - UNUSED(bx); - UNUSED(by); - UNUSED(bs); - assert(n % QK_MXFP8 == 0); - static_assert(QK_MXFP8 == QK8_0, "QK_MXFP8 and QK8_0 must be the same"); - - const block_mxfp8 * GGML_RESTRICT x = vx; - const block_q8_0 * GGML_RESTRICT y = vy; - - const int nb = n / QK_MXFP8; - - float sumf = 0; - - for (int ib = 0; ib < nb; ++ib) { - float sumq = 0; - for (int j = 0; j < QK_MXFP8; ++j) { - sumq += ggml_e4m3_to_fp32(x[ib].qs[j]) * y[ib].qs[j]; - } - sumf += GGML_CPU_FP16_TO_FP32(y[ib].d) * GGML_E8M0_TO_FP32(x[ib].e) * sumq; - } - *s = sumf; -} - // NVFP4: super-block of 64 elements = 4 sub-blocks of 16 = 2 q8_0 blocks void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(nrc == 1); diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index cc8b65fbe24a..93ea7eeffe5b 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -22,7 +22,6 @@ void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); -void quantize_row_mxfp8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); @@ -48,7 +47,6 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_mxfp8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index ab4f5b4ae571..2fc07bfe811d 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -352,9 +352,9 @@ void quantize_row_mxfp8_ref(const float * GGML_RESTRICT x, block_mxfp8 * GGML_RE assert(k % qk == 0); - const int nb = k / qk; + const int64_t nb = k / qk; - for (int i = 0; i < nb; i++) { + for (int64_t i = 0; i < nb; i++) { float amax = 0.0f; for (int j = 0; j < qk; j++) { amax = MAX(amax, fabsf(x[i*qk + j])); @@ -620,9 +620,9 @@ void dequantize_row_mxfp8(const block_mxfp8 * GGML_RESTRICT x, float * GGML_REST assert(k % qk == 0); - const int nb = k / qk; + const int64_t nb = k / qk; - for (int i = 0; i < nb; i++) { + for (int64_t i = 0; i < nb; i++) { // exact: e4m3 * 2^(e-127) is representable in fp32 for every code and scale const float d = x[i].e == 0xFF ? NAN : GGML_E8M0_TO_FP32(x[i].e); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 121aca1fb1eb..329a7f638bb0 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9811,7 +9811,6 @@ static std::vector> make_test_cases_eval() { } } test_cases.emplace_back(new test_get_rows(GGML_TYPE_MXFP8, 256, 1000, 24, 1, 1, false)); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP8, GGML_TYPE_F32, 16, 1, 256, {1, 1}, {1, 1})); test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 8, 2, 1, false)); test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 70000, 4, 1, false)); // row count > CUDA grid-y limit (65535) From a773f631b4c688978efeb740c0f24cd59d2ad43c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 05:13:10 -0700 Subject: [PATCH 13/49] convert : deepseek41 --outtype f32 keeps dense F32, Engram keys optional F32 output writes the dequantised FP8 and bf16 2D tensors as F32 for parity fixtures; a config without Engram layers writes no engram keys (GGUF has no empty arrays; the loader treats them as optional). --- conversion/deepseek41.py | 77 +++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py index e2f8d9d2d5db..6f32d2019043 100644 --- a/conversion/deepseek41.py +++ b/conversion/deepseek41.py @@ -172,10 +172,10 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) hp = self.hparams # dense (non-expert, non-Engram) linears follow --outtype: bf16 is lossless, q8_0 is the B2 Q8_0 companion - default_dense = "q8_0" if self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0 else "bf16" + default_dense = {gguf.LlamaFileType.MOSTLY_Q8_0: "q8_0", gguf.LlamaFileType.ALL_F32: "f32"}.get(self.ftype, "bf16") self._dense_type = os.environ.get("DSV41_DENSE_TYPE", default_dense).lower() self._engram_type = os.environ.get("DSV41_ENGRAM_TYPE", "mxfp8").lower() - if self._dense_type not in ("bf16", "q8_0"): + if self._dense_type not in ("bf16", "q8_0", "f32"): raise ValueError(f"DSV41_DENSE_TYPE={self._dense_type!r}") if self._engram_type not in ("mxfp8", "bf16", "f32"): raise ValueError(f"DSV41_ENGRAM_TYPE={self._engram_type!r}") @@ -260,39 +260,40 @@ def set_gguf_parameters(self): w.add_indexer_candidate_top_k_blocks(hp["candidate_topk_blocks"]) w.add_indexer_candidate_block_size(hp["candidate_block_size"]) - # Engram - layer_ids = list(hp["engram_layer_ids"]) - max_ngram, n_heads = hp["engram_max_ngram_size"], hp["engram_n_heads"] - primes = engram_primes(len(layer_ids), max_ngram, n_heads, hp["engram_vocab_size"]) - for li, layer_primes in enumerate(primes): - total = sum(p for per in layer_primes for p in per) - if total != hp["engram_num_embeddings"][li]: - raise ValueError(f"engram layer {layer_ids[li]}: prime sum {total} != num_embeddings {hp['engram_num_embeddings'][li]}") - token_map, cvocab = engram_compressed_token_map(self.dir_model if self.target_model_dir is None else self.target_model_dir) - if cvocab != hp["engram_compressed_vocab_size"]: - raise ValueError(f"compressed vocab {cvocab} != engram_compressed_vocab_size {hp['engram_compressed_vocab_size']}") - if len(token_map) != hp["vocab_size"]: - raise ValueError(f"token map has {len(token_map)} entries, vocab_size is {hp['vocab_size']}") - mults = engram_multipliers(layer_ids, max_ngram, cvocab) - w.add_engram_layers(layer_ids) - w.add_engram_num_embeddings(hp["engram_num_embeddings"]) - w.add_engram_max_ngram_size(max_ngram) - w.add_engram_vocab_size(hp["engram_vocab_size"]) - w.add_engram_head_count(n_heads) - w.add_engram_head_dim(hp["engram_head_dim"]) - w.add_engram_pad_token_id(hp["engram_pad_token_id"]) - w.add_engram_compressed_vocab_size(cvocab) - w.add_engram_token_map(token_map) - w.add_engram_multipliers([m for row in mults for m in row]) - w.add_engram_primes([p for layer in primes for per in layer for p in per]) - # per-layer row offsets of each (order, head) bucket range: exclusive cumsum of that layer's primes (order-major, - # head-minor) = NgramHashState.offsets; u64 so table addressing never narrows (tables are > 100 GB) - offsets = [] - for layer in primes: - flat = [p for per in layer for p in per] - offsets += [sum(flat[:i]) for i in range(len(flat))] - w.add_key_value(f"{self.gguf_writer.arch}.engram.offsets", offsets, gguf.GGUFValueType.ARRAY, gguf.GGUFValueType.UINT64) - logger.info(f"engram: compressed vocab {cvocab}, pad -> {token_map[hp['engram_pad_token_id']]}, multipliers {mults}") + # Engram (keys omitted when the model has no Engram layers: GGUF cannot hold empty arrays, the loader treats them as optional) + layer_ids = list(hp.get("engram_layer_ids") or []) + if layer_ids: + max_ngram, n_heads = hp["engram_max_ngram_size"], hp["engram_n_heads"] + primes = engram_primes(len(layer_ids), max_ngram, n_heads, hp["engram_vocab_size"]) + for li, layer_primes in enumerate(primes): + total = sum(p for per in layer_primes for p in per) + if total != hp["engram_num_embeddings"][li]: + raise ValueError(f"engram layer {layer_ids[li]}: prime sum {total} != num_embeddings {hp['engram_num_embeddings'][li]}") + token_map, cvocab = engram_compressed_token_map(self.dir_model if self.target_model_dir is None else self.target_model_dir) + if cvocab != hp["engram_compressed_vocab_size"]: + raise ValueError(f"compressed vocab {cvocab} != engram_compressed_vocab_size {hp['engram_compressed_vocab_size']}") + if len(token_map) != hp["vocab_size"]: + raise ValueError(f"token map has {len(token_map)} entries, vocab_size is {hp['vocab_size']}") + mults = engram_multipliers(layer_ids, max_ngram, cvocab) + w.add_engram_layers(layer_ids) + w.add_engram_num_embeddings(hp["engram_num_embeddings"]) + w.add_engram_max_ngram_size(max_ngram) + w.add_engram_vocab_size(hp["engram_vocab_size"]) + w.add_engram_head_count(n_heads) + w.add_engram_head_dim(hp["engram_head_dim"]) + w.add_engram_pad_token_id(hp["engram_pad_token_id"]) + w.add_engram_compressed_vocab_size(cvocab) + w.add_engram_token_map(token_map) + w.add_engram_multipliers([m for row in mults for m in row]) + w.add_engram_primes([p for layer in primes for per in layer for p in per]) + # per-layer row offsets of each (order, head) bucket range: exclusive cumsum of that layer's primes (order-major, + # head-minor) = NgramHashState.offsets; u64 so table addressing never narrows (tables are > 100 GB) + offsets = [] + for layer in primes: + flat = [p for per in layer for p in per] + offsets += [sum(flat[:i]) for i in range(len(flat))] + w.add_key_value(f"{self.gguf_writer.arch}.engram.offsets", offsets, gguf.GGUFValueType.ARRAY, gguf.GGUFValueType.UINT64) + logger.info(f"engram: compressed vocab {cvocab}, pad -> {token_map[hp['engram_pad_token_id']]}, multipliers {mults}") # DSpark (block layout only matters to the --mtp file, but the keys are cheap) if "dspark_block_size" in hp: @@ -543,11 +544,13 @@ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: # router, engram gate and confidence head stay F32 (exact from bf16; verify_types.py pins the router to F32) if new_name.endswith((".engram_gate.weight", ".ffn_gate_inp.weight")) or new_name == "conf_proj.weight": return gguf.GGMLQuantizationType.F32 + dense_qt = {"q8_0": gguf.GGMLQuantizationType.Q8_0, "f32": gguf.GGMLQuantizationType.F32}.get( + self._dense_type, gguf.GGMLQuantizationType.BF16) if name in self._dsv4_fp8_dequantized and n_dims >= 2: - return gguf.GGMLQuantizationType.Q8_0 if self._dense_type == "q8_0" else gguf.GGMLQuantizationType.BF16 + return dense_qt if name in self._dsv4_f32_tensors: return gguf.GGMLQuantizationType.F32 if name in self._dsv4_bf16_tensors and n_dims >= 2: - return gguf.GGMLQuantizationType.Q8_0 if self._dense_type == "q8_0" else gguf.GGMLQuantizationType.BF16 + return dense_qt return False From 91b8b86471ecf3aa0cf198446115994ca3903309 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 05:57:26 -0700 Subject: [PATCH 14/49] deepseek41: STATUS_A porting-workflow A-step record (A1, A2, A4) --- STATUS_A.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/STATUS_A.md b/STATUS_A.md index 413ffbded4e3..285764d1dd53 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -2,6 +2,35 @@ Branch `dsv41` in `$W/wt_dsv41` (off llama.cpp_pinset fd55cb98f). Contract for builder B: `TENSORS.md`. +## Porting workflow (unslothai/scripts #649 bundle, `temp/us_gguf`: gguf_build_workflow.md A1-A5, PORTING.md) +| step | state | evidence | +|---|---|---| +| A1 recon + reuse map + fixture + converter + graph | done | reuse map in deepseek41.cpp header (V4 HC/MLA/MoE/raw cache reused; new: Engram, lagged mHC, ratio 1/2 + sharing + candidates). Two fixtures: `tiny_fixture` (small dims, fast, seeded-bug harness) and `tiny_fixture_realdim` (A1 rule: REAL dims 5120 / 64x512 heads / rope 64 / q 1280 / o 8x1024 / expert 2304 / index 32x128 / Engram 8x256 n-gram 4 / window 128 / block 8 / YaRN 65536x16; only counts shrunk: 10 layers, 8 experts top-2, Engram bucket start 1009, index top-k 64, 16 candidate blocks). Rare-path thresholds: window 128, ratio-2 top-k binds > 128 tokens, candidate mask binds > 128 ratio-1 rows; prompt wiki1100 exceeds all. Reference = official model.py (no transformers implementation of deepseek_v41 exists in 4.57 / 5.x, so ref_logits.py --generic cannot load it); the fixture writes the ref_logits.py npz schema + full logits. Fixture generates under the reference (prefill vs decode max abs 2.7e-5 at real dims). | +| A2 parity (bundled tools/dump_logits + port/compare_ref.py, F32 gate --max-kl-mean 1e-6 --min-top1 1.0 --min-cos 0.99999) | done (CPU), CUDA calibrated | see "A2 numbers" below; 13/13 seeded bugs FAIL on the small fixture, 5/5 on real dims | +| A3 template + V4.1 DSML tool parser | not started (next) | template candidate exists: models/dsv41/chat_template.jinja (5/5 byte parity, Phase 0) | +| A4 never break other models | done | test-chat all passed; test-llama-archs CPU all 137 passed; CUDA: full suite exceeded its 2 h box (killed, no verdict), so the archs sharing touched code were run: deepseek4/41, deepseek2/32, glm-dsa, glm5-next, maple, qwen3moe, qwen35moe, qwen4exp on CUDA + CPU + Meta: 28/28 passed; deepseek41 on CPU / CUDA / Meta OK; test-backend-ops for used ops pass; V4 logits bitwise identical base vs head; no vocab change (vocab differential N/A) | +| A5 pin PR / pinset sim | not started (after A3 and B's GGUF) | | + +### A2 numbers +- CPU, F32 KV, FA off, single-shot / ub 7 / ub 1, swa_full 1 and 0: small fixture KL mean <= 4.9e-12, top-1 1.0, router top-k + exact on all 10 MoE layers; real-dims fixture (incl. wiki1100) KL mean <= 2.7e-11, max 3.0e-10, top-1 1.0, router exact + 40/40 (prompt x layer), l_out rel err <= 4e-6 on every layer. +- Seeded bugs (scripts/dsv41/seeded_bugs.py, every one must FAIL): Engram signed-sqrt dropped, Engram history order, + Engram ungated, mHC not lagged, output collapse with wrong pre, V4 per-head q norm, compressor softmax axis, index-K rope + dims, compressed-KV rope dims, indexer ReLU dropped, candidate pin dropped, candidate mask not applied, top-k reuse + replaced by dense: 13/13 FAIL (KL 4e-3 .. 2). On real dims: candidate pin / candidate mask / reuse / ReLU / Engram + history: 5/5 FAIL. Free oracle: below 16 visible ratio-1 rows the candidate bugs are inert (short16 passes), as the + architecture guarantees. +- CUDA (B200), `NVIDIA_TF32_OVERRIDE=0` alone: FAILS (KL 1e-3..7e-2): the override does not reach the mmf tf32 path for + F32 weights (PORTING.md Phase 5). With `GGML_CUDA_F32_EXACT=1` (debug knob in mmf.cu, default off) as well: small + fixture passes every cell (ub 0/7/1, graphs on and off); real dims pass everything except wiki1100 at ub 0/7 (KL mean + 2.3e-6, top-1 1.0, max 2.4e-3 at t=1035). Chased: one indexer near-tie at layer 2, t=1035, k-th vs (k+1)-th CPU score + gap 4.4e-5 relative while CUDA batched idx_score differs from CPU by 1.7e-5..2.6e-5 relative (score = signed sum of + relu terms, ~10x cancellation); the flip then propagates to that token's selections in later layers. Decode (ub 1) passes. + Classified as precision, not a defect; CUDA gate = CPU gate except one near-tie token. +- CUDA graphs on vs off (B10 A/B): identical results, no hang under the 1200 s watchdog (timeout -k 15 => SIGKILL). +- FA on (F16 attention) is an acceptance config, not a parity config (see older numbers below). + ## Milestones | # | state | commit | |---|---|---| @@ -20,7 +49,7 @@ reused in between, two-level candidate blocks (pool_1d max per block, pin newest no per-head q norm. KV cache: `llama-kv-cache-dsv4` slots are ratio-parametrised (V4: CSA 4 overlap + LID, HCA 128; V4.1: CSA slot = ratio 2, HCA slot = ratio 1, only kv-source layers own rows, no LID). -## Tiny fixture (`$W/models/dsv41/tiny_fixture`, GGUF `$W/models/dsv41/tiny_gguf/tiny-f32.gguf`) +## Tiny fixtures (`$W/models/dsv41/tiny_fixture` small dims; `tiny_fixture_realdim` + `tiny_gguf/realdim-f32.gguf` real dims, 24.6 GB F32). Small fixture (`$W/models/dsv41/tiny_fixture`, GGUF `$W/models/dsv41/tiny_gguf/tiny-f32.gguf`) 10 layers, dim 128, real tokenizer + real 99,092-id Engram token map, Engram on L1/L4, ratios [0,0,2,2,2,2,1,1,1,1], kv sources 2/4/6, index sources 2/4/6/8, candidate source 6 (4 blocks x 4), index_topk 8, window 16, YaRN orig 64. Checkpoint in official naming and storage formats (FP8 32x32 ue8m0, FP4 experts, Engram FP8 + per-row-32 scales). From 2f4055f002d2d080ac4257c1226628fa2040804a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 06:02:08 -0700 Subject: [PATCH 15/49] deepseek41: serialize Engram token history in KV state; router weight sum +1e-20 as the reference --- src/llama-graph.cpp | 9 +++++++-- src/llama-hparams.h | 1 + src/llama-kv-cache.cpp | 4 ++-- src/models/deepseek41.cpp | 1 + 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 902b0269e3ab..675ac5948088 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2265,8 +2265,13 @@ ggml_tensor * llm_graph_context::build_moe_ffn( ggml_tensor * weights_sum = ggml_sum_rows(ctx0, weights); // [1, n_tokens] cb(weights_sum, "ffn_moe_weights_sum", il); - // Avoid division by zero, clamp to smallest number representable by F16 - weights_sum = ggml_clamp(ctx0, weights_sum, 6.103515625e-5, INFINITY); + if (arch == LLM_ARCH_DEEPSEEK41) { + // reference: weights / (sum + 1e-20), not the F16-floor clamp (model.py Gate.forward) + weights_sum = ggml_scale_bias(ctx0, weights_sum, 1.0f, 1e-20f); + } else { + // Avoid division by zero, clamp to smallest number representable by F16 + weights_sum = ggml_clamp(ctx0, weights_sum, 6.103515625e-5, INFINITY); + } cb(weights_sum, "ffn_moe_weights_sum_clamped", il); weights = ggml_div(ctx0, weights, weights_sum); // [n_expert_used, n_tokens] diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 91d7940698b3..e879b84ac6b2 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -328,6 +328,7 @@ struct llama_hparams { std::array dsv41_kv_src; // layer whose compressed KV (and index K) this layer reads, -1 = none std::array dsv41_topk_src; // index-source layer whose top-k this layer attends with, -1 = none int32_t dsv41_cand_src = -1; // candidate_source_layer, -1 = two-level selection off + bool dsv41_tok_hist = false; // Engram: KV cells keep (and serialize) token ids for the n-gram history uint32_t dsv41_cand_topk_blocks = 0; uint32_t dsv41_cand_block_size = 0; diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index b7dfadef4638..b4c399b445a2 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1846,8 +1846,8 @@ void llama_kv_cache::set_input_v_rot(ggml_tensor * dst) const { } bool llama_kv_cache::has_cell_ext() const { - // M-RoPE needs the 2D position, the PLE n-gram hash needs the token id - return hparams.n_pos_per_embd() > 1 || hparams.ple_n_heads > 0; + // M-RoPE needs the 2D position, the PLE / Engram n-gram hashes need the token id + return hparams.n_pos_per_embd() > 1 || hparams.ple_n_heads > 0 || hparams.dsv41_tok_hist; } void llama_kv_cache::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector & res) const { diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index cf5365e06993..f996504434fe 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -167,6 +167,7 @@ void llama_model_deepseek41::load_arch_hparams(llama_model_loader & ml) { throw std::runtime_error("DeepSeek-V4.1 engram metadata is inconsistent"); } engram_layers.assign(e_layers.begin(), e_layers.end()); + hparams.dsv41_tok_hist = true; // state save/restore must carry the n-gram history engram_pad_cid = engram_token_map[pad_id]; engram_primes.assign(primes.begin(), primes.end()); engram_offsets.resize(primes.size()); From 54ec0d20df4ec171a88833374476e2b509aeb981 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 06:04:44 -0700 Subject: [PATCH 16/49] deepseek41: tag per-layer selected top-k (sel_top_k) for selection parity --- src/models/deepseek41.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index f996504434fe..aaf77def3cdf 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -774,6 +774,13 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( GGML_ASSERT(top_k && "DeepSeek-V4.1: top-k source layer produced no top-k"); } + { + // per-layer alias of the top-k this layer attends with (reused on consumers), for selection parity + ggml_tensor * sel = ggml_view_2d(ctx0, top_k, top_k->ne[0], top_k->ne[1]*top_k->ne[2]*top_k->ne[3], top_k->nb[1], 0); + cb(sel, "sel_top_k", il); + ggml_build_forward_expand(gf, sel); + } + // ---- sparse attention over [window raw KV | selected compressed rows] ggml_tensor * k_rot = inp_attn->self_k_rot; if (k_rot) { From 1d51c092fe7b820cb1b63306951feff437a6dabc Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 06:11:07 -0700 Subject: [PATCH 17/49] chat: DeepSeek-V4.1 template detection and DSML tool parser (leading-space tag names) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit V4.1 wraps tool calls in <｜DSML｜ calls> with <｜DSML｜ invoke> / <｜DSML｜ parameter> tags and otherwise keeps the V4 behaviour (bare in chat mode, tool results in call order, top-level response_format). Adds the V4.1-Flash-1017 template and the V4 parser tests ported to the V4.1 markup. --- common/chat.cpp | 7 +- common/parsers/deepseek.cpp | 18 +- ...deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja | 275 ++++++++++++++++++ tests/test-chat.cpp | 140 +++++++++ 4 files changed, 430 insertions(+), 10 deletions(-) create mode 100644 models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja diff --git a/common/chat.cpp b/common/chat.cpp index 4856713082ac..b79e32fb8c1d 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1195,11 +1195,12 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const // DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls. // The template source contains the token as a variable assignment, not as a literal in markup. // V3.2 names the tool call block "function_calls", V4 names it "tool_calls". - if (src.find("dsml_token") != std::string::npos && - src.find("DSML") != std::string::npos && + // V4.1 names it " calls" (leading-space tag names) and its template may not name the dsml_token variable. + if (((src.find("dsml_token") != std::string::npos && src.find("DSML") != std::string::npos) || + src.find("｜DSML｜ calls>") != std::string::npos) && (src.find("function_calls") != std::string::npos || src.find("tool_calls") != std::string::npos)) { - LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n"); + LOG_DBG("Using specialized template: DeepSeek V3.2/V4/V4.1\n"); return common_chat_params_init_deepseek_v3_2; } diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp index 7ee38ce6df74..1454ae9dbb54 100644 --- a/common/parsers/deepseek.cpp +++ b/common/parsers/deepseek.cpp @@ -75,6 +75,9 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ // non-thinking generation prompt ends with a bare instead of an empty // pair. const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos; + // V4.1 keeps the V4 behaviour but its DSML tag names carry a leading space: + // <｜DSML｜ calls>, <｜DSML｜ invoke name="...">, <｜DSML｜ parameter name="..." string="...">. + const bool is_v41 = is_v4 && tmpl.source().find("｜DSML｜ calls>") != std::string::npos; std::optional adjusted_messages; if (is_v4) { @@ -94,13 +97,14 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ const std::string DSML = "｜DSML｜"; const std::string THINK_START = ""; const std::string THINK_END = ""; - const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls"; - const std::string FC_START = "<" + DSML + TC_BLOCK + ">"; - const std::string FC_END = ""; - const std::string INVOKE_START = "<" + DSML + "invoke"; - const std::string INVOKE_END = ""; - const std::string PARAM_START = "<" + DSML + "parameter"; - const std::string PARAM_END = ""; + const std::string TAG_SP = is_v41 ? " " : ""; + const std::string TC_BLOCK = is_v41 ? "calls" : is_v4 ? "tool_calls" : "function_calls"; + const std::string FC_START = "<" + DSML + TAG_SP + TC_BLOCK + ">"; + const std::string FC_END = ""; + const std::string INVOKE_START = "<" + DSML + TAG_SP + "invoke"; + const std::string INVOKE_END = ""; + const std::string PARAM_START = "<" + DSML + TAG_SP + "parameter"; + const std::string PARAM_END = ""; const std::string GEN_PROMPT = "<｜Assistant｜>"; const std::string TC_SEPARATOR = "\n\n"; diff --git a/models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja b/models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja new file mode 100644 index 000000000000..aac8438a37df --- /dev/null +++ b/models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja @@ -0,0 +1,275 @@ +{#- DeepSeek-V4.1 chat template (for DeepSeek-V4.1-Flash-1017), ported from encoding/encoding.py in the model repo. + Based on the deepseek-ai/DeepSeek-V4.1-Flash template, with fixes so it matches encoding.py: + tool-result-only user turns do not start a new turn, tool_result content lists keep "[Unsupported x]" blocks, + top-level response_format is attached to the leading system message. + Extra kwargs: thinking_mode ("thinking" | "chat", default "thinking"; enable_thinking / thinking = true/false are + accepted as aliases), drop_thinking (default true), reasoning_effort (int in [1, 100] or "low"/"medium"/"high"/"max", + default "high"). Assistant reasoning is read from reasoning_content, or from thinking as an alias. + Tool call arguments must be objects (string arguments are rendered as a single "arguments" parameter). -#} +{%- set IMG = "<｜deepseek_image｜>" -%} +{%- set DSML = "｜DSML｜" -%} + +{%- if thinking_mode is not defined or thinking_mode is none -%} + {%- if enable_thinking is defined and enable_thinking is not none -%} + {%- set thinking_mode = "thinking" if enable_thinking else "chat" -%} + {%- elif thinking is defined and thinking is not none and thinking is not string -%} + {%- set thinking_mode = "thinking" if thinking else "chat" -%} + {%- else -%} + {%- set thinking_mode = "thinking" -%} + {%- endif -%} +{%- endif -%} +{%- if thinking_mode not in ["chat", "thinking"] -%} + {{- raise_exception("Invalid thinking_mode `" ~ thinking_mode ~ "`, should be \"chat\" or \"thinking\"") -}} +{%- endif -%} +{%- if drop_thinking is not defined or drop_thinking is none -%}{%- set drop_thinking = true -%}{%- endif -%} +{%- if add_generation_prompt is not defined -%}{%- set add_generation_prompt = false -%}{%- endif -%} +{%- if tools is not defined -%}{%- set tools = none -%}{%- endif -%} +{%- if response_format is not defined -%}{%- set response_format = none -%}{%- endif -%} + +{%- if reasoning_effort is not defined or reasoning_effort is none -%}{%- set reasoning_effort = "high" -%}{%- endif -%} +{%- set effort_map = {"low": 50, "medium": 62, "high": 75, "max": 100} -%} +{%- if reasoning_effort is string and reasoning_effort in effort_map -%} + {%- set budget = effort_map[reasoning_effort] -%} +{%- elif reasoning_effort is integer and reasoning_effort >= 1 and reasoning_effort <= 100 -%} + {%- set budget = reasoning_effort -%} +{%- else -%} + {{- raise_exception("Invalid reasoning effort: " ~ reasoning_effort ~ ", should be int within [1,100] or one of \"low\", \"medium\", \"high\", \"max\"") -}} +{%- endif -%} + +{#- Splits/validates "ns::name" against an explicit namespace and returns the qualified name. -#} +{%- macro qualified_name(name, ns) -%} + {%- if ns is mapping -%}{%- set ns = ns.name -%}{%- endif -%} + {%- set parts = name.split("::", 1) -%} + {%- if parts | length == 2 -%} + {%- if ns is not none and ns != parts[0] -%}{{- raise_exception("Conflicting tool namespaces: " ~ ns ~ " != " ~ parts[0]) -}}{%- endif -%} + {%- set ns = parts[0] -%} + {%- set name = parts[1] -%} + {%- endif -%} + {%- if "::" in name -%}{{- raise_exception("Tool name must not contain '::': " ~ name) -}}{%- endif -%} + {%- if ns is not none and "::" in ns -%}{{- raise_exception("Tool namespace must not contain '::': " ~ ns) -}}{%- endif -%} + {{- name if ns is none else ns ~ "::" ~ name -}} +{%- endmacro -%} + +{%- macro render_tools(tool_list) -%} + {%- set ts = namespace(schemas=[]) -%} + {%- for tool in tool_list -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set ns = tool.namespace if tool.namespace is defined and tool.namespace is not none else fn.get("namespace") -%} + {#- JSON assembled by hand (key order kept, name/description replaced in place): llama.cpp has no ** unpacking. -#} + {%- set qname = qualified_name(fn.name, ns) -%} + {%- set sch = namespace(parts=[]) -%} + {%- for k, v in fn.items() if k != "namespace" -%} + {%- if k == "name" -%}{%- set v = qname -%} + {%- elif k == "description" and ns is mapping and ns.description -%}{%- set v = ns.description ~ "\n" ~ (fn.description or "") -%} + {%- endif -%} + {%- set sch.parts = sch.parts + [(k | tojson) ~ ": " ~ (v | tojson)] -%} + {%- endfor -%} + {%- if "description" not in fn and ns is mapping and ns.description -%} + {%- set sch.parts = sch.parts + ['"description": ' ~ ((ns.description ~ "\n") | tojson)] -%} + {%- endif -%} + {%- set ts.schemas = ts.schemas + ["{" ~ sch.parts | join(", ") ~ "}"] -%} + {%- endfor -%} + {{- "## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜ calls>\" block like the following:\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"$TOOL_NAME\">\n<｜DSML｜ parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n...\n\n<｜DSML｜ invoke name=\"$TOOL_NAME2\">\n...\n\n\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response.\n\nOtherwise, output directly after with tool calls or final response.\n\n### Available Tool Schemas\n\n" -}} + {{- ts.schemas | join("\n") -}} + {{- "\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n" -}} +{%- endmacro -%} + +{#- 1. Merge tool results (and consecutive user turns) into user messages made of content blocks. -#} +{%- set mns = namespace(list=[]) -%} +{%- for msg in messages -%} + {%- set content = msg.get("content") -%} + {%- set is_blocks = content is iterable and content is not string and content is not mapping -%} + {%- if content is string and IMG in content -%} + {{- raise_exception("Message content contains image special token " ~ IMG ~ ". Images should be provided as image content blocks.") -}} + {%- endif -%} + {%- set rc = msg.reasoning_content if msg.get("reasoning_content") is not none else msg.get("thinking") -%} + {%- if rc is string and IMG in rc -%} + {{- raise_exception("reasoning_content contains image special token " ~ IMG) -}} + {%- endif -%} + {%- set prev = mns.list[-1] if mns.list else none -%} + {%- if msg.role == "tool" -%} + {%- set tblock = {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content if content is not none else ""} -%} + {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined -%} + {%- set mns.list = mns.list[:-1] + [dict(prev, content_blocks=prev.content_blocks + [tblock])] -%} + {%- else -%} + {%- set mns.list = mns.list + [{"role": "user", "content_blocks": [tblock]}] -%} + {%- endif -%} + {%- elif msg.role == "user" -%} + {%- set blocks = content if is_blocks else [{"type": "text", "text": content or ""}] -%} + {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined and prev.get("task") is none -%} + {%- set mns.list = mns.list[:-1] + [dict(prev, content_blocks=prev.content_blocks + blocks)] -%} + {%- else -%} + {%- set mns.list = mns.list + [dict(msg, content_blocks=blocks)] -%} + {%- endif -%} + {%- else -%} + {%- set mns.list = mns.list + [msg] -%} + {%- endif -%} +{%- endfor -%} + +{#- Top-level tools / response_format attach to a leading system message, or to a new empty one. -#} +{%- if tools or response_format -%} + {%- if not (mns.list and mns.list[0].role == "system") -%} + {%- set mns.list = [{"role": "system", "content": ""}] + mns.list -%} + {%- endif -%} + {%- set first = mns.list[0] -%} + {%- if tools and not first.get("tools") -%}{%- set first = dict(first, tools=tools) -%}{%- endif -%} + {%- if response_format and not first.get("response_format") -%}{%- set first = dict(first, response_format=response_format) -%}{%- endif -%} + {%- set mns.list = [first] + mns.list[1:] -%} +{%- endif -%} + +{#- 2. Reorder tool results to match the tool_calls order of the preceding assistant message. -#} +{%- set sns = namespace(list=[], ids=[]) -%} +{%- for m in mns.list -%} + {%- if m.role == "assistant" and m.get("tool_calls") -%} + {%- set sns.ids = [] -%} + {%- for tc in m.tool_calls -%} + {%- set sns.ids = sns.ids + [tc.get("id") or (tc.function.get("id", "") if tc.function is defined else "")] -%} + {%- endfor -%} + {%- set sns.list = sns.list + [m] -%} + {%- elif m.role == "user" and m.content_blocks -%} + {%- set tb = namespace(blocks=[]) -%} + {%- for b in m.content_blocks if b.type == "tool_result" -%} + {%- set order = namespace(idx=0) -%} + {%- for tc_id in sns.ids -%} + {%- if tc_id and tc_id == b.get("tool_use_id", "") -%}{%- set order.idx = loop.index0 -%}{%- endif -%} + {%- endfor -%} + {%- set tb.blocks = tb.blocks + [dict(b, _order=order.idx)] -%} + {%- endfor -%} + {%- if tb.blocks | length > 1 and sns.ids -%} + {%- set sorted_blocks = tb.blocks | sort(attribute="_order") -%} + {%- set rb = namespace(blocks=[], i=0) -%} + {%- for b in m.content_blocks -%} + {%- if b.type == "tool_result" -%} + {%- set rb.blocks = rb.blocks + [sorted_blocks[rb.i]] -%} + {%- set rb.i = rb.i + 1 -%} + {%- else -%} + {%- set rb.blocks = rb.blocks + [b] -%} + {%- endif -%} + {%- endfor -%} + {%- set sns.list = sns.list + [dict(m, content_blocks=rb.blocks)] -%} + {%- else -%} + {%- set sns.list = sns.list + [m] -%} + {%- endif -%} + {%- else -%} + {%- set sns.list = sns.list + [m] -%} + {%- endif -%} +{%- endfor -%} +{%- set msgs = sns.list -%} + +{#- Last user turn: users and mid-conversation systems count, except user turns made only of tool results. + Tools anywhere disable drop_thinking. -#} +{%- set state = namespace(last_user=-1, has_tools=false) -%} +{%- for m in msgs -%} + {%- if m.role == "user" or (m.role == "system" and loop.index0 > 0) -%} + {%- set real = namespace(v=true) -%} + {%- if m.get("content_blocks") -%} + {%- set real.v = false -%} + {%- for b in m.content_blocks if b.type in ["text", "image", "image_url"] -%}{%- set real.v = true -%}{%- endfor -%} + {%- endif -%} + {%- if real.v -%}{%- set state.last_user = loop.index0 -%}{%- endif -%} + {%- endif -%} + {%- if m.get("tools") -%}{%- set state.has_tools = true -%}{%- endif -%} +{%- endfor -%} +{%- set effective_drop = drop_thinking and not state.has_tools -%} +{%- set is_thinking = thinking_mode == "thinking" -%} + +{{- "<｜begin▁of▁sentence｜>" -}} +{#- 3. Render each message. -#} +{%- for m in msgs -%} + {%- set idx = loop.index0 -%} + {%- set content = m.get("content") -%} + {%- if content is iterable and content is not string and content is not mapping -%} + {%- set sp = namespace(parts=[]) -%} + {%- for b in content -%} + {%- if b.type == "text" -%}{%- set sp.parts = sp.parts + [b.text or ""] -%} + {%- elif b.type in ["image", "image_url"] -%}{%- set sp.parts = sp.parts + [IMG] -%} + {%- endif -%} + {%- endfor -%} + {%- set content = sp.parts | join("\n\n") -%} + {%- endif -%} + + {%- if idx == 0 -%} + {%- if is_thinking or m.role == "system" -%}{{- "<｜System｜>" -}}{%- endif -%} + {%- if is_thinking -%}{{- "Reasoning Effort: " ~ budget ~ " (range 1-100, the higher the value, the more thorough the reasoning)\n\n" -}}{%- endif -%} + {%- endif -%} + + {%- if m.role == "system" -%} + {%- if idx > 0 -%}{{- "<｜System｜>" -}}{%- endif -%} + {{- content or "" -}} + {%- if m.get("tools") -%}{{- "\n\n" ~ render_tools(m.tools) -}}{%- endif -%} + {%- if m.get("response_format") -%} + {{- "\n\n## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n" ~ (m.response_format | tojson) -}} + {%- endif -%} + + {%- elif m.role == "user" -%} + {{- "<｜User｜>" -}} + {%- set up = namespace(parts=[]) -%} + {%- for b in m.content_blocks -%} + {%- if b.type == "text" -%} + {%- if IMG in (b.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%} + {%- set up.parts = up.parts + [b.text or ""] -%} + {%- elif b.type in ["image", "image_url"] -%} + {%- set up.parts = up.parts + [IMG] -%} + {%- elif b.type == "tool_result" -%} + {%- set tc = b.get("content", "") -%} + {%- if tc is iterable and tc is not string and tc is not mapping -%} + {%- set sub = namespace(parts=[]) -%} + {%- for s in tc -%} + {%- if s.type == "text" -%} + {%- if IMG in (s.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%} + {%- set sub.parts = sub.parts + [s.get("text", "")] -%} + {%- elif s.type in ["image", "image_url"] -%}{%- set sub.parts = sub.parts + [IMG] -%} + {%- else -%}{%- set sub.parts = sub.parts + ["[Unsupported " ~ s.type ~ "]"] -%} + {%- endif -%} + {%- endfor -%} + {%- set tc = sub.parts | join("\n\n") -%} + {%- endif -%} + {%- set up.parts = up.parts + ["" ~ tc ~ ""] -%} + {%- else -%} + {%- set up.parts = up.parts + ["[Unsupported " ~ b.type ~ "]"] -%} + {%- endif -%} + {%- endfor -%} + {{- up.parts | join("\n\n") -}} + + {%- elif m.role == "latest_reminder" -%} + {{- "<｜latest_reminder｜>" ~ (content or "") -}} + + {%- elif m.role == "assistant" -%} + {%- set prev_has_task = idx > 0 and msgs[idx - 1].get("task") is not none -%} + {%- if is_thinking and not prev_has_task and (not effective_drop or idx > state.last_user) -%} + {{- ((m.reasoning_content if m.get("reasoning_content") is not none else m.get("thinking")) or "") ~ "" -}} + {%- endif -%} + {{- content or "" -}} + {%- if m.get("tool_calls") -%} + {%- set tcs = namespace(lines=[]) -%} + {%- for tc in m.tool_calls -%} + {%- set fn = tc.function if tc.function is defined else tc -%} + {%- set params = namespace(lines=[]) -%} + {%- set args = fn.arguments if fn.arguments is mapping else {"arguments": fn.arguments} -%} + {%- for k, v in args.items() -%} + {%- set params.lines = params.lines + ["<" ~ DSML ~ " parameter name=\"" ~ k ~ "\" string=\"" ~ ("true" if v is string else "false") ~ "\">" ~ (v if v is string else v | tojson) ~ ""] -%} + {%- endfor -%} + {%- set name = qualified_name(fn.name, tc.get("namespace") or fn.get("namespace")) -%} + {%- set tcs.lines = tcs.lines + ["<" ~ DSML ~ " invoke name=\"" ~ name ~ "\">\n" ~ params.lines | join("\n") ~ "\n"] -%} + {%- endfor -%} + {{- "\n\n<" ~ DSML ~ " calls>\n" ~ tcs.lines | join("\n") ~ "\n" -}} + {%- endif -%} + {%- if not m.get("wo_eos", false) -%}{{- "<｜end▁of▁sentence｜>" -}}{%- endif -%} + + {%- else -%} + {{- raise_exception("Unknown role: " ~ m.role) -}} + {%- endif -%} + + {#- Transition tokens, only when the next message is an assistant/latest_reminder turn or this is the last one. -#} + {%- if loop.last or msgs[idx + 1].role in ["assistant", "latest_reminder"] -%} + {%- set task = m.get("task") -%} + {%- if task is not none -%} + {%- set task_tokens = {"action": "<｜action｜>", "query": "<｜query｜>", "authority": "<｜authority｜>", "domain": "<｜domain｜>", "title": "<｜title｜>", "read_url": "<｜read_url｜>"} -%} + {%- if task not in task_tokens -%}{{- raise_exception("Invalid task: '" ~ task ~ "'") -}}{%- endif -%} + {%- if task == "action" -%}{{- "<｜Assistant｜>" ~ ("" if is_thinking else "") -}}{%- endif -%} + {{- task_tokens[task] -}} + {%- elif (m.role == "user" or (m.role == "system" and idx > 0)) and (add_generation_prompt or not loop.last) -%} + {{- "<｜Assistant｜>" ~ ("" if is_thinking and (not effective_drop or idx >= state.last_user) else "") -}} + {%- endif -%} + {%- endif -%} +{%- endfor -%} diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 01b2220da4e4..03f64a76f406 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -4381,6 +4381,146 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .run(); } + // DeepSeek V4.1 tests - V4 behaviour, DSML tag names with a leading space: + // <｜DSML｜ calls> / <｜DSML｜ invoke name="..."> / <｜DSML｜ parameter name="..." string="..."> + { + auto tst = peg_tester("models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja", detailed_debug); + + // Pure content (non-thinking mode; generation prompt ends with ) + tst.test("Hello, world!\nWhat's up?") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .expect(message_assist) + .run(); + + // Thinking + content + tst.test("I'm\nthinkingHello, world!\nWhat's up?") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .expect(message_assist_thoughts) + .run(); + + // Thinking + tool call (single, string param) + tst.test( + "Let me check the time\n\n" + "<｜DSML｜ calls>\n" + "<｜DSML｜ invoke name=\"get_time\">\n" + "<｜DSML｜ parameter name=\"city\" string=\"true\">Tokyo\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ get_time_tool }) + .expect(message_with_tool_calls_and_reasoning("get_time", R"({"city": "Tokyo"})", "Let me check the time")) + .run(); + + // Tool call without reasoning (non-thinking mode), integer param (string="false") + tst.test( + "<｜DSML｜ calls>\n" + "<｜DSML｜ invoke name=\"special_function\">\n" + "<｜DSML｜ parameter name=\"arg1\" string=\"false\">1\n" + "\n" + "") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ special_function_tool }) + .expect(message_assist_call) + .run(); + + // Multiple parallel tool calls with reasoning + tst.test( + "Calling both\n\n" + "<｜DSML｜ calls>\n" + "<｜DSML｜ invoke name=\"get_time\">\n" + "<｜DSML｜ parameter name=\"city\" string=\"true\">Paris\n" + "\n" + "<｜DSML｜ invoke name=\"get_weather\">\n" + "<｜DSML｜ parameter name=\"city\" string=\"true\">Paris\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .parallel_tool_calls(true) + .tools({ get_time_tool, get_weather_tool }) + .expect(message_with_reasoning_content_and_multiple_tool_calls( + "Calling both", "", + { { "get_time", R"({"city": "Paris"})" }, { "get_weather", R"({"city": "Paris"})" } })) + .run(); + + // Tool call with content before tool calls + tst.test( + "Thinking about it" + "Let me call the function.\n\n" + "<｜DSML｜ calls>\n" + "<｜DSML｜ invoke name=\"special_function\">\n" + "<｜DSML｜ parameter name=\"arg1\" string=\"false\">1\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ special_function_tool }) + .expect_reasoning("Thinking about it") + .expect_content("Let me call the function.") + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + }) + .expect_reconstruction() + .run(); + + // Tool call with multiple params (mixed types) + tst.test( + "Multi-arg call\n\n" + "<｜DSML｜ calls>\n" + "<｜DSML｜ invoke name=\"magic_int\">\n" + "<｜DSML｜ parameter name=\"ref\" string=\"false\">42\n" + "<｜DSML｜ parameter name=\"name\" string=\"true\">foo bar\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ magic_int_tool }) + .expect_reasoning("Multi-arg call") + .expect_tool_calls({ + { "magic_int", R"({"ref": 42, "name": "foo bar"})", {} }, + }) + .run(); + + // Continuation tests + tst.test("world!\nWhat's up?") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .enable_thinking(true) + .messages({ message_user, message_assist_prefill_content }) + .add_generation_prompt(false) + .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT) + .expect_reasoning("I'm thinking") + .expect_content("Hello, world!\nWhat's up?") + .run(); + + tst.test(" thinkingHello, world!\nWhat's up?") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .enable_thinking(true) + .messages({ message_user, message_assist_prefill_reasoning }) + .add_generation_prompt(false) + .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING) + .expect_reasoning("I'm thinking") + .expect_content("Hello, world!\nWhat's up?") + .run(); + + tst.test( + "Let me check the time\n\n" + "<｜DSML｜ calls>\n" + "<｜DSML｜ invoke name=\"get_time\">\n" + "<｜DSML｜ parameter name=\"city\" string=\"true\">Tokyo\n" + "\n" + "") // no after the TC close because the grammar will immediately constrain it to end + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ get_time_tool }) + .expect_reasoning("Let me check the time") + .expect_tool_calls({ { "get_time", R"({"city": "Tokyo"})", {} } }) + .run(); + } + { // The DSML separator belongs to the tool call block, not assistant content. auto tst = peg_tester("models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja", detailed_debug); From eeba836308083b0688bd1e131a2fb01a522b81fe Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 07:01:27 -0700 Subject: [PATCH 18/49] deepseek41: STATUS_A critique fold-in (boundary fixture, precision attribution, V4-Flash regression, A3) --- STATUS_A.md | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/STATUS_A.md b/STATUS_A.md index 285764d1dd53..8d2a19224b33 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -7,7 +7,7 @@ Branch `dsv41` in `$W/wt_dsv41` (off llama.cpp_pinset fd55cb98f). Contract for b |---|---|---| | A1 recon + reuse map + fixture + converter + graph | done | reuse map in deepseek41.cpp header (V4 HC/MLA/MoE/raw cache reused; new: Engram, lagged mHC, ratio 1/2 + sharing + candidates). Two fixtures: `tiny_fixture` (small dims, fast, seeded-bug harness) and `tiny_fixture_realdim` (A1 rule: REAL dims 5120 / 64x512 heads / rope 64 / q 1280 / o 8x1024 / expert 2304 / index 32x128 / Engram 8x256 n-gram 4 / window 128 / block 8 / YaRN 65536x16; only counts shrunk: 10 layers, 8 experts top-2, Engram bucket start 1009, index top-k 64, 16 candidate blocks). Rare-path thresholds: window 128, ratio-2 top-k binds > 128 tokens, candidate mask binds > 128 ratio-1 rows; prompt wiki1100 exceeds all. Reference = official model.py (no transformers implementation of deepseek_v41 exists in 4.57 / 5.x, so ref_logits.py --generic cannot load it); the fixture writes the ref_logits.py npz schema + full logits. Fixture generates under the reference (prefill vs decode max abs 2.7e-5 at real dims). | | A2 parity (bundled tools/dump_logits + port/compare_ref.py, F32 gate --max-kl-mean 1e-6 --min-top1 1.0 --min-cos 0.99999) | done (CPU), CUDA calibrated | see "A2 numbers" below; 13/13 seeded bugs FAIL on the small fixture, 5/5 on real dims | -| A3 template + V4.1 DSML tool parser | not started (next) | template candidate exists: models/dsv41/chat_template.jinja (5/5 byte parity, Phase 0) | +| A3 template + V4.1 DSML tool parser | done (1d51c092f) | template `models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja` (= `$W/models/dsv41/chat_template.jinja`, 5/5 byte parity with encoding.py). V4.1 tags carry a leading space (`<｜DSML｜ calls>`, `<｜DSML｜ invoke>`, `<｜DSML｜ parameter>`); detection in common/chat.cpp keys on `｜DSML｜ calls>` (the template never names `dsml_token`), parser = the V4 parser with the spaced tags. test-chat: the V4 parser suite ported to V4.1 markup (content, thinking, single / parallel / mixed-type calls, content before calls, continuation, reconstruction) all pass; negative control (V4.1 tag switch off) aborts on the V4.1 block. | | A4 never break other models | done | test-chat all passed; test-llama-archs CPU all 137 passed; CUDA: full suite exceeded its 2 h box (killed, no verdict), so the archs sharing touched code were run: deepseek4/41, deepseek2/32, glm-dsa, glm5-next, maple, qwen3moe, qwen35moe, qwen4exp on CUDA + CPU + Meta: 28/28 passed; deepseek41 on CPU / CUDA / Meta OK; test-backend-ops for used ops pass; V4 logits bitwise identical base vs head; no vocab change (vocab differential N/A) | | A5 pin PR / pinset sim | not started (after A3 and B's GGUF) | | @@ -31,6 +31,52 @@ Branch `dsv41` in `$W/wt_dsv41` (off llama.cpp_pinset fd55cb98f). Contract for b - CUDA graphs on vs off (B10 A/B): identical results, no hang under the 1200 s watchdog (timeout -k 15 => SIGKILL). - FA on (F16 attention) is an acceptance config, not a parity config (see older numbers below). +## Critique fold-in (plans/dsv41_support.md "Critique fold-in") +1. No per-head q RMS norm: the graph has none; "V4 per-head q norm" is seeded bug #6 and FAILS (KL 4e-3..2). +2. MATCHED oracle: the fixture ref/ IS the matched reference (shim act_quant / fp4_act_quant = identity); every gate above is + against it. NATIVE (in-graph FP8/FP4 activation quant emulated on CPU, `scripts/dsv41/native_gap.py`) vs MATCHED: + small KL 0.24..0.60, real dims 0.69..1.47 (random weights are chaotic); llama vs MATCHED 1.7e-11..2.7e-11 on the same + prompts. The gap belongs to the oracle, not to the port; on the real model gate against MATCHED (KL < 1e-3 + p99). +3. Harness uses full logits: `ref_forward` asserts `[1, T, vocab]` (ParallelHead full_logits path), dumps are [T, V]. +4. Epsilons kept separate: norms 1e-20 (as `RMSNorm.eps` in model.py), hc_eps 1e-6, router weight sum +1e-20 (2f4055f00; + V4 keeps its 6.1e-5 clamp); norm statistics in F32. +5. Boundaries, `tiny_fixture_budget` (real budgets on small dims: window 128, index top-k 512, candidate pool 2048 x 8 = + 16384 rows, YaRN 65536 x 16; one 17,000-token prompt, so every t is its own boundary: window 127/128/129, top-k + 511/512/513 at both ratios, pool 16383/16384/16385; 616 rows with real exclusion, up to 77 blocks excluded): + - candidate invariants (reference): newest partial block kept on every row, kept-block count = min(visible, 2048) on + every row, real exclusion present: PASS. + - CPU llama.cpp, single shot / ub 4096 / ub 1025 (odd ratio-2 groups split across ubatches): logits and every + selected set bitwise identical across the three splits. + - vs the fp32 reference: top-1 99.75%, KL median ~1e-7, mean 3.0e-4 (p99 3.9e-4, max 0.42), i.e. heavy-tailed, and 0 + below t=1100. Selection: L2 (first index layer) differs on 87 / 17000 rows, every one a single swap at rank 512/513; + llama-vs-reference index-score error (max over a row / row top score) median 1.3e-5, p99 4.7e-5; the reference's own + gap for each swapped pair is <= 2.4e-5 (median 2.3e-6), below the error p99 on 87 / 87 rows. Later layers inherit + the flips (L4 349, L6 1291, L8 2422 rows; picks outside the reference pool only on rows whose pool itself flipped). + The score error grows linearly with position (median 1.2e-6 at t < 1100 -> 2.5e-5 at t > 12000): float32 rope + angle rounding (|angle| up to ~1.7e4 rad, ~2e-4 rad rounding at t = 16000 on both sides, rounded differently: + ggml multiplies theta by theta_scale per pair, model.py multiplies t by a rounded freq). + Attribution (`scripts/dsv41/ref_precision_floor.py`, reference variants, same weights): + | reference variant | KL(variant ‖ ref32) | L2 / L4 / L6 / L8 rows differing from ref32 | llama vs variant: KL, L2 / L4 / L6 / L8 | + |---|---|---|---| + | float64 upcasts (fp32 angles) | 1.6e-4 | 8 / 53 / 616 / 1267 | 4.0e-4, 93 / 388 / 1721 / 3154 | + | exact (float64) rope angles | 6.6e-4 | 139 / 623 / 2519 / 4200 | 7.5e-4, 150 / 687 / 2781 / 4675 | + | ggml-rounded rope angles | 3.7e-4 | 87 / 338 / 1371 / 2648 | 1.1e-4, 8 / 40 / 274 / 701 | + With the reference rounding its rope angles the way ggml does, llama.cpp's L2 differences fall from 87 to 8 rows, + the same count as the reference's own fp32-vs-fp64 noise; with exact angles the fp32 reference is itself further + from the truth than llama.cpp is from it. Classified as precision (rope-angle rounding + fp32 arithmetic), no defect. + - CUDA (`GGML_CUDA_F32_EXACT=1`): same shape, more flips (L2 1539 rows, 97% single swaps; the >= 3-swap rows all sit + in the 128-token window after one L0 router near-tie at t=16017). KL mean 3.2e-3, top-1 98.2%; decode (ub 1, 17,000 + steps) gives the same numbers (KL 3.1e-3). CUDA rope rounds theta differently again (`pos*powf(theta_scale, i)`). + The 1e-6 F32 gate holds wherever the top-k does not bind on a near-tie (all of small / real-dims fixtures, and + t < 1100 here); at the real budgets the right gate is selection parity with gap evidence, as above. +6. Slot reuse / prefix restore (`scripts/dsv41/state_test.cpp`): save seq 0 after n1 tokens, clear memory, restore into + seq 0 or 1, continue: PASS (n1 201 / 17 / 164). Real defect found and fixed here: the Engram n-gram token history was + not in the saved state (2f4055f00, `has_cell_ext()`); negative control with the fix off FAILS (KL 0.115). + Compressor state rides the existing DSV4 snapshot path. +7. V4-Flash regression on the real model: unsloth/DeepSeek-V4-Flash-0731-GGUF UD-IQ1_S (77 GB, downloaded for this + check), CUDA, 600 tokens, F32 KV, FA off: logits bitwise identical base (fd55cb98f) vs head at ub 512 and ub 7; head + A/A also bitwise. Plus the random V4 model (bitwise, ub 256 / 7). + ## Milestones | # | state | commit | |---|---|---| @@ -101,6 +147,9 @@ Sizes: experts 289 GB (MXFP4), dense ~14 GB BF16 (FP8 dequant), embed+head 2.7 G fake-quant on a few prompts (`DSV41_FAKE_QUANT=1` in the shim emulates it on CPU). ## Not proven / open +- A5 (pin PR / pinset sim) not started; full-model parity waits for B's GGUF and the oracle. +- The F32 KL gate cannot be met at the real top-k budgets on any implementation (the fp32 reference misses its own + float64-rope version by KL 6.6e-4); long-context acceptance = selection parity with near-tie gap evidence. - Image tokens: Engram DEAD / gate-off for embd inputs is hashed as DEAD but the gate is not forced to 0 yet; vision not ported. - Multi-sequence batches (n_seq_max > 1) and rollback snapshots (n_rs_seq > 0) are wired like V4 but not parity-tested. - No ubatch shares a token across sequences (asserts, as the PLE path). From 8e3b617878f050f115ccf69cb7528460a8629b3f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 07:38:23 -0700 Subject: [PATCH 19/49] deepseek41: STATUS_A gate on the shared reference fixtures (F32 experts, all pass) --- STATUS_A.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/STATUS_A.md b/STATUS_A.md index 8d2a19224b33..6a209d85807a 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -31,6 +31,21 @@ Branch `dsv41` in `$W/wt_dsv41` (off llama.cpp_pinset fd55cb98f). Contract for b - CUDA graphs on vs off (B10 A/B): identical results, no hang under the 1200 s watchdog (timeout -k 15 => SIGKILL). - FA on (F16 attention) is an acceptance config, not a parity config (see older numbers below). +## Gate on agent C's fixtures (the parity oracle from now on; temp/dsv41_fixtures, refs/

/matched.npy) +Build: `temp/wt_dsv41_gate` = dsv41 + B's dsv41-mxfp8 (MXFP8 Engram), tools `temp/dsv41_tools/gate_cpu/`. GGUF: B's +`-F32.gguf` with MXFP4 experts dequantized to F32 (`models/dsv41/c_fx/-F32X.gguf`, scripts/dsv41/experts_to_f32.py; +with MXFP4 experts the CPU matmul quantizes activations to q8_0 and the gate fails at ~1e-3, as C found). Runner +`scripts/dsv41/gate_c.sh`. CPU, F32 KV, FA off. Gate KL(matched || llama) mean < 1e-3: PASS on every fixture. +| fixture | prompts | KL mean (range) | worst p99 | top-1 min | +|---|---|---|---|---| +| fx_base (Engram 1, 6; MXFP8 tables) | t100..t3001, c1600, sp1600 | 9e-12 .. 2.3e-5 | 8.5e-5 | 1.0000 | +| fx_noengram | same | 9e-12 .. 1.3e-5 | 8.9e-5 | 0.9987 | +| fx_engram0 (Engram on layer 0) | same | 9e-12 .. 2.7e-5 | 1.6e-4 | 0.9987 | +| fx_long (real 2048-block pool) | t2001 / sp20001 (ub 4096) | 1.2e-5 / 5.1e-4 | 1.1e-2 | 0.9861 | +| fx_base ub 7 / ub 1 | t129, t1027, sp1600 / t129, t700 | identical to one-shot | | 1.0000 | +llama.cpp vs matched is the same size as matched_f32 vs matched_f64 (the fp32 floor). sp20001 is the long-context +near-tie regime characterised above. CUDA on these fixtures not yet run. + ## Critique fold-in (plans/dsv41_support.md "Critique fold-in") 1. No per-head q RMS norm: the graph has none; "V4 per-head q norm" is seeded bug #6 and FAILS (KL 4e-3..2). 2. MATCHED oracle: the fixture ref/ IS the matched reference (shim act_quant / fp4_act_quant = identity); every gate above is From d7e38355f5a81edea33f9a25aad04faab1f39354 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 07:43:16 -0700 Subject: [PATCH 20/49] deepseek41: STATUS_A CUDA gate and full-model dump handoff --- STATUS_A.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/STATUS_A.md b/STATUS_A.md index 6a209d85807a..e58950524586 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -44,7 +44,10 @@ with MXFP4 experts the CPU matmul quantizes activations to q8_0 and the gate fai | fx_long (real 2048-block pool) | t2001 / sp20001 (ub 4096) | 1.2e-5 / 5.1e-4 | 1.1e-2 | 0.9861 | | fx_base ub 7 / ub 1 | t129, t1027, sp1600 / t129, t700 | identical to one-shot | | 1.0000 | llama.cpp vs matched is the same size as matched_f32 vs matched_f64 (the fp32 floor). sp20001 is the long-context -near-tie regime characterised above. CUDA on these fixtures not yet run. +near-tie regime characterised above. CUDA (`GGML_CUDA_F32_EXACT=1`, gate build `build-cuda`): fx_base t129 / t1027 / t3001 +KL 8.6e-12 / 4.5e-9 / 5.8e-5, top-1 1.0 / 1.0 / 0.999: PASS. +Full model: `scripts/dsv41/full_dump.sh` (3 B200 x 125 GB, ~45 min, window requested from main) dumps the run-of-record +pkg/BF16 GGUF on C's fx_base token files + real-text wiki1100 / long17k, for the TP8 MATCHED oracle. ## Critique fold-in (plans/dsv41_support.md "Critique fold-in") 1. No per-head q RMS norm: the graph has none; "V4 per-head q norm" is seeded bug #6 and FAILS (KL 4e-3..2). From c9786d7e0cd2630d5a18dd5a99d88ea4baa83023 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 07:53:44 -0700 Subject: [PATCH 21/49] deepseek41: STATUS_A flash-attention attribution, multi-sequence and rollback parity --- STATUS_A.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/STATUS_A.md b/STATUS_A.md index e58950524586..b86a436594bd 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -95,6 +95,26 @@ pkg/BF16 GGUF on C's fx_base token files + real-text wiki1100 / long17k, for the check), CUDA, 600 tokens, F32 KV, FA off: logits bitwise identical base (fd55cb98f) vs head at ub 512 and ub 7; head A/A also bitwise. Plus the random V4 model (bitwise, ub 256 / 7). +## Flash attention (fa=1) and multi-sequence / rollback (coordinator items 1, 2) +- FA graph is exact: with K/V kept F32 inside the FA op (debug-only patch, not committed) fa=1 matches the reference at + KL 5.0e-12 / 5.0e-12 / 4.6e-12 (short / wiki330 / rep300): mask concat (window + top-k -inf), attn_sinks, padding and + the compressed-row concat are right. FA accumulation is already GGML_PREC_F32 for every arch (llama-graph.cpp). +- The fa=1 excess is the F16 kernel: llama vs the reference with q/k/v rounded to F16 is still 1.9e-2 on short16 (floor + 7.9e-7, error from position 1 on, attn_raw-0 rel err 3e-4..2e-3 per row). The CPU single-chunk kernel also rounds Q to + F16 and accumulates V*softmax in an F16 VKQ16 buffer, which input rounding alone does not model; the small fixture + then amplifies it (routing flips). Not a V4.1 defect. +- What users see, on agent C's non-chaotic fx_base (F16 KV, fa=1, vs MATCHED): CPU KL 6.5e-8 (t129), 4.1e-5 (t1027), + 1.2e-4 (c1600), 3.4e-4 (t3001), top-1 >= 0.992; CUDA (default precision) 4.0e-4 / 3.0e-4 / 4.4e-4 / 9.3e-4, top-1 + >= 0.983. All < 1e-3 and 5-80x below C's bf16 floor (5.1e-3). +- Multi-sequence (`scripts/dsv41/seq_test.cpp multi`, n_seq_max 2, every ubatch interleaves wiki330 on seq 0 and rep300 + on seq 1, ub 7 and 64): each sequence vs its own single-sequence reference KL 4.6e-12..4.9e-12 (CPU), 1.6e-11..2.8e-11 + (CUDA exact), top-1 1.0. +- Rollback snapshots (`seq_test rollback`, n_rs_seq R; after each true chunk decode a draft whose tail is garbage tokens, + seq_rm the rejected tail, continue; odd chunk sizes so ratio-2 groups straddle the rollback point; garbage passes + through Engram history, compressor state and KV): R4/D4/acc1 (23 rollbacks), R4/D4/acc3, R2/D5/acc3: KL 4.9e-12 CPU, + 2.8e-11 CUDA, top-1 1.0. Negative control (keep one rejected token): KL 1.9, so the gate sees pollution. +- Slot reuse / prefix restore: state_test above (PASS). + ## Milestones | # | state | commit | |---|---|---| @@ -169,6 +189,5 @@ Sizes: experts 289 GB (MXFP4), dense ~14 GB BF16 (FP8 dequant), embed+head 2.7 G - The F32 KL gate cannot be met at the real top-k budgets on any implementation (the fp32 reference misses its own float64-rope version by KL 6.6e-4); long-context acceptance = selection parity with near-tie gap evidence. - Image tokens: Engram DEAD / gate-off for embd inputs is hashed as DEAD but the gate is not forced to 0 yet; vision not ported. -- Multi-sequence batches (n_seq_max > 1) and rollback snapshots (n_rs_seq > 0) are wired like V4 but not parity-tested. - No ubatch shares a token across sequences (asserts, as the PLE path). - DSpark drafting graph not implemented. From a60112c8b787726918212f3023dd137dac1afb0c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 07:58:46 -0700 Subject: [PATCH 22/49] deepseek41: STATUS_A selection-set gate vs shared traces --- STATUS_A.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/STATUS_A.md b/STATUS_A.md index b86a436594bd..c0017aeb8ecc 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -44,7 +44,12 @@ with MXFP4 experts the CPU matmul quantizes activations to q8_0 and the gate fai | fx_long (real 2048-block pool) | t2001 / sp20001 (ub 4096) | 1.2e-5 / 5.1e-4 | 1.1e-2 | 0.9861 | | fx_base ub 7 / ub 1 | t129, t1027, sp1600 / t129, t700 | identical to one-shot | | 1.0000 | llama.cpp vs matched is the same size as matched_f32 vs matched_f64 (the fp32 floor). sp20001 is the long-context -near-tie regime characterised above. CUDA (`GGML_CUDA_F32_EXACT=1`, gate build `build-cuda`): fx_base t129 / t1027 / t3001 +near-tie regime characterised above. +Selection sets vs C's matched_traces.pt (`scripts/dsv41/sel_vs_traces.py`, fx_base): t1027 L2-L7 0 rows, L8 2, L12 4, +pool 0; t3001 L2 5 .. L12 121 rows, pool 21 / 1977. Every mismatch is 1 swap; the earliest divergences (t=646, 877, 973, +998) have score gaps 7e-7..3.3e-6 of the row top score (near-ties); later ones follow earlier flips. Waiting on C's +f32-vs-f64 selection floor to close this gate. +CUDA (`GGML_CUDA_F32_EXACT=1`, gate build `build-cuda`): fx_base t129 / t1027 / t3001 KL 8.6e-12 / 4.5e-9 / 5.8e-5, top-1 1.0 / 1.0 / 0.999: PASS. Full model: `scripts/dsv41/full_dump.sh` (3 B200 x 125 GB, ~45 min, window requested from main) dumps the run-of-record pkg/BF16 GGUF on C's fx_base token files + real-text wiki1100 / long17k, for the TP8 MATCHED oracle. From 9f13d3534cf6be28c6cbdfae112d624eac76a594 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 08:18:05 -0700 Subject: [PATCH 23/49] deepseek41: STATUS_A first full-model results --- STATUS_A.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/STATUS_A.md b/STATUS_A.md index c0017aeb8ecc..88c7a5939cca 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -53,6 +53,10 @@ CUDA (`GGML_CUDA_F32_EXACT=1`, gate build `build-cuda`): fx_base t129 / t1027 / KL 8.6e-12 / 4.5e-9 / 5.8e-5, top-1 1.0 / 1.0 / 0.999: PASS. Full model: `scripts/dsv41/full_dump.sh` (3 B200 x 125 GB, ~45 min, window requested from main) dumps the run-of-record pkg/BF16 GGUF on C's fx_base token files + real-text wiki1100 / long17k, for the TP8 MATCHED oracle. +First full-model results (GPUs 0,2,3, ub 512, fa=0, F32 KV; load + 6 prompts in ~10 min): all finite; real-text +self-perplexity wiki1100 2.11 (next-token top-1 0.826), long17k 2.0 / 1.28 / 1.80 / 2.08 / 3.03 on [0,1k) / [1k,4k) / +[4k,8k) / [8k,16384) / [16384,17k); 256-token bands before the pool threshold already range 1.6..3.0 (text-dependent, no +jump at 16,392). ub 7 / ub 1 / fa=1 phases and the oracle comparison pending (outputs/dsv41_full/bf16_ub512). ## Critique fold-in (plans/dsv41_support.md "Critique fold-in") 1. No per-head q RMS norm: the graph has none; "V4 per-head q norm" is seeded bug #6 and FAILS (KL 4e-3..2). From 3fd0f278b17cd7b2e44126d96ecbf7aff2988b1a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 08:20:19 -0700 Subject: [PATCH 24/49] deepseek41: keep Engram tables in host memory under tensor overrides and split modes --- STATUS_A.md | 7 +++++++ src/llama-model-loader.cpp | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/STATUS_A.md b/STATUS_A.md index 88c7a5939cca..a787a754ec91 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -193,6 +193,13 @@ Sizes: experts 289 GB (MXFP4), dense ~14 GB BF16 (FP8 dequant), embed+head 2.7 G KV); the GGUF port does not, so expect KL above the fixture's 1e-11 even when correct. Measure the oracle with and without fake-quant on a few prompts (`DSV41_FAKE_QUANT=1` in the shim emulates it on CPU). +## Engram placement guard +`llama-model-loader.cpp`: an `engram_embd` placement on a non-host buffer (from `-ot`, or any split mode) is refused with a +warning and the table goes to the CPU buffer. Test `scripts/dsv41/engram_placement.cpp` (all layers offloaded, +override engram_embd -> CUDA0): with the guard, a warning per table, CUDA0 model buffer 78.90 MiB (= no override) and +logits bitwise equal to the run without the override; negative control (build without the guard): CUDA0 buffer +80.53 MiB, i.e. the tables were uploaded. + ## Not proven / open - A5 (pin PR / pinset sim) not started; full-model parity waits for B's GGUF and the oracle. - The F32 KL gate cannot be met at the real top-k budgets on any implementation (the fp32 reference misses its own diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index c5cab2d01279..8c3e43376139 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1270,6 +1270,15 @@ struct ggml_tensor * llama_model_loader::create_tensor( } } + // deepseek41 Engram tables (~100 GB each on the full model) are host-resident gather tables: the graph only + // get_rows a few rows per token on the CPU. Never let an override or a split mode put them in device memory. + if (tn.tensor == LLM_TENSOR_ENGRAM_EMBD && !ggml_backend_buft_is_host(buft)) { + LLAMA_LOG_WARN("llama_model_loader: %s (%zu MiB %s) must stay in host memory; ignoring placement on %s, using %s\n", + tn.str().c_str(), ggml_nbytes(t_meta) / 1024 / 1024, ggml_type_name(t_meta->type), + ggml_backend_buft_name(buft), ggml_backend_buft_name(ggml_backend_cpu_buffer_type())); + buft = ggml_backend_cpu_buffer_type(); + } + // avoid using a host buffer when using mmap auto * buft_dev = ggml_backend_buft_get_device(buft); if (use_mmap && buft_dev && buft == ggml_backend_dev_host_buffer_type(buft_dev)) { From 25c0ac367a9a590cd101815c4d51a84d25bc467c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 08:24:02 -0700 Subject: [PATCH 25/49] deepseek41: STATUS_A correct the stale MXFP4 fixture number --- STATUS_A.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/STATUS_A.md b/STATUS_A.md index a787a754ec91..2da30c5d2ad6 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -159,7 +159,11 @@ sweep: `bash scripts/dsv41/run_parity.sh "fa:ub .. itself with q/k/v rounded to F16 (4.5e-3 wiki, 1.7e-2 rep300): the random fixture's top-k / routing flips amplify F16 noise. - Mechanisms are live: KL(ref || ref without X) = Engram 3.1, candidate mask 0.17, top-k reuse (dense instead) 1.2, single-pass mHC (standard instead) 0.6; llama.cpp matches the full reference, not any ablation. -- MXFP4 experts (bit-exact repack verified): short16 KL 2.8e-4 from the Q8 activation quantisation of the MXFP4 matmul. +- MXFP4 experts (bit-exact repack verified): the 2.8e-4 short16 figure was from an older fixture and is stale. Current + small fixture: short 2.0e-2, wiki330 1.4e-1 (chaotic fixture). On C's fx_base (B's MXFP4-expert -F32 GGUF, CPU) vs + MATCHED: t129 3.2e-4, t1027 9.1e-4, t3001 2.1e-3 (top-1 0.965), vs the F32-expert copy at <= 2.3e-5: the CPU MXFP4 + matmul quantizes activations to q8_0 (C localised it: with q8_0 applied in the reference the expert outputs agree to + 5e-8). Kernel precision shared by every MXFP4 model, below the bf16 floor (5e-3), not a port defect. ## Other checks - test-llama-archs deepseek41: CPU OK, CUDA NMSE 3.3e-6, Meta (tensor split) OK, roundtrip OK. deepseek4: still OK. From a574da362f4abb9852473104be50295b1994689e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 09:22:17 -0700 Subject: [PATCH 26/49] deepseek41: STATUS_A full-model dump phases and batch-split attribution --- STATUS_A.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/STATUS_A.md b/STATUS_A.md index 2da30c5d2ad6..ee3b76f0bc42 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -56,7 +56,15 @@ pkg/BF16 GGUF on C's fx_base token files + real-text wiki1100 / long17k, for the First full-model results (GPUs 0,2,3, ub 512, fa=0, F32 KV; load + 6 prompts in ~10 min): all finite; real-text self-perplexity wiki1100 2.11 (next-token top-1 0.826), long17k 2.0 / 1.28 / 1.80 / 2.08 / 3.03 on [0,1k) / [1k,4k) / [4k,8k) / [8k,16384) / [16384,17k); 256-token bands before the pool threshold already range 1.6..3.0 (text-dependent, no -jump at 16,392). ub 7 / ub 1 / fa=1 phases and the oracle comparison pending (outputs/dsv41_full/bf16_ub512). +jump at 16,392). All phases done (outputs/dsv41_full/bf16_ub512/{ub512,ub7,ub1,fa1_ub512}), all finite; self-ppl +wiki1100 2.105 / 2.102 / 2.103 / 2.086 and long17k 1.862 / 1.862 / - / 1.858 for ub512 / ub7 / ub1 / fa1. +CUDA batch-split is not bitwise on the full model: ub7 vs ub512 KL mean 9.6e-3 (wiki1100), 3.4e-3 (long17k), 6e-2 on +the random-token t1025; fa1 vs ub512 7.4e-3 / 3.2e-3. Attribution: the full model on CPU (ngl 0, 12 wiki tokens, +~5 min) is bitwise identical at ub 512 and ub 7, and CPU vs each CUDA arm shows the same per-position profile (KL +3e-6 at t0, 0.08-0.2 at t5-t6) whichever ubatch / FA is used: those positions are numerically sensitive and CUDA's +kernels (MMQ / MMVQ, q8_1 activations) differ from CPU there. Released V4-Flash (UD-IQ1_S) on the same text: ub7 vs +ub512 KL mean 3.4e-2, 0.4 at t6. So the graph is batch-invariant; the CUDA spread is kernel numerics. Which side is +closer to the truth is for C's full-model MATCHED oracle (prompts: outputs/dsv41_full/bf16_ub512/tokens). ## Critique fold-in (plans/dsv41_support.md "Critique fold-in") 1. No per-head q RMS norm: the graph has none; "V4 per-head q norm" is seeded bug #6 and FAILS (KL 4e-3..2). From 47dd9807b2442ec91858f499b39fb568fecfd6c0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 09:23:21 -0700 Subject: [PATCH 27/49] models/templates: drop duplicate DeepSeek-V4.1 template (identical to the -Flash-1017 one) --- .../templates/deepseek-ai-DeepSeek-V4.1.jinja | 275 ------------------ 1 file changed, 275 deletions(-) delete mode 100644 models/templates/deepseek-ai-DeepSeek-V4.1.jinja diff --git a/models/templates/deepseek-ai-DeepSeek-V4.1.jinja b/models/templates/deepseek-ai-DeepSeek-V4.1.jinja deleted file mode 100644 index aac8438a37df..000000000000 --- a/models/templates/deepseek-ai-DeepSeek-V4.1.jinja +++ /dev/null @@ -1,275 +0,0 @@ -{#- DeepSeek-V4.1 chat template (for DeepSeek-V4.1-Flash-1017), ported from encoding/encoding.py in the model repo. - Based on the deepseek-ai/DeepSeek-V4.1-Flash template, with fixes so it matches encoding.py: - tool-result-only user turns do not start a new turn, tool_result content lists keep "[Unsupported x]" blocks, - top-level response_format is attached to the leading system message. - Extra kwargs: thinking_mode ("thinking" | "chat", default "thinking"; enable_thinking / thinking = true/false are - accepted as aliases), drop_thinking (default true), reasoning_effort (int in [1, 100] or "low"/"medium"/"high"/"max", - default "high"). Assistant reasoning is read from reasoning_content, or from thinking as an alias. - Tool call arguments must be objects (string arguments are rendered as a single "arguments" parameter). -#} -{%- set IMG = "<｜deepseek_image｜>" -%} -{%- set DSML = "｜DSML｜" -%} - -{%- if thinking_mode is not defined or thinking_mode is none -%} - {%- if enable_thinking is defined and enable_thinking is not none -%} - {%- set thinking_mode = "thinking" if enable_thinking else "chat" -%} - {%- elif thinking is defined and thinking is not none and thinking is not string -%} - {%- set thinking_mode = "thinking" if thinking else "chat" -%} - {%- else -%} - {%- set thinking_mode = "thinking" -%} - {%- endif -%} -{%- endif -%} -{%- if thinking_mode not in ["chat", "thinking"] -%} - {{- raise_exception("Invalid thinking_mode `" ~ thinking_mode ~ "`, should be \"chat\" or \"thinking\"") -}} -{%- endif -%} -{%- if drop_thinking is not defined or drop_thinking is none -%}{%- set drop_thinking = true -%}{%- endif -%} -{%- if add_generation_prompt is not defined -%}{%- set add_generation_prompt = false -%}{%- endif -%} -{%- if tools is not defined -%}{%- set tools = none -%}{%- endif -%} -{%- if response_format is not defined -%}{%- set response_format = none -%}{%- endif -%} - -{%- if reasoning_effort is not defined or reasoning_effort is none -%}{%- set reasoning_effort = "high" -%}{%- endif -%} -{%- set effort_map = {"low": 50, "medium": 62, "high": 75, "max": 100} -%} -{%- if reasoning_effort is string and reasoning_effort in effort_map -%} - {%- set budget = effort_map[reasoning_effort] -%} -{%- elif reasoning_effort is integer and reasoning_effort >= 1 and reasoning_effort <= 100 -%} - {%- set budget = reasoning_effort -%} -{%- else -%} - {{- raise_exception("Invalid reasoning effort: " ~ reasoning_effort ~ ", should be int within [1,100] or one of \"low\", \"medium\", \"high\", \"max\"") -}} -{%- endif -%} - -{#- Splits/validates "ns::name" against an explicit namespace and returns the qualified name. -#} -{%- macro qualified_name(name, ns) -%} - {%- if ns is mapping -%}{%- set ns = ns.name -%}{%- endif -%} - {%- set parts = name.split("::", 1) -%} - {%- if parts | length == 2 -%} - {%- if ns is not none and ns != parts[0] -%}{{- raise_exception("Conflicting tool namespaces: " ~ ns ~ " != " ~ parts[0]) -}}{%- endif -%} - {%- set ns = parts[0] -%} - {%- set name = parts[1] -%} - {%- endif -%} - {%- if "::" in name -%}{{- raise_exception("Tool name must not contain '::': " ~ name) -}}{%- endif -%} - {%- if ns is not none and "::" in ns -%}{{- raise_exception("Tool namespace must not contain '::': " ~ ns) -}}{%- endif -%} - {{- name if ns is none else ns ~ "::" ~ name -}} -{%- endmacro -%} - -{%- macro render_tools(tool_list) -%} - {%- set ts = namespace(schemas=[]) -%} - {%- for tool in tool_list -%} - {%- set fn = tool.function if tool.function is defined else tool -%} - {%- set ns = tool.namespace if tool.namespace is defined and tool.namespace is not none else fn.get("namespace") -%} - {#- JSON assembled by hand (key order kept, name/description replaced in place): llama.cpp has no ** unpacking. -#} - {%- set qname = qualified_name(fn.name, ns) -%} - {%- set sch = namespace(parts=[]) -%} - {%- for k, v in fn.items() if k != "namespace" -%} - {%- if k == "name" -%}{%- set v = qname -%} - {%- elif k == "description" and ns is mapping and ns.description -%}{%- set v = ns.description ~ "\n" ~ (fn.description or "") -%} - {%- endif -%} - {%- set sch.parts = sch.parts + [(k | tojson) ~ ": " ~ (v | tojson)] -%} - {%- endfor -%} - {%- if "description" not in fn and ns is mapping and ns.description -%} - {%- set sch.parts = sch.parts + ['"description": ' ~ ((ns.description ~ "\n") | tojson)] -%} - {%- endif -%} - {%- set ts.schemas = ts.schemas + ["{" ~ sch.parts | join(", ") ~ "}"] -%} - {%- endfor -%} - {{- "## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜ calls>\" block like the following:\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"$TOOL_NAME\">\n<｜DSML｜ parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n...\n\n<｜DSML｜ invoke name=\"$TOOL_NAME2\">\n...\n\n\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response.\n\nOtherwise, output directly after with tool calls or final response.\n\n### Available Tool Schemas\n\n" -}} - {{- ts.schemas | join("\n") -}} - {{- "\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n" -}} -{%- endmacro -%} - -{#- 1. Merge tool results (and consecutive user turns) into user messages made of content blocks. -#} -{%- set mns = namespace(list=[]) -%} -{%- for msg in messages -%} - {%- set content = msg.get("content") -%} - {%- set is_blocks = content is iterable and content is not string and content is not mapping -%} - {%- if content is string and IMG in content -%} - {{- raise_exception("Message content contains image special token " ~ IMG ~ ". Images should be provided as image content blocks.") -}} - {%- endif -%} - {%- set rc = msg.reasoning_content if msg.get("reasoning_content") is not none else msg.get("thinking") -%} - {%- if rc is string and IMG in rc -%} - {{- raise_exception("reasoning_content contains image special token " ~ IMG) -}} - {%- endif -%} - {%- set prev = mns.list[-1] if mns.list else none -%} - {%- if msg.role == "tool" -%} - {%- set tblock = {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content if content is not none else ""} -%} - {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined -%} - {%- set mns.list = mns.list[:-1] + [dict(prev, content_blocks=prev.content_blocks + [tblock])] -%} - {%- else -%} - {%- set mns.list = mns.list + [{"role": "user", "content_blocks": [tblock]}] -%} - {%- endif -%} - {%- elif msg.role == "user" -%} - {%- set blocks = content if is_blocks else [{"type": "text", "text": content or ""}] -%} - {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined and prev.get("task") is none -%} - {%- set mns.list = mns.list[:-1] + [dict(prev, content_blocks=prev.content_blocks + blocks)] -%} - {%- else -%} - {%- set mns.list = mns.list + [dict(msg, content_blocks=blocks)] -%} - {%- endif -%} - {%- else -%} - {%- set mns.list = mns.list + [msg] -%} - {%- endif -%} -{%- endfor -%} - -{#- Top-level tools / response_format attach to a leading system message, or to a new empty one. -#} -{%- if tools or response_format -%} - {%- if not (mns.list and mns.list[0].role == "system") -%} - {%- set mns.list = [{"role": "system", "content": ""}] + mns.list -%} - {%- endif -%} - {%- set first = mns.list[0] -%} - {%- if tools and not first.get("tools") -%}{%- set first = dict(first, tools=tools) -%}{%- endif -%} - {%- if response_format and not first.get("response_format") -%}{%- set first = dict(first, response_format=response_format) -%}{%- endif -%} - {%- set mns.list = [first] + mns.list[1:] -%} -{%- endif -%} - -{#- 2. Reorder tool results to match the tool_calls order of the preceding assistant message. -#} -{%- set sns = namespace(list=[], ids=[]) -%} -{%- for m in mns.list -%} - {%- if m.role == "assistant" and m.get("tool_calls") -%} - {%- set sns.ids = [] -%} - {%- for tc in m.tool_calls -%} - {%- set sns.ids = sns.ids + [tc.get("id") or (tc.function.get("id", "") if tc.function is defined else "")] -%} - {%- endfor -%} - {%- set sns.list = sns.list + [m] -%} - {%- elif m.role == "user" and m.content_blocks -%} - {%- set tb = namespace(blocks=[]) -%} - {%- for b in m.content_blocks if b.type == "tool_result" -%} - {%- set order = namespace(idx=0) -%} - {%- for tc_id in sns.ids -%} - {%- if tc_id and tc_id == b.get("tool_use_id", "") -%}{%- set order.idx = loop.index0 -%}{%- endif -%} - {%- endfor -%} - {%- set tb.blocks = tb.blocks + [dict(b, _order=order.idx)] -%} - {%- endfor -%} - {%- if tb.blocks | length > 1 and sns.ids -%} - {%- set sorted_blocks = tb.blocks | sort(attribute="_order") -%} - {%- set rb = namespace(blocks=[], i=0) -%} - {%- for b in m.content_blocks -%} - {%- if b.type == "tool_result" -%} - {%- set rb.blocks = rb.blocks + [sorted_blocks[rb.i]] -%} - {%- set rb.i = rb.i + 1 -%} - {%- else -%} - {%- set rb.blocks = rb.blocks + [b] -%} - {%- endif -%} - {%- endfor -%} - {%- set sns.list = sns.list + [dict(m, content_blocks=rb.blocks)] -%} - {%- else -%} - {%- set sns.list = sns.list + [m] -%} - {%- endif -%} - {%- else -%} - {%- set sns.list = sns.list + [m] -%} - {%- endif -%} -{%- endfor -%} -{%- set msgs = sns.list -%} - -{#- Last user turn: users and mid-conversation systems count, except user turns made only of tool results. - Tools anywhere disable drop_thinking. -#} -{%- set state = namespace(last_user=-1, has_tools=false) -%} -{%- for m in msgs -%} - {%- if m.role == "user" or (m.role == "system" and loop.index0 > 0) -%} - {%- set real = namespace(v=true) -%} - {%- if m.get("content_blocks") -%} - {%- set real.v = false -%} - {%- for b in m.content_blocks if b.type in ["text", "image", "image_url"] -%}{%- set real.v = true -%}{%- endfor -%} - {%- endif -%} - {%- if real.v -%}{%- set state.last_user = loop.index0 -%}{%- endif -%} - {%- endif -%} - {%- if m.get("tools") -%}{%- set state.has_tools = true -%}{%- endif -%} -{%- endfor -%} -{%- set effective_drop = drop_thinking and not state.has_tools -%} -{%- set is_thinking = thinking_mode == "thinking" -%} - -{{- "<｜begin▁of▁sentence｜>" -}} -{#- 3. Render each message. -#} -{%- for m in msgs -%} - {%- set idx = loop.index0 -%} - {%- set content = m.get("content") -%} - {%- if content is iterable and content is not string and content is not mapping -%} - {%- set sp = namespace(parts=[]) -%} - {%- for b in content -%} - {%- if b.type == "text" -%}{%- set sp.parts = sp.parts + [b.text or ""] -%} - {%- elif b.type in ["image", "image_url"] -%}{%- set sp.parts = sp.parts + [IMG] -%} - {%- endif -%} - {%- endfor -%} - {%- set content = sp.parts | join("\n\n") -%} - {%- endif -%} - - {%- if idx == 0 -%} - {%- if is_thinking or m.role == "system" -%}{{- "<｜System｜>" -}}{%- endif -%} - {%- if is_thinking -%}{{- "Reasoning Effort: " ~ budget ~ " (range 1-100, the higher the value, the more thorough the reasoning)\n\n" -}}{%- endif -%} - {%- endif -%} - - {%- if m.role == "system" -%} - {%- if idx > 0 -%}{{- "<｜System｜>" -}}{%- endif -%} - {{- content or "" -}} - {%- if m.get("tools") -%}{{- "\n\n" ~ render_tools(m.tools) -}}{%- endif -%} - {%- if m.get("response_format") -%} - {{- "\n\n## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n" ~ (m.response_format | tojson) -}} - {%- endif -%} - - {%- elif m.role == "user" -%} - {{- "<｜User｜>" -}} - {%- set up = namespace(parts=[]) -%} - {%- for b in m.content_blocks -%} - {%- if b.type == "text" -%} - {%- if IMG in (b.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%} - {%- set up.parts = up.parts + [b.text or ""] -%} - {%- elif b.type in ["image", "image_url"] -%} - {%- set up.parts = up.parts + [IMG] -%} - {%- elif b.type == "tool_result" -%} - {%- set tc = b.get("content", "") -%} - {%- if tc is iterable and tc is not string and tc is not mapping -%} - {%- set sub = namespace(parts=[]) -%} - {%- for s in tc -%} - {%- if s.type == "text" -%} - {%- if IMG in (s.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%} - {%- set sub.parts = sub.parts + [s.get("text", "")] -%} - {%- elif s.type in ["image", "image_url"] -%}{%- set sub.parts = sub.parts + [IMG] -%} - {%- else -%}{%- set sub.parts = sub.parts + ["[Unsupported " ~ s.type ~ "]"] -%} - {%- endif -%} - {%- endfor -%} - {%- set tc = sub.parts | join("\n\n") -%} - {%- endif -%} - {%- set up.parts = up.parts + ["" ~ tc ~ ""] -%} - {%- else -%} - {%- set up.parts = up.parts + ["[Unsupported " ~ b.type ~ "]"] -%} - {%- endif -%} - {%- endfor -%} - {{- up.parts | join("\n\n") -}} - - {%- elif m.role == "latest_reminder" -%} - {{- "<｜latest_reminder｜>" ~ (content or "") -}} - - {%- elif m.role == "assistant" -%} - {%- set prev_has_task = idx > 0 and msgs[idx - 1].get("task") is not none -%} - {%- if is_thinking and not prev_has_task and (not effective_drop or idx > state.last_user) -%} - {{- ((m.reasoning_content if m.get("reasoning_content") is not none else m.get("thinking")) or "") ~ "" -}} - {%- endif -%} - {{- content or "" -}} - {%- if m.get("tool_calls") -%} - {%- set tcs = namespace(lines=[]) -%} - {%- for tc in m.tool_calls -%} - {%- set fn = tc.function if tc.function is defined else tc -%} - {%- set params = namespace(lines=[]) -%} - {%- set args = fn.arguments if fn.arguments is mapping else {"arguments": fn.arguments} -%} - {%- for k, v in args.items() -%} - {%- set params.lines = params.lines + ["<" ~ DSML ~ " parameter name=\"" ~ k ~ "\" string=\"" ~ ("true" if v is string else "false") ~ "\">" ~ (v if v is string else v | tojson) ~ ""] -%} - {%- endfor -%} - {%- set name = qualified_name(fn.name, tc.get("namespace") or fn.get("namespace")) -%} - {%- set tcs.lines = tcs.lines + ["<" ~ DSML ~ " invoke name=\"" ~ name ~ "\">\n" ~ params.lines | join("\n") ~ "\n"] -%} - {%- endfor -%} - {{- "\n\n<" ~ DSML ~ " calls>\n" ~ tcs.lines | join("\n") ~ "\n" -}} - {%- endif -%} - {%- if not m.get("wo_eos", false) -%}{{- "<｜end▁of▁sentence｜>" -}}{%- endif -%} - - {%- else -%} - {{- raise_exception("Unknown role: " ~ m.role) -}} - {%- endif -%} - - {#- Transition tokens, only when the next message is an assistant/latest_reminder turn or this is the last one. -#} - {%- if loop.last or msgs[idx + 1].role in ["assistant", "latest_reminder"] -%} - {%- set task = m.get("task") -%} - {%- if task is not none -%} - {%- set task_tokens = {"action": "<｜action｜>", "query": "<｜query｜>", "authority": "<｜authority｜>", "domain": "<｜domain｜>", "title": "<｜title｜>", "read_url": "<｜read_url｜>"} -%} - {%- if task not in task_tokens -%}{{- raise_exception("Invalid task: '" ~ task ~ "'") -}}{%- endif -%} - {%- if task == "action" -%}{{- "<｜Assistant｜>" ~ ("" if is_thinking else "") -}}{%- endif -%} - {{- task_tokens[task] -}} - {%- elif (m.role == "user" or (m.role == "system" and idx > 0)) and (add_generation_prompt or not loop.last) -%} - {{- "<｜Assistant｜>" ~ ("" if is_thinking and (not effective_drop or idx >= state.last_user) else "") -}} - {%- endif -%} - {%- endif -%} -{%- endfor -%} From 8f6d7b618c13d5c8eaed46062fb83d96b18941ea Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 09:30:21 -0700 Subject: [PATCH 28/49] deepseek41: Engram gate off for embedding (image) tokens, NULL token history for embd cells --- STATUS_A.md | 11 ++++++++++- src/llama-kv-cache.cpp | 3 ++- src/models/deepseek41.cpp | 29 ++++++++++++++++++++++++++++- src/models/models.h | 1 + 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/STATUS_A.md b/STATUS_A.md index ee3b76f0bc42..a9e7fa37a62b 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -205,6 +205,15 @@ Sizes: experts 289 GB (MXFP4), dense ~14 GB BF16 (FP8 dequant), embed+head 2.7 G KV); the GGUF port does not, so expect KL above the fixture's 1e-11 even when correct. Measure the oracle with and without fake-quant on a few prompts (`DSV41_FAKE_QUANT=1` in the shim emulates it on CPU). +## Image-span Engram (embedding inputs) +model.py: image tokens are hashed DEAD (no n-gram spans them) and the Engram gate is 0 there; routing uses bias_vl. +llama.cpp: embd tokens hash DEAD, KV cells of embd tokens record LLAMA_TOKEN_NULL (dsv41_tok_hist), and embd ubatches +get an `engram_keep` input (0 for embd tokens) multiplied into the gate; text-only graphs are unchanged (no extra op). +Test (`scripts/dsv41/image_ref.py` + `embd_test.cpp`, small fixture, 48 wiki tokens, [16,24) fed as an embedding +ubatch of the same token rows, reference with token_types image mask and bias_vl): KL 4.4e-12 CPU, 5.2e-12 CUDA, +top-1 1.0; without the gate fix KL 0.97; image vs text reference differ by KL 1.0 (the test can fail). Text parity +unchanged (4.6e-12..4.9e-12 at ub 7), test-llama-archs CPU 137/137. + ## Engram placement guard `llama-model-loader.cpp`: an `engram_embd` placement on a non-host buffer (from `-ot`, or any split mode) is refused with a warning and the table goes to the CPU buffer. Test `scripts/dsv41/engram_placement.cpp` (all layers offloaded, @@ -216,6 +225,6 @@ logits bitwise equal to the run without the override; negative control (build wi - A5 (pin PR / pinset sim) not started; full-model parity waits for B's GGUF and the oracle. - The F32 KL gate cannot be met at the real top-k budgets on any implementation (the fp32 reference misses its own float64-rope version by KL 6.6e-4); long-context acceptance = selection parity with near-tie gap evidence. -- Image tokens: Engram DEAD / gate-off for embd inputs is hashed as DEAD but the gate is not forced to 0 yet; vision not ported. +- Vision tower not ported (the text side of image spans is done, see below). - No ubatch shares a token across sequences (asserts, as the PLE path). - DSpark drafting graph not implemented. diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index b4c399b445a2..40e9f16faa64 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1150,7 +1150,8 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & cells.pos_set(idx, ubatch.pos[i]); - if (ubatch.is_pos_2d() || ubatch.token || hparams.ple_n_heads > 0) { + // dsv41_tok_hist: embd cells must record LLAMA_TOKEN_NULL (Engram treats it as DEAD), never a stale id + if (ubatch.is_pos_2d() || ubatch.token || hparams.ple_n_heads > 0 || hparams.dsv41_tok_hist) { llama_kv_cell_ext ext; if (ubatch.is_pos_2d()) { diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index aaf77def3cdf..bbb4a850c70c 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -358,6 +358,7 @@ class llm_graph_input_dsv41 : public llm_graph_input_i { bool can_reuse(const llm_graph_params & params) override; std::vector engram_ids; // per engram layer: I32 [n_cols*n_tokens] + ggml_tensor * engram_keep = nullptr; // F32 [1, 1, n_tokens], 0 for embedding (image) tokens; only in embd ubatches ggml_tensor * cand_pin = nullptr; // F32 [n_blk, n_tokens/n_stream, 1, n_stream] bool cand_csa = false; // candidate source reads the CSA (ratio 2) slot, else HCA (ratio 1) @@ -367,6 +368,7 @@ class llm_graph_input_dsv41 : public llm_graph_input_i { std::vector prev; std::vector ids; std::vector pin; + std::vector keep; private: const llama_kv_cache_dsv4_context::comp_plan & cand_plan(const llama_ubatch & ubatch) const { @@ -390,6 +392,16 @@ void llm_graph_input_dsv41::set_input(const llama_ubatch * ubatch) { } } + // image tokens take no part in an n-gram (hashed DEAD above) and get no Engram contribution (gate 0), as model.py + if (engram_keep && engram_keep->buffer) { + keep.resize(ubatch->n_tokens); + for (uint32_t i = 0; i < ubatch->n_tokens; ++i) { + const bool is_embd = !ubatch->token || (ubatch->is_mixed() && ubatch->type[i]); + keep[i] = is_embd ? 0.0f : 1.0f; + } + ggml_backend_tensor_set(engram_keep, keep.data(), 0, keep.size()*sizeof(float)); + } + if (cand_pin && cand_pin->buffer) { const auto & plan = cand_plan(*ubatch); const int64_t n_blk = cand_pin->ne[0]; @@ -415,6 +427,10 @@ bool llm_graph_input_dsv41::can_reuse(const llm_graph_params & params) { for (auto * t : engram_ids) { res &= t->ne[0] == (int64_t) model.engram_n_cols()*params.ubatch.n_tokens; } + res &= (engram_keep != nullptr) == (!engram_ids.empty() && params.ubatch.embd != nullptr); + if (engram_keep) { + res &= engram_keep->ne[2] == (int64_t) params.ubatch.n_tokens; + } if (cand_pin) { const auto & plan = cand_plan(params.ubatch); const int64_t bs = model.hparams.dsv41_cand_block_size; @@ -493,6 +509,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_engram( const llama_model_deepseek41 & model, ggml_tensor * x, ggml_tensor * ids, + ggml_tensor * keep, int il) const { const auto & layer = model.layers[il]; const int64_t hc = hparams.dsv4_hc_mult; @@ -526,6 +543,9 @@ ggml_tensor * llama_model_deepseek41::graph::build_engram( // gate = sigmoid(sign(dot) * sqrt(max(|dot|, 1e-6))) ggml_tensor * mag = ggml_sqrt(ctx0, ggml_clamp(ctx0, ggml_abs(ctx0, dot), 1e-6f, INFINITY)); ggml_tensor * gate = ggml_sigmoid(ctx0, ggml_mul(ctx0, mag, ggml_sgn(ctx0, dot))); + if (keep) { + gate = ggml_mul(ctx0, gate, keep); // [1, hc, nt] * [1, 1, nt]: image tokens pass through untouched + } cb(gate, "engram_gate", il); ggml_tensor * upd = ggml_mul(ctx0, ggml_repeat_4d(ctx0, value, n_embd, hc, nt, 1), gate); @@ -866,6 +886,13 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g ggml_set_name(engram_ids[e], ("engram_ids-" + std::to_string(model.engram_layers[e])).c_str()); } inp41->engram_ids = engram_ids; + ggml_tensor * engram_keep = nullptr; + if (!engram_ids.empty() && ubatch.embd) { + engram_keep = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, 1, n_tokens); + ggml_set_input(engram_keep); + ggml_set_name(engram_keep, "engram_keep"); + inp41->engram_keep = engram_keep; + } ggml_tensor * cand_pin = nullptr; if (hparams.dsv41_cand_src >= 0) { const int32_t cs = hparams.dsv41_cand_src; @@ -895,7 +922,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g const int e = model.engram_index(il); if (e >= 0) { - inpL = build_engram(model, inpL, engram_ids[e], il); + inpL = build_engram(model, inpL, engram_ids[e], engram_keep, il); cb(inpL, "engram_out", il); } diff --git a/src/models/models.h b/src/models/models.h index d2d37ec8d448..e7cad1afd785 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1455,6 +1455,7 @@ struct llama_model_deepseek41 : public llama_model_base { const llama_model_deepseek41 & model, ggml_tensor * x, ggml_tensor * ids, + ggml_tensor * keep, int il) const; ggml_tensor * build_attention41( From 2f73c68aa9ab3895a2d90d493a23cbe249f64129 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 09:33:43 -0700 Subject: [PATCH 29/49] deepseek41: drop working notes and the F32-exact debug knob from the pin branch --- STATUS_A.md | 230 -------------------------------------- TENSORS.md | 98 ---------------- ggml/src/ggml-cuda/mmf.cu | 7 -- 3 files changed, 335 deletions(-) delete mode 100644 STATUS_A.md delete mode 100644 TENSORS.md diff --git a/STATUS_A.md b/STATUS_A.md deleted file mode 100644 index a9e7fa37a62b..000000000000 --- a/STATUS_A.md +++ /dev/null @@ -1,230 +0,0 @@ -# Builder A status (DeepSeek-V4.1-Flash-1017 llama.cpp port) - -Branch `dsv41` in `$W/wt_dsv41` (off llama.cpp_pinset fd55cb98f). Contract for builder B: `TENSORS.md`. - -## Porting workflow (unslothai/scripts #649 bundle, `temp/us_gguf`: gguf_build_workflow.md A1-A5, PORTING.md) -| step | state | evidence | -|---|---|---| -| A1 recon + reuse map + fixture + converter + graph | done | reuse map in deepseek41.cpp header (V4 HC/MLA/MoE/raw cache reused; new: Engram, lagged mHC, ratio 1/2 + sharing + candidates). Two fixtures: `tiny_fixture` (small dims, fast, seeded-bug harness) and `tiny_fixture_realdim` (A1 rule: REAL dims 5120 / 64x512 heads / rope 64 / q 1280 / o 8x1024 / expert 2304 / index 32x128 / Engram 8x256 n-gram 4 / window 128 / block 8 / YaRN 65536x16; only counts shrunk: 10 layers, 8 experts top-2, Engram bucket start 1009, index top-k 64, 16 candidate blocks). Rare-path thresholds: window 128, ratio-2 top-k binds > 128 tokens, candidate mask binds > 128 ratio-1 rows; prompt wiki1100 exceeds all. Reference = official model.py (no transformers implementation of deepseek_v41 exists in 4.57 / 5.x, so ref_logits.py --generic cannot load it); the fixture writes the ref_logits.py npz schema + full logits. Fixture generates under the reference (prefill vs decode max abs 2.7e-5 at real dims). | -| A2 parity (bundled tools/dump_logits + port/compare_ref.py, F32 gate --max-kl-mean 1e-6 --min-top1 1.0 --min-cos 0.99999) | done (CPU), CUDA calibrated | see "A2 numbers" below; 13/13 seeded bugs FAIL on the small fixture, 5/5 on real dims | -| A3 template + V4.1 DSML tool parser | done (1d51c092f) | template `models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja` (= `$W/models/dsv41/chat_template.jinja`, 5/5 byte parity with encoding.py). V4.1 tags carry a leading space (`<｜DSML｜ calls>`, `<｜DSML｜ invoke>`, `<｜DSML｜ parameter>`); detection in common/chat.cpp keys on `｜DSML｜ calls>` (the template never names `dsml_token`), parser = the V4 parser with the spaced tags. test-chat: the V4 parser suite ported to V4.1 markup (content, thinking, single / parallel / mixed-type calls, content before calls, continuation, reconstruction) all pass; negative control (V4.1 tag switch off) aborts on the V4.1 block. | -| A4 never break other models | done | test-chat all passed; test-llama-archs CPU all 137 passed; CUDA: full suite exceeded its 2 h box (killed, no verdict), so the archs sharing touched code were run: deepseek4/41, deepseek2/32, glm-dsa, glm5-next, maple, qwen3moe, qwen35moe, qwen4exp on CUDA + CPU + Meta: 28/28 passed; deepseek41 on CPU / CUDA / Meta OK; test-backend-ops for used ops pass; V4 logits bitwise identical base vs head; no vocab change (vocab differential N/A) | -| A5 pin PR / pinset sim | not started (after A3 and B's GGUF) | | - -### A2 numbers -- CPU, F32 KV, FA off, single-shot / ub 7 / ub 1, swa_full 1 and 0: small fixture KL mean <= 4.9e-12, top-1 1.0, router top-k - exact on all 10 MoE layers; real-dims fixture (incl. wiki1100) KL mean <= 2.7e-11, max 3.0e-10, top-1 1.0, router exact - 40/40 (prompt x layer), l_out rel err <= 4e-6 on every layer. -- Seeded bugs (scripts/dsv41/seeded_bugs.py, every one must FAIL): Engram signed-sqrt dropped, Engram history order, - Engram ungated, mHC not lagged, output collapse with wrong pre, V4 per-head q norm, compressor softmax axis, index-K rope - dims, compressed-KV rope dims, indexer ReLU dropped, candidate pin dropped, candidate mask not applied, top-k reuse - replaced by dense: 13/13 FAIL (KL 4e-3 .. 2). On real dims: candidate pin / candidate mask / reuse / ReLU / Engram - history: 5/5 FAIL. Free oracle: below 16 visible ratio-1 rows the candidate bugs are inert (short16 passes), as the - architecture guarantees. -- CUDA (B200), `NVIDIA_TF32_OVERRIDE=0` alone: FAILS (KL 1e-3..7e-2): the override does not reach the mmf tf32 path for - F32 weights (PORTING.md Phase 5). With `GGML_CUDA_F32_EXACT=1` (debug knob in mmf.cu, default off) as well: small - fixture passes every cell (ub 0/7/1, graphs on and off); real dims pass everything except wiki1100 at ub 0/7 (KL mean - 2.3e-6, top-1 1.0, max 2.4e-3 at t=1035). Chased: one indexer near-tie at layer 2, t=1035, k-th vs (k+1)-th CPU score - gap 4.4e-5 relative while CUDA batched idx_score differs from CPU by 1.7e-5..2.6e-5 relative (score = signed sum of - relu terms, ~10x cancellation); the flip then propagates to that token's selections in later layers. Decode (ub 1) passes. - Classified as precision, not a defect; CUDA gate = CPU gate except one near-tie token. -- CUDA graphs on vs off (B10 A/B): identical results, no hang under the 1200 s watchdog (timeout -k 15 => SIGKILL). -- FA on (F16 attention) is an acceptance config, not a parity config (see older numbers below). - -## Gate on agent C's fixtures (the parity oracle from now on; temp/dsv41_fixtures, refs/

/matched.npy) -Build: `temp/wt_dsv41_gate` = dsv41 + B's dsv41-mxfp8 (MXFP8 Engram), tools `temp/dsv41_tools/gate_cpu/`. GGUF: B's -`-F32.gguf` with MXFP4 experts dequantized to F32 (`models/dsv41/c_fx/-F32X.gguf`, scripts/dsv41/experts_to_f32.py; -with MXFP4 experts the CPU matmul quantizes activations to q8_0 and the gate fails at ~1e-3, as C found). Runner -`scripts/dsv41/gate_c.sh`. CPU, F32 KV, FA off. Gate KL(matched || llama) mean < 1e-3: PASS on every fixture. -| fixture | prompts | KL mean (range) | worst p99 | top-1 min | -|---|---|---|---|---| -| fx_base (Engram 1, 6; MXFP8 tables) | t100..t3001, c1600, sp1600 | 9e-12 .. 2.3e-5 | 8.5e-5 | 1.0000 | -| fx_noengram | same | 9e-12 .. 1.3e-5 | 8.9e-5 | 0.9987 | -| fx_engram0 (Engram on layer 0) | same | 9e-12 .. 2.7e-5 | 1.6e-4 | 0.9987 | -| fx_long (real 2048-block pool) | t2001 / sp20001 (ub 4096) | 1.2e-5 / 5.1e-4 | 1.1e-2 | 0.9861 | -| fx_base ub 7 / ub 1 | t129, t1027, sp1600 / t129, t700 | identical to one-shot | | 1.0000 | -llama.cpp vs matched is the same size as matched_f32 vs matched_f64 (the fp32 floor). sp20001 is the long-context -near-tie regime characterised above. -Selection sets vs C's matched_traces.pt (`scripts/dsv41/sel_vs_traces.py`, fx_base): t1027 L2-L7 0 rows, L8 2, L12 4, -pool 0; t3001 L2 5 .. L12 121 rows, pool 21 / 1977. Every mismatch is 1 swap; the earliest divergences (t=646, 877, 973, -998) have score gaps 7e-7..3.3e-6 of the row top score (near-ties); later ones follow earlier flips. Waiting on C's -f32-vs-f64 selection floor to close this gate. -CUDA (`GGML_CUDA_F32_EXACT=1`, gate build `build-cuda`): fx_base t129 / t1027 / t3001 -KL 8.6e-12 / 4.5e-9 / 5.8e-5, top-1 1.0 / 1.0 / 0.999: PASS. -Full model: `scripts/dsv41/full_dump.sh` (3 B200 x 125 GB, ~45 min, window requested from main) dumps the run-of-record -pkg/BF16 GGUF on C's fx_base token files + real-text wiki1100 / long17k, for the TP8 MATCHED oracle. -First full-model results (GPUs 0,2,3, ub 512, fa=0, F32 KV; load + 6 prompts in ~10 min): all finite; real-text -self-perplexity wiki1100 2.11 (next-token top-1 0.826), long17k 2.0 / 1.28 / 1.80 / 2.08 / 3.03 on [0,1k) / [1k,4k) / -[4k,8k) / [8k,16384) / [16384,17k); 256-token bands before the pool threshold already range 1.6..3.0 (text-dependent, no -jump at 16,392). All phases done (outputs/dsv41_full/bf16_ub512/{ub512,ub7,ub1,fa1_ub512}), all finite; self-ppl -wiki1100 2.105 / 2.102 / 2.103 / 2.086 and long17k 1.862 / 1.862 / - / 1.858 for ub512 / ub7 / ub1 / fa1. -CUDA batch-split is not bitwise on the full model: ub7 vs ub512 KL mean 9.6e-3 (wiki1100), 3.4e-3 (long17k), 6e-2 on -the random-token t1025; fa1 vs ub512 7.4e-3 / 3.2e-3. Attribution: the full model on CPU (ngl 0, 12 wiki tokens, -~5 min) is bitwise identical at ub 512 and ub 7, and CPU vs each CUDA arm shows the same per-position profile (KL -3e-6 at t0, 0.08-0.2 at t5-t6) whichever ubatch / FA is used: those positions are numerically sensitive and CUDA's -kernels (MMQ / MMVQ, q8_1 activations) differ from CPU there. Released V4-Flash (UD-IQ1_S) on the same text: ub7 vs -ub512 KL mean 3.4e-2, 0.4 at t6. So the graph is batch-invariant; the CUDA spread is kernel numerics. Which side is -closer to the truth is for C's full-model MATCHED oracle (prompts: outputs/dsv41_full/bf16_ub512/tokens). - -## Critique fold-in (plans/dsv41_support.md "Critique fold-in") -1. No per-head q RMS norm: the graph has none; "V4 per-head q norm" is seeded bug #6 and FAILS (KL 4e-3..2). -2. MATCHED oracle: the fixture ref/ IS the matched reference (shim act_quant / fp4_act_quant = identity); every gate above is - against it. NATIVE (in-graph FP8/FP4 activation quant emulated on CPU, `scripts/dsv41/native_gap.py`) vs MATCHED: - small KL 0.24..0.60, real dims 0.69..1.47 (random weights are chaotic); llama vs MATCHED 1.7e-11..2.7e-11 on the same - prompts. The gap belongs to the oracle, not to the port; on the real model gate against MATCHED (KL < 1e-3 + p99). -3. Harness uses full logits: `ref_forward` asserts `[1, T, vocab]` (ParallelHead full_logits path), dumps are [T, V]. -4. Epsilons kept separate: norms 1e-20 (as `RMSNorm.eps` in model.py), hc_eps 1e-6, router weight sum +1e-20 (2f4055f00; - V4 keeps its 6.1e-5 clamp); norm statistics in F32. -5. Boundaries, `tiny_fixture_budget` (real budgets on small dims: window 128, index top-k 512, candidate pool 2048 x 8 = - 16384 rows, YaRN 65536 x 16; one 17,000-token prompt, so every t is its own boundary: window 127/128/129, top-k - 511/512/513 at both ratios, pool 16383/16384/16385; 616 rows with real exclusion, up to 77 blocks excluded): - - candidate invariants (reference): newest partial block kept on every row, kept-block count = min(visible, 2048) on - every row, real exclusion present: PASS. - - CPU llama.cpp, single shot / ub 4096 / ub 1025 (odd ratio-2 groups split across ubatches): logits and every - selected set bitwise identical across the three splits. - - vs the fp32 reference: top-1 99.75%, KL median ~1e-7, mean 3.0e-4 (p99 3.9e-4, max 0.42), i.e. heavy-tailed, and 0 - below t=1100. Selection: L2 (first index layer) differs on 87 / 17000 rows, every one a single swap at rank 512/513; - llama-vs-reference index-score error (max over a row / row top score) median 1.3e-5, p99 4.7e-5; the reference's own - gap for each swapped pair is <= 2.4e-5 (median 2.3e-6), below the error p99 on 87 / 87 rows. Later layers inherit - the flips (L4 349, L6 1291, L8 2422 rows; picks outside the reference pool only on rows whose pool itself flipped). - The score error grows linearly with position (median 1.2e-6 at t < 1100 -> 2.5e-5 at t > 12000): float32 rope - angle rounding (|angle| up to ~1.7e4 rad, ~2e-4 rad rounding at t = 16000 on both sides, rounded differently: - ggml multiplies theta by theta_scale per pair, model.py multiplies t by a rounded freq). - Attribution (`scripts/dsv41/ref_precision_floor.py`, reference variants, same weights): - | reference variant | KL(variant ‖ ref32) | L2 / L4 / L6 / L8 rows differing from ref32 | llama vs variant: KL, L2 / L4 / L6 / L8 | - |---|---|---|---| - | float64 upcasts (fp32 angles) | 1.6e-4 | 8 / 53 / 616 / 1267 | 4.0e-4, 93 / 388 / 1721 / 3154 | - | exact (float64) rope angles | 6.6e-4 | 139 / 623 / 2519 / 4200 | 7.5e-4, 150 / 687 / 2781 / 4675 | - | ggml-rounded rope angles | 3.7e-4 | 87 / 338 / 1371 / 2648 | 1.1e-4, 8 / 40 / 274 / 701 | - With the reference rounding its rope angles the way ggml does, llama.cpp's L2 differences fall from 87 to 8 rows, - the same count as the reference's own fp32-vs-fp64 noise; with exact angles the fp32 reference is itself further - from the truth than llama.cpp is from it. Classified as precision (rope-angle rounding + fp32 arithmetic), no defect. - - CUDA (`GGML_CUDA_F32_EXACT=1`): same shape, more flips (L2 1539 rows, 97% single swaps; the >= 3-swap rows all sit - in the 128-token window after one L0 router near-tie at t=16017). KL mean 3.2e-3, top-1 98.2%; decode (ub 1, 17,000 - steps) gives the same numbers (KL 3.1e-3). CUDA rope rounds theta differently again (`pos*powf(theta_scale, i)`). - The 1e-6 F32 gate holds wherever the top-k does not bind on a near-tie (all of small / real-dims fixtures, and - t < 1100 here); at the real budgets the right gate is selection parity with gap evidence, as above. -6. Slot reuse / prefix restore (`scripts/dsv41/state_test.cpp`): save seq 0 after n1 tokens, clear memory, restore into - seq 0 or 1, continue: PASS (n1 201 / 17 / 164). Real defect found and fixed here: the Engram n-gram token history was - not in the saved state (2f4055f00, `has_cell_ext()`); negative control with the fix off FAILS (KL 0.115). - Compressor state rides the existing DSV4 snapshot path. -7. V4-Flash regression on the real model: unsloth/DeepSeek-V4-Flash-0731-GGUF UD-IQ1_S (77 GB, downloaded for this - check), CUDA, 600 tokens, F32 KV, FA off: logits bitwise identical base (fd55cb98f) vs head at ub 512 and ub 7; head - A/A also bitwise. Plus the random V4 model (bitwise, ub 256 / 7). - -## Flash attention (fa=1) and multi-sequence / rollback (coordinator items 1, 2) -- FA graph is exact: with K/V kept F32 inside the FA op (debug-only patch, not committed) fa=1 matches the reference at - KL 5.0e-12 / 5.0e-12 / 4.6e-12 (short / wiki330 / rep300): mask concat (window + top-k -inf), attn_sinks, padding and - the compressed-row concat are right. FA accumulation is already GGML_PREC_F32 for every arch (llama-graph.cpp). -- The fa=1 excess is the F16 kernel: llama vs the reference with q/k/v rounded to F16 is still 1.9e-2 on short16 (floor - 7.9e-7, error from position 1 on, attn_raw-0 rel err 3e-4..2e-3 per row). The CPU single-chunk kernel also rounds Q to - F16 and accumulates V*softmax in an F16 VKQ16 buffer, which input rounding alone does not model; the small fixture - then amplifies it (routing flips). Not a V4.1 defect. -- What users see, on agent C's non-chaotic fx_base (F16 KV, fa=1, vs MATCHED): CPU KL 6.5e-8 (t129), 4.1e-5 (t1027), - 1.2e-4 (c1600), 3.4e-4 (t3001), top-1 >= 0.992; CUDA (default precision) 4.0e-4 / 3.0e-4 / 4.4e-4 / 9.3e-4, top-1 - >= 0.983. All < 1e-3 and 5-80x below C's bf16 floor (5.1e-3). -- Multi-sequence (`scripts/dsv41/seq_test.cpp multi`, n_seq_max 2, every ubatch interleaves wiki330 on seq 0 and rep300 - on seq 1, ub 7 and 64): each sequence vs its own single-sequence reference KL 4.6e-12..4.9e-12 (CPU), 1.6e-11..2.8e-11 - (CUDA exact), top-1 1.0. -- Rollback snapshots (`seq_test rollback`, n_rs_seq R; after each true chunk decode a draft whose tail is garbage tokens, - seq_rm the rejected tail, continue; odd chunk sizes so ratio-2 groups straddle the rollback point; garbage passes - through Engram history, compressor state and KV): R4/D4/acc1 (23 rollbacks), R4/D4/acc3, R2/D5/acc3: KL 4.9e-12 CPU, - 2.8e-11 CUDA, top-1 1.0. Negative control (keep one rejected token): KL 1.9, so the gate sees pollution. -- Slot reuse / prefix restore: state_test above (PASS). - -## Milestones -| # | state | commit | -|---|---|---| -| 1 gguf-py constants, arch enums, keys, TENSORS.md | done | 7976d362d | -| 2 tiny fixture from official model.py + reference logits + fixture-only converter | done | scripts in `$W/scripts/dsv41/` | -| 3 graph + KV-cache generalisation, CPU + CUDA parity | done | bae963fb2 | -| 4 test-llama-archs (CPU, CUDA, Meta split), test-backend-ops, V4 regression | done | d29889d8f | -| 5 full-model handoff (below) | written, waiting for B's GGUF | | -| 6 DSpark / MTP graph | not started (tensors load as optional, graph throws on the MTP context) | | - -## What the graph implements (`src/models/deepseek41.cpp`) -Engram (host n-gram hash from the raw-cache token history, CPU get_rows on `engram_embd`, gate as model.py), single-pass mHC -(no hc_head), ratio-1/2 compressors (V4 HCA-style state, no ape/overlap), index K = k_norm(wk(latent)) stored next to the -compressed KV in one cache row, compressed KV / index K read from the kv source layer, top-k computed on index sources and -reused in between, two-level candidate blocks (pool_1d max per block, pin newest block, top blocks, mask later index layers), -no per-head q norm. KV cache: `llama-kv-cache-dsv4` slots are ratio-parametrised (V4: CSA 4 overlap + LID, HCA 128; -V4.1: CSA slot = ratio 2, HCA slot = ratio 1, only kv-source layers own rows, no LID). - -## Tiny fixtures (`$W/models/dsv41/tiny_fixture` small dims; `tiny_fixture_realdim` + `tiny_gguf/realdim-f32.gguf` real dims, 24.6 GB F32). Small fixture (`$W/models/dsv41/tiny_fixture`, GGUF `$W/models/dsv41/tiny_gguf/tiny-f32.gguf`) -10 layers, dim 128, real tokenizer + real 99,092-id Engram token map, Engram on L1/L4, ratios [0,0,2,2,2,2,1,1,1,1], -kv sources 2/4/6, index sources 2/4/6/8, candidate source 6 (4 blocks x 4), index_topk 8, window 16, YaRN orig 64. -Checkpoint in official naming and storage formats (FP8 32x32 ue8m0, FP4 experts, Engram FP8 + per-row-32 scales). -Reference = official model.py on CPU fp32 with activation fake-quant off (`scripts/dsv41/shim/kernel.py`). -Regenerate: `python scripts/dsv41/make_fixture.py`; convert: `python scripts/dsv41/fixture_to_gguf.py --out X.gguf [--experts mxfp4]`; -sweep: `bash scripts/dsv41/run_parity.sh "fa:ub ..."`; ablations: `python scripts/dsv41/ablate.py --got `. - -## Fixture parity (KL(ref||llama) mean / max, top-1; prompts wiki330, rep300, short16) -- CPU F32, fa=0, ub 512 / 7 / 1: KL mean <= 4.9e-12, max <= 1.5e-11, top-1 100% on all three (ub 7 differs from ub 512 by 5.7e-6 abs, still passes). -- CUDA (B200) F32, fa=0, `GGML_CUDA_F32_EXACT=1 NVIDIA_TF32_OVERRIDE=0`, ub 512 / 7 / 1: KL mean <= 2.8e-11, max <= 3.0e-10, top-1 100%. -- CUDA default precision fails the 1e-6 gate (F32 GEMMs run on TF32: hc_mixes rel err 9e-4 at layer 0); with the knob the - per-tensor CPU-vs-CUDA error is <= 1.3e-6 everywhere. -- fa=1 (CPU and CUDA): F16 flash attention. short16 KL 1e-6..2e-6; long prompts 4e-3..1e-1, the same scale as the reference - itself with q/k/v rounded to F16 (4.5e-3 wiki, 1.7e-2 rep300): the random fixture's top-k / routing flips amplify F16 noise. -- Mechanisms are live: KL(ref || ref without X) = Engram 3.1, candidate mask 0.17, top-k reuse (dense instead) 1.2, - single-pass mHC (standard instead) 0.6; llama.cpp matches the full reference, not any ablation. -- MXFP4 experts (bit-exact repack verified): the 2.8e-4 short16 figure was from an older fixture and is stale. Current - small fixture: short 2.0e-2, wiki330 1.4e-1 (chaotic fixture). On C's fx_base (B's MXFP4-expert -F32 GGUF, CPU) vs - MATCHED: t129 3.2e-4, t1027 9.1e-4, t3001 2.1e-3 (top-1 0.965), vs the F32-expert copy at <= 2.3e-5: the CPU MXFP4 - matmul quantizes activations to q8_0 (C localised it: with q8_0 applied in the reference the expert outputs agree to - 5e-8). Kernel precision shared by every MXFP4 model, below the bf16 floor (5e-3), not a port defect. - -## Other checks -- test-llama-archs deepseek41: CPU OK, CUDA NMSE 3.3e-6, Meta (tensor split) OK, roundtrip OK. deepseek4: still OK. -- V4 regression: random V4 model (ratios 4/128, 240 tokens) logits bitwise identical base vs head build, ub 256 and 7. -- test-backend-ops CUDA0: POOL_1D, TOP_K, SET_ROWS, CONCAT, FILL, GET_ROWS, SGN, ROPE, SOFT_MAX, SUM_ROWS all pass; - MUL_MAT type_a=f32 with GGML_CUDA_F32_EXACT=1 275/275. - -## For builder B -- Keys/tensors exactly as `TENSORS.md`. The loader derives sharing from `attention.kv_source_layers` / - `attention.indexer.source_layers` and validates ratios; Engram offsets are derived from `engram.primes`. -- `engram_embd` is placed like token_embd (host buffer); any type works if CPU `get_rows` supports it (MXFP8 must - implement CPU get_rows -> F32). Do not repack it. `engram_gate` must be F32 `q_weight*k_weight`. -- DSpark blocks may be written as blk.40-42 (loaded as optional, unused) or left out. -- Smoke test for a converted fixture: `python scripts/dsv41/fixture_to_gguf.py` is the reference converter; B's output on - `$W/models/dsv41/tiny_fixture` should give the same parity numbers (compare with `run_parity.sh`). - -## Full-model parity run (once B's reference GGUF exists) -Sizes: experts 289 GB (MXFP4), dense ~14 GB BF16 (FP8 dequant), embed+head 2.7 GB, Engram 203 GB (MXFP8, host). -1. Engram tables stay on the host automatically (input-layer placement); host RAM 2 TB is enough. mmap first-touch is - random reads (48 rows per token): pre-warm the file (`vmtouch -t` or one `cat > /dev/null`) before timing. -2. GPU: ~306 GB weights -> 2 B200 with ~50 GB headroom (3 to be safe). Lease all cards at once: - `python $W/gpu_pool_lease.py take --owner $$ --key $$:dsv41 --gpus A,B --gb 178 --est-s 3600`, run with - `CUDA_VISIBLE_DEVICES=A,B`, then `gpu_pool_lease.py drop --key $$:dsv41 --gpus A,B`. -3. Logits: `$W/wt_dsv41/build-cuda/bin/llama_dump_logits -m DSV41.gguf -o OUT -c 8192 -ub 512 -ngl 99 -fa 0 --kv-f32 - prompts/*.tokens.bin` (the Phase-1 oracle writes the same int32 token files); compare with - `python $W/scripts/dsv41/compare.py --ref ORACLE --got OUT --kl-max 1e-3`. Repeat with `-ub 1` and `-ub 7` - (batch-split invariance) and once with fa=1 (expected higher KL, F16 attention). -4. Long-context gate: a prompt > 16,384 + 8 tokens (candidate pool) and > 512 compressed rows per ratio, so the - candidate mask, top-k and reuse all bind. -5. Oracle caveat: model.py on GPU quantises activations (FP8 before every FP8/FP4 GEMM, FP8 window KV, FP4 index/compressed - KV); the GGUF port does not, so expect KL above the fixture's 1e-11 even when correct. Measure the oracle with and without - fake-quant on a few prompts (`DSV41_FAKE_QUANT=1` in the shim emulates it on CPU). - -## Image-span Engram (embedding inputs) -model.py: image tokens are hashed DEAD (no n-gram spans them) and the Engram gate is 0 there; routing uses bias_vl. -llama.cpp: embd tokens hash DEAD, KV cells of embd tokens record LLAMA_TOKEN_NULL (dsv41_tok_hist), and embd ubatches -get an `engram_keep` input (0 for embd tokens) multiplied into the gate; text-only graphs are unchanged (no extra op). -Test (`scripts/dsv41/image_ref.py` + `embd_test.cpp`, small fixture, 48 wiki tokens, [16,24) fed as an embedding -ubatch of the same token rows, reference with token_types image mask and bias_vl): KL 4.4e-12 CPU, 5.2e-12 CUDA, -top-1 1.0; without the gate fix KL 0.97; image vs text reference differ by KL 1.0 (the test can fail). Text parity -unchanged (4.6e-12..4.9e-12 at ub 7), test-llama-archs CPU 137/137. - -## Engram placement guard -`llama-model-loader.cpp`: an `engram_embd` placement on a non-host buffer (from `-ot`, or any split mode) is refused with a -warning and the table goes to the CPU buffer. Test `scripts/dsv41/engram_placement.cpp` (all layers offloaded, -override engram_embd -> CUDA0): with the guard, a warning per table, CUDA0 model buffer 78.90 MiB (= no override) and -logits bitwise equal to the run without the override; negative control (build without the guard): CUDA0 buffer -80.53 MiB, i.e. the tables were uploaded. - -## Not proven / open -- A5 (pin PR / pinset sim) not started; full-model parity waits for B's GGUF and the oracle. -- The F32 KL gate cannot be met at the real top-k budgets on any implementation (the fp32 reference misses its own - float64-rope version by KL 6.6e-4); long-context acceptance = selection parity with near-tie gap evidence. -- Vision tower not ported (the text side of image spans is done, see below). -- No ubatch shares a token across sequences (asserts, as the PLE path). -- DSpark drafting graph not implemented. diff --git a/TENSORS.md b/TENSORS.md deleted file mode 100644 index 598eb3db93ec..000000000000 --- a/TENSORS.md +++ /dev/null @@ -1,98 +0,0 @@ -# DeepSeek-V4.1 (`deepseek41`) GGUF contract - -Owner: builder A (arch, graph). Builder B (converter) writes exactly this. Shapes are GGUF `ne` order (fastest dim first), -i.e. the reverse of the HF `[out, in]` shape. HF names are the official checkpoint names (`layers.N.*`, `mtp.M.*`). - -## Arch -`general.architecture = "deepseek41"` (`gguf.MODEL_ARCH.DEEPSEEK41`, `LLM_ARCH_DEEPSEEK41`). -`block_count` = 40 (backbone only) or 43 with `nextn_predict_layers = 3` when the DSpark blocks `mtp.0..2` are written as -`blk.40..42` (optional, later). - -## Tensors -Unchanged from deepseek4 (same names and shapes): -`token_embd`, `output_norm`, `output`; per block `attn_norm`, `attn_sinks` (F32 `{n_head}`), `attn_q_a`, `attn_q_a_norm`, -`attn_q_b`, `attn_kv`, `attn_kv_a_norm`, `attn_output_a`, `attn_output_b`, `hc_{attn,ffn}_{fn,base,scale}` (F32), -`ffn_norm`, `ffn_gate_inp`, `exp_probs_b` (F32 `.bias`), `exp_probs_b_vl` (F32 `.bias`, optional), -`ffn_{gate,up,down}_exps` (MXFP4), `ffn_{gate,up,down}_shexp`. - -Removed vs deepseek4: `output_hc_{fn,base,scale}` (no hc_head: the final collapse uses the last FFN pre-mix), -`attn_compressor_ape`, `indexer.compressor_*`, `ffn_gate_tid2eid`, `nextn.*`. - -| GGUF name | HF name | ne | dtype | layers | -|---|---|---|---|---| -| `blk.N.attn_compressor_kv.weight` | `layers.N.attn.compressor.wkv.weight` | `{n_embd, head_dim}` | BF16 | kv sources (2,8,14,20) | -| `blk.N.attn_compressor_gate.weight` | `layers.N.attn.compressor.wgate.weight` | `{n_embd, head_dim}` | BF16 | kv sources with ratio > 1 (2,8,14) | -| `blk.N.attn_compressor_norm.weight` | `layers.N.attn.compressor.norm.weight` | `{head_dim}` | F32/BF16 | kv sources | -| `blk.N.indexer.attn_k.weight` | `layers.N.attn.indexer.wk.weight` | `{head_dim, index_head_dim}` | BF16 | kv sources | -| `blk.N.indexer.k_norm.weight` | `layers.N.attn.indexer.k_norm.weight` | `{index_head_dim}` | F32/BF16 | kv sources | -| `blk.N.indexer.attn_q_b.weight` | `layers.N.attn.indexer.wq_b.weight` | `{q_lora_rank, index_n_head*index_head_dim}` | BF16 (FP8 dequant) | index sources (2,8,...,36) | -| `blk.N.indexer.proj.weight` | `layers.N.attn.indexer.weights_proj.weight` | `{n_embd, index_n_head}` | BF16 | index sources | -| `blk.N.engram_embd.weight` | `layers.N.engram.embed.weight` + `.scale` | `{engram_head_dim, num_embeddings[i]}` | MXFP8 (B) / BF16 / F32 | engram layers (1,14) | -| `blk.N.engram_kv.weight` | `layers.N.engram.wkv.weight` | `{n_hash_cols*engram_head_dim, n_embd*(hc+1)}` | BF16 (FP8 dequant) | engram layers | -| `blk.N.engram_gate.weight` | `layers.N.engram.q_weight * layers.N.engram.k_weight` (folded, elementwise) | `{n_embd, hc}` | F32 | engram layers | -| `blk.40.dspark_main_proj.weight` | `mtp.0.main_proj.weight` | `{n_embd*len(target_layers), n_embd}` | BF16 | DSpark stage 0 (optional) | -| `blk.40.dspark_main_norm.weight` | `mtp.0.main_norm.weight` | `{n_embd}` | F32 | DSpark stage 0 | -| `blk.42.dspark_norm.weight` | `mtp.2.norm.weight` | `{n_embd}` | F32 | last DSpark stage | -| `markov_w1.weight` | `mtp.2.markov_head.embed.weight` | `{markov_rank, n_vocab}` | BF16 | DSpark | -| `markov_w2.weight` | `mtp.2.markov_head.head.weight` | `{markov_rank, n_vocab}` | BF16 | DSpark | -| `conf_proj.weight` | `mtp.2.confidence_head.proj.weight` | `{n_embd+markov_rank, 1}` | F32 | DSpark | - -Notes -- `n_hash_cols = (engram.max_ngram_size - 1) * engram.head_count` (24). `engram_kv` output rows keep the HF order: - rows `[0, hc*n_embd)` are the `hc` keys (copy-major), rows `[hc*n_embd, (hc+1)*n_embd)` the shared value. -- `engram_embd` rows are written exactly as stored (no TP padding rows: exactly `engram_num_embeddings[i]` rows). The - row for hash id r is `float(w[r]).view(8,32) * 2^(scale[r]-127)`. Never repack, never offload: the graph gathers it on - the host with `ggml_get_rows`. Any type with get_rows on CPU works for the graph. -- FP8 linear weights use 32x32 `ue8m0` blocks (derive the block from the scale shape, do not hard-code 128). -- Keep F32 in every output file (checked by gguf_build `eval/verify_types.py`): `ffn_gate_inp` (router), `exp_probs_b`, - `exp_probs_b_vl`, every `*norm.weight` (incl. `attn_compressor_norm`, `indexer.k_norm`), `hc_*`, `attn_sinks`, `engram_gate`. -- Every layer `0..39` has all backbone attention/HC/MoE tensors; DSpark blocks have 128 experts (`dspark.expert_count`). - -## Keys (`deepseek41.` prefix) -Same as deepseek4: `context_length`, `embedding_length`, `block_count`, `feed_forward_length` (unused), -`expert_feed_forward_length`, `expert_count` (384), `expert_used_count` (6), `expert_shared_count`, `expert_weights_scale` -(1.5), `expert_weights_norm` (true), `expert_gating_func` (`gguf.ExpertGatingFuncType.SQRTSOFTPLUS` = 4), -`swiglu_clamp_exp` / `swiglu_clamp_shexp` (f32[block_count], 10.0), `attention.head_count` (64), `attention.head_count_kv` (1), -`attention.key_length` = `attention.value_length` (512), `rope.dimension_count` (64), `attention.q_lora_rank`, -`attention.sliding_window` (128), `attention.layer_norm_rms_epsilon` (1e-20), `attention.output_group_count`, -`attention.output_lora_rank`, `attention.compress_ratios` (u32[>= block_count], all 43 entries ok), -`attention.compress_rope_freq_base` (160000), `rope.freq_base` (10000), `rope.scaling.type` yarn, `rope.scaling.factor` 16, -`rope.scaling.original_context_length` 65536, `rope.scaling.yarn_beta_fast` 32, `rope.scaling.yarn_beta_slow` 1, -`attention.indexer.head_count` (32), `attention.indexer.key_length` (128), `attention.indexer.top_k` (512), -`hyper_connection.count` (4), `hyper_connection.sinkhorn_iterations` (20), `hyper_connection.epsilon` (1e-6), -`embedding_length_out` = n_embd*hc (as deepseek4). -Not written: `hash_layer_count`. - -New: -| key | type | value (V4.1-Flash-1017) | -|---|---|---| -| `attention.kv_source_layers` | u32[] | [2, 8, 14, 20] | -| `attention.indexer.source_layers` | u32[] | [2, 8, 14, 20, 24, 28, 32, 36] | -| `attention.indexer.candidate_source_layer` | i32 | 20 (-1 = off) | -| `attention.indexer.candidate_top_k_blocks` | u32 | 2048 | -| `attention.indexer.candidate_block_size` | u32 | 8 | -| `engram.layers` | u32[] | [1, 14] | -| `engram.num_embeddings` | u32[] | [384006168, 384016682] | -| `engram.max_ngram_size` | u32 | 4 | -| `engram.vocab_size` | u32 | 16000000 (prime search start, informational) | -| `engram.head_count` | u32 | 8 | -| `engram.head_dim` | u32 | 256 | -| `engram.pad_token_id` | u32 | 2 (raw id; the graph maps it through token_map) | -| `engram.compressed_vocab_size` | u32 | 99092 | -| `engram.token_map` | i32[n_vocab] | `build_compressed_token_map(tokenizer)` from inference/engram.py | -| `engram.multipliers` | u64[n_engram*max_ngram] | layer-major, `compute_hash_multipliers` | -| `engram.primes` | u32[n_engram*(max_ngram-1)*head_count] | layer, n-gram order (2..max), head; `EngramLayout.primes` | -| `dspark.block_size` | u32 | 5 | -| `dspark.noise_token_id` | u32 | 128799 | -| `dspark.target_layers` | u32[] | [37, 38, 39] | -| `dspark.markov_rank` | u32 | 256 | -| `dspark.expert_count` | u32 | 128 | -| `dspark.expert_used_count` | u32 | 3 | - -Engram offsets are not stored: per layer they are the exclusive cumulative sum of that layer's 24 primes (order-major, -head-minor), exactly `NgramHashState.offsets`. - -gguf-py writer helpers: `add_attention_kv_source_layers`, `add_indexer_source_layers`, `add_indexer_candidate_source_layer`, -`add_indexer_candidate_top_k_blocks`, `add_indexer_candidate_block_size`, `add_engram_{layers,num_embeddings,max_ngram_size, -vocab_size,head_count,head_dim,pad_token_id,compressed_vocab_size,token_map,multipliers,primes}`, -`add_dspark_{block_size,noise_token_id,target_layers,markov_rank,expert_count,expert_used_count}`. diff --git a/ggml/src/ggml-cuda/mmf.cu b/ggml/src/ggml-cuda/mmf.cu index e67b46d47ae5..646a5899c803 100644 --- a/ggml/src/ggml-cuda/mmf.cu +++ b/ggml/src/ggml-cuda/mmf.cu @@ -136,13 +136,6 @@ bool ggml_cuda_should_use_mmf(enum ggml_type type, int cc, int warp_size, const return false; } - // debug: GGML_CUDA_F32_EXACT=1 keeps F32 weights off the TF32 tensor-core path (together with - // NVIDIA_TF32_OVERRIDE=0 for cuBLAS), for exact-F32 parity tests against a CPU / PyTorch reference - static const bool f32_exact = getenv("GGML_CUDA_F32_EXACT") != nullptr; - if (type == GGML_TYPE_F32 && f32_exact) { - return false; - } - const size_t ts = ggml_type_size(type); if (src0_ne[0] % (warp_size * (4/ts)) != 0) { return false; From cd1714ad9429a0d380826177f0610aaa78cb88a7 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 09:39:23 -0700 Subject: [PATCH 30/49] models/templates: DeepSeek-V4.1 template namespace attribute list -> msgs (newer jinja resolves .list as the builtin) --- ...deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja b/models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja index aac8438a37df..c597891ba11e 100644 --- a/models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja +++ b/models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja @@ -75,7 +75,7 @@ {%- endmacro -%} {#- 1. Merge tool results (and consecutive user turns) into user messages made of content blocks. -#} -{%- set mns = namespace(list=[]) -%} +{%- set mns = namespace(msgs=[]) -%} {%- for msg in messages -%} {%- set content = msg.get("content") -%} {%- set is_blocks = content is iterable and content is not string and content is not mapping -%} @@ -86,46 +86,46 @@ {%- if rc is string and IMG in rc -%} {{- raise_exception("reasoning_content contains image special token " ~ IMG) -}} {%- endif -%} - {%- set prev = mns.list[-1] if mns.list else none -%} + {%- set prev = mns.msgs[-1] if mns.msgs else none -%} {%- if msg.role == "tool" -%} {%- set tblock = {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content if content is not none else ""} -%} {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined -%} - {%- set mns.list = mns.list[:-1] + [dict(prev, content_blocks=prev.content_blocks + [tblock])] -%} + {%- set mns.msgs = mns.msgs[:-1] + [dict(prev, content_blocks=prev.content_blocks + [tblock])] -%} {%- else -%} - {%- set mns.list = mns.list + [{"role": "user", "content_blocks": [tblock]}] -%} + {%- set mns.msgs = mns.msgs + [{"role": "user", "content_blocks": [tblock]}] -%} {%- endif -%} {%- elif msg.role == "user" -%} {%- set blocks = content if is_blocks else [{"type": "text", "text": content or ""}] -%} {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined and prev.get("task") is none -%} - {%- set mns.list = mns.list[:-1] + [dict(prev, content_blocks=prev.content_blocks + blocks)] -%} + {%- set mns.msgs = mns.msgs[:-1] + [dict(prev, content_blocks=prev.content_blocks + blocks)] -%} {%- else -%} - {%- set mns.list = mns.list + [dict(msg, content_blocks=blocks)] -%} + {%- set mns.msgs = mns.msgs + [dict(msg, content_blocks=blocks)] -%} {%- endif -%} {%- else -%} - {%- set mns.list = mns.list + [msg] -%} + {%- set mns.msgs = mns.msgs + [msg] -%} {%- endif -%} {%- endfor -%} {#- Top-level tools / response_format attach to a leading system message, or to a new empty one. -#} {%- if tools or response_format -%} - {%- if not (mns.list and mns.list[0].role == "system") -%} - {%- set mns.list = [{"role": "system", "content": ""}] + mns.list -%} + {%- if not (mns.msgs and mns.msgs[0].role == "system") -%} + {%- set mns.msgs = [{"role": "system", "content": ""}] + mns.msgs -%} {%- endif -%} - {%- set first = mns.list[0] -%} + {%- set first = mns.msgs[0] -%} {%- if tools and not first.get("tools") -%}{%- set first = dict(first, tools=tools) -%}{%- endif -%} {%- if response_format and not first.get("response_format") -%}{%- set first = dict(first, response_format=response_format) -%}{%- endif -%} - {%- set mns.list = [first] + mns.list[1:] -%} + {%- set mns.msgs = [first] + mns.msgs[1:] -%} {%- endif -%} {#- 2. Reorder tool results to match the tool_calls order of the preceding assistant message. -#} -{%- set sns = namespace(list=[], ids=[]) -%} -{%- for m in mns.list -%} +{%- set sns = namespace(msgs=[], ids=[]) -%} +{%- for m in mns.msgs -%} {%- if m.role == "assistant" and m.get("tool_calls") -%} {%- set sns.ids = [] -%} {%- for tc in m.tool_calls -%} {%- set sns.ids = sns.ids + [tc.get("id") or (tc.function.get("id", "") if tc.function is defined else "")] -%} {%- endfor -%} - {%- set sns.list = sns.list + [m] -%} + {%- set sns.msgs = sns.msgs + [m] -%} {%- elif m.role == "user" and m.content_blocks -%} {%- set tb = namespace(blocks=[]) -%} {%- for b in m.content_blocks if b.type == "tool_result" -%} @@ -146,15 +146,15 @@ {%- set rb.blocks = rb.blocks + [b] -%} {%- endif -%} {%- endfor -%} - {%- set sns.list = sns.list + [dict(m, content_blocks=rb.blocks)] -%} + {%- set sns.msgs = sns.msgs + [dict(m, content_blocks=rb.blocks)] -%} {%- else -%} - {%- set sns.list = sns.list + [m] -%} + {%- set sns.msgs = sns.msgs + [m] -%} {%- endif -%} {%- else -%} - {%- set sns.list = sns.list + [m] -%} + {%- set sns.msgs = sns.msgs + [m] -%} {%- endif -%} {%- endfor -%} -{%- set msgs = sns.list -%} +{%- set msgs = sns.msgs -%} {#- Last user turn: users and mid-conversation systems count, except user turns made only of tool results. Tools anywhere disable drop_thinking. -#} From 1c9a39a25bc0f92e357abdbf2a64b958c144c7d8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 10:11:27 -0700 Subject: [PATCH 31/49] deepseek41: DSpark speculative draft (DFLASH single-pass mHC variant, --dspark converter, block window mask) --- conversion/deepseek41.py | 98 ++++++++++++- convert_hf_to_gguf.py | 12 +- gguf-py/gguf/constants.py | 2 + gguf-py/gguf/gguf_writer.py | 3 + src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-hparams.h | 1 + src/models/deepseek41.cpp | 5 +- src/models/dflash.cpp | 269 +++++++++++++++++++++++++++++++++++- src/models/models.h | 7 + 10 files changed, 389 insertions(+), 10 deletions(-) diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py index 6f32d2019043..2d2b19b245ae 100644 --- a/conversion/deepseek41.py +++ b/conversion/deepseek41.py @@ -186,7 +186,7 @@ def __init__(self, *args, **kwargs): raise ValueError(f"unsupported FP8 scale format {qc.get('scale_fmt')!r}") # the V4 constructor installed the V4 template; V4.1 has its own - template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.1.jinja" + template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja" if template_path.is_file(): self.gguf_writer.add_chat_template(template_path.read_text(encoding="utf-8")) @@ -554,3 +554,99 @@ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: return dense_qt return False + + +class DeepseekV41DSparkModel(DeepseekV41Model): + """DeepSeek-V4.1 DSpark speculative draft (mtp.0..2 of the checkpoint) as a DFLASH GGUF (`--dspark`). + + Layout = the V4 DSpark draft (conversion/deepseek.py DeepseekV4DSparkModel) with V4.1's differences flagged by + `dflash.hyper_connection.single_pass`: stages blk.0..2 (128 experts top-3, MXFP4 bit-exact, dense BF16), + mtp.0.main_proj / main_norm -> fc / enc.output_norm, mtp.2.norm -> output_norm, markov_head.embed / head -> + markov_w1 / markov_w2, confidence_head.proj -> conf_proj (F32), no hc_head. Token embeddings and the lm head are + the target's (shared at runtime through ctx_other). target_layers are the V4.1 ids as-is: the draft reads the + attention input of those layers (model.py Transformer.forward), which is what llama_set_embeddings_layer_inp + extracts. + """ + model_arch = gguf.MODEL_ARCH.DFLASH + mtp_only = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + hp = self.hparams + self._n_main = hp["num_hidden_layers"] + self._n_stage = hp["num_nextn_predict_layers"] + if self._n_stage <= 0 or "dspark_block_size" not in hp: + raise ValueError("DeepSeek-V4.1 DSpark export needs num_nextn_predict_layers > 0 and the dspark_* config") + # the main-model rekey (mtp.M -> layers.n_main+M) is kept for tensor selection and the MXFP4 expert source + # names; format_tensor_name() maps the block ids back to 0..n_stage-1 + self.block_count = self._n_main + self._n_stage + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self._n_stage) + + def format_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str: + if bid is not None and bid >= self._n_main: + bid -= self._n_main + return super().format_tensor_name(key, bid, suffix) + + def set_vocab(self): + super().set_vocab() + self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"]) + + def set_gguf_parameters(self): + hp = self.hparams + n = self._n_stage + w = self.gguf_writer + w.add_block_count(n) + w.add_context_length(hp["max_position_embeddings"]) + w.add_embedding_length(hp["hidden_size"]) + w.add_feed_forward_length(hp["moe_intermediate_size"]) + w.add_head_count(hp["num_attention_heads"]) + w.add_head_count_kv(1) + w.add_key_length(hp["head_dim"]) + w.add_value_length(hp["head_dim"]) + w.add_rope_dimension_count(hp["qk_rope_head_dim"]) + # DSpark stages are compress-ratio-0 layers: plain rope at rope_theta, no YaRN + w.add_rope_freq_base(float(hp["rope_theta"])) + w.add_layer_norm_rms_eps(hp["rms_norm_eps"]) + w.add_q_lora_rank(hp["q_lora_rank"]) + w.add_sliding_window(hp["sliding_window"]) + w.add_expert_count(hp["dspark_n_routed_experts"]) + w.add_expert_used_count(hp["dspark_num_experts_per_tok"]) + w.add_expert_feed_forward_length(hp["moe_intermediate_size"]) + w.add_expert_shared_count(hp["n_shared_experts"]) + w.add_expert_weights_scale(hp["routed_scaling_factor"]) + w.add_expert_weights_norm(hp["norm_topk_prob"]) + w.add_expert_gating_func(gguf.ExpertGatingFuncType.SQRTSOFTPLUS) + w.add_swiglu_clamp_exp([hp["swiglu_limit"]] * n) + w.add_swiglu_clamp_shexp([hp["swiglu_limit"]] * n) + w.add_attention_output_group_count(hp["o_groups"]) + w.add_attention_output_lora_rank(hp["o_lora_rank"]) + w.add_attention_compress_ratios([0] * n) + w.add_hyper_connection_count(hp["hc_mult"]) + w.add_hyper_connection_sinkhorn_iterations(hp["hc_sinkhorn_iters"]) + w.add_hyper_connection_epsilon(hp["hc_eps"]) + w.add_hyper_connection_single_pass(True) + w.add_block_size(hp["dspark_block_size"]) + w.add_target_layers(hp["dspark_target_layer_ids"]) + w.add_sample_from_anchor(True) + w.add_has_confidence_head(True) + w.add_file_type(self.ftype) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + # stage experts only (no Engram tables in the draft) + return super().generate_extra_tensors() + + _DSPARK_ROOT: dict[str, gguf.MODEL_TENSOR] = { + "main_proj.weight": gguf.MODEL_TENSOR.FC, + "main_norm.weight": gguf.MODEL_TENSOR.ENC_OUTPUT_NORM, + "norm.weight": gguf.MODEL_TENSOR.OUTPUT_NORM, + } + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name in ("embed.weight", "head.weight", "norm.weight"): + return [] # the target's (ctx_other) + if name.endswith(".ffn.gate.bias_vl"): + return [] # drafts are text-only + m = re.match(r"layers\.(\d+)\.(main_proj\.weight|main_norm\.weight|norm\.weight)$", name) + if m is not None and int(m.group(1)) >= self._n_main: + return [(self.format_tensor_name(self._DSPARK_ROOT[m.group(2)], None), data_torch)] + return super().modify_tensors(data_torch, name, bid) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index e09616b190cf..5e263fd16fa7 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -267,11 +267,15 @@ def main() -> None: sys.exit(1) if args.dspark: - if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM": - logger.error("--dspark is only supported for DeepseekV4ForCausalLM") + if is_mistral_format or model_architecture not in ("DeepseekV4ForCausalLM", "DeepseekV41ForCausalLM"): + logger.error("--dspark is only supported for DeepseekV4ForCausalLM and DeepseekV41ForCausalLM") sys.exit(1) - from conversion.deepseek import DeepseekV4DSparkModel - model_class = DeepseekV4DSparkModel + if model_architecture == "DeepseekV41ForCausalLM": + from conversion.deepseek41 import DeepseekV41DSparkModel + model_class = DeepseekV41DSparkModel + else: + from conversion.deepseek import DeepseekV4DSparkModel + model_class = DeepseekV4DSparkModel if args.mtp or args.no_mtp: if not model_class.supports_mtp_export: diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 5a3d7bc23310..1baf9cb6ff8f 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -248,6 +248,8 @@ class HyperConnection: MAGNITUDE = "{arch}.hyper_connection.magnitude" # absent means the mix projection is full rank (DeepSeek-V4 behaviour) LOW_RANK = "{arch}.hyper_connection.low_rank" + # single-pass mHC: each sublayer collapses with the previous sublayer's pre-mix (DeepSeek-V4.1) + SINGLE_PASS = "{arch}.hyper_connection.single_pass" class Engram: # deepseek41 hashed n-gram memory LAYERS = "{arch}.engram.layers" # u32[n_engram] diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 79b488f4efe6..759322f0bc81 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1158,6 +1158,9 @@ def add_hyper_connection_magnitude(self, value: float) -> None: def add_hyper_connection_low_rank(self, value: int) -> None: self.add_uint32(Keys.HyperConnection.LOW_RANK.format(arch=self.arch), value) + def add_hyper_connection_single_pass(self, value: bool) -> None: + self.add_bool(Keys.HyperConnection.SINGLE_PASS.format(arch=self.arch), value) + def add_ple_layers(self, values: Sequence[int]) -> None: self.add_array(Keys.PerLayerEmbedding.LAYERS.format(arch=self.arch), values) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index e80ac1299c1d..379b5def0305 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -322,6 +322,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_HYPER_CONNECTION_EPSILON, "%s.hyper_connection.epsilon" }, { LLM_KV_HYPER_CONNECTION_MAGNITUDE, "%s.hyper_connection.magnitude" }, { LLM_KV_HYPER_CONNECTION_LOW_RANK, "%s.hyper_connection.low_rank" }, + { LLM_KV_HYPER_CONNECTION_SINGLE_PASS, "%s.hyper_connection.single_pass" }, { LLM_KV_ENGRAM_LAYERS, "%s.engram.layers" }, { LLM_KV_ENGRAM_NUM_EMBEDDINGS, "%s.engram.num_embeddings" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index e49643df283d..75f000771c16 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -327,6 +327,7 @@ enum llm_kv { LLM_KV_HYPER_CONNECTION_EPSILON, LLM_KV_HYPER_CONNECTION_MAGNITUDE, LLM_KV_HYPER_CONNECTION_LOW_RANK, + LLM_KV_HYPER_CONNECTION_SINGLE_PASS, LLM_KV_ENGRAM_LAYERS, LLM_KV_ENGRAM_NUM_EMBEDDINGS, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index e879b84ac6b2..f14c9b7b2ef1 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -318,6 +318,7 @@ struct llama_hparams { uint32_t dsv4_o_group_count = 0; uint32_t dsv4_o_lora_rank = 0; uint32_t dsv4_hc_mult = 0; + bool dsv4_hc_single_pass = false; // DeepSeek-V4.1 DSpark draft: lagged pre-mix, no hc_head uint32_t dsv4_hc_sinkhorn_iters = 0; uint32_t dsv4_hash_layer_count = 0; float dsv4_compress_rope_base = 0.0f; diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index bbb4a850c70c..263f85ed7fc7 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -208,7 +208,7 @@ void llama_model_deepseek41::load_arch_tensors(llama_model_loader & ml) { const int64_t n_idx_embd = hparams.indexer_head_size; const int64_t n_idx_head = hparams.indexer_n_head; - // DSpark stages (blk.n_layer..) are loaded as optional tensors; the graph does not use them yet + // DSpark stages (blk.n_layer..) are loaded as optional tensors and unused here: the draft runs as a DFLASH model uint32_t n_expert_dspark = n_expert; ml.get_key(LLM_KV_DSPARK_EXPERT_COUNT, n_expert_dspark, false); @@ -288,7 +288,8 @@ void llama_model_deepseek41::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_deepseek41::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - throw std::runtime_error("DeepSeek-V4.1 DSpark drafting is not implemented yet"); + throw std::runtime_error("DeepSeek-V4.1 has no MTP graph: its DSpark draft is a separate DFLASH GGUF " + "(convert_hf_to_gguf.py --dspark), used with --spec-type draft-dspark"); } return std::make_unique(*this, params); } diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 982351dc860a..e9b3d4f2d989 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -81,6 +81,7 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, hparams.dsv4_hc_sinkhorn_iters); ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps); ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, false); + ml.get_key(LLM_KV_HYPER_CONNECTION_SINGLE_PASS, hparams.dsv4_hc_single_pass, false); GGML_ASSERT(hparams.dsv4_o_group_count > 0); // avoid div by zero @@ -94,6 +95,12 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) { } GGML_ASSERT(hparams.n_swa > 0); + if (hparams.dsv4_hc_single_pass) { + // DeepSeek-V4.1 DSpark: every block query sees the whole main-KV ring (the window_size positions up to the + // anchor's predecessor) plus the block. The graph gives all queries of a block the mask row of its first + // query, whose standard-SWA window must then reach back window_size positions before the anchor. + hparams.n_swa += 1; + } hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.set_swa_pattern(0); for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { @@ -256,9 +263,12 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { const int64_t hc_dim = hc_mult * n_embd; const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult; - hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc_dim, hc_mult}, 0); - hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0); - hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0); + // single-pass mHC (DeepSeek-V4.1) collapses with the last FFN pre-mix: no hc_head + if (!hparams.dsv4_hc_single_pass) { + hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc_dim, hc_mult}, 0); + hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0); + hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0); + } for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; @@ -1279,12 +1289,265 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ } } +// DeepSeek-V4.1 DSpark decoder (mtp.0..2 of the V4.1 checkpoint, model.py DSparkBlock / forward_spec), dual-mode +// by batch type like graph_dsv4: +// * embd batch -> main_x = main_norm(main_proj(mean-hc attention inputs of the target layers)), every stage's +// kv_norm(wkv(main_x)) roped at the target positions is written into its window ring +// * token batch -> [anchor, noise...] block through the stages; V4.1 blocks use single-pass mHC (attention +// collapses with the previous sublayer's pre-mix, identity copy 0 for the first stage, the FFN with +// this attention's), no per-head q norm, plain rope; collapse with the last FFN pre-mix (no +// hc_head); markov + confidence heads as V4 +llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const llm_graph_params & params) : + llama_model_deepseek41::graph(params) { + const int64_t n_embd_inp = hparams.n_embd_inp_enc(); + const int64_t n_embd_head = hparams.n_embd_head_k(); + const int64_t n_embd_head_rope = hparams.n_rot(); + const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope; + + GGML_ASSERT(hparams.dsv4_hc_single_pass); + + ggml_tensor * inp_pos = build_inp_pos(); + + llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa(); + + // stages are ratio-0 layers: plain rope at rope_theta, no YaRN (model.py precompute_freqs_cis, original_seq_len 0) + const auto rope = [&](ggml_tensor * t) { + t = ggml_rope_ext(ctx0, t, inp_pos, nullptr, n_embd_head_rope, rope_type, 0, + freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + return ggml_rope_set_offset(t, n_embd_head_nope); + }; + + ASSERT_EMBD_OR_TOKEN(ubatch); + if (ubatch.embd) { + auto inp = std::make_unique(n_embd_inp); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens); + ggml_set_input(inp->embd); + + ggml_tensor * inp_target = inp->embd; + cb(inp_target, "inp_target_features", -1); + + res->add_input(std::move(inp)); + + ggml_tensor * inp_g = build_lora_mm(model.fc, inp_target, model.fc_s); + inp_g = build_norm(inp_g, model.output_norm_enc, nullptr, LLM_NORM_RMS, -1); + cb(inp_g, "main_x", -1); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + ggml_tensor * kv = build_lora_mm(layer.wkv, inp_g); + kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il); + kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens); + kv = rope(kv); + cb(kv, "kv_injected", il); + + if (inp_attn->self_k_rot_swa) { + kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa); + } + ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il)); + } + + res->t_embd = inp_g; + + ggml_build_forward_expand(gf, inp_g); + return; + } + + auto * tok_embd = model.tok_embd; + if (tok_embd == nullptr) { + GGML_ASSERT(cparams.ctx_other != nullptr); + const auto * model_other = llama_get_model(cparams.ctx_other); + GGML_ASSERT(model_other->tok_embd != nullptr && "DSpark decoder requires the target model's token embeddings"); + tok_embd = model_other->tok_embd; + } + + auto inp = std::make_unique(n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + ggml_tensor * inp_tokens = inp->tokens; + + ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens); + cb(inpL, "inp_noise_embd", -1); + + res->add_input(std::move(inp)); + + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t n_groups = hparams.dsv4_o_group_count; + const int64_t n_heads_group = n_head / n_groups; + const int64_t o_lora_rank = hparams.dsv4_o_lora_rank; + const int64_t o_group_dim = n_heads_group*n_embd_head; + const float kq_scale = 1.0f/sqrtf(float(n_embd_head)); + + GGML_ASSERT(n_embd_head == n_embd_head_v); + GGML_ASSERT(n_head % n_groups == 0); + + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, 1, n_tokens); + inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1); + cb(inpL, "hc_init", -1); + + // window of a block = window of its first query (get_dspark_topk_idxs: ring + block for every query), see n_swa + { + ggml_tensor * m = inp_attn->self_kq_mask_swa_cnv; + const int64_t n_blocks = ubatch.n_seqs_unq; + const int64_t bd = n_blocks > 0 ? n_tokens / n_blocks : 0; + if (m && bd > 1 && n_tokens % n_blocks == 0 && m->ne[2] == 1 && m->ne[1] % bd == 0 && ggml_is_contiguous(m)) { + ggml_tensor * m4 = ggml_reshape_4d(ctx0, m, m->ne[0], bd, m->ne[1]/bd, m->ne[3]); + ggml_tensor * r0 = ggml_view_4d(ctx0, m4, m4->ne[0], 1, m4->ne[2], m4->ne[3], m4->nb[1], m4->nb[2], m4->nb[3], 0); + m4 = ggml_repeat(ctx0, r0, m4); + inp_attn->self_kq_mask_swa_cnv = ggml_reshape_4d(ctx0, m4, m->ne[0], m->ne[1], 1, m->ne[3]); + cb(inp_attn->self_kq_mask_swa_cnv, "dspark_block_mask", -1); + } + } + + ggml_tensor * pre_prev = nullptr; + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + ggml_tensor * residual = inpL; + ggml_tensor * post = nullptr; + ggml_tensor * comb = nullptr; + + ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il); + + ggml_tensor * cur; + if (pre_prev == nullptr) { + // identity pre-mix of the first stage (make_identity_pre_mix): copy 0 + cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0)); + } else { + cur = build_hc_pre(inpL, pre_prev, il); + } + cb(cur, "hc_attn_pre", il); + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // DSparkAttention: block queries see the main-KV ring plus every block key (non-causal draft context) + { + ggml_tensor * qr = build_lora_mm(layer.wq_a, cur); + qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(qr, "qr_norm", il); + + ggml_tensor * q = build_lora_mm(layer.wq_b, qr); + q = ggml_reshape_3d(ctx0, q, n_embd_head, n_head, n_tokens); + q = rope(q); + cb(q, "q", il); + + ggml_tensor * kv = build_lora_mm(layer.wkv, cur); + kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il); + kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens); + kv = rope(kv); + cb(kv, "kv", il); + + ggml_tensor * out = build_attn(inp_attn, + nullptr, nullptr, nullptr, + q, kv, kv, + nullptr, layer.attn_sinks, nullptr, + kq_scale, il); + cb(out, "attn_raw", il); + + out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, n_tokens); + out = ggml_rope_ext_back(ctx0, out, inp_pos, nullptr, n_embd_head_rope, rope_type, 0, + freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + out = ggml_rope_set_offset(out, n_embd_head_nope); + cb(out, "attn_derope", il); + + out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, n_tokens); + out = ggml_permute(ctx0, out, 0, 2, 1, 3); + ggml_tensor * oa = ggml_mul_mat(ctx0, layer.wo_a, out); + cb(oa, "attn_wo_a", il); + oa = ggml_permute(ctx0, oa, 0, 2, 1, 3); + oa = ggml_cont_2d(ctx0, oa, o_lora_rank*n_groups, n_tokens); + + cur = build_lora_mm(layer.wo_b, oa); + cb(cur, "attn_out", il); + } + + inpL = build_hc_post(cur, residual, post, comb, il); + cb(inpL, "hc_attn_post", il); + + residual = inpL; + ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il); + + cur = build_hc_pre(inpL, attn_pre, il); + cb(cur, "hc_ffn_pre", il); + + cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, hparams.n_expert_used(), + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il); + cb(moe_out, "ffn_moe_out", il); + + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, nullptr, nullptr, + layer.ffn_gate_shexp, nullptr, nullptr, + layer.ffn_down_shexp, nullptr, nullptr, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + + inpL = build_hc_post(cur, residual, post, comb, il); + + ggml_tensor * l_out = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens); + cb(l_out, "l_out", il); + ggml_build_forward_expand(gf, l_out); + + pre_prev = ffn_pre; + } + + // no hc_head: collapse with the last FFN's pre-mix (forward_head: hc_pre(x, pre_mix)) + ggml_tensor * cur = build_hc_pre(inpL, pre_prev, -1); + cb(cur, "hc_out", -1); + + // confidence head input: the un-normed collapsed hidden state + res->t_embd = cur; + + cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + + auto * output = model.output; + auto * output_s = model.output_s; + if (output == nullptr) { + GGML_ASSERT(cparams.ctx_other != nullptr); + const auto * model_other = llama_get_model(cparams.ctx_other); + GGML_ASSERT(model_other->output != nullptr && "DSpark decoder requires the target model's output projection"); + output = model_other->output; + output_s = model_other->output_s; + } + + cur = build_lora_mm(output, cur, output_s); + cb(cur, "result_output", -1); + res->t_logits = cur; + + if (model.dspark_markov_w1) { + build_dspark_markov_head(*this, model, inp_tokens); + } +} + std::unique_ptr llama_model_dflash::build_arch_graph(const llm_graph_params & params) const { switch (params.gtype) { case LLM_GRAPH_TYPE_ENCODER: return std::make_unique>(*this, params); case LLM_GRAPH_TYPE_DEFAULT: case LLM_GRAPH_TYPE_DECODER: + if (hparams.dsv4_hc_mult > 0 && hparams.dsv4_hc_single_pass) { + return std::make_unique(*this, params); + } if (hparams.dsv4_hc_mult > 0) { return std::make_unique(*this, params); } diff --git a/src/models/models.h b/src/models/models.h index e7cad1afd785..fe7f0424d2a4 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1440,6 +1440,8 @@ struct llama_model_deepseek41 : public llama_model_base { struct graph : public llama_model_deepseek4::graph { graph(const llama_model & model, const llm_graph_params & params); + // helpers only (no graph built): base of the DSpark draft graph (llama_model_dflash::graph_dsv41) + explicit graph(const llm_graph_params & params) : llama_model_deepseek4::graph(params) {} // HC coefficients of one sublayer: returns pre [hc, n_tokens], fills post / comb ggml_tensor * build_hc_mixes( @@ -1534,6 +1536,11 @@ struct llama_model_dflash : public llama_model_base { graph_dsv4(const llama_model & model, const llm_graph_params & params); }; + // DeepSeek-V4.1 DSpark: single-pass mHC (lagged pre-mix, no hc_head), no per-head q norm + struct graph_dsv41 : public llama_model_deepseek41::graph { + graph_dsv41(const llama_model & model, const llm_graph_params & params); + }; + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; From 7a582830e00504717a9ed91192b21d0f7d4a8a1d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 10:13:22 -0700 Subject: [PATCH 32/49] deepseek41: STATUS_A A2 results (CPU vs MATCHED, A5 pin sim, DSpark) --- STATUS_A.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/STATUS_A.md b/STATUS_A.md index a9e7fa37a62b..cdc354f6c846 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -9,7 +9,7 @@ Branch `dsv41` in `$W/wt_dsv41` (off llama.cpp_pinset fd55cb98f). Contract for b | A2 parity (bundled tools/dump_logits + port/compare_ref.py, F32 gate --max-kl-mean 1e-6 --min-top1 1.0 --min-cos 0.99999) | done (CPU), CUDA calibrated | see "A2 numbers" below; 13/13 seeded bugs FAIL on the small fixture, 5/5 on real dims | | A3 template + V4.1 DSML tool parser | done (1d51c092f) | template `models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja` (= `$W/models/dsv41/chat_template.jinja`, 5/5 byte parity with encoding.py). V4.1 tags carry a leading space (`<｜DSML｜ calls>`, `<｜DSML｜ invoke>`, `<｜DSML｜ parameter>`); detection in common/chat.cpp keys on `｜DSML｜ calls>` (the template never names `dsml_token`), parser = the V4 parser with the spaced tags. test-chat: the V4 parser suite ported to V4.1 markup (content, thinking, single / parallel / mixed-type calls, content before calls, continuation, reconstruction) all pass; negative control (V4.1 tag switch off) aborts on the V4.1 block. | | A4 never break other models | done | test-chat all passed; test-llama-archs CPU all 137 passed; CUDA: full suite exceeded its 2 h box (killed, no verdict), so the archs sharing touched code were run: deepseek4/41, deepseek2/32, glm-dsa, glm5-next, maple, qwen3moe, qwen35moe, qwen4exp on CUDA + CPU + Meta: 28/28 passed; deepseek41 on CPU / CUDA / Meta OK; test-backend-ops for used ops pass; V4 logits bitwise identical base vs head; no vocab change (vocab differential N/A) | -| A5 pin PR / pinset sim | not started (after A3 and B's GGUF) | | +| A5 pin PR / pinset sim | local branch `dsv41-pin` (worktree `wt_dsv41_pin`), not pushed | see "A5" below | ### A2 numbers - CPU, F32 KV, FA off, single-shot / ub 7 / ub 1, swa_full 1 and 0: small fixture KL mean <= 4.9e-12, top-1 1.0, router top-k @@ -140,7 +140,7 @@ closer to the truth is for C's full-model MATCHED oracle (prompts: outputs/dsv41 | 3 graph + KV-cache generalisation, CPU + CUDA parity | done | bae963fb2 | | 4 test-llama-archs (CPU, CUDA, Meta split), test-backend-ops, V4 regression | done | d29889d8f | | 5 full-model handoff (below) | written, waiting for B's GGUF | | -| 6 DSpark / MTP graph | not started (tensors load as optional, graph throws on the MTP context) | | +| 6 DSpark / MTP graph | done (DFLASH draft) | 1c9a39a25, see "DSpark" below | ## What the graph implements (`src/models/deepseek41.cpp`) Engram (host n-gram hash from the raw-cache token history, CPU get_rows on `engram_embd`, gate as model.py), single-pass mHC @@ -228,3 +228,54 @@ logits bitwise equal to the run without the override; negative control (build wi - Vision tower not ported (the text side of image spans is done, see below). - No ubatch shares a token across sequences (asserts, as the PLE path). - DSpark drafting graph not implemented. + +## A2 additions (2026-10-17) +### CPU full model vs official MATCHED (`scripts/dsv41/score_arms.py `, dumps `outputs/dsv41_full/cpu_wiki1100`) +CPU ngl 0, ub 512, F32 KV, fa 0, 96 threads (`scripts/dsv41/dump_logits_mt.cpp`, DUMP_THREADS), ~220 s per prompt. +KL(matched || arm) mean: wiki1100 CPU 6.21e-3, CUDA ub512 6.11e-3, ub7 9.9e-3, ub1 6.2e-3, fa1 6.3e-3, native 2.6e-2; +at the 30 CPU-vs-CUDA sensitive positions (KL > 0.05) CPU 7.4e-2 vs CUDA 8.1e-2, CPU closer on 60%: no systematic +winner. t1025 (random tokens): CPU 3.95e-2, CUDA 2.26e-2, native 9.4e-2; CPU is already off at t0 (KL 0.84 vs CUDA +0.058), i.e. per-token kernel numerics amplified by a chaotic input. Both arms sit ~6x over 1e-3, at C's bf16 floor. + +### A5 (pin PR = upstream + this change; local only) +Branch `dsv41-pin` = dsv41 minus STATUS_A.md / TENSORS.md / the GGML_CUDA_F32_EXACT debug knob, plus +`git merge --no-ff b11707` ("Fix merge conflicts": test-llama-archs gating switch, upstream added MELLUM, DEEPSEEK41 added +to the DEEPSEEK4 case), plus later dsv41 merges. Upstream #30670 (Beam) added a jinja object builtin `list` that shadows +the template's `namespace(list=...)` attribute ("Function is not a bool value" in test-chat): fixed in the template +(attribute renamed to msgs, cd1714ad9; 5/5 encoding.py parity re-verified; `models/dsv41/chat_template.jinja` updated, +old copy kept as `.pre_msgs_rename`). GGUFs built before this embed the old template and break on the nightly (which +pins #30670). Converter template path fixed (`deepseek-ai-DeepSeek-V4.1.jinja` was deleted in 47dd9807b, so the +converter silently kept the V4 template). +b11714 + pin (CPU): test-llama-archs 140/140, test-chat all pass, DSpark gate PASS (below). +`llama_pinset.py sim --tags window` with the pin appended (placeholder `danielhanchen#99999`, `temp/dsv41_pinset/pr-set.json`): +REFUSED on every tag b11707..b11714 on `ggml/include/ggml.h` + `gguf-py/gguf/constants.py` against unslothai#61 +(IQ1_XS/XXS/XXXS at 64..66, GGML_TYPE_COUNT 67) vs MXFP8 = 43 / COUNT 44: both pins edit the GGML_TYPE_COUNT line, not +additive. Needs a decision (fold MXFP8 into the pinset next to #61, or order + a resolved type list). Baseline without +the pin already FAILs: ggml-org#30670 REFUSED (common/parsers/*, + llama-arch/model on b11714), pre-existing. + +### DSpark (MTP) draft +DeepSeek-V4.1 DSpark = upstream's DFLASH DSpark draft (V4 `graph_dsv4`, driver `--spec-type draft-dspark`) with +`dflash.hyper_connection.single_pass = true` selecting `llama_model_dflash::graph_dsv41`: single-pass mHC (identity copy 0 +for stage 0, FFN uses the attention pre-mix), no hc_head (collapse with the last FFN pre-mix), no per-head q norm, plain +rope; target_layers = the V4.1 ids (attention INPUT of layers 37-39, after Engram, as model.py). Exact ring window: every +block query sees the 128-position main-KV ring plus the block (get_dspark_topk_idxs); the graph gives all block queries +the mask row of the first one and n_swa = window + 1. Converter: `convert_hf_to_gguf.py --dspark` -> +`DeepseekV41DSparkModel` (fixture output value-identical, 78/78 tensors, to `scripts/dsv41/dspark_fixture_gguf.py`). +Reference: `scripts/dsv41/dspark_ref.py` (C's ref_run.build matched, forward_spec at temperature 0, fx_base t700, draft +points s = 64 / 127 / 128 / 300 / 699) -> `outputs/dsv41_dspark/ref`. Gate `scripts/dsv41/dspark_gate.py` + harness +`dspark_test.cpp` (target prefill with layer-input extraction, embd injection, [anchor, noise x4] block, logits incl. +markov bias, sigmoid confidence): +| build | KL mean (all points) | chain top-1 | conf abs err | +|---|---|---|---| +| CPU F32 | 9.6e-13 .. 2.3e-11 | 25/25 | <= 6.5e-7 | +| CUDA F32 exact | 1.0e-12 .. 4.3e-12 | 25/25 | <= 5.2e-7 | +| b11714 + pin, CPU | same as CPU | 25/25 | | +| converter GGUF (MXFP4 experts), CPU | 2.8e-5 .. 3.0e-5 (q8_0 activations) | 10/10 (s64, s300) | 1.2e-3 | +Negative control: before the block-window mask (standard SWA 128) s >= 127 FAILED (KL 3e-3 .. 5e-3, chain 3/5 .. 4/5). +Server loop (`scripts/dsv41/dspark_server_smoke.sh`, llama-server, greedy 300 tokens): with F32 KV + fa off, output with +the draft is identical to no draft (881 drafted, 4 accepted: random fixture weights). With server defaults (F16 KV, +FA auto) both draft-dspark and draft-simple (self-draft, 97% accepted) diverge at the same token, a 0.009-logit near-tie: +F16 attention numerics of the batched verify. Target rollback via the API (seq_test rollback on fx_base, D 4, R 3, +acc 1..4) is exact: max abs logit diff 9.5e-6 vs one-shot. +Not proven: real-model DSpark acceptance (needs the 7.9 GB mtp.* converted + a GPU window); multi-sequence DSpark +blocks; KV eviction of ring cells in a long running loop with swa_full off (the smoke ran 300 tokens past the window). From da47a97a93b7f6c9fd0e113d057486367fea78aa Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 10:15:49 -0700 Subject: [PATCH 33/49] ggml: MXFP8 -> fork-local type id 67 (42..63 reserved for upstream), GGML_TYPE_COUNT 68 --- ggml/include/ggml.h | 6 ++++-- gguf-py/gguf/constants.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index d3d484ee7d19..17e2918290ab 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -430,8 +430,10 @@ extern "C" { GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, GGML_TYPE_Q2_0 = 42, - GGML_TYPE_MXFP8 = 43, // MXFP8 E4M3 (32 elements, E8M0 scale) - GGML_TYPE_COUNT = 44, + // fork-local (unslothai): 42..63 stay reserved for upstream, so a stock build rejects these files + // instead of misreading them + GGML_TYPE_MXFP8 = 67, // MXFP8 E4M3 (32 elements, E8M0 scale) + GGML_TYPE_COUNT = 68, }; // [TAG_GGML_PREC] diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 1baf9cb6ff8f..5f4cbdf376b2 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -6198,7 +6198,7 @@ class GGMLQuantizationType(IntEnum): NVFP4 = 40 Q1_0 = 41 Q2_0 = 42 - MXFP8 = 43 + MXFP8 = 67 # fork-local id (42..63 reserved for upstream) class ExpertGatingFuncType(IntEnum): From 3ac687a2ba3fa9de62d4cce5ce04502d66478737 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 10:44:30 -0700 Subject: [PATCH 34/49] deepseek41: STATUS_A MXFP8 67, header patcher, pin after #61, real DSpark draft --- STATUS_A.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/STATUS_A.md b/STATUS_A.md index cdc354f6c846..f6bb9175b5c2 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -279,3 +279,25 @@ F16 attention numerics of the batched verify. Target rollback via the API (seq_t acc 1..4) is exact: max abs logit diff 9.5e-6 vs one-shot. Not proven: real-model DSpark acceptance (needs the 7.9 GB mtp.* converted + a GPU window); multi-sequence DSpark blocks; KV eviction of ring cells in a long running loop with swa_full off (the smoke ran 300 tokens past the window). + +### MXFP8 renumber to fork-local 67 (main's decision, da47a97a9) +GGML_TYPE_MXFP8 = 67, GGML_TYPE_COUNT = 68 (42..63 reserved for upstream, as #61). No other literal 43 / COUNT-sized +table depends on it (type_traits designated initialisers; type loops already skip NULL names / blck_size 0). +New build rejects old (43) files and old builds reject 67 files (`invalid ggml type`), both checked. +Header patcher `scripts/dsv41/gguf_header_patch.py` (dry run default; --apply backs up header bytes; pwrite inside the +header only): type 43 -> 67 for `blk.N.engram_embd.weight` (2-D, ne0 % 32, size checked), template list -> msgs +(same length; patched template == the repo jinja byte for byte). On copies (temp/dsv41_patch_test): fx_base, fx_engram0, +pkg shard-1 copy, pkg shard-8 header + 256 MiB: data sha256 unchanged; cmp vs original = 126 bytes, all below +data_start. Dry run on pkg/BF16 (read only): shard 1 template, shards 8 and 9 one Engram table each, mtp-*.gguf template. +NOT applied to pkg/* (main applies after D's grid). verify_all.py accepts type 67 (or 43) for Engram MXFP8. +Fixture gate on the patched fx_base with the new build: unchanged (t129 9.2e-12, t3001 2.30e-5); DSpark gate unchanged; +CUDA test-backend-ops GET_ROWS 225/225 (MXFP8 incl.). +Pin: `dsv41-pin` (f8cfdaedb) = upstream + change (MXFP8 67) still REFUSED after #61 (both edit the COUNT line). +`dsv41-pin-after61` (71752e6e0) = dsv41-pin + #61 merged, COUNT pre-resolved to 68: merges clean on b11707..b11714; +pin_contract FAILs only unslothai#61 (`GGML_TYPE_COUNT = 67` not kept: replaced by 68) -> needs the fold (#241/#253 +precedent). Mix tree b11714 + pins (temp/dsv41_mix_b11714): test-llama-archs 142/142, test-chat pass, fx_base gate PASS +(t3001 2.44e-5), DSpark gate PASS. ggml-org#30670 REFUSED on every tag before us: its squash 01fc3aae8 is already in +the aged base b11707, so the pin re-applies upstream code (pr-set doc: delete the entry once a base tag carries it). +Real DSpark draft: models/dsv41/gguf/dspark/DeepSeek-V4.1-Flash-1017-DSpark-DFLASH-BF16.gguf (8.5 GB, 78 tensors, +target_layers 37/38/39, 128 experts top-3, 13 s). Acceptance run: scripts/dsv41/dspark_accept_real.sh with the MXFP8=43 +build temp/wt_dsv41_43 (7a582830e) because pkg is not patched yet; gpu_queue --gb 36 (--cpu-moe target). From 70c3c18fd41077735603f3b1bd2f872789e1b6f9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 12:31:24 -0700 Subject: [PATCH 35/49] deepseek41: STATUS_A full-model selection sets vs MATCHED --- STATUS_A.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/STATUS_A.md b/STATUS_A.md index f6bb9175b5c2..465df4ea381c 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -301,3 +301,17 @@ the aged base b11707, so the pin re-applies upstream code (pr-set doc: delete th Real DSpark draft: models/dsv41/gguf/dspark/DeepSeek-V4.1-Flash-1017-DSpark-DFLASH-BF16.gguf (8.5 GB, 78 tensors, target_layers 37/38/39, 128 experts top-3, 13 s). Acceptance run: scripts/dsv41/dspark_accept_real.sh with the MXFP8=43 build temp/wt_dsv41_43 (7a582830e) because pkg is not patched yet; gpu_queue --gb 36 (--cpu-moe target). + +### Full-model selection sets vs official MATCHED traces (CPU BF16 GGUF, ub 512; logits bitwise = the CPU arm) +Capture: `scripts/dsv41/dump_sel.cpp` (chunked per ubatch: sel_top_k's width min(n_comp, 512) changes while +n_comp < 512, so plain concatenation misaligns rows; outputs/dsv41_full/sel_ub512 sel_top_k files are affected). +Analysis `scripts/dsv41/sel_full.py`: per layer rows whose visible selected set differs; for index layers the gap +(lowest llama-only pick minus highest reference-only pick, llama's own scores) relative to the row's top score. +wiki1100 (T 1100): L2 12 / 75 binding rows (all single swaps, gap max 1.1e-3), L8 26, L14 57 (gap max 8.4e-2), +L20-L36 579-587 / 588 binding rows (median 6 swaps; gap <= 1e-2 on 526-555 rows, > 1e-1 on 5-21); candidate pool 0 +mismatches (no exclusion below 16k). long17k (T 17000): L2 13092 / 15975 binding rows (median 1 swap, gap <= 1e-3 on +11861, max 9.7e-3); L8 15760, L14 15942, ratio-1 index layers ~16480 / 16488 (median 13-15 swaps, gap <= 1e-2 on +99.5%, > 1e-1 on <= 21 rows); candidate pool 558 / 17000 rows (616 with real exclusion), median 3 block swaps, gap +median 2.3e-3, <= 1e-2 on 461. Reuse layers equal their source. Every first-index-layer flip is a near-tie at the +BF16 activation precision of the CPU path (vs 87 / 17000 L2 rows on the F32 budget fixture); later layers inherit. +No reference scores exist in the traces, so large-gap rows (cascades) cannot be classified further. From f947debe6b9360229cb9e7b5f46329739e304ecd Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 12:53:46 -0700 Subject: [PATCH 36/49] deepseek41: STATUS_A full-model gate against the official MATCHED oracle --- STATUS_A.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/STATUS_A.md b/STATUS_A.md index 465df4ea381c..6b3a6e793c59 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -65,6 +65,18 @@ the random-token t1025; fa1 vs ub512 7.4e-3 / 3.2e-3. Attribution: the full mode kernels (MMQ / MMVQ, q8_1 activations) differ from CPU there. Released V4-Flash (UD-IQ1_S) on the same text: ub7 vs ub512 KL mean 3.4e-2, 0.4 at t6. So the graph is batch-invariant; the CUDA spread is kernel numerics. Which side is closer to the truth is for C's full-model MATCHED oracle (prompts: outputs/dsv41_full/bf16_ub512/tokens). +Full-model gate vs C's official MATCHED (TP4 model.py, outputs/dsv41_full/official/refs; `scripts/dsv41/gate_full.py`), +KL mean (top-1), with native-vs-MATCHED for scale: +| prompt | ub512 | ub7 | ub1 | fa1 | native | +|---|---|---|---|---|---| +| wiki1100 (real text) | 6.1e-3 (0.977) | 9.9e-3 (0.976) | 6.2e-3 (0.983) | 6.3e-3 (0.982) | 2.6e-2 | +| long17k (real, > pool) | 2.8e-3 (0.987) | 3.3e-3 (0.987) | - | 3.0e-3 (0.987) | 1.5e-2 | +| t3001 (random ids) | 1.5e-2 (0.901) | - | - | 1.7e-2 (0.905) | 3.7e-2 | +| c1600 / t1025 / t100 (random ids) | 1.9e-2 / 2.3e-2 / 1.5e-1 | - / 5.7e-2 / - | - / - / 2.1e-1 | | 4.9e-2 / 9.4e-2 / 2.1e-1 | +llama.cpp (BF16 dense + MXFP4 experts, CUDA) is 2.5-5x closer to MATCHED than the official native kernels on every +prompt but t100 ub1 (equal); real text sits at the bf16 / q8-activation floor (C: bf16 floor 5e-3 on fixtures). +Random-id prompts are an out-of-distribution, chaotic regime where native itself is 0.05-0.2 away. First 12 wiki +tokens: CPU and CUDA llama are equally close to MATCHED at the sensitive positions (t6: 0.2 / 0.1; native 0.9). ## Critique fold-in (plans/dsv41_support.md "Critique fold-in") 1. No per-head q RMS norm: the graph has none; "V4 per-head q norm" is seeded bug #6 and FAILS (KL 4e-3..2). From 9d110cc6dfcfe7555b7126e3c84e96c22305f4f5 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 13:39:11 -0700 Subject: [PATCH 37/49] deepseek41: keep F32-weight matmuls off TF32 on CUDA (ubatch-dependent routing) CUDA ran F32 x F32 MUL_MAT exactly for <= 3 columns (MMVF), in TF32 on tensor cores for 4..16 (MMF) and with cuBLAS tensor-op math above, so the DSV4.1 router, mHC mixes and indexer scores changed with the ubatch size and the 384-expert top-6 routing flipped (full model ub7 KL vs MATCHED 9.9e-3 vs 6.1e-3 at ub512). - ggml-cuda: honour ggml_prec_set_src(GGML_PREC_F32, 1) for F32 weights: MMVF up to 8 columns, else cuBLAS SGEMM with CUBLAS_DEFAULT_MATH - deepseek41: request it on router logits, hc mixes, indexer scores and kq / kqv (arch-gated in llama-graph) - test-backend-ops: MUL_MAT prec-F32 cases at NMSE 1e-10 across the MMVF / MMF / cuBLAS ranges --- STATUS_A.md | 27 ++++++++++++++++++++++++ ggml/src/ggml-cuda/ggml-cuda.cu | 23 +++++++++++++++++--- src/llama-graph.cpp | 12 +++++++++++ src/models/deepseek41.cpp | 3 +++ tests/test-backend-ops.cpp | 37 +++++++++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 3 deletions(-) diff --git a/STATUS_A.md b/STATUS_A.md index 6b3a6e793c59..0a88c6e43751 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -327,3 +327,30 @@ mismatches (no exclusion below 16k). long17k (T 17000): L2 13092 / 15975 binding median 2.3e-3, <= 1e-2 on 461. Reuse layers equal their source. Every first-index-layer flip is a near-tie at the BF16 activation precision of the CPU path (vs 87 / 17000 L2 rows on the F32 budget fixture); later layers inherit. No reference scores exist in the traces, so large-gap rows (cascades) cannot be classified further. + +## CUDA small-ubatch divergence (A3, 2026-10-17): root cause TF32 on F32 weights, fixed +- Symptom (full model, CUDA, vs official MATCHED): wiki1100 KL mean ub512 6.1e-3 / ub7 9.9e-3 (max 1.94 at t=406); + t1025 2.3e-2 / 5.7e-2. CPU is ub-invariant (bitwise). +- Not a leak: per-layer dumps at wiki t=406 (ub7 vs ub512, `scripts/dsv41/ubsweep/full_an.py`) show l_out drifting by + 5e-4..7.5e-3 relative over L0-L4, then a routed-expert flip at L5 (173 vs 81) and more flips downstream; no + mask / selection / state anomaly. The drift source: F32-weight matmuls (MoE router `ffn_gate_inp`, mHC `hc_*_fn`, + indexer score `idx_k x idx_q`, and K/Q/V with an F32 KV cache) run in TF32 on CUDA, with a precision that depends + on the ubatch size: ncols <= 3 MMVF (exact F32), 4..16 MMF tensor cores (TF32, max err 7e-5 of sum|a*b|), >= 17 + cuBLAS with CUBLAS_TF32_TENSOR_OP_MATH (2.4e-5) (`scripts/dsv41/ubsweep/mm_prec.cpp`). The 384-expert top-6 router + is near-tie sensitive, so ubatch size changed routing. Full-model A/B: ub7 with GGML_CUDA_F32_EXACT=1 + + NVIDIA_TF32_OVERRIDE=0 = 6.0e-3 (excess gone); ub8 6.2e-3, ub9 6.7e-3 (same TF32, luckier tails). +- Fix (commit below): CUDA honours `ggml_prec_set_src(mm, GGML_PREC_F32, 1)` for F32 x F32 MUL_MAT (no TF32: MMVF up + to 8 columns, else cuBLAS SGEMM with CUBLAS_DEFAULT_MATH); the DSV4.1 graph requests it on the router logits, the + hc mixes, the indexer scores and kq / kqv (no-op for F16 / quantized KV caches). Arch-gated: other models unchanged + (random V4 model bitwise identical base vs fix at ub 7 / 256; test-llama-archs deepseek2/4/41, glm-dsa 12/12). +- After (full model, same GPUs/tokens, 2 x B200): wiki1100 ub512 5.74e-3, ub7 5.75e-3, ub5 5.56e-3, ub8 6.91e-3 + (CPU 6.21e-3); medians 4.6e-6..9e-6. t1025 ub512 3.3e-2, ub7 3.0e-2: t1025 position 0 is degenerate (random first + token): CPU 0.84, fixed CUDA 0.92 / 1.07, old TF32 ub512 0.058 by luck; random-id prompts stay uninformative. +- Gates: new `scripts/dsv41/gate_ub_cuda.sh` (A: F32X exact, ub 1..17 vs one-shot KL <= 1.7e-8 mean, one near-tie + position; B: hc_mixes-0 token 0 at every ub vs ub1 <= 2.7e-7 rel, real dtypes, no env) PASS; negative control (pre-fix 7a582830e): A passes + (the graph itself was always ub-invariant), B FAILS at 2.6e-4 for every ub >= 4 and the one-shot; test-backend-ops new + `MUL_MAT(m,n,k,bs)` prec-F32 cases 18/18 OK (negative control without the CUDA change: FAIL, NMSE 4.7e-7 MMF / + 6.8e-8 cuBLAS vs 1e-10); fixture gate_c CPU t129 / t1027 9.2e-12 / 1.1e-7, CUDA exact t129 / t1027 / t3001 + 8.6e-12 / 4.5e-9 / 5.8e-5 (unchanged). +- Upstream V4-Flash: same code path (F32 router / hc_fn on CUDA go through MMF TF32 at 4..16 columns and cuBLAS TF32 + above; ubatch-dependent routing). Not changed here (arch gate). Candidate upstream issue, not filed. diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 2feea3840fd9..3074617445e7 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -1959,13 +1959,30 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor // Therefore, in such cases use cuBLAS. const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) && src0->view_src; - if (bad_padding_clear || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + + const int cc = ggml_cuda_info().devices[ctx.device].cc; + const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; + + // [TAG_GGML_PREC] src1 requested at GGML_PREC_F32 with F32 weights: keep both operands in F32, i.e. no TF32 + // rounding (the MMF tensor-core path and cuBLAS tensor-op math both round to a 10-bit mantissa). Small batches use + // the F32 matrix-vector kernel, larger ones cuBLAS SGEMM with the default (non-TF32) math mode. + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && + ggml_get_op_params_i32(dst, 3) == GGML_PREC_F32) { + if (!bad_padding_clear && ne11 <= MMVF_MAX_BATCH_SIZE && + ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, /*ne11 =*/ 1)) { + ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst); + return; + } + CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_DEFAULT_MATH)); ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst); + CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_TF32_TENSOR_OP_MATH)); return; } - const int cc = ggml_cuda_info().devices[ctx.device].cc; - const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; + if (bad_padding_clear || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst); + return; + } if (ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11)) { // The custom F16 vector kernel can be used over batched cuBLAS GEMM. diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 675ac5948088..c5545e83bb4d 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2154,6 +2154,11 @@ ggml_tensor * llm_graph_context::build_moe_ffn( if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) { ggml_prec_set_acc(logits, GGML_PREC_F32); } + if (arch == LLM_ARCH_DEEPSEEK41) { + // F32 router: keep the logits off TF32 on CUDA (top-6 of 384 is near-tie sensitive; with TF32 the routing + // depended on the ubatch size, 4..16-token batches being the worst) + ggml_prec_set_src(logits, GGML_PREC_F32, 1); + } cb(logits, "ffn_moe_logits", il); } else { logits = probs_in; @@ -2936,6 +2941,10 @@ ggml_tensor * llm_graph_context::build_attn_mha( // note: this op tends to require high floating point range // while for some models F16 is enough, for others it is not, so we default to F32 here ggml_prec_set_acc(kq, GGML_PREC_F32); + if (arch == LLM_ARCH_DEEPSEEK41) { + // with an F32 KV cache: no TF32 rounding of K / Q on CUDA (no-op for F16 / quantized caches) + ggml_prec_set_src(kq, GGML_PREC_F32, 1); + } if (arch == LLM_ARCH_GROK) { // need to do the following: @@ -2975,6 +2984,9 @@ ggml_tensor * llm_graph_context::build_attn_mha( } ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq); + if (arch == LLM_ARCH_DEEPSEEK41) { + ggml_prec_set_src(kqv, GGML_PREC_F32, 1); + } cb(kqv, "kqv", il); // for MLA with the absorption optimization, we need to "decompress" from MQA back to MHA diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index 263f85ed7fc7..4f64b8fc42f9 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -473,6 +473,8 @@ ggml_tensor * llama_model_deepseek41::graph::build_hc_mixes( ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc_dim, nt); ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, norm_rms_eps); ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm); + // F32 weights (model.py hc_pre runs in fp32): no TF32 rounding on CUDA, else the mixes depend on the ubatch size + ggml_prec_set_src(mixes, GGML_PREC_F32, 1); cb(mixes, "hc_mixes", il); ggml_tensor * pre = dsv41_view_2d(ctx0, mixes, hc, nt, 0); @@ -745,6 +747,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( ggml_tensor * kq = ggml_mul_mat(ctx0, ggml_permute(ctx0, ik, 0, 2, 1, 3), // [d, n_comp, 1, ns] ggml_permute(ctx0, iq, 0, 2, 1, 3)); // [d, nt/ns, n_ih, ns] -> [n_comp, nt/ns, n_ih, ns] + ggml_prec_set_src(kq, GGML_PREC_F32, 1); // F32 index keys: no TF32 rounding of the near-tie scores kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3)); // [n_ih, nt/ns, n_comp, ns] ggml_tensor * score = ggml_sum_rows(ctx0, ggml_mul(ctx0, ggml_relu(ctx0, kq), iw)); // [1, nt/ns, n_comp, ns] score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); // [n_comp, nt/ns, 1, ns] diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 329a7f638bb0..0301a0bbe06a 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -5192,6 +5192,37 @@ static bool graph_mul_mat_hi_prec_act(ggml_cgraph * gf, ggml_op op) { return false; } +// GGML_OP_MUL_MAT with src1 requested at GGML_PREC_F32 ([TAG_GGML_PREC]): F32 weights must not be rounded to a +// lower precision (e.g. TF32 tensor cores) at any batch size, so the tolerance is at the F32 level, not 5e-4 +struct test_mul_mat_prec_f32 : public test_case { + const int64_t m; + const int64_t n; + const int64_t k; + const int64_t bs; + + std::string vars() override { + return VARS_TO_STR4(m, n, k, bs); + } + + double max_nmse_err() override { + return 1e-10; // F32 accumulation-order noise is ~1e-14; TF32 inputs give ~1e-7 + } + + test_mul_mat_prec_f32(int64_t m = 128, int64_t n = 7, int64_t k = 1024, int64_t bs = 1) + : m(m), n(n), k(k), bs(bs) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, m, bs); + ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, n, bs); + ggml_set_name(a, "a"); + ggml_set_name(b, "b"); + ggml_tensor * out = ggml_mul_mat(ctx, a, b); + ggml_prec_set_src(out, GGML_PREC_F32, 1); + ggml_set_name(out, "out"); + return out; + } +}; + // GGML_OP_MUL_MAT struct test_mul_mat : public test_case { const ggml_type type_a; @@ -10740,6 +10771,12 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); + // src1 at GGML_PREC_F32: no TF32 at any batch size (mmvf / mmf / cuBLAS ranges), incl. a batched (attention) shape + for (int64_t n : {1, 3, 4, 7, 8, 9, 16, 17, 64}) { + test_cases.emplace_back(new test_mul_mat_prec_f32(384, n, 5120, 1)); + test_cases.emplace_back(new test_mul_mat_prec_f32(256, n, 512, 4)); + } + // m == 1, with n on both sides of MMVF_MAX_BATCH_SIZE (8): mmvf below, operand swap above for (int64_t n : {1, 7, 8, 9, 16, 127, 128, 511, 512}) { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1, n, 2048, {1, 1}, {1, 1})); From 0d25f726d47e449ad4eb1a33cf6abc81852ef3da Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 14:04:03 -0700 Subject: [PATCH 38/49] deepseek41: STATUS_A selection-trace layout, DSpark identity check --- STATUS_A.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/STATUS_A.md b/STATUS_A.md index 0a88c6e43751..3900d5d05f34 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -354,3 +354,16 @@ No reference scores exist in the traces, so large-gap rows (cascades) cannot be 8.6e-12 / 4.5e-9 / 5.8e-5 (unchanged). - Upstream V4-Flash: same code path (F32 router / hc_fn on CUDA go through MMF TF32 at 4..16 columns and cuBLAS TF32 above; ubatch-dependent routing). Not changed here (arch gate). Candidate upstream issue, not filed. +- Full-model selection traces (main's item): the multi-ubatch sel_top_k dump is a concatenation with a per-ubatch row + width k = min(n_comp, 512) (ratio-2 layers: k = 256 in the first 512-token ubatch), so shape.txt's [T-256, 512] is a + misparse; `scripts/dsv41/ubsweep/sel_full.py` parses per ubatch. Correctly parsed, t3001 L2 still differs on + 1515 / 3001 rows (1458 with the TF32 fix), mostly single swaps, from the first binding row (t = 1041). Not a split + defect: the fixture at ONE ubatch with the real dtypes (fx_base BF16 + MXFP4) gives L2 1426 / 3001, L8 2377, L12 + 2406, while F32X exact gives 12 / 129 / 174. The gap is llama.cpp's bf16 / q8 activation rounding vs the f32 + MATCHED oracle at near-tie top-k boundaries (score the selections against a bf16-activation oracle to gate them). +- DSpark draft vs no-draft (F32 KV, fa off, greedy): fixture F32X with GGML_CUDA_F32_EXACT=1 + NVIDIA_TF32_OVERRIDE=0: + IDENTICAL (300 tokens, pre- and post-fix), so the speculative verify path is exact. Default precision: differs + (fixture: dense F32 weights stay TF32 by design; real model, --cpu-moe, fixed build: first diff at token 50, + base top-2 " all" -0.87 vs " these" -1.17). Bitwise draft == no-draft needs batch-invariant dense kernels: ub 1 runs + BF16 weights through MMVF with F32 activations, a 6-token verify batch through MMF with BF16-rounded activations. + Not a defect; the TF32 fix cannot and does not claim to close it. From 44d2f263f36d67e47010a91725430c944b6c6362 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 14:14:19 -0700 Subject: [PATCH 39/49] ggml-cuda: deterministic tie selection in the radix top-k Elements equal to the k-th key were placed in atomicAdd order, so with exact ties at the cut the selected set changed from run to run (DSV4.1 indexer: one swapped compressed row at long context, nondeterministic logits). Ties now take the lowest column indices via an ordered per-row scan; elements above the cut are unchanged. --- STATUS_A.md | 11 +++++++ ggml/src/ggml-cuda/top-k.cu | 61 ++++++++++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/STATUS_A.md b/STATUS_A.md index 3900d5d05f34..91333ada023e 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -367,3 +367,14 @@ No reference scores exist in the traces, so large-gap rows (cascades) cannot be base top-2 " all" -0.87 vs " these" -1.17). Bitwise draft == no-draft needs batch-invariant dense kernels: ub 1 runs BF16 weights through MMVF with F32 activations, a 6-token verify batch through MMF with BF16-rounded activations. Not a defect; the TF32 fix cannot and does not claim to close it. +- CUDA run-to-run nondeterminism (C's long17k finding), reproduced on ONE card on fixture fx_long (sp20001 truncated to + 1536 tokens, ub 512): row 1085 bimodal, 2 of 4 runs differ. First differing tensor between runs: attn_comp-10 (l_out + -9 identical); the visible selection at layer 10 (index source, level two under the layer-8 candidate mask) swaps + one row (787 vs 601) at row 1085. Cause: the CUDA radix top-k (nrows > SM count or ncols > 1024) placed elements + equal to the k-th key via atomicAdd order, so with exact ties at the cut (ReLU-zeroed / masked scores) the chosen + set changed run to run. Fix (commit below): ties now take the lowest column indices (ordered per-row scan with a + block prefix count); the greater-than-cut elements are unchanged. After: 6/6 runs bitwise equal at 1536 tokens and + 2/2 at the full 20,001 tokens (`scripts/dsv41/gate_det_cuda.sh`); test-backend-ops TOP_K OK; gate_c CUDA exact + fx_long t2001 9.5e-6, fx_base t1027 / t3001 4.5e-9 / 5.8e-5; random V4 bitwise unchanged; test-llama-archs 8/8. + Upstream: the radix top-k is upstream code, so V4-Flash's indexer (and any top-k with ties beyond the bitonic + range) is run-to-run nondeterministic on CUDA too. Candidate upstream issue, not filed. diff --git a/ggml/src/ggml-cuda/top-k.cu b/ggml/src/ggml-cuda/top-k.cu index b0f60999021f..620490607eda 100644 --- a/ggml/src/ggml-cuda/top-k.cu +++ b/ggml/src/ggml-cuda/top-k.cu @@ -200,12 +200,63 @@ static __global__ void top_k_radix_gather( if (key > state->prefix) { const int pos = atomicAdd(&state->greater_count, 1); row_dst[pos] = col; - } else if (key == state->prefix) { - const int pos = atomicAdd(&state->equal_count, 1); - if (pos < state->rank) { - row_dst[k - state->rank + pos] = col; + } + } +} + +// Elements equal to the k-th key fill the last `rank` slots, taking the lowest column indices first. With the atomic +// slot order used for the greater elements, which of several tied elements made the cut changed from run to run +// (exactly tied scores are common, e.g. masked or ReLU-zeroed rows), so the selected set was not reproducible. +// One block per row scans the columns in order with a block-wide prefix count, stopping once the slots are full. +template +static __global__ void top_k_radix_gather_equal( + const float * __restrict__ src, + int * __restrict__ dst, + const top_k_radix_state * __restrict__ states, + int ncols, + int k) { + static_assert(BLOCK_SIZE % WARP_SIZE == 0, "BLOCK_SIZE must be a multiple of WARP_SIZE"); + constexpr int NWARPS = BLOCK_SIZE / WARP_SIZE; + __shared__ int warp_sums[NWARPS]; + + const int row = blockIdx.x; + const int tid = threadIdx.x; + const int lane = tid % WARP_SIZE; + const int warp = tid / WARP_SIZE; + const float * row_src = src + (size_t) row * ncols; + int * row_dst = dst + (size_t) row * k; + const top_k_radix_state state = states[row]; + + int base = 0; + for (int64_t col0 = 0; col0 < ncols && base < state.rank; col0 += BLOCK_SIZE) { + const int64_t col = col0 + tid; + const int eq = col < ncols && top_k_float_to_ordered(row_src[col]) == state.prefix; + + // inclusive prefix count within the warp, then across warps + int incl = eq; +#pragma unroll + for (int off = 1; off < WARP_SIZE; off *= 2) { + const int y = __shfl_up_sync(0xFFFFFFFF, incl, off, WARP_SIZE); + if (lane >= off) { + incl += y; } } + if (lane == WARP_SIZE - 1) { + warp_sums[warp] = incl; + } + __syncthreads(); + int warp_off = 0; + int total = 0; + for (int w = 0; w < NWARPS; ++w) { + warp_off += w < warp ? warp_sums[w] : 0; + total += warp_sums[w]; + } + const int pos = base + warp_off + incl - eq; + if (eq && pos < state.rank) { + row_dst[k - state.rank + pos] = col; + } + base += total; + __syncthreads(); } } @@ -244,6 +295,8 @@ static void top_k_radix_cuda( top_k_radix_gather <<>>( src, dst, states, ncols, k, blocks_per_row); + top_k_radix_gather_equal + <<>>(src, dst, states, ncols, k); src += (size_t) ncols * iter_nrows; dst += (size_t) k * iter_nrows; From 6b00e2579646f9ea4ed04f65502a04ae6522b524 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 13:58:34 -0700 Subject: [PATCH 40/49] chat: DeepSeek V4.1 keeps non-OpenAI message keys and media block separators The V4.1 template reads keys the OpenAI message struct drops (task, per-message tools, tool call namespace) and joins content blocks with blank lines around images. Keep unknown message keys on common_chat_msg and restore them, with media markers as separate text blocks, only for templates using the V4.1 DSML tags. Fixes encoding.py golden cases 3-5 on the common_chat apply path. --- common/chat.cpp | 84 ++++++++++++++++++++++++++++++++++++++++++++- common/chat.h | 3 ++ tests/test-chat.cpp | 37 ++++++++++++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) diff --git a/common/chat.cpp b/common/chat.cpp index b79e32fb8c1d..28556c352430 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -428,6 +428,41 @@ std::vector common_chat_msgs_parse_oaicompat(const json & messa if (message.contains("tool_call_id")) { msg.tool_call_id = message.at("tool_call_id"); } + { + json extra = json::object(); + for (const auto & [k, v] : message.items()) { + if (k != "role" && k != "content" && k != "tool_calls" && k != "reasoning_content" && + k != "name" && k != "tool_call_id") { + extra[k] = v; + } + } + if (has_tool_calls) { + // per tool call: keys besides type/function/id, and function keys besides name/arguments + json tc_extra = json::array(); + bool any = false; + for (const auto & tool_call : message.at("tool_calls")) { + json e = json::object(); + for (const auto & [k, v] : tool_call.items()) { + if (k != "type" && k != "function" && k != "id") { + e[k] = v; + } + } + for (const auto & [k, v] : tool_call.at("function").items()) { + if (k != "name" && k != "arguments") { + e["function"][k] = v; + } + } + any = any || !e.empty(); + tc_extra.push_back(e); + } + if (any) { + extra["__tool_call_extras"] = tc_extra; + } + } + if (!extra.empty()) { + msg.extra_json = extra.dump(); + } + } msgs.push_back(msg); } @@ -516,6 +551,51 @@ static json render_message_to_json(const std::vector & msgs, co return messages_inp_normalizer(c).normalize(messages); } +// DeepSeek V4.1: its template reads non-OpenAI message keys ("task", per-message "tools", tool call +// "namespace"), and joins content blocks with "\n\n" around images, so keep media markers as typed text +// parts instead of the default glued string and restore the extra keys. +static json render_message_to_json_deepseek_v4_1(const std::vector & msgs, const jinja::caps & c) { + json messages = json::array(); + for (const auto & msg : msgs) { + json jmsg; + if (msg.contains_media() && msg.content.empty()) { + common_chat_msg typed = msg; + for (auto & part : typed.content_parts) { + part.type = "text"; + } + jmsg = typed.to_json_oaicompat(/* concat_typed_text= */ false); + } else { + jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false); + } + if (!msg.extra_json.empty()) { + const json extra = json::parse(msg.extra_json); + for (const auto & [k, v] : extra.items()) { + if (k == "__tool_call_extras") { + if (!jmsg.contains("tool_calls")) { + continue; + } + auto & tcs = jmsg.at("tool_calls"); + for (size_t i = 0; i < tcs.size() && i < v.size(); i++) { + for (const auto & [tk, tv] : v[i].items()) { + if (tk == "function") { + for (const auto & [fk, fv] : tv.items()) { + tcs[i]["function"][fk] = fv; + } + } else { + tcs[i][tk] = tv; + } + } + } + } else if (!jmsg.contains(k)) { + jmsg[k] = v; + } + } + } + messages.push_back(std::move(jmsg)); + } + return messages_inp_normalizer(c).normalize(messages); +} + // DEPRECATED: only used in tests json common_chat_msgs_to_json_oaicompat(const std::vector & msgs, bool concat_typed_text) { jinja::caps c; @@ -1301,7 +1381,9 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ workaround::trim_all_content(trimmed_messages); messages_to_render = &trimmed_messages; } - params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps()); + params.messages = src.find("｜DSML｜ calls>") != std::string::npos ? + render_message_to_json_deepseek_v4_1(*messages_to_render, tmpl.original_caps()) : + render_message_to_json(*messages_to_render, tmpl.original_caps()); params.tool_choice = inputs.tool_choice; params.reasoning_format = inputs.reasoning_format; params.enable_thinking = inputs.enable_thinking; diff --git a/common/chat.h b/common/chat.h index dcff94919819..4569f8d9feef 100644 --- a/common/chat.h +++ b/common/chat.h @@ -94,6 +94,9 @@ struct common_chat_msg { std::string reasoning_content; std::string tool_name; std::string tool_call_id; + // Message keys the OpenAI fields above do not cover (e.g. "task", per-message "tools", tool call + // "namespace"), kept as a JSON object string. Only templates that read them get them (DeepSeek V4.1). + std::string extra_json; common_json to_json_oaicompat(bool concat_typed_text = false) const; diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 03f64a76f406..269cbf021bfc 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -7921,6 +7921,42 @@ static void test_deepseek_v4_tool_result_ordering() { } } +// DeepSeek V4.1: non-OpenAI message keys reach the template (encoding.py golden cases 3 and 4) and +// content blocks around media markers are joined with "\n\n" like encoding.py does around images. +static void test_deepseek_v4_1_message_fields() { + LOG_DBG("%s\n", __func__); + + auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja"); + auto render = [&](const char * messages_json, bool add_generation_prompt) { + common_chat_templates_inputs inputs; + inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages_json)); + inputs.add_generation_prompt = add_generation_prompt; + inputs.chat_template_kwargs["thinking_mode"] = "\"chat\""; + return common_chat_templates_apply(tmpls.get(), inputs).prompt.str(); + }; + + // "task" on a user turn selects the task token + assert_equals(std::string("<｜begin▁of▁sentence｜><｜User｜>q<｜Assistant｜><｜action｜>"), + render(R"([{"role": "user", "content": "q", "task": "action"}])", false)); + + // per-message "tools" on a later system message, tool call "namespace" + { + auto prompt = render(R"([ + {"role": "system", "content": "a"}, + {"role": "system", "content": "b", "tools": [{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}]}, + {"role": "user", "content": "q"}, + {"role": "assistant", "content": "", "tool_calls": [{"type": "function", "namespace": "ns", "function": {"name": "f", "arguments": {"x": 1}}}]} + ])", false); + assert_contains(prompt, "<｜System｜>b\n\n## Tools"); + assert_contains(prompt, "{\"name\": \"f\", \"parameters\": {\"type\": \"object\"}}"); + assert_contains(prompt, "<｜DSML｜ invoke name=\"ns::f\">"); + } + + // media markers stay separate blocks + assert_equals(std::string("<｜begin▁of▁sentence｜><｜User｜>a\n\n<__media__>\n\nb<｜Assistant｜>"), + render(R"([{"role": "user", "content": [{"type": "text", "text": "a"}, {"type": "media_marker", "text": "<__media__>"}, {"type": "text", "text": "b"}]}])", true)); +} + static void test_prompt_parts() { LOG_DBG("%s\n", __func__); const std::string tmpl_src = @@ -8271,6 +8307,7 @@ int main(int argc, char ** argv) { test_translate_gemma(); test_deepseek_v4_thinking_retention(); test_deepseek_v4_tool_result_ordering(); + test_deepseek_v4_1_message_fields(); test_template_generation_prompt(); test_reasoning_effort_caps(); test_prompt_parts(); From a9df79a4495c1443e2a91ade5f3bdd4e016602e3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 14:05:34 -0700 Subject: [PATCH 41/49] chat: DeepSeek V4.1 maps OpenAI reasoning_effort values, parses system-message tools The V4.1 template only accepts low/high/max or 1-100 and raises otherwise, so map minimal -> low, xhigh -> max and medium -> unset (template default) instead of failing the request. Tools declared only on a system message now also feed the tool call parser and grammar. Both are V4.1 only. --- common/parsers/deepseek.cpp | 42 +++++++++++++++++++++++++++++++++++-- tests/test-chat.cpp | 32 ++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp index 1454ae9dbb54..96e70eaa1174 100644 --- a/common/parsers/deepseek.cpp +++ b/common/parsers/deepseek.cpp @@ -1,4 +1,5 @@ #include "parsers.h" +#include "log.h" // The DeepSeek V4 reference implementation renders consecutive tool results into a single // user block, ordered by the tool call order of the preceding assistant message (matched @@ -84,7 +85,22 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages); } - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + // V4.1 can declare tools on a system message instead of top-level "tools": parse calls to those too. + json tools = inputs.tools; + if (is_v41 && !(tools.is_array() && !tools.empty()) && inputs.messages.is_array()) { + for (const auto & msg : inputs.messages) { + if (msg.value("role", "") == "system" && msg.contains("tools") && msg.at("tools").is_array()) { + if (!tools.is_array()) { + tools = json::array(); + } + for (const auto & tool : msg.at("tools")) { + tools.push_back(tool); + } + } + } + } + + auto has_tools = tools.is_array() && !tools.empty(); auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); @@ -93,6 +109,28 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ if (is_v4 && has_response_format) { additional_context = json{ { "response_format", inputs.json_schema } }; } + // V4.1 accepts low/high/max or 1-100 and raises on anything else: map the remaining OpenAI values + // (minimal -> low, xhigh -> max, medium -> unset = template default) instead of failing the request. + if (is_v41 && inputs.extra_context.is_object() && inputs.extra_context.contains("reasoning_effort") && + inputs.extra_context.at("reasoning_effort").is_string()) { + const std::string effort = inputs.extra_context.at("reasoning_effort").get(); + json mapped; + if (effort == "minimal") { + mapped = "low"; + } else if (effort == "xhigh") { + mapped = "max"; + } else if (effort == "medium") { + mapped = nullptr; + } + if (effort == "minimal" || effort == "xhigh" || effort == "medium") { + LOG_DBG("%s: DeepSeek V4.1 reasoning_effort '%s' -> %s\n", __func__, effort.c_str(), + mapped.is_null() ? "unset (template default)" : mapped.get().c_str()); + if (!additional_context.has_value()) { + additional_context = json::object(); + } + (*additional_context)["reasoning_effort"] = mapped; + } + } const std::string DSML = "｜DSML｜"; const std::string THINK_START = ""; @@ -156,7 +194,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ // build tool call section first since we might need it in reasoning auto tool_choice = p.choice(); if (has_tool_calls) { - foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) { + foreach_function(tools, [&](size_t tool_index, const json & tool) { const auto & function = tool.at("function"); std::string name = function.at("name"); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 269cbf021bfc..870627bef45f 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -7955,6 +7955,38 @@ static void test_deepseek_v4_1_message_fields() { // media markers stay separate blocks assert_equals(std::string("<｜begin▁of▁sentence｜><｜User｜>a\n\n<__media__>\n\nb<｜Assistant｜>"), render(R"([{"role": "user", "content": [{"type": "text", "text": "a"}, {"type": "media_marker", "text": "<__media__>"}, {"type": "text", "text": "b"}]}])", true)); + + // OpenAI reasoning_effort values never fail: minimal -> low, medium -> unset (template default high), xhigh -> max + for (const auto & [effort, budget] : std::vector>{ + { "minimal", "50" }, { "low", "50" }, { "medium", "75" }, { "high", "75" }, { "xhigh", "100" }, { "max", "100" } }) { + common_chat_templates_inputs inputs; + inputs.messages = { message_user }; + inputs.add_generation_prompt = true; + inputs.chat_template_kwargs["reasoning_effort"] = json(effort).dump(); + auto prompt = common_chat_templates_apply(tmpls.get(), inputs).prompt.str(); + assert_contains(prompt, "<｜System｜>Reasoning Effort: " + budget + " (range 1-100"); + } + + // tools declared only on a system message are still parsed as tool calls + { + common_chat_templates_inputs inputs; + inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(R"([ + {"role": "system", "content": "s", "tools": [{"type": "function", "function": {"name": "get_time", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}]}, + {"role": "user", "content": "q"} + ])")); + inputs.add_generation_prompt = true; + inputs.chat_template_kwargs["thinking_mode"] = "\"chat\""; + inputs.enable_thinking = false; + inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; + common_chat_session session(tmpls.get(), nullptr, inputs, {}); + const auto & msg = session.finish(common_chat_input(std::string( + "\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_time\">\n" + "<｜DSML｜ parameter name=\"city\" string=\"true\">Tokyo\n\n"))); + assert_equals(size_t(1), msg.tool_calls.size()); + assert_equals(std::string("get_time"), msg.tool_calls[0].name); + assert_equals(std::string("{\"city\":\"Tokyo\"}"), msg.tool_calls[0].arguments); + } } static void test_prompt_parts() { From 07d0fa0c8d7cb8e4c3c36c85114d3deb791874c9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 17:53:45 -0700 Subject: [PATCH 42/49] deepseek41: STATUS_A CUDA matmul kernel check --- STATUS_A.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/STATUS_A.md b/STATUS_A.md index 91333ada023e..7e176a12cfc7 100644 --- a/STATUS_A.md +++ b/STATUS_A.md @@ -77,6 +77,9 @@ llama.cpp (BF16 dense + MXFP4 experts, CUDA) is 2.5-5x closer to MATCHED than th prompt but t100 ub1 (equal); real text sits at the bf16 / q8-activation floor (C: bf16 floor 5e-3 on fixtures). Random-id prompts are an out-of-distribution, chaotic regime where native itself is 0.05-0.2 away. First 12 wiki tokens: CPU and CUDA llama are equally close to MATCHED at the sensitive positions (t6: 0.2 / 0.1; native 0.9). +CUDA kernel check for the small-batch question: test-backend-ops vs CPU, MUL_MAT_ID mxfp4/bf16 81/81 OK and MUL_MAT +type_a mxfp4/bf16 303/303 OK (0 failures). Open: CUDA long17k is not run-to-run reproducible (C: first divergence at +rows 1639 / 13451), so ub7-vs-ub512 deltas on the full model are inside run-to-run noise until that is pinned down. ## Critique fold-in (plans/dsv41_support.md "Critique fold-in") 1. No per-head q RMS norm: the graph has none; "V4 per-head q norm" is seeded bug #6 and FAILS (KL 4e-3..2). From d0ff0cc27bde8866ee9fc740f1c1b69d416b0a61 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 19:23:41 -0700 Subject: [PATCH 43/49] deepseek41: tokens shared by several sequences (llama-batched) instead of GGML_ASSERT llama-batched decodes the prompt once with n_seq_id = n_parallel. The Engram input of such a token asserted (process abort). A shared token is well defined when every one of its sequences has the same n-gram predecessors: llama_kv_cache_dsv4::init_batch now checks that (batch first, then the cached cells via the new llama_kv_cache::seq_pos_tok_le) and refuses the batch cleanly otherwise (decode returns 1, caches untouched). Shared prompts then match the single-sequence reference (CPU F32 KV KL <= 4.4e-12, the ubatch-split floor). --- src/llama-hparams.h | 1 + src/llama-kv-cache-dsv4.cpp | 66 +++++++++++++++++++++++++++++++++++++ src/llama-kv-cache.cpp | 8 +++++ src/llama-kv-cache.h | 3 ++ src/models/deepseek41.cpp | 7 ++-- 5 files changed, 82 insertions(+), 3 deletions(-) diff --git a/src/llama-hparams.h b/src/llama-hparams.h index f14c9b7b2ef1..b3af7a7d23c6 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -330,6 +330,7 @@ struct llama_hparams { std::array dsv41_topk_src; // index-source layer whose top-k this layer attends with, -1 = none int32_t dsv41_cand_src = -1; // candidate_source_layer, -1 = two-level selection off bool dsv41_tok_hist = false; // Engram: KV cells keep (and serialize) token ids for the n-gram history + uint32_t dsv41_hist_len = 0; // Engram: number of predecessor tokens an n-gram hash reads (max_ngram - 1) uint32_t dsv41_cand_topk_blocks = 0; uint32_t dsv41_cand_block_size = 0; diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp index 9046ddde41b6..c04c9bd58073 100644 --- a/src/llama-kv-cache-dsv4.cpp +++ b/src/llama-kv-cache-dsv4.cpp @@ -1379,12 +1379,78 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( clear_compressed(-1, true); } +// DeepSeek-V4.1 Engram hashes the n-gram ending at each token, so a token that belongs to several sequences +// (llama-batched shares the prompt that way) has one well-defined input only if all of its sequences agree on +// its predecessors. Predecessors come from the same batch first, then from the cached cells. Returns false (and +// logs) for the first ambiguous token. +static bool dsv41_shared_history_ok(const llama_batch & batch, const llama_kv_cache * kv, uint32_t n_hist) { + if (n_hist == 0 || !batch.token || !batch.n_seq_id) { + return true; + } + + bool any_shared = false; + for (int32_t i = 0; i < batch.n_tokens && !any_shared; ++i) { + any_shared = batch.n_seq_id[i] > 1; + } + if (!any_shared) { + return true; + } + + std::map, llama_token> in_batch; + for (int32_t i = 0; i < batch.n_tokens; ++i) { + for (int32_t s = 0; s < batch.n_seq_id[i]; ++s) { + in_batch[{ batch.seq_id[i][s], batch.pos[i] }] = batch.token[i]; + } + } + + const auto tok_at = [&](llama_seq_id seq_id, llama_pos p) -> llama_token { + // largest position <= p of this sequence, batch first (the batch continues the cache) + auto it = in_batch.upper_bound({ seq_id, p }); + if (it != in_batch.begin()) { + --it; + if (it->first.first == seq_id) { + return it->second; + } + } + return kv ? kv->seq_pos_tok_le(seq_id, p) : LLAMA_TOKEN_NULL; + }; + + for (int32_t i = 0; i < batch.n_tokens; ++i) { + if (batch.n_seq_id[i] <= 1) { + continue; + } + for (uint32_t d = 1; d <= n_hist; ++d) { + const llama_pos p = batch.pos[i] - (llama_pos) d; + if (p < 0) { + break; + } + const llama_token t0 = tok_at(batch.seq_id[i][0], p); + for (int32_t s = 1; s < batch.n_seq_id[i]; ++s) { + const llama_token ts = tok_at(batch.seq_id[i][s], p); + if (ts != t0) { + LLAMA_LOG_ERROR("%s: token %d at pos %d is shared by seq %d and seq %d, whose Engram n-gram histories " + "differ at pos %d (%d vs %d); decode such tokens per sequence\n", __func__, + i, batch.pos[i], batch.seq_id[i][0], batch.seq_id[i][s], p, t0, ts); + return false; + } + } + } + } + + return true; +} + llama_memory_context_ptr llama_kv_cache_dsv4::init_batch( llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) { GGML_UNUSED(embd_all); + if (hparams_raw.dsv41_tok_hist && + !dsv41_shared_history_ok(balloc.get_batch(), kv_raw->get_swa(), hparams_raw.dsv41_hist_len)) { + return std::make_unique(LLAMA_MEMORY_STATUS_FAILED_PREPARE); + } + const bool raw_per_seq = kv_raw->get_base()->get_n_stream() != 1; const bool comp_per_seq = csa_state->get_n_stream() > 1; const bool has_coupled = dsv4_batch_has_coupled(balloc.get_batch()); diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 40e9f16faa64..77651694677e 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1910,6 +1910,14 @@ void llama_kv_cache::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, st } } +llama_token llama_kv_cache::seq_pos_tok_le(llama_seq_id seq_id, llama_pos p) const { + if (!has_cell_ext() || seq_id < 0 || (size_t) seq_id >= seq_to_stream.size() || p < 0) { + return LLAMA_TOKEN_NULL; + } + + return v_cells[seq_to_stream[seq_id]].seq_pos_tok_le(seq_id, p); +} + size_t llama_kv_cache::total_size() const { size_t size = 0; diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index 9bfb8f231fe6..9d19f772ee65 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -250,6 +250,9 @@ class llama_kv_cache : public llama_memory_i { // note: used by n-gram input embeddings void get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector & res) const; + // the token of sequence seq_id at the largest cached position <= p (LLAMA_TOKEN_NULL if none or no cell ext) + llama_token seq_pos_tok_le(llama_seq_id seq_id, llama_pos p) const; + private: const llama_model & model; const llama_hparams & hparams; diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index 4f64b8fc42f9..deadd63adb0f 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -168,6 +168,7 @@ void llama_model_deepseek41::load_arch_hparams(llama_model_loader & ml) { } engram_layers.assign(e_layers.begin(), e_layers.end()); hparams.dsv41_tok_hist = true; // state save/restore must carry the n-gram history + hparams.dsv41_hist_len = engram_max_ngram > 0 ? (uint32_t) engram_max_ngram - 1 : 0; engram_pad_cid = engram_token_map[pad_id]; engram_primes.assign(primes.begin(), primes.end()); engram_offsets.resize(primes.size()); @@ -379,9 +380,9 @@ class llm_graph_input_dsv41 : public llm_graph_input_i { void llm_graph_input_dsv41::set_input(const llama_ubatch * ubatch) { if (!engram_ids.empty()) { - for (uint32_t i = 0; i < ubatch->n_tokens; ++i) { - GGML_ASSERT(ubatch->n_seq_id[i] == 1 && "DeepSeek-V4.1 Engram n-grams do not support tokens shared by multiple sequences"); - } + // a token shared by several sequences (n_seq_id > 1) reads the history of its first sequence: + // llama_kv_cache_dsv4::init_batch rejected the batch unless every sequence of the token has the same + // predecessors, so the first one is representative // predecessors come from the raw window cells, which already hold this ubatch mctx->get_raw()->get_prev_tokens(*ubatch, model.engram_max_ngram - 1, prev); for (size_t e = 0; e < engram_ids.size(); ++e) { From db044c8d18fedbcc21773ebabd367a2acf174a1a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 19:23:41 -0700 Subject: [PATCH 44/49] deepseek41: embedding contexts return the flat HC state (n_embd_out wide) n_embd_out is hc_mult*n_embd (the MTP / DSpark heads read the flat HC state through it) but t_embd was the n_embd-wide result_norm, so llama_get_embeddings* / server --embeddings read past the tensor (GGML_ASSERT tensor read out of bounds) for pooling none, and pooled vectors were short. With cparams.embeddings the graph now exposes the flat HC state; logits are unchanged. --- src/models/deepseek41.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index deadd63adb0f..bde39cdb9253 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -1033,7 +1033,11 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); cb(cur, "result_norm", -1); - res->t_embd = cur; + // llama_model_n_embd_out() is hc_mult*n_embd (the MTP / DSpark heads read the flat HC state through it), and + // embedding extraction / pooling copy n_embd_out floats per row: an n_embd-wide result_norm here read past the + // tensor end (GGML_ASSERT in ggml_backend_tensor_get). Embedding contexts therefore return the flat HC state + // [hc_mult*n_embd] per token (pooled as usual); logits are unaffected. + res->t_embd = cparams.embeddings ? flat : cur; cur = ggml_mul_mat(ctx0, model.output, cur); cb(cur, "result_output", -1); From 094024b1865cfad6de240ba9f90f8f2b6ced9770 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 19:23:41 -0700 Subject: [PATCH 45/49] llama-context: re-reserve the scheduler on every embeddings toggle llama-server --embeddings --pooling none toggles llama_set_embeddings per request; with pooling NONE the toggle did not trigger a re-reserve and the first /completion segfaulted in ggml_gallocr (ggml_vbuffer_tensor_alloc on a NULL buffer). Reproduced on tiny-random Llama as well, so not model specific. --- src/llama-context.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 4629ce08f1d1..442b1df8b64c 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1323,8 +1323,11 @@ void llama_context::set_abort_callback(bool (*abort_callback)(void * data), void void llama_context::set_embeddings(bool value) { LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value); - // a pooled context computes the logits only without embeddings, so the toggle changes the graph shape - if (cparams.embeddings != value && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE) { + // a pooled context computes the logits only without embeddings, so the toggle changes the graph shape. + // with pooling NONE the reserved graph (all tokens output) can also differ from the generation graph: + // llama-server --embeddings --pooling none segfaulted in ggml_gallocr on the first /completion (any arch), + // so re-reserve on every toggle + if (cparams.embeddings != value) { sched_need_reserve = true; } From 7704fce5e11cb5d68f8029da205276211c0518a8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 19:28:06 -0700 Subject: [PATCH 46/49] chat: DeepSeek V4/V4.1 thinking-mode content continuation consumes the bare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With a prefilled assistant message (continue_final_message) and thinking enabled, V4 renders the generation prompt as <｜Assistant｜> + content, but the thinking branch of the parser only accepted ..., so the final (non-streamed) message carried a stray "" in content while the streamed deltas (which start after the prefill) did not: stream != non-stream for the same request. Accept the bare THINK_END in the V4 thinking branch. test-chat passes. --- common/parsers/deepseek.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp index 96e70eaa1174..93b5d5270be5 100644 --- a/common/parsers/deepseek.cpp +++ b/common/parsers/deepseek.cpp @@ -265,7 +265,11 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ } if (extract_reasoning && inputs.enable_thinking) { - reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); + // V4 renders a content continuation without reasoning as "<｜Assistant｜>" + content even in + // thinking mode (generation_prompt above), so the bare THINK_END must be consumed here as well, or it + // leaks into the final message content (the streamed deltas, which start after the prefill, omit it) + reasoning = is_v4 ? p.optional((THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END) | p.literal(THINK_END)) + : p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); reasoning_with_tc = THINK_START + p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) + p.space() + obligatory_tool_calls; From 4ebca5e9aa3d4ba56ac7147b5d651e638dad837c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 19:44:12 -0700 Subject: [PATCH 47/49] deepseek41: skip the F32-precision matmul flag for quantized hc_fn, router and index keys Third-party GGUFs quantize hc_attn_fn / hc_ffn_fn (stock llama-quantize), and a quantized src0 with GGML_PREC_F32 routes to MMQ, which asserts (mmq.cu:126). F32 / F16 / BF16 sources keep the exact path. --- src/llama-graph.cpp | 3 ++- src/models/deepseek41.cpp | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index c5545e83bb4d..33d1e4f21869 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2154,9 +2154,10 @@ ggml_tensor * llm_graph_context::build_moe_ffn( if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) { ggml_prec_set_acc(logits, GGML_PREC_F32); } - if (arch == LLM_ARCH_DEEPSEEK41) { + if (arch == LLM_ARCH_DEEPSEEK41 && !ggml_is_quantized(gate_inp->type)) { // F32 router: keep the logits off TF32 on CUDA (top-6 of 384 is near-tie sensitive; with TF32 the routing // depended on the ubatch size, 4..16-token batches being the worst) + // skipped for quantized routers (third-party GGUFs): a quantized src0 with PREC_F32 asserts in MMQ ggml_prec_set_src(logits, GGML_PREC_F32, 1); } cb(logits, "ffn_moe_logits", il); diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index 4f64b8fc42f9..e257b0c71ff3 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -473,8 +473,11 @@ ggml_tensor * llama_model_deepseek41::graph::build_hc_mixes( ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc_dim, nt); ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, norm_rms_eps); ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm); - // F32 weights (model.py hc_pre runs in fp32): no TF32 rounding on CUDA, else the mixes depend on the ubatch size - ggml_prec_set_src(mixes, GGML_PREC_F32, 1); + // F32 weights (model.py hc_pre runs in fp32): no TF32 rounding on CUDA, else the mixes depend on the ubatch size. + // Skipped for quantized weights: third-party GGUFs quantize hc_*_fn, and a quantized src0 with PREC_F32 asserts in MMQ. + if (!ggml_is_quantized(hc_fn->type)) { + ggml_prec_set_src(mixes, GGML_PREC_F32, 1); + } cb(mixes, "hc_mixes", il); ggml_tensor * pre = dsv41_view_2d(ctx0, mixes, hc, nt, 0); @@ -747,7 +750,9 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( ggml_tensor * kq = ggml_mul_mat(ctx0, ggml_permute(ctx0, ik, 0, 2, 1, 3), // [d, n_comp, 1, ns] ggml_permute(ctx0, iq, 0, 2, 1, 3)); // [d, nt/ns, n_ih, ns] -> [n_comp, nt/ns, n_ih, ns] - ggml_prec_set_src(kq, GGML_PREC_F32, 1); // F32 index keys: no TF32 rounding of the near-tie scores + if (!ggml_is_quantized(ik->type)) { + ggml_prec_set_src(kq, GGML_PREC_F32, 1); // F32 index keys: no TF32 rounding of the near-tie scores + } kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3)); // [n_ih, nt/ns, n_comp, ns] ggml_tensor * score = ggml_sum_rows(ctx0, ggml_mul(ctx0, ggml_relu(ctx0, kq), iw)); // [1, nt/ns, n_comp, ns] score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); // [n_comp, nt/ns, 1, ns] From 328c85a9ab33602dc3a661a0071a866f22c5708b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 19:56:23 -0700 Subject: [PATCH 48/49] ggml-cuda: add the GGML_PREC_F32 matmul path without moving the cc / warp_size lookups (merges beside the Inkling pin) --- ggml/src/ggml-cuda/ggml-cuda.cu | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index b3d753a36770..9b0ff9397a77 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -1966,16 +1966,15 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) && src0->view_src; - const int cc = ggml_cuda_info().devices[ctx.device].cc; - const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; - // [TAG_GGML_PREC] src1 requested at GGML_PREC_F32 with F32 weights: keep both operands in F32, i.e. no TF32 // rounding (the MMF tensor-core path and cuBLAS tensor-op math both round to a 10-bit mantissa). Small batches use // the F32 matrix-vector kernel, larger ones cuBLAS SGEMM with the default (non-TF32) math mode. if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && ggml_get_op_params_i32(dst, 3) == GGML_PREC_F32) { + const int cc_f32 = ggml_cuda_info().devices[ctx.device].cc; + const int warp_size_f32 = ggml_cuda_info().devices[ctx.device].warp_size; if (!bad_padding_clear && ne11 <= MMVF_MAX_BATCH_SIZE && - ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, /*ne11 =*/ 1)) { + ggml_cuda_should_use_mmvf(src0->type, cc_f32, warp_size_f32, src0->ne, src0->nb, /*ne11 =*/ 1)) { ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst); return; } @@ -1990,6 +1989,9 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor return; } + const int cc = ggml_cuda_info().devices[ctx.device].cc; + const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; + if (ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11)) { // The custom F16 vector kernel can be used over batched cuBLAS GEMM. // But this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention) From 6172bb1d6fff5929907a6b76f9ca8b1451066cf8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 17 Oct 2026 19:28:58 -0700 Subject: [PATCH 49/49] ggml-cuda: radix top-k writes its indices in a reproducible order The elements above the cut were placed with an atomicAdd slot counter, so the selected set was stable but its order changed from run to run. Consumers that gather rows in top-k order (DSV4.1 sparse attention over the selected compressed rows for decode-sized ubatches) then accumulate in a different order every run and the logits are not reproducible. Each block now owns a contiguous column chunk: one pass counts its elements above the cut, a second writes them at the earlier chunks' total plus an ordered in-block prefix count. Result order: elements above the cut by ascending column, then the tied elements by ascending column. The single shared counter is gone, so the gather itself no longer serialises on one atomic. (cherry picked from commit 984bb2f38a1204e60cdb79e6ad2032bc966eed11) --- ggml/src/ggml-cuda/top-k.cu | 147 ++++++++++++++++++++++++------------ 1 file changed, 98 insertions(+), 49 deletions(-) diff --git a/ggml/src/ggml-cuda/top-k.cu b/ggml/src/ggml-cuda/top-k.cu index 620490607eda..08a48132265c 100644 --- a/ggml/src/ggml-cuda/top-k.cu +++ b/ggml/src/ggml-cuda/top-k.cu @@ -92,14 +92,12 @@ struct top_k_radix_state { uint32_t prefix; uint32_t prefix_mask; int rank; - int greater_count; - int equal_count; }; static __global__ void top_k_radix_init(top_k_radix_state * states, int nrows, int k) { const int row = blockIdx.x * blockDim.x + threadIdx.x; if (row < nrows) { - states[row] = {0, 0, k, 0, 0}; + states[row] = {0, 0, k}; } } @@ -170,11 +168,77 @@ static __global__ void top_k_radix_select( } } -static __global__ void top_k_radix_reset_counters(top_k_radix_state * states, int nrows) { - const int row = blockIdx.x * blockDim.x + threadIdx.x; - if (row < nrows) { - states[row].greater_count = 0; - states[row].equal_count = 0; +// Block-wide inclusive prefix count of `flag` (0/1) over the threads of a block; returns this thread's inclusive count +// and the block total. Must be called by every thread of the block. +template +static __device__ __forceinline__ int top_k_block_prefix_count(const int flag, int & total) { + static_assert(BLOCK_SIZE % WARP_SIZE == 0, "BLOCK_SIZE must be a multiple of WARP_SIZE"); + constexpr int NWARPS = BLOCK_SIZE / WARP_SIZE; + __shared__ int warp_sums[NWARPS]; + + const int lane = threadIdx.x % WARP_SIZE; + const int warp = threadIdx.x / WARP_SIZE; + + int incl = flag; +#pragma unroll + for (int off = 1; off < WARP_SIZE; off *= 2) { + const int y = __shfl_up_sync(0xFFFFFFFF, incl, off, WARP_SIZE); + if (lane >= off) { + incl += y; + } + } + if (lane == WARP_SIZE - 1) { + warp_sums[warp] = incl; + } + __syncthreads(); + int warp_off = 0; + total = 0; + for (int w = 0; w < NWARPS; ++w) { + warp_off += w < warp ? warp_sums[w] : 0; + total += warp_sums[w]; + } + __syncthreads(); + return warp_off + incl; +} + +// Elements above the cut are written in ascending column order (not atomicAdd order), so the index order of the +// result is reproducible run to run. Consumers that gather rows in top-k order (sparse attention over the selected +// rows) otherwise accumulate in a different order every run. Each block owns a contiguous column chunk: pass one +// counts its elements above the cut, pass two writes them at the sum of the earlier chunks' counts plus an ordered +// in-block prefix count. +template +static __global__ void top_k_radix_count_greater( + const float * __restrict__ src, + const top_k_radix_state * __restrict__ states, + int * __restrict__ chunk_counts, + int ncols, + int chunk_size, + int blocks_per_row) { + const int row = blockIdx.x / blocks_per_row; + const int row_block = blockIdx.x % blocks_per_row; + const float * row_src = src + (size_t) row * ncols; + const uint32_t prefix = states[row].prefix; + + const int64_t col_begin = (int64_t) row_block * chunk_size; + const int64_t col_end = col_begin + chunk_size < ncols ? col_begin + chunk_size : (int64_t) ncols; + + int count = 0; + for (int64_t col = col_begin + threadIdx.x; col < col_end; col += BLOCK_SIZE) { + count += top_k_float_to_ordered(row_src[col]) > prefix; + } + count = warp_reduce_sum(count); + + __shared__ int warp_counts[BLOCK_SIZE / WARP_SIZE]; + if (threadIdx.x % WARP_SIZE == 0) { + warp_counts[threadIdx.x / WARP_SIZE] = count; + } + __syncthreads(); + if (threadIdx.x == 0) { + int total = 0; + for (int w = 0; w < BLOCK_SIZE / WARP_SIZE; ++w) { + total += warp_counts[w]; + } + chunk_counts[blockIdx.x] = total; } } @@ -182,25 +246,34 @@ template static __global__ void top_k_radix_gather( const float * __restrict__ src, int * __restrict__ dst, - top_k_radix_state * __restrict__ states, + const top_k_radix_state * __restrict__ states, + const int * __restrict__ chunk_counts, int ncols, int k, + int chunk_size, int blocks_per_row) { const int row = blockIdx.x / blocks_per_row; const int row_block = blockIdx.x % blocks_per_row; - const int tid = threadIdx.x; const float * row_src = src + (size_t) row * ncols; int * row_dst = dst + (size_t) row * k; - top_k_radix_state * state = &states[row]; + const uint32_t prefix = states[row].prefix; - for (int64_t col = row_block * BLOCK_SIZE + tid; - col < ncols; - col += blocks_per_row * BLOCK_SIZE) { - const uint32_t key = top_k_float_to_ordered(row_src[col]); - if (key > state->prefix) { - const int pos = atomicAdd(&state->greater_count, 1); - row_dst[pos] = col; + int base = 0; + for (int j = 0; j < row_block; ++j) { + base += chunk_counts[row * blocks_per_row + j]; + } + + const int64_t col_begin = (int64_t) row_block * chunk_size; + const int64_t col_end = col_begin + chunk_size < ncols ? col_begin + chunk_size : (int64_t) ncols; + for (int64_t col0 = col_begin; col0 < col_end; col0 += BLOCK_SIZE) { + const int64_t col = col0 + threadIdx.x; + const int gt = col < col_end && top_k_float_to_ordered(row_src[col]) > prefix; + int total; + const int incl = top_k_block_prefix_count(gt, total); + if (gt) { + row_dst[base + incl - 1] = col; } + base += total; } } @@ -215,14 +288,8 @@ static __global__ void top_k_radix_gather_equal( const top_k_radix_state * __restrict__ states, int ncols, int k) { - static_assert(BLOCK_SIZE % WARP_SIZE == 0, "BLOCK_SIZE must be a multiple of WARP_SIZE"); - constexpr int NWARPS = BLOCK_SIZE / WARP_SIZE; - __shared__ int warp_sums[NWARPS]; - const int row = blockIdx.x; const int tid = threadIdx.x; - const int lane = tid % WARP_SIZE; - const int warp = tid / WARP_SIZE; const float * row_src = src + (size_t) row * ncols; int * row_dst = dst + (size_t) row * k; const top_k_radix_state state = states[row]; @@ -231,32 +298,12 @@ static __global__ void top_k_radix_gather_equal( for (int64_t col0 = 0; col0 < ncols && base < state.rank; col0 += BLOCK_SIZE) { const int64_t col = col0 + tid; const int eq = col < ncols && top_k_float_to_ordered(row_src[col]) == state.prefix; - - // inclusive prefix count within the warp, then across warps - int incl = eq; -#pragma unroll - for (int off = 1; off < WARP_SIZE; off *= 2) { - const int y = __shfl_up_sync(0xFFFFFFFF, incl, off, WARP_SIZE); - if (lane >= off) { - incl += y; - } - } - if (lane == WARP_SIZE - 1) { - warp_sums[warp] = incl; - } - __syncthreads(); - int warp_off = 0; - int total = 0; - for (int w = 0; w < NWARPS; ++w) { - warp_off += w < warp ? warp_sums[w] : 0; - total += warp_sums[w]; - } - const int pos = base + warp_off + incl - eq; + int total; + const int pos = base + top_k_block_prefix_count(eq, total) - eq; if (eq && pos < state.rank) { row_dst[k - state.rank + pos] = col; } base += total; - __syncthreads(); } } @@ -267,6 +314,7 @@ static void top_k_radix_cuda( constexpr int RADIX_BITS = 8; constexpr int NBINS = 1 << RADIX_BITS; const int blocks_per_row = (int) std::min(((int64_t) ncols + 1023) / 1024, 64); + const int chunk_size = (int) (((int64_t) ncols + blocks_per_row - 1) / blocks_per_row); // chunk the rows to bound the histogram memory to 64 MB const int64_t chunk_nrows = ggml_cuda_chunk_nrows((size_t) blocks_per_row * NBINS * sizeof(int), nrows); @@ -290,11 +338,12 @@ static void top_k_radix_cuda( <<>>(histograms, states, blocks_per_row, shift); } - top_k_radix_reset_counters - <<<(iter_nrows + BLOCK_SIZE - 1) / BLOCK_SIZE, BLOCK_SIZE, 0, stream>>>(states, iter_nrows); + // the last select pass has consumed the histograms: reuse them for the per-chunk counts + top_k_radix_count_greater + <<>>(src, states, histograms, ncols, chunk_size, blocks_per_row); top_k_radix_gather <<>>( - src, dst, states, ncols, k, blocks_per_row); + src, dst, states, histograms, ncols, k, chunk_size, blocks_per_row); top_k_radix_gather_equal <<>>(src, dst, states, ncols, k);