Skip to content

feat(inference): parse Qwen3.5 vision config fields (ADR-069 S1) #670

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ohdearquant wants to merge 1 commit into
base: feat/vision
Choose a base branch
from
Draft

feat(inference): parse Qwen3.5 vision config fields (ADR-069 S1) #670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions crates/inference/src/forward/batch_prefill.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1663,6 +1663,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
}
}

Expand Down
15 changes: 15 additions & 0 deletions crates/inference/src/forward/cpu_f16.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1120,6 +1120,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

let rope_dim = cfg.rope_dim(); // = 16
Expand Down Expand Up @@ -1505,6 +1510,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

// embed_tokens is [vocab * hidden] packed u16 (f16 zeros = 0u16).
Expand Down Expand Up @@ -1652,6 +1662,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

// The negative gamma at dim-0 flips the sign of that component after
Expand Down
20 changes: 20 additions & 0 deletions crates/inference/src/forward/cpu_q8.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -934,6 +934,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

let rope_dim = cfg.rope_dim(); // = 16
Expand Down Expand Up @@ -1204,6 +1209,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

let rope = RopeTable::new(cfg.rope_dim(), 512, cfg.rope_theta);
Expand Down Expand Up @@ -1377,6 +1387,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

// embed_tokens is [vocab, hidden] and also serves as the tied LM head.
Expand Down Expand Up @@ -1620,6 +1635,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

let rope_dim = (head_dim as f32 * cfg.partial_rotary_factor) as usize; // 16
Expand Down
20 changes: 20 additions & 0 deletions crates/inference/src/forward/metal_qwen35.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,11 @@ mod mtp_resolve_tests {
eos_token_id: 63,
max_position_embeddings: 128,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
}
}

Expand Down Expand Up @@ -19617,6 +19622,11 @@ kernel void decode_attention_reference(
eos_token_id: (vocab - 1) as u32,
max_position_embeddings: 128,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

let mut embed_tokens = vec![0.0f32; vocab * hidden];
Expand Down Expand Up @@ -22375,6 +22385,11 @@ kernel void decode_attention_reference(
eos_token_id: (vocab - 1) as u32,
max_position_embeddings: 128,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

let make_common = || CommonLayerWeights {
Expand Down Expand Up @@ -22721,6 +22736,11 @@ kernel void decode_attention_reference(
eos_token_id: (vocab - 1) as u32,
max_position_embeddings: 128,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

let embed_tokens = vec![0.0f32; vocab * hidden];
Expand Down
30 changes: 30 additions & 0 deletions crates/inference/src/forward/neon_forward.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1153,6 +1153,11 @@ pub mod bench_support {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

let rope_dim = (head_dim as f32 * cfg.partial_rotary_factor) as usize; // 32
Expand Down Expand Up @@ -1832,6 +1837,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

let rope_dim = cfg.rope_dim(); // = 16
Expand Down Expand Up @@ -2025,6 +2035,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

let rope_dim = (head_dim as f32 * cfg.partial_rotary_factor) as usize; // 16
Expand Down Expand Up @@ -2292,6 +2307,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};
let rope = RopeTable::new(cfg.rope_dim(), 512, cfg.rope_theta);

Expand Down Expand Up @@ -2529,6 +2549,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

// All-zero packed weights: scale=0, all i8=0 → logits all 0 → greedy picks 0.
Expand Down Expand Up @@ -2632,6 +2657,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

// The negative gamma at dim-0 creates a "bounce" each decode step.
Expand Down
5 changes: 5 additions & 0 deletions crates/inference/src/model/qwen35/eval.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -545,6 +545,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
}
}

Expand Down
10 changes: 10 additions & 0 deletions crates/inference/src/model/qwen35/generation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1302,6 +1302,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

// Deterministic xorshift for reproducible synthetic weights.
Expand Down Expand Up @@ -1524,6 +1529,11 @@ mod tests {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
};

let z = |len: usize| vec![0.0_f32; len];
Expand Down
5 changes: 5 additions & 0 deletions crates/inference/src/model/qwen35/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,11 @@ mod lora_serving {
mtp_num_hidden_layers: 0,
mtp_use_dedicated_embeddings: false,
quarot_rotation_seed: None,
vision_config: None,
image_token_id: None,
video_token_id: None,
vision_start_token_id: None,
vision_end_token_id: None,
}
}

Expand Down
Loading
Loading