diff --git a/llm/llama_server.go b/llm/llama_server.go index 3cb00f761d1..082e3c0bc8f 100644 --- a/llm/llama_server.go +++ b/llm/llama_server.go @@ -175,6 +175,7 @@ type llamaServerRunner struct { type llamaServerLaunchConfig struct { modelPath string modelArch string + visionArch string draftType string projectors []string mmprojMemory uint64 @@ -385,7 +386,7 @@ func startLlamaServer(launch llamaServerLaunchConfig, out io.Writer) (cmd *exec. params = appendMMProjArgs(params, launch) params = appendDraftArgs(params, launch.draftType, launch.config.DraftModelPath, launch.opts) - params = append(params, visionServerArgs(launch.modelArch, launch.opts)...) + params = append(params, visionServerArgs(launch.visionArch, launch.opts)...) for _, adapter := range launch.adapters { slog.Warn("LoRA adapters are deprecated and will be removed in a future release", "adapter", adapter) @@ -1001,6 +1002,7 @@ func NewLlamaServerRunner( launch := llamaServerLaunchConfig{ modelPath: splitModel.modelPath, modelArch: arch, + visionArch: VisionBudgetArch(arch, externalProjectorType(modelPath, projectors)), draftType: draftType, projectors: slices.Clone(splitModel.projectors), mmprojMemory: mmprojMemory, @@ -1165,6 +1167,42 @@ func applyArchServerEnvs(serverEnvs map[string]string, modelArch string) { } } +// VisionBudgetArch returns the architecture whose image-token budget rules +// apply to a model, given its text architecture and the projector type of a +// separate projector file ("" when there is none, or when the vision tensors +// are inline). +// +// It differs from modelArch in one case. v0.40.2 converts a legacy +// nemotron_h_omni GGUF into a llamacpp child (compatmigrate/nemotron3.go): a +// text model with general.architecture nemotron_h_moe (or nemotron_h) and a +// separate clip projector of type nemotron_v2_vl. The payload's dynamic +// resolution keys on that projector type (llama/compat/002), so the budget +// flags stay live, but every switch here keyed on nemotron_h_omni and the +// converted child launched without them (MaxusAI/ollama#480). A text-only +// nemotron_h_moe has no projector and keeps its own architecture. +func VisionBudgetArch(modelArch, projectorType string) string { + if projectorType == "nemotron_v2_vl" && (modelArch == "nemotron_h" || modelArch == "nemotron_h_moe") { + return "nemotron_h_omni" + } + return modelArch +} + +// externalProjectorType reads clip.vision.projector_type from the first +// projector when it is a separate file. An inline projector (the model file +// itself) and an unreadable one both report "", which leaves VisionBudgetArch +// on the text architecture. +func externalProjectorType(modelPath string, projectors []string) string { + if len(projectors) == 0 || projectors[0] == modelPath { + return "" + } + projector, err := LoadModel(projectors[0], 1024) + if err != nil { + slog.Debug("read projector type", "projector", projectors[0], "error", err) + return "" + } + return projector.KV().String("vision.projector_type") +} + func visionServerArgs(modelArch string, opts api.Options) []string { switch modelArch { case "qwen2vl", "qwen25vl", "qwen3vl", "qwen3vlmoe", "qwen35", "qwen35moe": diff --git a/llm/llama_server_test.go b/llm/llama_server_test.go index 84a274abe48..016c27ee06d 100644 --- a/llm/llama_server_test.go +++ b/llm/llama_server_test.go @@ -2931,6 +2931,59 @@ func TestMMProjFitTargetExtraEnvs(t *testing.T) { }) } +// TestVisionBudgetArch pins #480: v0.40.2 converts a legacy nemotron_h_omni +// GGUF into a nemotron_h_moe model plus a separate nemotron_v2_vl projector, +// and the converted child must launch with the same image-budget flags the +// legacy model did. A text-only nemotron_h_moe must launch with none. +func TestVisionBudgetArch(t *testing.T) { + for _, tc := range []struct { + arch, projector, want string + }{ + {"nemotron_h_moe", "nemotron_v2_vl", "nemotron_h_omni"}, + {"nemotron_h", "nemotron_v2_vl", "nemotron_h_omni"}, + {"nemotron_h_moe", "", "nemotron_h_moe"}, + {"nemotron_h_omni", "", "nemotron_h_omni"}, + {"gemma4", "", "gemma4"}, + {"qwen35moe", "nemotron_v2_vl", "qwen35moe"}, + } { + if got := VisionBudgetArch(tc.arch, tc.projector); got != tc.want { + t.Errorf("VisionBudgetArch(%q, %q) = %q, want %q", tc.arch, tc.projector, got, tc.want) + } + } + + opts := api.DefaultOptions() + opts.ImageMinTokens, opts.ImageMaxTokens = 256, 1024 + converted := visionServerArgs(VisionBudgetArch("nemotron_h_moe", "nemotron_v2_vl"), opts) + if want := []string{"--image-min-tokens", "256", "--image-max-tokens", "1024"}; !slices.Equal(converted, want) { + t.Errorf("converted nemotron3 child launches with %v, want %v", converted, want) + } + if textOnly := visionServerArgs(VisionBudgetArch("nemotron_h_moe", ""), opts); textOnly != nil { + t.Errorf("text-only nemotron_h_moe launches with %v, want no image flags", textOnly) + } + + modelPath, _ := writeTestGGUF(t, gguftest.KV{"general.architecture": "nemotron_h_moe"}, []*gguftest.Tensor{ + testGGUFTensor("blk.0.attn_q.weight", gguf.TensorTypeF32, []uint64{4}), + }) + projectorPath, _ := writeTestGGUF(t, gguftest.KV{ + "general.architecture": "clip", + "clip.vision.projector_type": "nemotron_v2_vl", + }, []*gguftest.Tensor{ + testGGUFTensor("v.patch_embd.weight", gguf.TensorTypeF16, []uint64{16}), + }) + if got := externalProjectorType(modelPath, []string{projectorPath}); got != "nemotron_v2_vl" { + t.Errorf("externalProjectorType(projector file) = %q, want nemotron_v2_vl", got) + } + if got := externalProjectorType(modelPath, []string{modelPath}); got != "" { + t.Errorf("externalProjectorType(inline) = %q, want \"\"", got) + } + if got := externalProjectorType(modelPath, nil); got != "" { + t.Errorf("externalProjectorType(none) = %q, want \"\"", got) + } + if got := externalProjectorType(modelPath, []string{filepath.Join(t.TempDir(), "missing.gguf")}); got != "" { + t.Errorf("externalProjectorType(unreadable) = %q, want \"\"", got) + } +} + func TestMMProjMemoryRequirement(t *testing.T) { if got, err := mmprojMemoryRequirement("model.gguf", nil, nil); err != nil || got != 0 { t.Fatalf("no projector memory = %d, %v; want 0, nil", got, err) diff --git a/server/images.go b/server/images.go index 689b009c31b..bf35962f588 100644 --- a/server/images.go +++ b/server/images.go @@ -529,6 +529,24 @@ func suppressAudioCapability(m *Model, arch string) bool { return false } +// projectorType returns the first separate projector's +// clip.vision.projector_type, or "" when the model has none. It reads the +// metadata GetModel already extracted, so it costs no file access. +// llm.VisionBudgetArch uses it to tell a converted nemotron3 GGUF (a +// nemotron_h_moe text model plus a nemotron_v2_vl projector, #480) from a +// text-only nemotron_h_moe. +func (m *Model) projectorType() string { + if m == nil { + return "" + } + for _, md := range m.projectorMetadata { + if t := md.String("vision.projector_type"); t != "" { + return t + } + } + return "" +} + func isNemotronSafetensors(m *Model) bool { return isNemotronSafetensorsConfig(m.Config) } diff --git a/server/prompt.go b/server/prompt.go index e7331c8d5eb..51935c0ab26 100644 --- a/server/prompt.go +++ b/server/prompt.go @@ -183,7 +183,7 @@ func visionTokenArch(m *Model) (string, bool) { return "", false } - return arch, true + return llm.VisionBudgetArch(arch, m.projectorType()), true } // maxImageTokens returns the worst-case per-image token cost the chat diff --git a/server/prompt_test.go b/server/prompt_test.go index cf819378cb4..54bd192e828 100644 --- a/server/prompt_test.go +++ b/server/prompt_test.go @@ -749,6 +749,20 @@ func TestRenderPromptResolvesDynamicGemma4Renderer(t *testing.T) { } } +// convertedNemotron is what v0.40.2's compatmigrate leaves for a legacy +// nemotron_h_omni GGUF: a nemotron_h_moe text model with a separate clip +// projector of type nemotron_v2_vl (#480). +func convertedNemotron() *Model { + return &Model{ + ProjectorPaths: []string{"projector"}, + Config: model.ConfigV2{ModelFamily: "nemotron_h_moe", ModelFamilies: []string{"nemotron_h_moe", "clip"}}, + projectorMetadata: []ggufMetadata{{KV: map[string]any{ + "general.architecture": "clip", + "clip.vision.projector_type": "nemotron_v2_vl", + }}}, + } +} + func TestMaxImageTokensModelGating(t *testing.T) { opts := &api.Options{Runner: api.Runner{NumCtx: 4096}} @@ -802,6 +816,27 @@ func TestMaxImageTokensModelGating(t *testing.T) { opts: opts, want: 3330, }, + { + // #480: the converted child must charge what the legacy omni GGUF did. + name: "converted nemotron3 child charges the omni ceiling", + m: convertedNemotron(), + opts: opts, + want: 3330, + }, + { + name: "converted nemotron3 child resolves a pinned ceiling", + m: convertedNemotron(), + opts: &api.Options{Runner: api.Runner{ImageMinTokens: 256, ImageMaxTokens: 1024}}, + want: 1026, + }, + { + // Nemotron 3 Nano without vision is also nemotron_h_moe, with no + // projector: it has no image input path at all. + name: "text-only nemotron_h_moe has no image input path", + m: &Model{Config: model.ConfigV2{ModelFamily: "nemotron_h_moe", ModelFamilies: []string{"nemotron_h_moe"}}}, + opts: opts, + want: 0, + }, { name: "inline gemma4 resolves ceiling from opts", m: &Model{Config: model.ConfigV2{ModelFamily: "gemma4", ModelFamilies: []string{"gemma4"}}}, @@ -841,6 +876,12 @@ func TestImageTokenCosts(t *testing.T) { t.Errorf("nemotron costs mismatch (-got +want):\n%s", diff) } + // #480: the converted nemotron3 child sizes images exactly as the legacy + // omni GGUF does. + if diff := cmp.Diff(imageTokenCosts(convertedNemotron(), opts, msgs), []int{302 + 3330, 0, 3330}); diff != "" { + t.Errorf("converted nemotron costs mismatch (-got +want):\n%s", diff) + } + textOnly := &Model{Config: model.ConfigV2{ModelFamily: "llama"}} if diff := cmp.Diff(imageTokenCosts(textOnly, opts, msgs), []int{0, 0, 0}); diff != "" { t.Errorf("text-only costs mismatch (-got +want):\n%s", diff) diff --git a/server/sched.go b/server/sched.go index 4a3fc38e988..3aede6336a7 100644 --- a/server/sched.go +++ b/server/sched.go @@ -1557,8 +1557,9 @@ func (runner *runnerRef) needsReload(ctx context.Context, req *LlmRequest) bool // one that names those bounds explicitly launch identical flags. Compare // what the runner would be started with, not what the caller happened to // send, or the second request reloads the model for nothing. - llm.NormalizeImageTokenBudget(req.model.Config.ModelFamily, &optsExisting) - llm.NormalizeImageTokenBudget(req.model.Config.ModelFamily, &optsNew) + visionArch := llm.VisionBudgetArch(req.model.Config.ModelFamily, req.model.projectorType()) + llm.NormalizeImageTokenBudget(visionArch, &optsExisting) + llm.NormalizeImageTokenBudget(visionArch, &optsNew) // The same holds for the prompt-cache size: an invalid prompt_cache_ram is // ignored at launch, so it starts the same llama-server as an unset one. llm.NormalizePromptCacheRAM(&optsExisting) diff --git a/server/sched_test.go b/server/sched_test.go index a406683c310..535b520068f 100644 --- a/server/sched_test.go +++ b/server/sched_test.go @@ -2615,6 +2615,7 @@ func TestSchedNeedsReloadImageTokenBudget(t *testing.T) { for _, tc := range []struct { name string family string + projector string minTok int maxTok int wantReload bool @@ -2631,6 +2632,27 @@ func TestSchedNeedsReloadImageTokenBudget(t *testing.T) { minTok: 256, maxTok: 512, wantReload: true, }, + { + // #480: v0.40.2 converts a legacy nemotron_h_omni GGUF into a + // nemotron_h_moe model plus a nemotron_v2_vl projector. Its budget + // must still reach the launch, so a different ceiling reloads. + name: "converted nemotron3 genuinely different ceiling reloads", + family: "nemotron_h_moe", projector: "nemotron_v2_vl", + minTok: 256, maxTok: 512, + wantReload: true, + }, + { + name: "converted nemotron3 explicit native bounds match the unset sentinel", + family: "nemotron_h_moe", projector: "nemotron_v2_vl", + minTok: 256, maxTok: 3328, + wantReload: false, + }, + { + name: "text-only nemotron_h_moe ignores the bounds", + family: "nemotron_h_moe", + minTok: 256, maxTok: 512, + wantReload: false, + }, { name: "gemma4 explicit defaults do not reload", family: "gemma4", @@ -2651,15 +2673,26 @@ func TestSchedNeedsReloadImageTokenBudget(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { + newModel := func() *Model { + m := &Model{Config: model.ConfigV2{ModelFamily: tc.family}} + if tc.projector != "" { + m.ProjectorPaths = []string{"projector"} + m.projectorMetadata = []ggufMetadata{{KV: map[string]any{ + "general.architecture": "clip", + "clip.vision.projector_type": tc.projector, + }}} + } + return m + } do := api.DefaultOptions() runner := &runnerRef{ - model: &Model{Config: model.ConfigV2{ModelFamily: tc.family}}, + model: newModel(), Options: &do, llama: &mockLlm{vramByGPU: map[ml.DeviceID]uint64{}}, numParallel: 1, } req := &LlmRequest{ - model: &Model{Config: model.ConfigV2{ModelFamily: tc.family}}, + model: newModel(), opts: api.DefaultOptions(), } req.opts.ImageMinTokens, req.opts.ImageMaxTokens = tc.minTok, tc.maxTok