fix(vision): a converted nemotron3 GGUF keeps its per-request image budget (#480)

llm/llama_server.go

@@ -175,6 +175,7 @@ type llamaServerRunner struct {
 type llamaServerLaunchConfig struct {
 	modelPath            string
 	modelArch            string
+	visionArch           string
 	draftType            string
 	projectors           []string
 	mmprojMemory         uint64
@@ -385,7 +386,7 @@ func startLlamaServer(launch llamaServerLaunchConfig, out io.Writer) (cmd *exec.
 	params = appendMMProjArgs(params, launch)
 	params = appendDraftArgs(params, launch.draftType, launch.config.DraftModelPath, launch.opts)
 
-	params = append(params, visionServerArgs(launch.modelArch, launch.opts)...)
+	params = append(params, visionServerArgs(launch.visionArch, launch.opts)...)
 
 	for _, adapter := range launch.adapters {
 		slog.Warn("LoRA adapters are deprecated and will be removed in a future release", "adapter", adapter)
@@ -1001,6 +1002,7 @@ func NewLlamaServerRunner(
 	launch := llamaServerLaunchConfig{
 		modelPath:    splitModel.modelPath,
 		modelArch:    arch,
+		visionArch:   VisionBudgetArch(arch, externalProjectorType(modelPath, projectors)),
 		draftType:    draftType,
 		projectors:   slices.Clone(splitModel.projectors),
 		mmprojMemory: mmprojMemory,
@@ -1165,6 +1167,42 @@ func applyArchServerEnvs(serverEnvs map[string]string, modelArch string) {
 	}
 }
 
+// VisionBudgetArch returns the architecture whose image-token budget rules
+// apply to a model, given its text architecture and the projector type of a
+// separate projector file ("" when there is none, or when the vision tensors
+// are inline).
+//
+// It differs from modelArch in one case. v0.40.2 converts a legacy
+// nemotron_h_omni GGUF into a llamacpp child (compatmigrate/nemotron3.go): a
+// text model with general.architecture nemotron_h_moe (or nemotron_h) and a
+// separate clip projector of type nemotron_v2_vl. The payload's dynamic
+// resolution keys on that projector type (llama/compat/002), so the budget
+// flags stay live, but every switch here keyed on nemotron_h_omni and the
+// converted child launched without them (MaxusAI/ollama#480). A text-only
+// nemotron_h_moe has no projector and keeps its own architecture.
+func VisionBudgetArch(modelArch, projectorType string) string {
+	if projectorType == "nemotron_v2_vl" && (modelArch == "nemotron_h" || modelArch == "nemotron_h_moe") {
+		return "nemotron_h_omni"
+	}
+	return modelArch
+}
+
+// externalProjectorType reads clip.vision.projector_type from the first
+// projector when it is a separate file. An inline projector (the model file
+// itself) and an unreadable one both report "", which leaves VisionBudgetArch
+// on the text architecture.
+func externalProjectorType(modelPath string, projectors []string) string {
+	if len(projectors) == 0 || projectors[0] == modelPath {
+		return ""
+	}
+	projector, err := LoadModel(projectors[0], 1024)
+	if err != nil {
+		slog.Debug("read projector type", "projector", projectors[0], "error", err)
+		return ""
+	}
+	return projector.KV().String("vision.projector_type")
+}
+
 func visionServerArgs(modelArch string, opts api.Options) []string {
 	switch modelArch {
 	case "qwen2vl", "qwen25vl", "qwen3vl", "qwen3vlmoe", "qwen35", "qwen35moe":

llm/llama_server_test.go

@@ -2931,6 +2931,59 @@ func TestMMProjFitTargetExtraEnvs(t *testing.T) {
 	})
 }
 
+// TestVisionBudgetArch pins #480: v0.40.2 converts a legacy nemotron_h_omni
+// GGUF into a nemotron_h_moe model plus a separate nemotron_v2_vl projector,
+// and the converted child must launch with the same image-budget flags the
+// legacy model did. A text-only nemotron_h_moe must launch with none.
+func TestVisionBudgetArch(t *testing.T) {
+	for _, tc := range []struct {
+		arch, projector, want string
+	}{
+		{"nemotron_h_moe", "nemotron_v2_vl", "nemotron_h_omni"},
+		{"nemotron_h", "nemotron_v2_vl", "nemotron_h_omni"},
+		{"nemotron_h_moe", "", "nemotron_h_moe"},
+		{"nemotron_h_omni", "", "nemotron_h_omni"},
+		{"gemma4", "", "gemma4"},
+		{"qwen35moe", "nemotron_v2_vl", "qwen35moe"},
+	} {
+		if got := VisionBudgetArch(tc.arch, tc.projector); got != tc.want {
+			t.Errorf("VisionBudgetArch(%q, %q) = %q, want %q", tc.arch, tc.projector, got, tc.want)
+		}
+	}
+
+	opts := api.DefaultOptions()
+	opts.ImageMinTokens, opts.ImageMaxTokens = 256, 1024
+	converted := visionServerArgs(VisionBudgetArch("nemotron_h_moe", "nemotron_v2_vl"), opts)
+	if want := []string{"--image-min-tokens", "256", "--image-max-tokens", "1024"}; !slices.Equal(converted, want) {
+		t.Errorf("converted nemotron3 child launches with %v, want %v", converted, want)
+	}
+	if textOnly := visionServerArgs(VisionBudgetArch("nemotron_h_moe", ""), opts); textOnly != nil {
+		t.Errorf("text-only nemotron_h_moe launches with %v, want no image flags", textOnly)
+	}
+
+	modelPath, _ := writeTestGGUF(t, gguftest.KV{"general.architecture": "nemotron_h_moe"}, []*gguftest.Tensor{
+		testGGUFTensor("blk.0.attn_q.weight", gguf.TensorTypeF32, []uint64{4}),
+	})
+	projectorPath, _ := writeTestGGUF(t, gguftest.KV{
+		"general.architecture":       "clip",
+		"clip.vision.projector_type": "nemotron_v2_vl",
+	}, []*gguftest.Tensor{
+		testGGUFTensor("v.patch_embd.weight", gguf.TensorTypeF16, []uint64{16}),
+	})
+	if got := externalProjectorType(modelPath, []string{projectorPath}); got != "nemotron_v2_vl" {
+		t.Errorf("externalProjectorType(projector file) = %q, want nemotron_v2_vl", got)
+	}
+	if got := externalProjectorType(modelPath, []string{modelPath}); got != "" {
+		t.Errorf("externalProjectorType(inline) = %q, want \"\"", got)
+	}
+	if got := externalProjectorType(modelPath, nil); got != "" {
+		t.Errorf("externalProjectorType(none) = %q, want \"\"", got)
+	}
+	if got := externalProjectorType(modelPath, []string{filepath.Join(t.TempDir(), "missing.gguf")}); got != "" {
+		t.Errorf("externalProjectorType(unreadable) = %q, want \"\"", got)
+	}
+}
+
 func TestMMProjMemoryRequirement(t *testing.T) {
 	if got, err := mmprojMemoryRequirement("model.gguf", nil, nil); err != nil || got != 0 {
 		t.Fatalf("no projector memory = %d, %v; want 0, nil", got, err)

server/images.go

@@ -529,6 +529,24 @@ func suppressAudioCapability(m *Model, arch string) bool {
 	return false
 }
 
+// projectorType returns the first separate projector's
+// clip.vision.projector_type, or "" when the model has none. It reads the
+// metadata GetModel already extracted, so it costs no file access.
+// llm.VisionBudgetArch uses it to tell a converted nemotron3 GGUF (a
+// nemotron_h_moe text model plus a nemotron_v2_vl projector, #480) from a
+// text-only nemotron_h_moe.
+func (m *Model) projectorType() string {
+	if m == nil {
+		return ""
+	}
+	for _, md := range m.projectorMetadata {
+		if t := md.String("vision.projector_type"); t != "" {
+			return t
+		}
+	}
+	return ""
+}
+
 func isNemotronSafetensors(m *Model) bool {
 	return isNemotronSafetensorsConfig(m.Config)
 }

server/prompt.go

@@ -183,7 +183,7 @@ func visionTokenArch(m *Model) (string, bool) {
 		return "", false
 	}
 
-	return arch, true
+	return llm.VisionBudgetArch(arch, m.projectorType()), true
 }
 
 // maxImageTokens returns the worst-case per-image token cost the chat

server/prompt_test.go

@@ -749,6 +749,20 @@ func TestRenderPromptResolvesDynamicGemma4Renderer(t *testing.T) {
 	}
 }
 
+// convertedNemotron is what v0.40.2's compatmigrate leaves for a legacy
+// nemotron_h_omni GGUF: a nemotron_h_moe text model with a separate clip
+// projector of type nemotron_v2_vl (#480).
+func convertedNemotron() *Model {
+	return &Model{
+		ProjectorPaths: []string{"projector"},
+		Config:         model.ConfigV2{ModelFamily: "nemotron_h_moe", ModelFamilies: []string{"nemotron_h_moe", "clip"}},
+		projectorMetadata: []ggufMetadata{{KV: map[string]any{
+			"general.architecture":       "clip",
+			"clip.vision.projector_type": "nemotron_v2_vl",
+		}}},
+	}
+}
+
 func TestMaxImageTokensModelGating(t *testing.T) {
 	opts := &api.Options{Runner: api.Runner{NumCtx: 4096}}
 
@@ -802,6 +816,27 @@ func TestMaxImageTokensModelGating(t *testing.T) {
 			opts: opts,
 			want: 3330,
 		},
+		{
+			// #480: the converted child must charge what the legacy omni GGUF did.
+			name: "converted nemotron3 child charges the omni ceiling",
+			m:    convertedNemotron(),
+			opts: opts,
+			want: 3330,
+		},
+		{
+			name: "converted nemotron3 child resolves a pinned ceiling",
+			m:    convertedNemotron(),
+			opts: &api.Options{Runner: api.Runner{ImageMinTokens: 256, ImageMaxTokens: 1024}},
+			want: 1026,
+		},
+		{
+			// Nemotron 3 Nano without vision is also nemotron_h_moe, with no
+			// projector: it has no image input path at all.
+			name: "text-only nemotron_h_moe has no image input path",
+			m:    &Model{Config: model.ConfigV2{ModelFamily: "nemotron_h_moe", ModelFamilies: []string{"nemotron_h_moe"}}},
+			opts: opts,
+			want: 0,
+		},
 		{
 			name: "inline gemma4 resolves ceiling from opts",
 			m:    &Model{Config: model.ConfigV2{ModelFamily: "gemma4", ModelFamilies: []string{"gemma4"}}},
@@ -841,6 +876,12 @@ func TestImageTokenCosts(t *testing.T) {
 		t.Errorf("nemotron costs mismatch (-got +want):\n%s", diff)
 	}
 
+	// #480: the converted nemotron3 child sizes images exactly as the legacy
+	// omni GGUF does.
+	if diff := cmp.Diff(imageTokenCosts(convertedNemotron(), opts, msgs), []int{302 + 3330, 0, 3330}); diff != "" {
+		t.Errorf("converted nemotron costs mismatch (-got +want):\n%s", diff)
+	}
+
 	textOnly := &Model{Config: model.ConfigV2{ModelFamily: "llama"}}
 	if diff := cmp.Diff(imageTokenCosts(textOnly, opts, msgs), []int{0, 0, 0}); diff != "" {
 		t.Errorf("text-only costs mismatch (-got +want):\n%s", diff)

server/sched.go

@@ -1557,8 +1557,9 @@ func (runner *runnerRef) needsReload(ctx context.Context, req *LlmRequest) bool
 	// one that names those bounds explicitly launch identical flags. Compare
 	// what the runner would be started with, not what the caller happened to
 	// send, or the second request reloads the model for nothing.
-	llm.NormalizeImageTokenBudget(req.model.Config.ModelFamily, &optsExisting)
-	llm.NormalizeImageTokenBudget(req.model.Config.ModelFamily, &optsNew)
+	visionArch := llm.VisionBudgetArch(req.model.Config.ModelFamily, req.model.projectorType())
+	llm.NormalizeImageTokenBudget(visionArch, &optsExisting)
+	llm.NormalizeImageTokenBudget(visionArch, &optsNew)
 	// The same holds for the prompt-cache size: an invalid prompt_cache_ram is
 	// ignored at launch, so it starts the same llama-server as an unset one.
 	llm.NormalizePromptCacheRAM(&optsExisting)

server/sched_test.go

@@ -2615,6 +2615,7 @@ func TestSchedNeedsReloadImageTokenBudget(t *testing.T) {
 	for _, tc := range []struct {
 		name       string
 		family     string
+		projector  string
 		minTok     int
 		maxTok     int
 		wantReload bool
@@ -2631,6 +2632,27 @@ func TestSchedNeedsReloadImageTokenBudget(t *testing.T) {
 			minTok: 256, maxTok: 512,
 			wantReload: true,
 		},
+		{
+			// #480: v0.40.2 converts a legacy nemotron_h_omni GGUF into a
+			// nemotron_h_moe model plus a nemotron_v2_vl projector. Its budget
+			// must still reach the launch, so a different ceiling reloads.
+			name:   "converted nemotron3 genuinely different ceiling reloads",
+			family: "nemotron_h_moe", projector: "nemotron_v2_vl",
+			minTok: 256, maxTok: 512,
+			wantReload: true,
+		},
+		{
+			name:   "converted nemotron3 explicit native bounds match the unset sentinel",
+			family: "nemotron_h_moe", projector: "nemotron_v2_vl",
+			minTok: 256, maxTok: 3328,
+			wantReload: false,
+		},
+		{
+			name:   "text-only nemotron_h_moe ignores the bounds",
+			family: "nemotron_h_moe",
+			minTok: 256, maxTok: 512,
+			wantReload: false,
+		},
 		{
 			name:   "gemma4 explicit defaults do not reload",
 			family: "gemma4",
@@ -2651,15 +2673,26 @@ func TestSchedNeedsReloadImageTokenBudget(t *testing.T) {
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
+			newModel := func() *Model {
+				m := &Model{Config: model.ConfigV2{ModelFamily: tc.family}}
+				if tc.projector != "" {
+					m.ProjectorPaths = []string{"projector"}
+					m.projectorMetadata = []ggufMetadata{{KV: map[string]any{
+						"general.architecture":       "clip",
+						"clip.vision.projector_type": tc.projector,
+					}}}
+				}
+				return m
+			}
 			do := api.DefaultOptions()
 			runner := &runnerRef{
-				model:       &Model{Config: model.ConfigV2{ModelFamily: tc.family}},
+				model:       newModel(),
 				Options:     &do,
 				llama:       &mockLlm{vramByGPU: map[ml.DeviceID]uint64{}},
 				numParallel: 1,
 			}
 			req := &LlmRequest{
-				model: &Model{Config: model.ConfigV2{ModelFamily: tc.family}},
+				model: newModel(),
 				opts:  api.DefaultOptions(),
 			}
 			req.opts.ImageMinTokens, req.opts.ImageMaxTokens = tc.minTok, tc.maxTok