From ef5dc5f26cb3ae318ce63a342d3a902da680f43b Mon Sep 17 00:00:00 2001 From: thatguywob Date: Wed, 28 Oct 2026 18:11:18 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(inference):=20reject=20reque?= =?UTF-8?q?sts=20without=20a=20configured=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the configured gateway default for a blank route model. When neither is set, return an actionable local translation error and retain it for the agent dashboard instead of forwarding an empty model to the provider. Hive-Run: hivecommons/hive#15032 Hive-Plan: prevent-empty-inference-model Hive-Spec: issue-15032#empty-model-evidence Signed-off-by: thatguywob --- .../fixed-15032-empty-inference-model.md | 1 + src/crux/proxy/anthropic_translate.go | 5 + src/crux/proxy/github_proxy_inference.go | 12 +- .../proxy/inference_missing_model_test.go | 103 ++++++++++++++++++ 4 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-15032-empty-inference-model.md create mode 100644 src/crux/proxy/inference_missing_model_test.go diff --git a/changelog.d/fixed-15032-empty-inference-model.md b/changelog.d/fixed-15032-empty-inference-model.md new file mode 100644 index 0000000000..c394b198fc --- /dev/null +++ b/changelog.d/fixed-15032-empty-inference-model.md @@ -0,0 +1 @@ +- Inference requests inherit a configured gateway default when the agent model is blank; if both are blank, Hive rejects the request locally and shows an actionable model configuration error instead of repeatedly sending an empty model to the provider ([#15032](https://github.com/hivecommons/hive/issues/15032)). diff --git a/src/crux/proxy/anthropic_translate.go b/src/crux/proxy/anthropic_translate.go index dd2df44378..66e2751fe3 100644 --- a/src/crux/proxy/anthropic_translate.go +++ b/src/crux/proxy/anthropic_translate.go @@ -16,6 +16,11 @@ import ( // open-source models toward agentic tool-calling behaviour. Pass "" to // leave the system prompt unchanged (used by tests and non-inference paths). func translateAnthropicToOpenAI(body []byte, targetModel string, maxContextLen int, preamble string) ([]byte, error) { + // The CLI's model is a Claude alias, not necessarily a gateway model ID. + // Never use it as a fallback or send an empty model to the provider. + if strings.TrimSpace(targetModel) == "" { + return nil, fmt.Errorf("no inference model configured: select an agent model or set the gateway default_model in Model Gateways") + } var req anthropicRequest if err := json.Unmarshal(body, &req); err != nil { return nil, fmt.Errorf("unmarshal anthropic request: %w", err) diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go index 85803f15eb..42d7bf2da7 100644 --- a/src/crux/proxy/github_proxy_inference.go +++ b/src/crux/proxy/github_proxy_inference.go @@ -243,6 +243,14 @@ func (p *GitHubProxy) recordInferenceError(route *InferenceRoute, agentName stri // and concurrent requests race on nothing. Unknown entitlements, verbatim // matches, and ambiguous candidates all pass the route through unchanged. func (p *GitHubProxy) routeWithEntitledModel(route *InferenceRoute, agentName string) *InferenceRoute { + // A blank per-agent selection inherits the configured backend default, + // even before an entitlement probe has completed. Keep the stored snapshot + // untouched, as with entitlement rewrites below. + if route != nil && strings.TrimSpace(route.Model) == "" && strings.TrimSpace(route.DefaultModel) != "" { + rewritten := *route + rewritten.Model = route.DefaultModel + route = &rewritten + } if route == nil || route.Backend != "litellm" || p.entitlements == nil { return route } @@ -429,7 +437,8 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { openaiBody, err := translateAnthropicToOpenAI(body, route.Model, route.MaxContextLen, resolveInferencePreamble(route, agentName)) if err != nil { p.logger.Error("inference translate request failed", "agent", agentName, "error", err) - http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"translation error: %s"}}`, err.Error()), http.StatusBadGateway) + p.recordAgentInferenceError(agentName, "translation error: "+err.Error(), 0) + http.Error(w, fmt.Sprintf(`{"type":"error","error":{"type":"api_error","message":"translation error: %s"}}`, jsonEscape(err.Error())), http.StatusBadGateway) return } @@ -645,6 +654,7 @@ func (p *GitHubProxy) handleInferenceRequest(conn net.Conn, req *http.Request, a openaiBody, err := translateAnthropicToOpenAI(body, route.Model, route.MaxContextLen, resolveInferencePreamble(route, agentName)) if err != nil { p.logger.Error("inference translate request failed", "agent", agentName, "error", err) + p.recordAgentInferenceError(agentName, "translation error: "+err.Error(), 0) p.writeHTTPError(conn, http.StatusBadGateway, "translation error: "+err.Error()) return } diff --git a/src/crux/proxy/inference_missing_model_test.go b/src/crux/proxy/inference_missing_model_test.go new file mode 100644 index 0000000000..135298e058 --- /dev/null +++ b/src/crux/proxy/inference_missing_model_test.go @@ -0,0 +1,103 @@ +package proxy + +import ( + "bufio" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +// Both entry points must keep blank models away from the provider, expose the +// configuration remedy, and recover when a gateway default is supplied. +func TestInferenceMissingModel(t *testing.T) { + for _, entry := range []string{"translator", "mitm"} { + for _, tc := range []struct { + name string + model string + defaultModel string + wantModel string + }{ + {name: "empty"}, + {name: "whitespace", model: " \t", defaultModel: " \n"}, + {name: "gateway default", defaultModel: "Azure/gpt-4o", wantModel: "Azure/gpt-4o"}, + {name: "whitespace inherits default", model: " \t", defaultModel: "Azure/gpt-4o", wantModel: "Azure/gpt-4o"}, + {name: "explicit selection", model: "aws/claude-sonnet-4-6", defaultModel: "Azure/gpt-4o", wantModel: "aws/claude-sonnet-4-6"}, + } { + t.Run(entry+"/"+tc.name, func(t *testing.T) { + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + var req openaiRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("decode upstream request: %v", err) + } + if req.Model == "" || req.Model != tc.wantModel { + t.Errorf("upstream model = %q, want %q", req.Model, tc.wantModel) + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"id":"test","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}]}`) + })) + defer upstream.Close() + p := newInferenceTestProxy() + p.inferenceLastErrors = newInferenceLastErrorState() + route := &InferenceRoute{Backend: "litellm", Endpoint: upstream.URL, Model: tc.model, DefaultModel: tc.defaultModel} + const body = `{"model":"claude-cli-alias","max_tokens":8,"messages":[{"role":"user","content":"hi"}]}` + var resp *http.Response + if entry == "translator" { + p.inference.Set("scanner", route) + req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body)) + req.Header.Set("x-api-key", "sk-hive-scanner") + w := httptest.NewRecorder() + p.inferenceTranslatorHandler().ServeHTTP(w, req) + resp = w.Result() + } else { + client, server := net.Pipe() + defer client.Close() + done := make(chan struct{}) + go func() { + defer close(done) + defer server.Close() + req, _ := http.NewRequest(http.MethodPost, "https://api.anthropic.com/v1/messages", strings.NewReader(body)) + p.handleInferenceRequest(server, req, "scanner", route) + }() + var err error + resp, err = http.ReadResponse(bufio.NewReader(client), nil) + if err != nil { + t.Fatal(err) + } + defer func() { client.Close(); <-done }() + } + defer resp.Body.Close() + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !json.Valid(responseBody) { + t.Fatalf("invalid response JSON: %s", responseBody) + } + if tc.wantModel == "" { + if resp.StatusCode != http.StatusBadGateway || !strings.Contains(string(responseBody), "no inference model configured") || !strings.Contains(string(responseBody), "default_model") { + t.Fatalf("missing configuration response: %d %s", resp.StatusCode, responseBody) + } + if hits.Load() != 0 { + t.Fatal("missing model was forwarded upstream") + } + msg, status, _, ok := p.LastInferenceError("scanner") + if !ok || status != 0 || !strings.Contains(msg, "default_model") { + t.Fatalf("missing dashboard error: %q, %d, %v", msg, status, ok) + } + } else if resp.StatusCode != http.StatusOK || hits.Load() != 1 { + t.Fatalf("configured request: status=%d hits=%d body=%s", resp.StatusCode, hits.Load(), responseBody) + } + if route.Model != tc.model { + t.Fatalf("stored model mutated: %q, want %q", route.Model, tc.model) + } + }) + } + } +}