From 3b563aa26873c8a82f4e38139dca68ebee553bc9 Mon Sep 17 00:00:00 2001 From: Dean Eckert Date: Tue, 1 Sep 2026 14:09:46 +0200 Subject: [PATCH] feat(metrics): normalized model names and added configurable `REQUEST_HEALTH_MODEL_CAPTURE_LIMIT_BYTES` for changing capture size limit for requests. Default 3 MiB --- .env.example | 3 ++- apiproxy/proxy.go | 8 ++++++- apiproxy/request_health.go | 24 +++++++++++++++----- apiproxy/request_health_test.go | 22 ++++++++++++++++++ apiproxy/rerank.go | 4 ++-- app/src/server/api/routers/service-health.ts | 6 +++-- local-dev/app.env.example | 1 + 7 files changed, 56 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index 5d6ca4b..7070d41 100644 --- a/.env.example +++ b/.env.example @@ -10,5 +10,6 @@ DEFAULT_BACKEND=azure RERANK_TIMEOUT_MS=30000 RERANK_MAX_REQUEST_BYTES=10485760 RERANK_MAX_RESPONSE_BYTES=10485760 +REQUEST_HEALTH_MODEL_CAPTURE_LIMIT_BYTES=3145728 -DATABASE_PATH=postgresql://user:password@server:5432/db?sslmode=disable \ No newline at end of file +DATABASE_PATH=postgresql://user:password@server:5432/db?sslmode=disable diff --git a/apiproxy/proxy.go b/apiproxy/proxy.go index 24ac179..d15effd 100644 --- a/apiproxy/proxy.go +++ b/apiproxy/proxy.go @@ -23,6 +23,7 @@ type AzureConfig struct { RerankTimeout time.Duration RerankMaxRequestBytes int64 RerankMaxResponseBytes int64 + ModelCaptureLimitBytes int64 } var ( @@ -49,6 +50,7 @@ func Init(mux *http.ServeMux, db *db.Database) *requestHealthRecorder { RerankTimeout: rerankTimeoutFromEnv(), RerankMaxRequestBytes: rerankMaxRequestBytesFromEnv(), RerankMaxResponseBytes: rerankMaxResponseBytesFromEnv(), + ModelCaptureLimitBytes: requestHealthModelCaptureLimitFromEnv(), } defaultBackend = os.Getenv("DEFAULT_BACKEND") rc := &ResponseConf{ @@ -152,7 +154,11 @@ func (h *baseHandle) HandleAzure(w http.ResponseWriter, r *http.Request, backend // `/openai` base will produce `/openai/v1/responses` as desired. r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api") ensureStreamUsageForChatCompletions(r) - model, streaming := captureRequestHealthRequest(r) + captureLimit := h.az.ModelCaptureLimitBytes + if captureLimit <= 0 { + captureLimit = requestHealthModelCaptureLimit + } + model, streaming := captureRequestHealthRequestWithLimit(r, captureLimit) r = withRequestHealthMetadata(r, requestHealthMetadata{ Endpoint: originalEndpoint, Model: model, diff --git a/apiproxy/request_health.go b/apiproxy/request_health.go index 7f42f98..0f4e1fd 100644 --- a/apiproxy/request_health.go +++ b/apiproxy/request_health.go @@ -273,7 +273,7 @@ func (t *requestHealthTransport) RoundTrip(req *http.Request) (*http.Response, e RequestID: requestID, Endpoint: endpoint, Upstream: req.URL.Host, - Model: metadata.Model, + Model: normalizeTelemetryModel(metadata.Model), Method: req.Method, StartedAt: started, Streaming: metadata.Streaming || requestAcceptsEventStream(req), @@ -303,7 +303,15 @@ func (t *requestHealthTransport) RoundTrip(req *http.Request) (*http.Response, e return response, nil } -const requestHealthModelCaptureLimit = 1024 * 1024 +const requestHealthModelCaptureLimit = 3 * 1024 * 1024 + +func requestHealthModelCaptureLimitFromEnv() int64 { + return positiveInt64Env("REQUEST_HEALTH_MODEL_CAPTURE_LIMIT_BYTES", requestHealthModelCaptureLimit) +} + +func normalizeTelemetryModel(model string) string { + return strings.ToLower(strings.TrimSpace(model)) +} func captureRequestHealthModel(req *http.Request) string { model, _ := captureRequestHealthRequest(req) @@ -311,11 +319,15 @@ func captureRequestHealthModel(req *http.Request) string { } func captureRequestHealthRequest(req *http.Request) (string, bool) { - if req == nil || req.Body == nil || (req.ContentLength >= 0 && req.ContentLength > requestHealthModelCaptureLimit) { + return captureRequestHealthRequestWithLimit(req, requestHealthModelCaptureLimit) +} + +func captureRequestHealthRequestWithLimit(req *http.Request, limit int64) (string, bool) { + if limit <= 0 || req == nil || req.Body == nil || (req.ContentLength >= 0 && req.ContentLength > limit) { return "", false } - body, err := io.ReadAll(io.LimitReader(req.Body, requestHealthModelCaptureLimit+1)) - if err != nil || int64(len(body)) > requestHealthModelCaptureLimit { + body, err := io.ReadAll(io.LimitReader(req.Body, limit+1)) + if err != nil || int64(len(body)) > limit { req.Body = io.NopCloser(io.MultiReader(bytes.NewReader(body), req.Body)) return "", false } @@ -327,7 +339,7 @@ func captureRequestHealthRequest(req *http.Request) (string, bool) { if err := json.Unmarshal(body, &payload); err != nil { return "", false } - return strings.TrimSpace(payload.Model), payload.Stream != nil && *payload.Stream + return normalizeTelemetryModel(payload.Model), payload.Stream != nil && *payload.Stream } type requestHealthBody struct { diff --git a/apiproxy/request_health_test.go b/apiproxy/request_health_test.go index fc08b4d..8b857ce 100644 --- a/apiproxy/request_health_test.go +++ b/apiproxy/request_health_test.go @@ -5,6 +5,7 @@ import ( "errors" "io" "net/http" + "net/http/httptest" "strings" "sync" "testing" @@ -287,3 +288,24 @@ func TestCaptureRequestHealthRequestUsesStreamField(t *testing.T) { }) } } + +func TestCaptureRequestHealthRequestNormalizesModel(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "https://example.test", strings.NewReader(`{"model":" GPT-5-Mini "}`)) + + model, _ := captureRequestHealthRequest(req) + if model != "gpt-5-mini" { + t.Fatalf("got model %q, want %q", model, "gpt-5-mini") + } +} + +func TestRequestHealthModelCaptureLimitFromEnv(t *testing.T) { + t.Setenv("REQUEST_HEALTH_MODEL_CAPTURE_LIMIT_BYTES", "1234") + if got := requestHealthModelCaptureLimitFromEnv(); got != 1234 { + t.Fatalf("got limit %d, want 1234", got) + } + + t.Setenv("REQUEST_HEALTH_MODEL_CAPTURE_LIMIT_BYTES", "0") + if got := requestHealthModelCaptureLimitFromEnv(); got != requestHealthModelCaptureLimit { + t.Fatalf("got invalid-value fallback %d, want %d", got, requestHealthModelCaptureLimit) + } +} diff --git a/apiproxy/rerank.go b/apiproxy/rerank.go index e38902c..37cc32e 100644 --- a/apiproxy/rerank.go +++ b/apiproxy/rerank.go @@ -193,7 +193,7 @@ func (h *baseHandle) HandleRerank(w http.ResponseWriter, r *http.Request) { upstreamRequest.Header.Set("X-Request-ID", requestID) upstreamRequest = withRequestHealthMetadata(upstreamRequest, requestHealthMetadata{ Endpoint: r.URL.Path, - Model: request.Model, + Model: normalizeTelemetryModel(request.Model), }) client := h.rerankClient @@ -287,7 +287,7 @@ func (h *baseHandle) HandleRerank(w http.ResponseWriter, r *http.Request) { ApiKeyID: apiKeyID, RequestType: db.RequestTypeRerank, SearchUnits: &searchUnits, - Model: request.Model, + Model: normalizeTelemetryModel(request.Model), IsApproximated: false, } if err := h.db.WriteRerankRequest(&rq); err != nil { diff --git a/app/src/server/api/routers/service-health.ts b/app/src/server/api/routers/service-health.ts index e703b2f..874a02e 100644 --- a/app/src/server/api/routers/service-health.ts +++ b/app/src/server/api/routers/service-health.ts @@ -264,13 +264,15 @@ export const serviceHealthRouter = createTRPCRouter({ .limit(maxBreakdownRows), ctx.db .select({ - model: sql`coalesce(${requestHealthAttempts.model}, 'Unknown')`, + model: sql`coalesce(nullif(lower(trim(${requestHealthAttempts.model})), ''), 'Unknown')`, count: sql`count(*)::int`, unsuccessful: sql`count(*) filter (where ${unsuccessfulCondition})::int`, }) .from(requestHealthAttempts) .where(filter) - .groupBy(sql`coalesce(${requestHealthAttempts.model}, 'Unknown')`) + .groupBy( + sql`coalesce(nullif(lower(trim(${requestHealthAttempts.model})), ''), 'Unknown')`, + ) .orderBy(desc(sql`count(*)`)) .limit(maxBreakdownRows), (() => { diff --git a/local-dev/app.env.example b/local-dev/app.env.example index e3ac1e6..bdc75de 100644 --- a/local-dev/app.env.example +++ b/local-dev/app.env.example @@ -22,6 +22,7 @@ DEFAULT_BACKEND=azure RERANK_TIMEOUT_MS=30000 RERANK_MAX_REQUEST_BYTES=10485760 RERANK_MAX_RESPONSE_BYTES=10485760 +REQUEST_HEALTH_MODEL_CAPTURE_LIMIT_BYTES=3145728 # Optional: override the generated DATABASE_PATH # DATABASE_PATH=postgresql://openai:openai_pass@db/openai_proxy