feat(compute-plane): scaffold request-trace-uploader

.github/workflows/bazel.yml

@@ -146,6 +146,7 @@ jobs:
           worker-init|src/compute-plane-services/worker-init|false|go-root|build-container
           worker-llm-credentials|src/compute-plane-services/worker-llm-credentials|false|go-root|build-container
           worker-task|src/compute-plane-services/worker-task|false|go-root|build-container
+          request-trace-uploader|src/compute-plane-services/request-trace-uploader|false|go-root|build-container
           worker-utils|src/compute-plane-services/worker-utils|false|go-root|build-container
           function-autoscaler|src/control-plane-services/function-autoscaler|false|go-root|build-container
           helm-reval|src/control-plane-services/helm-reval|false|go-root|build-container

go.work.bazel

@@ -41,6 +41,7 @@ use (
 	./src/compute-plane-services/worker-init
 	./src/compute-plane-services/worker-llm-credentials
 	./src/compute-plane-services/worker-task
+	./src/compute-plane-services/request-trace-uploader
 	./src/compute-plane-services/worker-utils
 	./src/invocation-plane-services/grpc-proxy
 	./src/invocation-plane-services/llm-api-gateway

src/compute-plane-services/request-trace-uploader/AGENTS.md

@@ -0,0 +1,30 @@
+# AGENTS.md - request-trace-uploader
+
+Native Go sidecar scaffold for closed Dynamo request-trace segments. It does
+not publish or delete source segments until a supported upload adapter lands.
+
+## Layout
+
+- `cmd/`: process entrypoint and OCI image target
+- `internal/config/`: current sidecar contract and bounded policy parsing
+- `internal/segment/`: closed trace/audit segment discovery
+- `internal/health/`: liveness and readiness handlers
+- `internal/metrics/`: Prometheus metrics and handler
+- `internal/upload/`: future upload-client boundary
+- `internal/service/`: startup, recovery scan, and HTTP server
+
+## Build and test
+
+```bash
+bazel test //src/compute-plane-services/request-trace-uploader/...
+bazel build //src/compute-plane-services/request-trace-uploader/cmd:image
+```
+
+Run `bazel run //:gazelle` after changing Go imports or Bazel metadata.
+
+## Rules
+
+- Preserve the existing `trace` and `audit` capture-type names.
+- Treat the highest indexed segment for each prefix as active.
+- Do not add a release entry until an approved upload adapter exists.
+- Do not log request payloads, credentials, paths, or remote upload IDs.

src/compute-plane-services/request-trace-uploader/BUILD.bazel

@@ -0,0 +1,5 @@
+load("@gazelle//:def.bzl", "gazelle")
+
+# gazelle:prefix github.com/NVIDIA/nvcf/src/compute-plane-services/request-trace-uploader
+# gazelle:go_naming_convention import_alias
+gazelle(name = "gazelle")

src/compute-plane-services/request-trace-uploader/CLAUDE.md

@@ -0,0 +1 @@
+@AGENTS.md

src/compute-plane-services/request-trace-uploader/README.md

@@ -0,0 +1,52 @@
+# request-trace-uploader
+
+`request-trace-uploader` is the NVCF sidecar for Dynamo request tracing.
+Dynamo calls the captured objects `RequestTraceRecord` values. The records that
+contain input and output payloads have event type `request_payload`.
+
+The current deployment writes request-trace segments to rotating `.jsonl.gz`
+files. It uses two capture types, `trace` and `audit`. This service discovers
+only closed segments: the highest indexed segment for each prefix remains owned
+by the Dynamo writer.
+
+## Initial scaffold
+
+This initial implementation validates the sidecar configuration, verifies its
+secret-file mount, creates state and quarantine directories, exposes health and
+Prometheus metrics, and discovers the current backlog. It intentionally does
+not transform records, submit uploads, poll remote status, delete source files,
+or publish a release image.
+
+The `internal/upload` package defines the future upload-client boundary. The
+real adapter and durable journal are separate follow-up work.
+
+## Configuration
+
+The scaffold retains the current file contract:
+
+- `TRACE_DIR`: absolute directory containing trace and audit segments
+- `TRACE_FILE_PREFIX`: trace segment prefix
+- `AUDIT_FILE_PREFIX`: audit segment prefix
+- `REQUEST_TRACE_UPLOADER_DROP_NCA_IDS`: optional CSV NCA ID drop list for audit
+  payloads. Bare IDs and `nca-<id>-nca` are equivalent. The future
+  transform retains correlation metadata and the normalized NCA ID, but removes
+  request and response payloads plus non-NCA headers before upload.
+- `KRATOS_SECRETS_FILE`: readable mounted secret file; default
+  `/var/secrets/secrets.json`
+
+It also accepts these bounded operational settings:
+
+- `METRICS_ADDR`: default `:8011`
+- `UPLOAD_INTERVAL_SECONDS`: default `30`
+- `STATUS_INTERVAL_SECONDS`: default `5`
+- `STATUS_TIMEOUT_SECONDS`: default `900`
+- `REQUEST_TRACE_UPLOADER_ATTEMPT_TIMEOUT`: default `30s`
+- `REQUEST_TRACE_UPLOADER_OPERATION_TIMEOUT`: default `90s`
+- `REQUEST_TRACE_UPLOADER_MAX_RETRIES`: default `2`
+- `REQUEST_TRACE_UPLOADER_RETRY_INITIAL_BACKOFF`: default `100ms`
+- `REQUEST_TRACE_UPLOADER_RETRY_MAX_BACKOFF`: default `15s`
+- `REQUEST_TRACE_UPLOADER_RETRY_MULTIPLIER`: default `2.0`
+
+Invalid policy values fall back to defaults and produce a safe startup warning.
+Missing paths, unreadable secret files, and incompatible required values prevent
+readiness.

src/compute-plane-services/request-trace-uploader/cmd/BUILD.bazel

@@ -0,0 +1,28 @@
+load("@rules_go//go:def.bzl", "go_binary", "go_library")
+load("//rules/oci:defs.bzl", "go_oci_image")
+
+go_library(
+    name = "cmd_lib",
+    srcs = ["main.go"],
+    importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/request-trace-uploader/cmd",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//src/compute-plane-services/request-trace-uploader/internal/config",
+        "//src/compute-plane-services/request-trace-uploader/internal/service",
+        "@com_github_prometheus_client_golang//prometheus",
+    ],
+)
+
+go_binary(
+    name = "request-trace-uploader",
+    embed = [":cmd_lib"],
+    visibility = ["//visibility:public"],
+)
+
+go_oci_image(
+    name = "image",
+    base = "@distroless_go",
+    binary = ":request-trace-uploader",
+    tags = ["nvcf-request-trace-uploader"],
+    visibility = ["//visibility:public"],
+)

src/compute-plane-services/request-trace-uploader/cmd/main.go

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// request-trace-uploader validates and observes Dynamo request-trace segments.
+package main
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/NVIDIA/nvcf/src/compute-plane-services/request-trace-uploader/internal/config"
+	"github.com/NVIDIA/nvcf/src/compute-plane-services/request-trace-uploader/internal/service"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func main() {
+	cfg, warnings, err := config.LoadFromEnv()
+	if err != nil {
+		slog.Error("invalid request trace uploader configuration", "error", err)
+		os.Exit(1)
+	}
+	for _, warning := range warnings {
+		slog.Warn("request trace uploader configuration fallback", "setting", warning)
+	}
+
+	svc, err := service.New(cfg, prometheus.NewRegistry())
+	if err != nil {
+		slog.Error("create request trace uploader service", "error", err)
+		os.Exit(1)
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	if err := svc.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
+		slog.Error("request trace uploader stopped", "error", err)
+		os.Exit(1)
+	}
+}

src/compute-plane-services/request-trace-uploader/go.mod

@@ -0,0 +1,18 @@
+module github.com/NVIDIA/nvcf/src/compute-plane-services/request-trace-uploader
+
+go 1.26.5
+
+require github.com/prometheus/client_golang v1.23.2
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	google.golang.org/protobuf v1.36.8 // indirect
+)

src/compute-plane-services/request-trace-uploader/go.sum

@@ -0,0 +1,46 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
+google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

src/compute-plane-services/request-trace-uploader/internal/config/BUILD.bazel

@@ -0,0 +1,20 @@
+load("@rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "config",
+    srcs = ["config.go"],
+    importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/request-trace-uploader/internal/config",
+    visibility = ["//src/compute-plane-services/request-trace-uploader:__subpackages__"],
+)
+
+go_test(
+    name = "config_test",
+    srcs = ["config_test.go"],
+    embed = [":config"],
+)
+
+alias(
+    name = "go_default_library",
+    actual = ":config",
+    visibility = ["//src/compute-plane-services/request-trace-uploader:__subpackages__"],
+)