feat(llm-routing): support per-replica worker endpoints

deploy/helm/llm-request-router/README.md

@@ -83,6 +83,53 @@ Important settings to review before deployment:
 
 The default values include development-oriented placeholders. Override them before using the chart in any shared or production environment.
 
+## Split-Cluster Worker Access
+
+Remote pylons need one shared seed and one address per Stargate replica. The
+shared Service is only the `WatchStargates` bootstrap address. After discovery,
+each pylon registers directly with every returned replica over TCP and opens a
+reverse QUIC tunnel to that same replica over UDP.
+
+Enable per-replica Services with:
+
+```yaml
+llmRequestRouter:
+  service:
+    annotations: {}
+  externalAccess:
+    enabled: true
+    domain: router.region-a.example
+    service:
+      type: LoadBalancer
+      annotations: {}
+  discovery:
+    remoteStargateURLs:
+      - http://router-seed.region-b.example:50071
+```
+
+For a StatefulSet pod named `llm-request-router-0`, this configuration
+advertises these dial addresses:
+
+- TCP registration: `llm-request-router-0.router.region-a.example:50071`
+- UDP reverse QUIC: `llm-request-router-0.router.region-a.example:50072`
+
+The external names are dial-only. The internal advertised hostname remains
+the gRPC authority and the QUIC SNI. Keep the internal exact and wildcard names
+in `certificate.dnsNames`; do not add the external domain to the router
+certificate.
+
+The infrastructure provider must create the DNS records and transparently
+forward both protocols to the matching per-pod Service. Do not terminate TLS,
+change the HTTP/2 authority, or route several replicas behind one endpoint.
+Use a region-unique external domain if `remoteStargateURLs` connects router
+meshes from more than one region. The chart does not create DNS records or
+provider load balancers beyond the requested Kubernetes Service type.
+
+In the self-managed stack,
+`global.workerEndpoints.llmRequestRouterAddress` remains the single shared seed.
+Configure `addons.llm.requestRouter.externalAccess` separately for the
+per-replica paths.
+
 ## Load Balancer Configuration
 
 The chart can pass a Stargate load-balancer config in either of two ways:

deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl

@@ -149,6 +149,41 @@ comparing suffixes.
 {{- end -}}
 {{- end -}}
 
+{{/*
+External access gives every replica a distinct TCP and UDP dial address. The
+internal advertised hostname remains the gRPC authority and QUIC certificate
+identity, so the external domain must not be added to certificate SANs.
+*/}}
+{{- define "llm-request-router.validateExternalAccess" -}}
+{{- $externalAccess := .Values.llmRequestRouter.externalAccess | default dict -}}
+{{- if $externalAccess.enabled -}}
+{{- $domain := $externalAccess.domain | default "" | toString | lower -}}
+{{- if not $domain -}}
+{{- fail "llmRequestRouter.externalAccess.domain is required when llmRequestRouter.externalAccess.enabled is true" -}}
+{{- end -}}
+{{- $labels := splitList "." $domain -}}
+{{- $validDomain := and
+      (gt (len $domain) 0)
+      (le (len $domain) 253)
+      (not (hasPrefix "." $domain))
+      (not (hasSuffix "." $domain)) -}}
+{{- range $label := $labels -}}
+{{- if not (regexMatch "^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$" $label) -}}
+{{- $validDomain = false -}}
+{{- end -}}
+{{- end -}}
+{{- if regexMatch "^[0-9]+$" (last $labels) -}}
+{{- $validDomain = false -}}
+{{- end -}}
+{{- if not $validDomain -}}
+{{- fail (printf "llmRequestRouter.externalAccess.domain %q is not a valid DNS name" $domain) -}}
+{{- end -}}
+{{- if not .Values.llmRequestRouter.transport.reverseTunnelListenAddr -}}
+{{- fail "llmRequestRouter.transport.reverseTunnelListenAddr is required when llmRequestRouter.externalAccess.enabled is true" -}}
+{{- end -}}
+{{- end -}}
+{{- end }}
+
 {{- define "llm-request-router.serviceAccountName" -}}
 {{- if .Values.llmRequestRouter.serviceAccount.create }}
 {{- default (include "llm-request-router.fullname" .) .Values.llmRequestRouter.serviceAccount.name }}

deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml

@@ -36,6 +36,8 @@ The identity guard lives here, not in certificate.yaml: existingSecret mode
 renders no Certificate, so a guard in that template would never run.
 */}}
 {{- include "llm-request-router.validateTlsIdentity" . }}
+{{- $externalAccess := .Values.llmRequestRouter.externalAccess | default dict }}
+{{- include "llm-request-router.validateExternalAccess" . }}
 {{- $advertisedHostnameTemplate := include "llm-request-router.advertisedHostnameTemplate" . }}
 spec:
   serviceName: {{ .Values.llmRequestRouter.service.headlessName }}
@@ -87,6 +89,9 @@ spec:
             {{- with dig "discovery" "watchHeartbeatMs" "" .Values.llmRequestRouter }}
             - --watch-heartbeat-ms={{ . }}
             {{- end }}
+            {{- range (dig "discovery" "remoteStargateURLs" (list) .Values.llmRequestRouter) }}
+            - {{ printf "--remote-stargate-url=%s" . | quote }}
+            {{- end }}
             - --shutdown-drain-timeout-ms={{ .Values.llmRequestRouter.shutdown.drainTimeoutMs }}
             - --quic-connect-timeout-ms={{ .Values.llmRequestRouter.transport.quicConnectTimeoutMs }}
             - --quic-request-timeout-ms={{ .Values.llmRequestRouter.transport.quicRequestTimeoutMs }}
@@ -95,7 +100,12 @@ spec:
             - --backend-connectivity=reverse
             - --reverse-tunnel-listen-addr={{ .Values.llmRequestRouter.transport.reverseTunnelListenAddr }}
             {{- end }}
-            {{- if and .Values.llmRequestRouter.transport.reverseTunnelListenAddr (gt $replicaCount 1) }}
+            {{- if $externalAccess.enabled }}
+            - --grpc-pylon-dial-addr={stargate_id}.{{ $externalAccess.domain }}:{{ .Values.llmRequestRouter.service.grpcPort }}
+            {{- end }}
+            {{- if and .Values.llmRequestRouter.transport.reverseTunnelListenAddr $externalAccess.enabled }}
+            - --reverse-tunnel-pylon-dial-addr=$(POD_NAME).{{ $externalAccess.domain }}:{{ .Values.llmRequestRouter.service.reverseTunnelPort }}
+            {{- else if and .Values.llmRequestRouter.transport.reverseTunnelListenAddr (gt $replicaCount 1) }}
             - --reverse-tunnel-pylon-dial-addr=$(POD_IP):{{ .Values.llmRequestRouter.service.reverseTunnelPort }}
             {{- end }}
             {{- if .Values.llmRequestRouter.transport.reverseTunnelConnectTimeoutMs }}

deploy/helm/llm-request-router/llm-request-router/templates/service-per-pod.yaml

@@ -0,0 +1,49 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- $externalAccess := .Values.llmRequestRouter.externalAccess | default dict }}
+{{- if $externalAccess.enabled }}
+{{- include "llm-request-router.validateExternalAccess" . }}
+{{- $service := $externalAccess.service | default dict }}
+{{- $fullname := include "llm-request-router.fullname" . }}
+{{- range $ordinal := until (int $.Values.llmRequestRouter.replicaCount) }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ printf "%s-%d" $fullname $ordinal }}
+  namespace: {{ include "llm-request-router.namespace" $ }}
+  labels:
+    {{- include "llm-request-router.labels" $ | nindent 4 }}
+  {{- with $service.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  type: {{ $service.type | default "LoadBalancer" }}
+  selector:
+    {{- include "llm-request-router.selectorLabels" $ | nindent 4 }}
+    statefulset.kubernetes.io/pod-name: {{ printf "%s-%d" $fullname $ordinal }}
+  ports:
+    - name: grpc
+      port: {{ $.Values.llmRequestRouter.service.grpcPort }}
+      targetPort: grpc
+      protocol: TCP
+    - name: quic
+      port: {{ $.Values.llmRequestRouter.service.reverseTunnelPort }}
+      targetPort: reverse
+      protocol: UDP
+{{- end }}
+{{- end }}

deploy/helm/llm-request-router/llm-request-router/templates/service.yaml

@@ -20,6 +20,10 @@ metadata:
   namespace: {{ include "llm-request-router.namespace" . }}
   labels:
     {{- include "llm-request-router.labels" . | nindent 4 }}
+  {{- with .Values.llmRequestRouter.service.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   type: {{ .Values.llmRequestRouter.service.type }}
   selector:

deploy/helm/llm-request-router/llm-request-router/values.yaml

@@ -65,6 +65,7 @@ llmRequestRouter:
 
   service:
     type: ClusterIP
+    annotations: {}
     httpPort: 8000
     grpcPort: 50071
     metricsPort: 9090
@@ -121,8 +122,8 @@ llmRequestRouter:
     # suffixes). REQUIRED when enabled. Typically:
     #   "<customer-domain>,cluster.local"
     # so the signing role accepts both the customer's external DNS and
-    # in-cluster service names. Missing value → script aborts non-zero →
-    # entrypoint accumulator → Job fails on backoff exhaustion.
+    # in-cluster service names. Missing value -> script aborts non-zero ->
+    # entrypoint accumulator -> Job fails on backoff exhaustion.
     allowedDomains: ""
     # nvcf-openbao-migrations image. The chart picks up the same image
     # that the k8s-openbao Helm hook uses; supply registry/repository/tag
@@ -154,13 +155,29 @@ llmRequestRouter:
 
   discovery:
     disableDnsDiscovery: false
+    # Additional Stargate WatchStargates endpoints. Each item renders as one
+    # --remote-stargate-url argument.
+    remoteStargateURLs: []
 
   transport:
     quicConnectTimeoutMs: 2000
     quicRequestTimeoutMs: 30000
     reverseTunnelListenAddr: "0.0.0.0:50072"
     reverseTunnelConnectTimeoutMs: 10000
 
+  # Per-replica TCP registration and UDP reverse-tunnel Services. Enable this
+  # when pylons run outside the router cluster. Each external hostname is
+  # "<stargate-id>.<domain>". The provider owns DNS and transparent L4
+  # forwarding for these names; the chart does not create DNS records. Use a
+  # region-unique domain. These names are dial-only and do not belong in the
+  # router certificate SANs.
+  externalAccess:
+    enabled: false
+    domain: ""
+    service:
+      type: LoadBalancer
+      annotations: {}
+
   shutdown:
     drainTimeoutMs: 30000
 

deploy/helm/llm-request-router/scripts/check-multi-replica-render.sh

@@ -40,6 +40,25 @@ statefulset_args() {
   yq -r 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.containers[0].args[]' "${manifest}"
 }
 
+per_pod_service_names() {
+  local manifest="$1"
+  yq -r 'select(.kind == "Service" and (.metadata.name | test("^llm-request-router-[0-9]+$"))) | .metadata.name' "${manifest}" \
+    | grep -v '^---$' \
+    | sort
+}
+
+service_field() {
+  local manifest="$1"
+  local name="$2"
+  local expression="$3"
+  yq -r "select(.kind == \"Service\" and .metadata.name == \"${name}\") | (${expression})" "${manifest}" | head -n1
+}
+
+certificate_dns_names() {
+  local manifest="$1"
+  yq -r 'select(.kind == "Certificate") | .spec.dnsNames[]' "${manifest}"
+}
+
 default_manifest="${tmp_dir}/default.yaml"
 render "${default_manifest}"
 
@@ -83,4 +102,95 @@ render "${custom_template_manifest}" \
 custom_template_args="$(statefulset_args "${custom_template_manifest}")"
 printf '%s\n' "${custom_template_args}" | grep -qx -- '--reverse-tunnel-pylon-dial-addr=$(POD_IP):50072' || fail "custom multi-replica advertised hostname template missing reverse tunnel pylon dial addr"
 
+[ -z "$(per_pod_service_names "${default_manifest}")" ] || fail "default render unexpectedly created per-pod Services"
+if printf '%s\n' "${default_args}" | grep -q -- '--grpc-pylon-dial-addr'; then
+  fail "default render unexpectedly configured an external gRPC dial address"
+fi
+
+external_domain="router.region.example"
+external_manifest="${tmp_dir}/external-access.yaml"
+render "${external_manifest}" \
+  --set llmRequestRouter.replicaCount=2 \
+  --set llmRequestRouter.service.type=NodePort \
+  --set-string llmRequestRouter.service.annotations.shared=seed \
+  --set llmRequestRouter.externalAccess.enabled=true \
+  --set-string llmRequestRouter.externalAccess.domain="${external_domain}" \
+  --set llmRequestRouter.externalAccess.service.type=NodePort \
+  --set-string llmRequestRouter.externalAccess.service.annotations.scope=replica \
+  --set-string 'llmRequestRouter.discovery.remoteStargateURLs[0]=https://watch-a.example:50071' \
+  --set-string 'llmRequestRouter.discovery.remoteStargateURLs[1]=https://watch-b.example:50071'
+
+external_args="$(statefulset_args "${external_manifest}")"
+printf '%s\n' "${external_args}" | grep -qx -- "--grpc-pylon-dial-addr={stargate_id}.${external_domain}:50071" || fail "external render missing templated per-replica gRPC dial address"
+printf '%s\n' "${external_args}" | grep -qx -- "--reverse-tunnel-pylon-dial-addr=\$(POD_NAME).${external_domain}:50072" || fail "external render missing per-replica QUIC dial address"
+[ "$(printf '%s\n' "${external_args}" | grep -cx -- '--remote-stargate-url=https://watch-a.example:50071')" = "1" ] || fail "first remote Stargate URL was not rendered exactly once"
+[ "$(printf '%s\n' "${external_args}" | grep -cx -- '--remote-stargate-url=https://watch-b.example:50071')" = "1" ] || fail "second remote Stargate URL was not rendered exactly once"
+
+expected_services="$(printf 'llm-request-router-0\nllm-request-router-1\n')"
+[ "$(per_pod_service_names "${external_manifest}")" = "${expected_services}" ] || fail "external render did not create exactly two per-pod Services"
+[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.selector."statefulset.kubernetes.io/pod-name"')" = "llm-request-router-1" ] || fail "per-pod Service is not pinned to its StatefulSet replica"
+[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.type')" = "NodePort" ] || fail "per-pod Service type override was not applied"
+[ "$(service_field "${external_manifest}" llm-request-router-1 '.metadata.annotations.scope')" = "replica" ] || fail "per-pod Service annotations were not applied"
+[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.ports[] | select(.name == "grpc") | .port')" = "50071" ] || fail "per-pod Service missing registration port 50071"
+[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.ports[] | select(.name == "grpc") | .protocol')" = "TCP" ] || fail "per-pod registration port is not TCP"
+[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.ports[] | select(.name == "quic") | .port')" = "50072" ] || fail "per-pod Service missing reverse-tunnel port 50072"
+[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.ports[] | select(.name == "quic") | .protocol')" = "UDP" ] || fail "per-pod reverse-tunnel port is not UDP"
+[ "$(service_field "${external_manifest}" llm-request-router '.metadata.annotations.shared')" = "seed" ] || fail "shared seed Service annotations were not applied"
+
+scaled_manifest="${tmp_dir}/external-access-scaled.yaml"
+render "${scaled_manifest}" \
+  --set llmRequestRouter.replicaCount=4 \
+  --set llmRequestRouter.externalAccess.enabled=true \
+  --set-string llmRequestRouter.externalAccess.domain="${external_domain}"
+[ "$(per_pod_service_names "${scaled_manifest}" | wc -l | tr -d ' ')" = "4" ] || fail "per-pod Service count does not follow replicaCount"
+
+assert_render_fails() {
+  local description="$1"
+  local expected="$2"
+  shift 2
+  local error_file="${tmp_dir}/external-invalid.err"
+  if helm template "${release}" "${chart_dir}" \
+    --namespace "${namespace}" \
+    --values "${chart_dir}/values.yaml" \
+    "$@" \
+    > "${tmp_dir}/external-invalid.yaml" 2> "${error_file}"; then
+    fail "${description} unexpectedly succeeded"
+  fi
+  grep -Fq "${expected}" "${error_file}" || fail "${description} did not return the expected guard message"
+}
+
+assert_render_fails \
+  "external access without a domain" \
+  "llmRequestRouter.externalAccess.domain is required" \
+  --set llmRequestRouter.externalAccess.enabled=true
+
+assert_render_fails \
+  "external access with an invalid domain" \
+  "is not a valid DNS name" \
+  --set llmRequestRouter.externalAccess.enabled=true \
+  --set-string llmRequestRouter.externalAccess.domain='not_a_domain.example'
+
+assert_render_fails \
+  "external access without a reverse tunnel listener" \
+  "llmRequestRouter.transport.reverseTunnelListenAddr is required" \
+  --set llmRequestRouter.externalAccess.enabled=true \
+  --set-string llmRequestRouter.externalAccess.domain="${external_domain}" \
+  --set-string llmRequestRouter.transport.reverseTunnelListenAddr=''
+
+pki_manifest="${tmp_dir}/external-access-pki.yaml"
+render "${pki_manifest}" \
+  --set llmRequestRouter.replicaCount=2 \
+  --set llmRequestRouter.externalAccess.enabled=true \
+  --set-string llmRequestRouter.externalAccess.domain="${external_domain}" \
+  --set llmRequestRouter.certificate.enabled=true \
+  --set-string llmRequestRouter.certificate.issuerRef.name=test-issuer \
+  --set-string 'llmRequestRouter.certificate.dnsNames[0]=llm-request-router.nvcf.svc.cluster.local' \
+  --set-string 'llmRequestRouter.certificate.dnsNames[1]=*.llm-request-router-headless.nvcf.svc.cluster.local'
+pki_dns_names="$(certificate_dns_names "${pki_manifest}")"
+printf '%s\n' "${pki_dns_names}" | grep -qx -- 'llm-request-router.nvcf.svc.cluster.local' || fail "PKI render missing exact internal Service SAN"
+printf '%s\n' "${pki_dns_names}" | grep -Fqx -- '*.llm-request-router-headless.nvcf.svc.cluster.local' || fail "PKI render missing wildcard internal headless Service SAN"
+if printf '%s\n' "${pki_dns_names}" | grep -Fq -- "${external_domain}"; then
+  fail "PKI render unexpectedly added the external dial domain to certificate SANs"
+fi
+
 echo "multi-replica render checks passed"

deploy/stacks/self-managed/environments/base.yaml

@@ -257,6 +257,26 @@ addons:
   # LLM addon: gateway + request router (stargate) for LLM function invocation
   llm:
     enabled: false
+    requestRouter:
+      replicaCount: 3
+      # The shared Service is the single worker bootstrap address. It is
+      # separate from the per-replica external Services below.
+      service:
+        type: ClusterIP
+        annotations: {}
+        grpcPort: 50071
+      discovery:
+        # Additional WatchStargates seeds, typically one or more per region.
+        remoteStargateURLs: []
+      # Provider-owned DNS and transparent L4 forwarding expose one TCP and
+      # UDP endpoint per StatefulSet replica when workers are remote. Use a
+      # region-unique domain. It is dial-only and is not a certificate SAN.
+      externalAccess:
+        enabled: false
+        domain: ""
+        service:
+          type: LoadBalancer
+          annotations: {}
     # QUIC TLS certificate for the request router (Stargate). Disabled by
     # default; opt in per env.
     #