Experimental Native Protocol Rewrite

.env.example

@@ -186,15 +186,23 @@
 #
 # TTL and max entries for the sequential strategy's in-memory session->credential
 # sticky cache. These do not enable disk persistence.
-#SESSION_STICKY_ENTRY_TTL_SECONDS=3600
-#SESSION_STICKY_ENTRY_TTL_SECONDS_GEMINI_CLI=3600
+# Default idle cache-affinity lifetime is 300 seconds. Override per provider
+# when its upstream prompt-cache invalidation period differs. Set to 0 to disable
+# sticky credential bindings for that provider.
+#SESSION_STICKY_ENTRY_TTL_SECONDS=300
+#SESSION_STICKY_ENTRY_TTL_SECONDS_GEMINI_CLI=300
 #SESSION_STICKY_MAX_ENTRIES=10000
 #SESSION_STICKY_MAX_ENTRIES_GEMINI_CLI=10000
 #
 # Optional: promote explicit request fields to trusted strong session evidence.
 # Leave unset unless your clients send stable conversation IDs.
 #TRUSTED_SESSION_ID_FIELDS="conversation_id,thread_id"
 
+# Persist inferred session lineage across proxy restarts. Disabled by default;
+# state is stored as content-free schema-3 hashes in session_stickiness.json.
+#SESSION_PERSISTENCE_ENABLED=true
+#SESSION_PERSISTENCE_FLUSH_INTERVAL_SECONDS=5.0
+
 # --- Priority-Based Concurrency Multipliers ---
 # Credentials can be assigned to priority tiers (1=highest, 2, 3, etc.).
 # Each tier can have a concurrency multiplier that increases the effective
@@ -265,6 +273,15 @@
 # EXHAUSTION_COOLDOWN_THRESHOLD_GEMINI_CLI=300
 # EXHAUSTION_COOLDOWN_THRESHOLD=300  # Global fallback for all providers
 
+# No-reset quota exhaustion policy. Some quota APIs report an exhausted bucket
+# without a reset timestamp when the account cannot access that model/group.
+# Policies: warn_only | cooldown | disable_scope
+# Providers default to warn_only unless they explicitly opt into a fallback.
+# QUOTA_NO_RESET_EXHAUSTION_POLICY_PROVIDER=cooldown
+# QUOTA_NO_RESET_COOLDOWN_SECONDS_PROVIDER=86400
+# QUOTA_NO_RESET_EXHAUSTION_POLICY=warn_only
+# QUOTA_NO_RESET_COOLDOWN_SECONDS=0
+
 # ------------------------------------------------------------------------------
 # | [ADVANCED] Custom Caps                                                      |
 # ------------------------------------------------------------------------------
@@ -333,6 +350,98 @@
 # Default: false
 # STREAM_RETRY_ON_REASONING_ONLY=false
 
+# --- Optional Structured Config File ---
+# Optional JSON config for structured experimental settings such as fallback
+# groups, model routes, advisory pricing, streaming observability, and field
+# cache rules. Environment variables override JSON config. Do not put API keys,
+# OAuth tokens, bearer tokens, or authorization headers in this file.
+# Supported top-level sections include: routing, pricing, streaming, retry,
+# responses, field_cache, and providers. The providers section is for non-secret
+# provider tuning and config-defined custom providers; credentials must stay in
+# env or provider credential files. A custom provider can declare api_base,
+# protocol_name (openai_chat, responses, anthropic_messages, or gemini),
+# endpoint_paths, auth_mode (bearer, x-api-key, x-goog-api-key, custom, or none),
+# auth_header_name, models, default_output_protocol, adapter_names,
+# adapter_config, native_streaming_supported, field_cache, and model_quota_groups.
+# Endpoint paths stay on api_base and may use {model}, {operation}, or {provider};
+# transport selectors such as ?alt=sse are allowed, but credential-like query
+# keys, URL credentials, fragments, and cross-origin endpoint URLs are rejected.
+# Custom provider definitions are discovered and snapshotted at process startup.
+# Supply credentials separately as <PROVIDER>_API_KEY variables or through the
+# existing provider credential facilities. Never put credentials or OAuth data
+# in the JSON file.
+# LLM_PROXY_CONFIG_FILE=./config/llm-proxy.json
+
+# Non-streaming generation routes accept X-Proxy-Output-Protocol to select an
+# independent response format. Explicit library arguments take priority, then
+# this header, then a provider default, then the input protocol. Cross-protocol
+# streaming is rejected until canonical stream conversion is enabled.
+
+# --- Fallback Groups and Model Routes ---
+# Ordered fallback groups try targets in order when the previous target fails
+# with a retryable provider/category error. Execution suffixes:
+#   @auto              Let the provider choose custom/native/LiteLLM behavior
+#   @native            Require native protocol execution
+#   @custom            Require provider custom execution
+#   @litellm_fallback  Explicitly use LiteLLM fallback
+# FALLBACK_GROUPS=code_chain
+# FALLBACK_GROUP_CODE_CHAIN=codex/gpt-5.1-codex@native,openai/gpt-5.1@litellm_fallback
+# MODEL_ROUTE_CODE=group:code_chain
+
+# --- Provider Cooldown Activation ---
+# Provider-level cooldown is conservative and only intended for large/global
+# retry-after events, not every per-credential quota error. Model-capacity
+# errors can start a model-scoped cooldown without blocking unrelated models.
+# PROVIDER_COOLDOWN_MIN_SECONDS=10
+# PROVIDER_COOLDOWN_DEFAULT_SECONDS=30
+# PROVIDER_COOLDOWN_ON_QUOTA=false
+# Repeated transient provider/model failures can increase bounded backoff.
+# PROVIDER_BACKOFF_WINDOW_SECONDS=60
+# PROVIDER_BACKOFF_THRESHOLD=3
+# PROVIDER_BACKOFF_BASE_SECONDS=0  # 0/unset means use provider cooldown default
+# PROVIDER_BACKOFF_MAX_SECONDS=300
+# FAILURE_HISTORY_MAX_ENTRIES=200
+
+# --- Responses API Store Policy ---
+# Responses are stored in-memory by default. Use provider_cache for durable JSON
+# storage via the existing provider-cache layer. TTL/max limits are disabled
+# when unset or <= 0. Failed stream responses are stored by default; in-progress
+# streaming state is disabled unless explicitly enabled.
+# RESPONSES_STORE_BACKEND=memory
+# RESPONSES_STORE_CACHE_NAME=responses
+# RESPONSES_STORE_CACHE_PREFIX=responses
+# RESPONSES_STORE_CACHE_DIR=
+# RESPONSES_STORE_CACHE_MEMORY_TTL_SECONDS=3600
+# RESPONSES_STORE_CACHE_DISK_TTL_SECONDS=172800
+# RESPONSES_STORE_TTL_SECONDS=0
+# RESPONSES_STORE_MAX_ITEMS=0
+# RESPONSES_STORE_FAILED=true
+# RESPONSES_STORE_IN_PROGRESS=false
+
+# --- Streaming Observability ---
+# Stream lifecycle metrics are traced by default. TTFB/stall timeout values are
+# active when set to >0; keep at 0 to disable. Heartbeats are SSE comments and
+# do not count as visible output.
+# STREAM_TRACE_METRICS=true
+# STREAM_TTFB_TIMEOUT_SECONDS=0
+# STREAM_STALL_TIMEOUT_SECONDS=0
+# STREAM_HEARTBEAT_INTERVAL_SECONDS=0
+# STREAM_HEARTBEAT_SECONDS=0  # Legacy alias
+# STREAM_CANCEL_UPSTREAM_ON_DISCONNECT=true
+
+# --- Advisory Model Pricing ---
+# Per-token advisory prices used only when providers do not report actual cost.
+# Precedence: skip-cost provider setting > provider-reported cost/SSE cost event
+# > provider explicit pricing > env pricing > JSON pricing > LiteLLM metadata.
+# Streaming providers can report actual cost with `: cost {...}` comments or
+# `event: cost` frames.
+# Env names sanitize provider/model by replacing non-alphanumerics with `_`.
+# MODEL_PRICE_OPENAI_GPT_5_1_INPUT=0.000001
+# MODEL_PRICE_OPENAI_GPT_5_1_OUTPUT=0.00001
+# MODEL_PRICE_OPENAI_GPT_5_1_CACHE_READ=0.0000001
+# MODEL_PRICE_OPENAI_GPT_5_1_CACHE_WRITE=0.000001
+# MODEL_PRICE_OPENAI_GPT_5_1_REASONING=0.00001
+
 # ------------------------------------------------------------------------------
 # | [ADVANCED] HTTP Timeout Configuration                                       |
 # ------------------------------------------------------------------------------

.gitignore

@@ -139,6 +139,7 @@ cache/antigravity/thought_signatures.json
 !tests/
 tests/*
 !tests/test_classifier_scoped_routing.py
+!tests/test_nvidia_provider.py
 !tests/test_session_tracking.py
 !tests/test_selection_engine.py
 docs/ignored/