From 37f5062df26b7e055a6eb18efcf584cb92a0ed91 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Fri, 6 Nov 2026 05:08:55 +0800 Subject: [PATCH] fix(accounts): try alternatives after a rejected API key cools down Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../loopx-codex-provider-routing/NATIVE.md | 9 ++++++ .../extension.toml | 2 +- .../pyproject.toml | 2 +- .../smoke/openai-policy.test.mjs | 17 +++++++++++ .../native-extension.toml | 2 +- .../openai_keys.py | 5 ++-- .../openai_policy.ts | 13 +++++++-- .../tests/test_api_stream_outcome.py | 28 +++++++++++++++++++ 8 files changed, 71 insertions(+), 7 deletions(-) diff --git a/packages/loopx-codex-provider-routing/NATIVE.md b/packages/loopx-codex-provider-routing/NATIVE.md index 12add4fdeb..79f5039036 100644 --- a/packages/loopx-codex-provider-routing/NATIVE.md +++ b/packages/loopx-codex-provider-routing/NATIVE.md @@ -154,6 +154,15 @@ balance query or permission to add credits. See the official For an HTTP 429 with this code, the existing quota cooldown (300 seconds when no `Retry-After` is supplied) replaces the generic rate-limit cooldown. +Version 0.15.3 tries eligible alternatives before a previously rejected key, +including when the next independently admitted request arrives after its +cooldown. Among rejected keys, the oldest observation is tried first; ties and +keys without a known rejection retain configured order. Model-specific +permission failures affect only that model. An expired rejection remains +eligible and a subsequent accepted request clears its preference penalty. +Selection reuses the existing private runtime observations, adds no probes, +and never retries an admitted stream or overrides the subscription/API role. + **Check connection** only requests the official [`GET /v1/models`](https://developers.openai.com/api/reference/resources/models/methods/list). It makes no inference call and does not establish spendable quota. Its 401, 403, diff --git a/packages/loopx-codex-provider-routing/extension.toml b/packages/loopx-codex-provider-routing/extension.toml index 9441d25573..19c4d0d7de 100644 --- a/packages/loopx-codex-provider-routing/extension.toml +++ b/packages/loopx-codex-provider-routing/extension.toml @@ -1,6 +1,6 @@ schema_version = "loopx_extension_manifest_v0" id = "loopx-codex-provider-routing" -version = "0.15.2" +version = "0.15.3" requires_loopx_api = ">=1,<2" permissions = [] diff --git a/packages/loopx-codex-provider-routing/pyproject.toml b/packages/loopx-codex-provider-routing/pyproject.toml index 6c746e4a77..43727653c7 100644 --- a/packages/loopx-codex-provider-routing/pyproject.toml +++ b/packages/loopx-codex-provider-routing/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "loopx-codex-provider-routing" -version = "0.15.2" +version = "0.15.3" description = "Native Codex account slots and exhausted-account handoff for LoopX" readme = "README.md" requires-python = ">=3.11" diff --git a/packages/loopx-codex-provider-routing/smoke/openai-policy.test.mjs b/packages/loopx-codex-provider-routing/smoke/openai-policy.test.mjs index 49e5d7c8e1..dda98d2308 100644 --- a/packages/loopx-codex-provider-routing/smoke/openai-policy.test.mjs +++ b/packages/loopx-codex-provider-routing/smoke/openai-policy.test.mjs @@ -18,6 +18,23 @@ test("cooldowns and model-scoped permission failures retain key order without ma assert.deepEqual(apiKeys({now:1000,model:"selected-model",keys}),["b","c"]); assert.deepEqual(apiKeys({now:1100,model:"selected-model",keys}),["a","b","c"]); }); +test("expired rejections yield to alternatives instead of starving a spaced recovery request", () => { + const rejected={id:"a",state:"quota_exhausted",observed_at:1000,blocked_until:1300}; + const keys=[rejected,{id:"b",state:"accepted",observed_at:900},{id:"c"}]; + assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys}),["b","c","a"]); + assert.deepEqual(apiKeys({now:1200,model:"selected-model",keys:[rejected]}),[]); + assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys:[rejected]}),["a"],"Expiry permits a future attempt, not a capacity claim"); + const failures=[rejected,{id:"b",state:"rate_limited",observed_at:800,blocked_until:1200}, + {id:"c",state:"authentication_failed",observed_at:800,blocked_until:1200}]; + assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys:failures}),["b","c","a"],"Oldest failures remain eligible with stable ties"); + const permission={...rejected,state:"permission_denied",blocked_model:"other-model"}; + assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys:[permission,{id:"b"}]}),["a","b"]); + assert.deepEqual(apiKeys({now:1400,model:"other-model",keys:[permission,{id:"b"}]}),["b","a"]); + for (const state of ["accepted","failed",undefined]) { + assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys:[{...rejected,state},{id:"b"}]}),["a","b"]); + } + assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys:[{...rejected,observed_at:undefined},{id:"b"}]}),["a","b"]); +}); test("only structured streamed quota failures update future key availability", () => { for (const code of ["insufficient_quota", "credit_balance_exhausted", "organization_spend_limit_exceeded", "project_spend_limit_exceeded", "organization_usage_limit_exceeded", "usage_limit_exceeded", diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native-extension.toml b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native-extension.toml index 99f440d6bb..b6706285c9 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native-extension.toml +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native-extension.toml @@ -1,6 +1,6 @@ schema_version = "loopx_extension_manifest_v0" id = "loopx-native-account-provider" -version = "0.15.2" +version = "0.15.3" requires_loopx_api = ">=1,<2" permissions = ["model-provider.local-read", "model-provider.transport-manage"] diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_keys.py b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_keys.py index 3d587e2321..f25cc001a3 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_keys.py +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_keys.py @@ -132,8 +132,9 @@ def candidates(self, model: str) -> list[dict]: state = self._load() ids = api_policy({"operation": "candidates", "now": self.owner.clock(), "model": model, "keys": [{"id": row["id"], **{k: row.get("runtime", {}).get(k) - for k in ("blocked_until", "blocked_model")}} for row in state["keys"]]}) - return [row for row in state["keys"] if row["id"] in ids] + for k in ("blocked_until", "blocked_model", "state", "observed_at")}} for row in state["keys"]]}) + rows = {row["id"]: row for row in state["keys"]} + return [rows[key_id] for key_id in ids if key_id in rows] def record(self, key_id: str, *, status: int, code: str | None, model: str, headers: dict) -> dict: try: diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_policy.ts b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_policy.ts index 43fb2036cb..379035e011 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_policy.ts +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_policy.ts @@ -1,5 +1,6 @@ /** OpenAI transport outcomes. No credentials, request contents or IO. */ -export type KeyRuntime = { id: string; blocked_until?: number; blocked_model?: string | null }; +export type KeyRuntime = { id: string; blocked_until?: number; blocked_model?: string | null; + state?: string; observed_at?: number }; const QUOTA_CODES = ["insufficient_quota", "credit_balance_exhausted", "organization_spend_limit_exceeded", "project_spend_limit_exceeded", "organization_usage_limit_exceeded", "usage_limit_exceeded"]; @@ -24,8 +25,16 @@ export function apiStreamOutcome(value: unknown): {status: number; code: string} return null; } export function apiKeys(input: { now: number; model: string; keys: KeyRuntime[] }) { + // Expiry permits a future attempt; it does not establish recovered capacity. + // Try alternatives before a known rejection, even when requests are spaced + // farther apart than the cooldown. Oldest failures are reconsidered first. + const rejectionTime = (key: KeyRuntime) => ["quota_exhausted", "authentication_failed", + "permission_denied", "rate_limited"].includes(key.state ?? "") + && typeof key.observed_at === "number" && Number.isFinite(key.observed_at) && key.observed_at >= 0 + && (!key.blocked_model || key.blocked_model === input.model) ? key.observed_at : -1; return input.keys.filter(key => !(typeof key.blocked_until === "number" - && key.blocked_until > input.now && (!key.blocked_model || key.blocked_model === input.model))).map(key => key.id); + && key.blocked_until > input.now && (!key.blocked_model || key.blocked_model === input.model))) + .sort((a,b) => rejectionTime(a) - rejectionTime(b)).map(key => key.id); } export function apiOutcome(input: { now: number; status: number; code: string | null; retry_after: number | null; model: string }) { const quota = input.status === 429 && QUOTA_CODES.includes(input.code ?? ""); diff --git a/packages/loopx-codex-provider-routing/tests/test_api_stream_outcome.py b/packages/loopx-codex-provider-routing/tests/test_api_stream_outcome.py index 5ac3fa78b5..2da7fa7982 100644 --- a/packages/loopx-codex-provider-routing/tests/test_api_stream_outcome.py +++ b/packages/loopx-codex-provider-routing/tests/test_api_stream_outcome.py @@ -40,6 +40,34 @@ def test_stream_failure_is_forwarded_once_and_only_next_request_uses_next_key(re assert keys.status()["balance_available"] is False +@pytest.mark.parametrize("failure", [ + {"type": "error", "code": "usage_limit_exceeded"}, + {"type": "response.failed", "response": {"status": "failed", "error": {"code": "insufficient_quota"}}}, +]) +def test_spaced_request_after_cooldown_tries_alternative_without_replaying_failed_stream(relay, failure): + owner, keys, calls, outcomes, post, _ = relay + now = [1000] + owner.clock = lambda: now[0] + stream = event(failure) + outcomes.append(Response(stream)) + assert post() == (200, stream) and len(calls) == 1 + rejected_key = calls[0].get_header("Authorization") + now[0] = 1400 # The next original request arrives after the five-minute cooldown. + success = event({"type": "response.completed", "response": {"status": "completed"}}) + outcomes.append(Response(success)) + assert post() == (200, success) and len(calls) == 2 + assert calls[1].get_header("Authorization") != rejected_key + now[0] = 1800 + outcomes.append(Response(success)) + assert post() == (200, success) and len(calls) == 3 + assert calls[2].get_header("Authorization") == calls[1].get_header("Authorization") + # An expired rejection is a lower-priority candidate, not a permanent ban. + keys.remove(expected_revision=2, key_id=keys.status()["keys"][1]["id"]) + outcomes.append(Response(success)) + assert post() == (200, success) and len(calls) == 4 + assert calls[3].get_header("Authorization") == rejected_key + + @pytest.mark.parametrize("stream", [ event({"type": "response.output_text.delta", "delta": '{"type":"error","code":"insufficient_quota"}'}), event({"type": "error", "message": "Quota exceeded"}),