Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions packages/loopx-codex-provider-routing/NATIVE.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,15 @@ balance query or permission to add credits. See the official
For an HTTP 429 with this code, the existing quota cooldown (300 seconds when
no `Retry-After` is supplied) replaces the generic rate-limit cooldown.

Version 0.15.3 tries eligible alternatives before a previously rejected key,
including when the next independently admitted request arrives after its
cooldown. Among rejected keys, the oldest observation is tried first; ties and
keys without a known rejection retain configured order. Model-specific
permission failures affect only that model. An expired rejection remains
eligible and a subsequent accepted request clears its preference penalty.
Selection reuses the existing private runtime observations, adds no probes,
and never retries an admitted stream or overrides the subscription/API role.

**Check connection** only requests the official
[`GET /v1/models`](https://developers.openai.com/api/reference/resources/models/methods/list).
It makes no inference call and does not establish spendable quota. Its 401, 403,
Expand Down
2 changes: 1 addition & 1 deletion packages/loopx-codex-provider-routing/extension.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
schema_version = "loopx_extension_manifest_v0"
id = "loopx-codex-provider-routing"
version = "0.15.2"
version = "0.15.3"
requires_loopx_api = ">=1,<2"
permissions = []

Expand Down
2 changes: 1 addition & 1 deletion packages/loopx-codex-provider-routing/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "loopx-codex-provider-routing"
version = "0.15.2"
version = "0.15.3"
description = "Native Codex account slots and exhausted-account handoff for LoopX"
readme = "README.md"
requires-python = ">=3.11"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,23 @@ test("cooldowns and model-scoped permission failures retain key order without ma
assert.deepEqual(apiKeys({now:1000,model:"selected-model",keys}),["b","c"]);
assert.deepEqual(apiKeys({now:1100,model:"selected-model",keys}),["a","b","c"]);
});
test("expired rejections yield to alternatives instead of starving a spaced recovery request", () => {
const rejected={id:"a",state:"quota_exhausted",observed_at:1000,blocked_until:1300};
const keys=[rejected,{id:"b",state:"accepted",observed_at:900},{id:"c"}];
assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys}),["b","c","a"]);
assert.deepEqual(apiKeys({now:1200,model:"selected-model",keys:[rejected]}),[]);
assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys:[rejected]}),["a"],"Expiry permits a future attempt, not a capacity claim");
const failures=[rejected,{id:"b",state:"rate_limited",observed_at:800,blocked_until:1200},
{id:"c",state:"authentication_failed",observed_at:800,blocked_until:1200}];
assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys:failures}),["b","c","a"],"Oldest failures remain eligible with stable ties");
const permission={...rejected,state:"permission_denied",blocked_model:"other-model"};
assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys:[permission,{id:"b"}]}),["a","b"]);
assert.deepEqual(apiKeys({now:1400,model:"other-model",keys:[permission,{id:"b"}]}),["b","a"]);
for (const state of ["accepted","failed",undefined]) {
assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys:[{...rejected,state},{id:"b"}]}),["a","b"]);
}
assert.deepEqual(apiKeys({now:1400,model:"selected-model",keys:[{...rejected,observed_at:undefined},{id:"b"}]}),["a","b"]);
});
test("only structured streamed quota failures update future key availability", () => {
for (const code of ["insufficient_quota", "credit_balance_exhausted", "organization_spend_limit_exceeded",
"project_spend_limit_exceeded", "organization_usage_limit_exceeded", "usage_limit_exceeded",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
schema_version = "loopx_extension_manifest_v0"
id = "loopx-native-account-provider"
version = "0.15.2"
version = "0.15.3"
requires_loopx_api = ">=1,<2"
permissions = ["model-provider.local-read", "model-provider.transport-manage"]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,9 @@ def candidates(self, model: str) -> list[dict]:
state = self._load()
ids = api_policy({"operation": "candidates", "now": self.owner.clock(), "model": model,
"keys": [{"id": row["id"], **{k: row.get("runtime", {}).get(k)
for k in ("blocked_until", "blocked_model")}} for row in state["keys"]]})
return [row for row in state["keys"] if row["id"] in ids]
for k in ("blocked_until", "blocked_model", "state", "observed_at")}} for row in state["keys"]]})
rows = {row["id"]: row for row in state["keys"]}
return [rows[key_id] for key_id in ids if key_id in rows]

def record(self, key_id: str, *, status: int, code: str | None, model: str, headers: dict) -> dict:
try:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/** OpenAI transport outcomes. No credentials, request contents or IO. */
export type KeyRuntime = { id: string; blocked_until?: number; blocked_model?: string | null };
export type KeyRuntime = { id: string; blocked_until?: number; blocked_model?: string | null;
state?: string; observed_at?: number };
const QUOTA_CODES = ["insufficient_quota", "credit_balance_exhausted", "organization_spend_limit_exceeded",
"project_spend_limit_exceeded", "organization_usage_limit_exceeded",
"usage_limit_exceeded"];
Expand All @@ -24,8 +25,16 @@ export function apiStreamOutcome(value: unknown): {status: number; code: string}
return null;
}
export function apiKeys(input: { now: number; model: string; keys: KeyRuntime[] }) {
// Expiry permits a future attempt; it does not establish recovered capacity.
// Try alternatives before a known rejection, even when requests are spaced
// farther apart than the cooldown. Oldest failures are reconsidered first.
const rejectionTime = (key: KeyRuntime) => ["quota_exhausted", "authentication_failed",
"permission_denied", "rate_limited"].includes(key.state ?? "")
&& typeof key.observed_at === "number" && Number.isFinite(key.observed_at) && key.observed_at >= 0
&& (!key.blocked_model || key.blocked_model === input.model) ? key.observed_at : -1;
return input.keys.filter(key => !(typeof key.blocked_until === "number"
&& key.blocked_until > input.now && (!key.blocked_model || key.blocked_model === input.model))).map(key => key.id);
&& key.blocked_until > input.now && (!key.blocked_model || key.blocked_model === input.model)))
.sort((a,b) => rejectionTime(a) - rejectionTime(b)).map(key => key.id);
}
export function apiOutcome(input: { now: number; status: number; code: string | null; retry_after: number | null; model: string }) {
const quota = input.status === 429 && QUOTA_CODES.includes(input.code ?? "");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,34 @@ def test_stream_failure_is_forwarded_once_and_only_next_request_uses_next_key(re
assert keys.status()["balance_available"] is False


@pytest.mark.parametrize("failure", [
{"type": "error", "code": "usage_limit_exceeded"},
{"type": "response.failed", "response": {"status": "failed", "error": {"code": "insufficient_quota"}}},
])
def test_spaced_request_after_cooldown_tries_alternative_without_replaying_failed_stream(relay, failure):
owner, keys, calls, outcomes, post, _ = relay
now = [1000]
owner.clock = lambda: now[0]
stream = event(failure)
outcomes.append(Response(stream))
assert post() == (200, stream) and len(calls) == 1
rejected_key = calls[0].get_header("Authorization")
now[0] = 1400 # The next original request arrives after the five-minute cooldown.
success = event({"type": "response.completed", "response": {"status": "completed"}})
outcomes.append(Response(success))
assert post() == (200, success) and len(calls) == 2
assert calls[1].get_header("Authorization") != rejected_key
now[0] = 1800
outcomes.append(Response(success))
assert post() == (200, success) and len(calls) == 3
assert calls[2].get_header("Authorization") == calls[1].get_header("Authorization")
# An expired rejection is a lower-priority candidate, not a permanent ban.
keys.remove(expected_revision=2, key_id=keys.status()["keys"][1]["id"])
outcomes.append(Response(success))
assert post() == (200, success) and len(calls) == 4
assert calls[3].get_header("Authorization") == rejected_key


@pytest.mark.parametrize("stream", [
event({"type": "response.output_text.delta", "delta": '{"type":"error","code":"insufficient_quota"}'}),
event({"type": "error", "message": "Quota exceeded"}),
Expand Down
Loading