Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/codex/routing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -612,6 +612,15 @@ export function tryAcquireCodexQuotaScopeProbeLease(
return probeLeaseId;
}

/** Side-effect-free check for a confirmed model-specific quota probe. */
export function canAcquireCodexQuotaScopeProbeLease(
accountId: string,
scope: CodexQuotaScope,
now = Date.now(),
): boolean {
return canAcquireQuotaProbeLease(scopedHealthFor(accountId, scope), now);
}

/**
* Hand a probe lease back without recording an upstream outcome. Used by paths
* that take a lease and then fail before any request reaches upstream.
Expand Down
102 changes: 90 additions & 12 deletions src/codex/subagent-model-fallback.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@ import { CODEX_HOME, getCodexHome } from "./paths";
import { CODEX_UNKNOWN_USAGE_SCORE, getAccountQuota } from "./quota";
import {
canAcquireCodexQuotaProbeLease,
canAcquireCodexQuotaScopeProbeLease,
codexQuotaScopeForModel,
computeCodexUsageScore,
getCodexQuotaHealthSnapshot,
getEffectiveActiveCodexAccountId,
getPoolAccountPlan,
isCodexAccountInCooldown,
} from "./routing";
import {
isCodexAccountUsable,
Expand All @@ -38,6 +38,7 @@ import {
import { routeModel, type RouteResult } from "../router";
import { sweepExpiredOnWrite } from "../lib/state-store-sweeper";
import { codexAccountNamespaceForModel } from "./account-namespace-match";
import { ACCOUNT_GATED_NATIVE_OPENAI_MODELS } from "./catalog/native-models";
import {
getUpstreamHostHealth,
normalizeUpstreamHostCircuitThreshold,
Expand All @@ -48,6 +49,15 @@ export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000;
const CODEX_FORWARD_ORIGIN = new URL(CODEX_FORWARD_BASE_URL).origin.toLowerCase();

type SubagentQuotaPrimeFn = (config: OcxConfig, reason: string) => Promise<void>;
/** Side-effect-free pool account preview for one resolved candidate model. */
export type SubagentPoolAccountPreview = (
modelId: string | undefined,
now: number,
modelEligibleAccountIds?: ReadonlySet<string>,
) => string | null;
export type SubagentModelEligibleAccountIds = (
modelId: string | undefined,
) => ReadonlySet<string> | undefined;
let subagentQuotaPrimeForTests: SubagentQuotaPrimeFn | null = null;
let quotaPrimeInFlight: Promise<void> | null = null;

Expand Down Expand Up @@ -177,8 +187,15 @@ function resolveRouteFallbackAccountId(
route: RouteResult | null,
config: OcxConfig,
accountId?: string | null,
now = Date.now(),
poolAccountPreview?: SubagentPoolAccountPreview,
modelEligibleAccountIds?: ReadonlySet<string>,
): string | null {
return route?.codexAccountId ?? resolvePoolFallbackAccountId(config, accountId);
if (route?.codexAccountId !== undefined) return route.codexAccountId;
if (route && isPoolCodexRoute(route) && poolAccountPreview) {
return poolAccountPreview(route.modelId, now, modelEligibleAccountIds);
}
return resolvePoolFallbackAccountId(config, accountId);
}

function isRoutableFallbackModel(model: string, config: OcxConfig): boolean {
Expand Down Expand Up @@ -231,39 +248,62 @@ export function isModelHealthBlocked(
return !!health && health.unavailableUntil > now;
}

/**
* Check one fallback candidate against the pool account its resolved model scope would use.
* The optional preview must not bind affinity, move an account cursor, or acquire a probe lease.
*/
export function isSubagentModelUnavailable(
model: string,
config: OcxConfig,
accountId?: string | null,
now = Date.now(),
accountUsabilityOptions?: CodexAccountUsabilityOptions,
poolAccountPreview?: SubagentPoolAccountPreview,
modelEligibleAccountIdsForModel?: SubagentModelEligibleAccountIds,
): boolean {
if (isDisabledFallbackModel(model, config)) return true;
if (!isRoutableFallbackModel(model, config)) return true;
const route = tryRouteFallbackModel(config, model);
if (!route || route.provider.disabled === true) return true;
if (isModelHealthBlocked(model, config, accountId, now)) return true;
const candidateAccountUsabilityOptions = modelEligibleAccountIdsForModel
? {
...accountUsabilityOptions,
modelEligibleAccountIds: modelEligibleAccountIdsForModel(route.modelId),
}
: accountUsabilityOptions;
const resolvedAccountId = resolveRouteFallbackAccountId(
route,
config,
accountId,
now,
poolAccountPreview,
candidateAccountUsabilityOptions?.modelEligibleAccountIds,
);
if (isModelHealthBlocked(model, config, resolvedAccountId, now)) return true;
if (!isPoolCodexRoute(route)) return false;

// Pool candidates need a usable account. Derive requirement from the resolved
// route (canonical openai defaults to pool even when codexAccountMode is omitted).
const resolvedAccountId = resolveRouteFallbackAccountId(route, config, accountId);
if (!resolvedAccountId) return true;
if (isCodexAccountPaused(config, resolvedAccountId)) return true;
if (!isCodexAccountUsable(config, resolvedAccountId, accountUsabilityOptions)) return true;
if (!isCodexAccountUsable(config, resolvedAccountId, candidateAccountUsabilityOptions)) return true;
if (route.codexAccountId !== undefined) {
// An account-qualified route is pinned and cannot consume Pool's recovery-probe
// escape hatch. Honor both account-wide and model-scoped cooldowns so fallback
// advances instead of selecting a candidate that exact auth will reject.
const quotaScope = codexQuotaScopeForModel(route.modelId);
if (getCodexQuotaHealthSnapshot(resolvedAccountId, quotaScope, now) !== null) return true;
} else if (
isCodexAccountInCooldown(resolvedAccountId, now)
&& !canAcquireCodexQuotaProbeLease(resolvedAccountId, now)
) {
return true;
} else {
const quotaScope = codexQuotaScopeForModel(route.modelId);
const cooldown = getCodexQuotaHealthSnapshot(resolvedAccountId, quotaScope, now);
if (cooldown !== null) {
const probeAvailable = cooldown.quotaScope
? canAcquireCodexQuotaScopeProbeLease(resolvedAccountId, cooldown.quotaScope, now)
Comment thread
luvs01 marked this conversation as resolved.
: canAcquireCodexQuotaProbeLease(resolvedAccountId, now);
if (!probeAvailable) return true;
}
}
return isNativeModelQuotaExhausted(model, config, accountId, now);
return isNativeModelQuotaExhausted(model, config, resolvedAccountId, now);
}

export function selectAvailableSubagentModel(
Expand All @@ -275,6 +315,8 @@ export function selectAvailableSubagentModel(
nativeFallbackOnly = false,
accountUsabilityOptions?: CodexAccountUsabilityOptions,
trailingFallback: readonly string[] = [],
poolAccountPreview?: SubagentPoolAccountPreview,
modelEligibleAccountIdsForModel?: SubagentModelEligibleAccountIds,
): { model: string; rewritten: boolean; skipped: string[] } {
const chain = normalizedChain(primary, config, extraFallback, trailingFallback);
const skipped: string[] = [];
Expand All @@ -286,7 +328,15 @@ export function selectAvailableSubagentModel(
continue;
}
}
if (isSubagentModelUnavailable(candidate, config, accountId, now, accountUsabilityOptions)) {
if (isSubagentModelUnavailable(
candidate,
config,
accountId,
now,
accountUsabilityOptions,
poolAccountPreview,
modelEligibleAccountIdsForModel,
)) {
skipped.push(candidate);
continue;
}
Expand Down Expand Up @@ -515,6 +565,8 @@ export function applySubagentModelFallback(
now = Date.now(),
nativeFallbackOnly = false,
accountUsabilityOptions?: CodexAccountUsabilityOptions,
poolAccountPreview?: SubagentPoolAccountPreview,
modelEligibleAccountIdsForModel?: SubagentModelEligibleAccountIds,
): { from?: string; to?: string; skipped?: string[] } | null {
if (!isThreadSpawnRequest(headers)) return null;
const tomlRoleFallback = resolveAgentModelFallbackForPrimary(
Expand All @@ -536,6 +588,8 @@ export function applySubagentModelFallback(
nativeFallbackOnly,
accountUsabilityOptions,
tomlRoleFallback,
poolAccountPreview,
modelEligibleAccountIdsForModel,
);
if (!selection.rewritten) return selection.skipped.length > 0
? { from: parsed.modelId, to: parsed.modelId, skipped: selection.skipped }
Expand All @@ -545,6 +599,30 @@ export function applySubagentModelFallback(
return { from, to: selection.model, skipped: selection.skipped };
}

/** Whether this request's configured fallback chain contains an account-gated native model. */
export function subagentFallbackNeedsModelEntitlements(
parsed: OcxParsedRequest,
config: OcxConfig,
): boolean {
const tomlRoleFallback = resolveAgentModelFallbackForPrimary(
parsed.modelId,
getCodexHome(),
config.codexAccountNamespaces,
);
const configuredFallback = resolveConfiguredModelFallbackForPrimary(parsed.modelId, config);
const globalFallback = config.subagentModelFallback ?? [];
if (globalFallback.length === 0 && configuredFallback.length === 0 && tomlRoleFallback.length === 0) {
return false;
}
return normalizedChain(parsed.modelId, config, configuredFallback, tomlRoleFallback)
.some((candidate) => {
const route = tryRouteFallbackModel(config, candidate);
return !!route
&& isPoolCodexRoute(route)
&& ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(route.modelId);
});
}

export function subagentFallbackGuidanceText(config: OcxConfig): string {
const chain = config.subagentModelFallback ?? [];
if (chain.length === 0) return "";
Expand Down
49 changes: 43 additions & 6 deletions src/server/responses/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,9 @@ import {
applySubagentModelFallback,
maybePrimeSubagentQuota,
recordSubagentQuotaFailureForThreadSpawn,
subagentFallbackNeedsModelEntitlements,
type SubagentModelEligibleAccountIds,
type SubagentPoolAccountPreview,
} from "../../codex/subagent-model-fallback";
import { isNativeMainTrafficBlocked } from "../../codex/native-profile-startup";
import {
Expand Down Expand Up @@ -1224,6 +1227,8 @@ export interface HandleResponsesOptions {
/** One-shot TTFT callback: first non-empty model output observed (WP4). */
onFirstOutput?: () => void;
onCodexAuthContextResolved?: (context: CodexAuthContext | undefined) => void;
/** Internal deterministic seam for account-gated native fallback tests. */
resolveCodexModelEntitlements?: typeof resolveCodexModelEntitlements;
recordTerminalOutcomes?: boolean;
setTerminalOutcomeRecorder?: (recorder: ((status: ResponsesTerminalStatus, httpStatusOverride?: number) => void) | undefined) => void;
onNativePassthroughTerminal?: (status: ResponsesTerminalStatus) => void;
Expand Down Expand Up @@ -1515,6 +1520,7 @@ async function resolveResponsesCodexAuth(
modelId: route.modelId,
substituteMainCredentialForDirect: substituteMainCredential,
beginCodexAccountSelection: codexAccountSelectionForTurn(options.turnAdmissionLease),
resolveCodexModelEntitlements: options.resolveCodexModelEntitlements,
});
options.onCodexAuthContextResolved?.(authCtx);
} else {
Expand Down Expand Up @@ -2361,6 +2367,8 @@ async function handleResponsesInner(
let selectedForwardHeaders = req.headers;
let subagentFallbackAccountId = config.activeCodexAccountId ?? null;
let subagentFallbackPreviewAccountId: string | null | undefined;
let subagentFallbackAccountPreview: SubagentPoolAccountPreview | undefined;
let subagentFallbackModelEligibleAccountIds: SubagentModelEligibleAccountIds | undefined;
let subagentQuotaFailureModel = parsed.modelId;
const parentThreadId = req.headers.get("x-codex-parent-thread-id")?.trim() ?? null;
const poolAffinityKey = codexPoolAffinityKey(req.headers) ?? null;
Expand All @@ -2374,6 +2382,20 @@ async function handleResponsesInner(
await maybePrimeSubagentQuota(config, Date.now(), { nativeMainReadsForbidden });
}

if (
threadSpawn
&& !options.comboAttempt
&& route.codexAccountId === undefined
&& subagentFallbackNeedsModelEntitlements(parsed, config)
) {
const entitlementSnapshot = await (options.resolveCodexModelEntitlements
?? resolveCodexModelEntitlements)(config);
subagentFallbackModelEligibleAccountIds = (modelId) => entitledCodexAccountIdsForModel(
entitlementSnapshot,
modelId,
);
}

// Subagent fallback must settle the final model/provider BEFORE route-dependent
// normalization (virtual models, effort caps, service tier, wire protocol).
// Preview the preferred Codex account without acquiring a probe lease or refreshing
Expand All @@ -2383,12 +2405,22 @@ async function handleResponsesInner(
// so the preview must read the same scope slot — an undefined scope would map to the
// "legacy" affinity bucket and never find a binding made under "shared" or a native
// model scope, making the preview diverge from the account that actually authenticates.
const previewAccountId = previewCodexAccountForRequest(
const fallbackNow = Date.now();
subagentFallbackAccountPreview = (
modelId,
previewNow,
modelEligibleAccountIds,
) => previewCodexAccountForRequest(
poolAffinityKey,
config,
Date.now(),
codexQuotaScopeForModel(route.modelId),
previewSelectionOptions,
previewNow,
codexQuotaScopeForModel(modelId),
{ ...previewSelectionOptions, modelEligibleAccountIds },
);
const previewAccountId = subagentFallbackAccountPreview(
route.modelId,
fallbackNow,
subagentFallbackModelEligibleAccountIds?.(route.modelId),
);
subagentFallbackPreviewAccountId = previewAccountId;
subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null;
Expand All @@ -2397,9 +2429,11 @@ async function handleResponsesInner(
req.headers,
config,
previewAccountId,
Date.now(),
fallbackNow,
unreadableEncryptedAgentTask,
previewSelectionOptions,
subagentFallbackAccountPreview,
subagentFallbackModelEligibleAccountIds,
);
if (fallback) {
(logCtx as unknown as Record<string, unknown>).subagentModelFallbackFrom = fallback.from;
Expand Down Expand Up @@ -2482,14 +2516,17 @@ async function handleResponsesInner(
// The ciphertext-only pass intentionally excludes routed candidates. Once recovery
// makes the assignment readable, run selection again with the full configured chain
// and keep the route in sync with any newly selected fallback.
const fallbackNow = Date.now();
const fallback = applySubagentModelFallback(
parsed,
req.headers,
config,
subagentFallbackPreviewAccountId,
Date.now(),
fallbackNow,
false,
previewSelectionOptions,
subagentFallbackAccountPreview,
subagentFallbackModelEligibleAccountIds,
);
if (fallback) {
(logCtx as unknown as Record<string, unknown>).subagentModelFallbackFrom = fallback.from;
Expand Down
Loading
Loading