Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions crates/switchyard-server/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -267,13 +267,17 @@ enum RouteConfig {
context_window: Option<u32>,
#[serde(default)]
tool_calling: Option<bool>,
#[serde(default)]
reasoning: Option<bool>,
},
Random {
id: String,
#[serde(default)]
context_window: Option<u32>,
#[serde(default)]
tool_calling: Option<bool>,
#[serde(default)]
reasoning: Option<bool>,
targets: Vec<String>,
weights: Option<Vec<f64>>,
seed: Option<u64>,
Expand All @@ -284,6 +288,8 @@ enum RouteConfig {
context_window: Option<u32>,
#[serde(default)]
tool_calling: Option<bool>,
#[serde(default)]
reasoning: Option<bool>,
target: String,
},
LlmClassifier {
Expand All @@ -292,6 +298,8 @@ enum RouteConfig {
context_window: Option<u32>,
#[serde(default)]
tool_calling: Option<bool>,
#[serde(default)]
reasoning: Option<bool>,
classifier_target: String,
#[serde(default)]
mode: Option<ClassifierMode>,
Expand Down Expand Up @@ -330,6 +338,8 @@ enum RouteConfig {
context_window: Option<u32>,
#[serde(default)]
tool_calling: Option<bool>,
#[serde(default)]
reasoning: Option<bool>,
capable_target: String,
efficient_target: String,
/// Tier a turn falls back to when the signals are not confident.
Expand Down Expand Up @@ -405,30 +415,36 @@ impl RouteConfig {
Noop {
context_window,
tool_calling,
reasoning,
..
}
| Random {
context_window,
tool_calling,
reasoning,
..
}
| Passthrough {
context_window,
tool_calling,
reasoning,
..
}
| LlmClassifier {
context_window,
tool_calling,
reasoning,
..
}
| StageRouter {
context_window,
tool_calling,
reasoning,
..
} => ModelCapabilities {
context_window: *context_window,
tool_calling: *tool_calling,
reasoning: *reasoning,
},
}
}
Expand Down
83 changes: 82 additions & 1 deletion crates/switchyard-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,16 @@ pub type ServerResult<T> = std::result::Result<T, ServerError>;

/// Capabilities that one route advertises on `GET /v1/models`.
///
/// An unset capability is undeclared and serializes as `null`.
/// An unset capability is undeclared: it serializes as `null` in the OpenAI
/// `data` entry, and the Codex entry falls back to a safe default for it.
#[derive(Clone, Copy, Default)]
struct ModelCapabilities {
context_window: Option<u32>,
tool_calling: Option<bool>,
// Whether the routed model takes reasoning controls. The server cannot probe
// this, so a route opts in via config; undeclared routes advertise as
// non-reasoning to Codex (fail closed).
reasoning: Option<bool>,
}

/// A registered route: the libsy algorithm that serves it and the capabilities
Expand Down Expand Up @@ -1045,6 +1050,11 @@ fn model_list_payload<'a>(
json!({
"object": "list",
"data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::<Vec<_>>(),
"models": entries
.iter()
.enumerate()
.map(|(priority, (model, caps))| codex_model_entry_json(model, *caps, priority))
.collect::<Vec<_>>(),
"first_id": first_id,
"last_id": last_id,
"has_more": false,
Expand Down Expand Up @@ -1074,6 +1084,77 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value {
})
}

// Builds the metadata Codex requires when it discovers models from a direct provider.
//
// This mirrors Codex's `ModelInfo` card. The launcher path builds the same card in
// `switchyard/cli/launchers/codex_model_catalog.py`; keep the two in sync when Codex
// changes the shape. Every field below is either derived from the route's declared
// capabilities or a required `ModelInfo` field the server has no better value for.
//
// Two kinds of fields live here. context_window, tool_calling, and reasoning are model
// facts a backend can publish; the route declares them in config today. The rest
// (shell_type, apply_patch_tool_type, base_instructions, the reasoning-level presets,
// truncation_policy) are Codex client conventions no backend returns, so they stay
// constant.
//
// TODO: source context_window, tool_calling, and reasoning from the backend, not route
// config. Switchyard is a proxy, so it should re-publish what the backend advertises
// when it can — OpenRouter's /api/v1/models exposes context_length and
// supported_parameters — and fall back to the route's declared value. Some backends
// publish nothing (the NVIDIA gateway returns id-only models and blocks /model/info),
// so keep failing closed to config.
fn codex_model_entry_json(model: &str, capabilities: ModelCapabilities, priority: usize) -> Value {
// Codex is non-functional without shell and apply_patch, so an undeclared tool
// capability defaults to enabled here; the OpenAI `data` entry reports the raw
// Option separately for clients that want the undeclared state.
let tool_calling = capabilities.tool_calling.unwrap_or(true);
let reasoning = capabilities.reasoning.unwrap_or(false);
json!({
"slug": model,
"display_name": model,
"description": "Switchyard-routed model.",
"default_reasoning_level": if reasoning { json!("xhigh") } else { Value::Null },
"supported_reasoning_levels": if reasoning { reasoning_levels() } else { json!([]) },
"shell_type": if tool_calling { "shell_command" } else { "disabled" },
"visibility": "list",
"supported_in_api": true,
// Catalog list position (routes are listed in sorted id order), not a quality rank.
"priority": priority,
"additional_speed_tiers": [],
"availability_nux": null,
"upgrade": null,
// Required `ModelInfo` string. Unlike the launcher, the server cannot read
// Codex's bundled prompt, so it sends a minimal stub.
"base_instructions": "You are Codex, a coding agent.",
"supports_reasoning_summaries": reasoning,
"default_reasoning_summary": "none",
"support_verbosity": reasoning,
"default_verbosity": if reasoning { json!("low") } else { Value::Null },
"apply_patch_tool_type": if tool_calling { Some("freeform") } else { None },
"web_search_tool_type": "text",
"truncation_policy": {"mode": "tokens", "limit": 10_000},
"supports_parallel_tool_calls": tool_calling,
"supports_image_detail_original": false,
"context_window": capabilities.context_window,
"max_context_window": capabilities.context_window,
"effective_context_window_percent": 95,
"experimental_supported_tools": [],
"input_modalities": ["text"],
"supports_search_tool": false,
})
}

// The reasoning-effort presets Codex offers for a reasoning-capable route. Kept in
// step with the launcher template in `codex_model_catalog.py`.
fn reasoning_levels() -> Value {
json!([
{"effort": "low", "description": "Fast responses with lighter reasoning"},
{"effort": "medium", "description": "Balances speed and reasoning depth"},
{"effort": "high", "description": "Greater reasoning depth"},
{"effort": "xhigh", "description": "Extra high reasoning depth"},
])
Comment thread
grahamking marked this conversation as resolved.
}

fn startup_banner(options: &ServerRunOptions, state: &ServerState, color: bool) -> String {
let scheme = if options.is_tls() { "https" } else { "http" };
let listen_url = url_for_addr(scheme, options.addr);
Expand Down
89 changes: 89 additions & 0 deletions crates/switchyard-server/tests/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1140,6 +1140,12 @@ target = "shared"
context_window = 262000
tool_calling = false

[routes.reasoning]
id = "reasoning"
type = "passthrough"
target = "shared"
reasoning = true

[routes.undeclared]
id = "undeclared"
type = "passthrough"
Expand All @@ -1161,6 +1167,89 @@ target = "shared"
assert_eq!(capabilities["restricted"]["tool_calling"], json!(false));
assert_eq!(capabilities["undeclared"]["context_window"], json!(null));
assert_eq!(capabilities["undeclared"]["tool_calling"], json!(null));

let codex_models = body["models"].as_array().cloned().unwrap_or_default();
let codex_metadata = codex_models
.iter()
.filter_map(|entry| entry["slug"].as_str().map(|slug| (slug, entry)))
.collect::<BTreeMap<_, _>>();
// This checks the shape the server emits. That Codex 0.144.5 actually decodes it
// (context_window: null included) is verified by a live Codex run in SWITCH-1225.
assert_eq!(codex_metadata.len(), 4);
assert_eq!(
codex_metadata["declared"]["context_window"],
json!(1_000_000)
);
assert_eq!(codex_metadata["declared"]["shell_type"], "shell_command");
assert_eq!(
codex_metadata["declared"]["apply_patch_tool_type"],
"freeform"
);
// Constant fields Codex requires: a typo here would fail its decode, so pin them.
assert_eq!(codex_metadata["declared"]["visibility"], "list");
assert_eq!(codex_metadata["declared"]["supported_in_api"], json!(true));
assert_eq!(codex_metadata["declared"]["web_search_tool_type"], "text");
assert_eq!(
codex_metadata["declared"]["input_modalities"],
json!(["text"])
);
assert_eq!(
codex_metadata["declared"]["truncation_policy"],
json!({"mode": "tokens", "limit": 10_000})
);
assert_eq!(
codex_metadata["restricted"]["context_window"],
json!(262_000)
);
assert_eq!(codex_metadata["restricted"]["shell_type"], "disabled");
assert_eq!(
codex_metadata["restricted"]["apply_patch_tool_type"],
json!(null)
);
// A reasoning route advertises the effort presets and reasoning controls.
assert_eq!(
codex_metadata["reasoning"]["default_reasoning_level"],
"xhigh"
);
assert_eq!(
codex_metadata["reasoning"]["supported_reasoning_levels"]
.as_array()
.map(Vec::len),
Some(4)
);
assert_eq!(
codex_metadata["reasoning"]["supports_reasoning_summaries"],
json!(true)
);
assert_eq!(
codex_metadata["reasoning"]["support_verbosity"],
json!(true)
);
assert_eq!(codex_metadata["reasoning"]["default_verbosity"], "low");
// An undeclared route: null context window, non-reasoning, but tools default on
// so `switchyard launch codex` stays usable out of the box.
assert_eq!(codex_metadata["undeclared"]["context_window"], json!(null));
assert_eq!(
codex_metadata["undeclared"]["supported_reasoning_levels"],
json!([])
);
assert_eq!(
codex_metadata["undeclared"]["default_reasoning_level"],
json!(null)
);
assert_eq!(
codex_metadata["undeclared"]["supports_reasoning_summaries"],
json!(false)
);
assert_eq!(codex_metadata["undeclared"]["shell_type"], "shell_command");
assert_eq!(
codex_metadata["undeclared"]["apply_patch_tool_type"],
"freeform"
);
assert_eq!(
codex_metadata["undeclared"]["supports_parallel_tool_calls"],
json!(true)
);
Ok(())
}

Expand Down
5 changes: 4 additions & 1 deletion docs/core_concepts.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,10 @@ inside the TOML file. Their `id` fields have different external meanings:

The server lists route IDs on `GET /v1/models`. A request selects a route by
putting that ID in its `model` field. The native Rust server does not discover
or register additional provider models automatically.
or register additional provider models automatically. The same response also
carries a Codex-compatible `models` array so Codex can use the server as a direct
provider; each entry reflects the route's declared context window, tool support,
and reasoning.

## Routing Algorithms

Expand Down
1 change: 1 addition & 0 deletions docs/reference/toml_schema.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ Every route takes the common keys below, plus the keys for its type.
| `type` | Yes | — | Routing algorithm for this route. |
| `context_window` | No | unset | Positive token count advertised for this route by `GET /v1/models`. Unset values appear as `null`. This does not enforce a request limit. |
| `tool_calling` | No | unset | Whether `GET /v1/models` advertises tool-calling support for this route. Unset values appear as `null`. |
| `reasoning` | No | unset | Whether `GET /v1/models` advertises reasoning support to Codex direct-provider discovery. Unset routes are advertised as non-reasoning. |

### `noop`

Expand Down
Loading