Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion hermes-plugin/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -672,7 +672,8 @@ def _request_base_url(request: Dict[str, Any], context: Dict[str, Any]) -> Optio

def context_window_for(model: Any, base_url: Any = None) -> Optional[int]:
"""#3261: the host's own known context window for a model, read from
HERMES_HOME/context_length_cache.yaml (flat mapping "<model>@<base_url>" -> tokens).
HERMES_HOME/context_length_cache.yaml (context_lengths mapping
"<model>@<base_url>" -> tokens; legacy flat mappings are also accepted).
Fail-open to None on any miss/error so the proxy falls back to its normal window chain.
A stage matching several DIFFERENT windows yields nothing rather than a guess: an
overestimate risks preflight overflow, an underestimate over-compresses — both worse
Expand All @@ -687,6 +688,8 @@ def context_window_for(model: Any, base_url: Any = None) -> Optional[int]:
return None
if not isinstance(data, dict):
return None
if isinstance(data.get("context_lengths"), dict):
data = data["context_lengths"]
pairs: List[Tuple[str, int]] = []
for key, value in data.items():
k = str(key)
Expand Down
55 changes: 30 additions & 25 deletions tests/hermes-native.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -453,10 +453,13 @@ elif SCENARIO.startswith("attach-window"):
except Exception:
out["yaml_ok"] = False
home = os.environ["HERMES_HOME"]
if SCENARIO == "attach-window":
if SCENARIO in ("attach-window", "attach-window-flat"):
cache = {"group/sol@http://127.0.0.1:3425/v1": 500000,
"unrelated@https://other.example/v1": 123456}
if SCENARIO == "attach-window":
cache = {"context_lengths": cache}
with open(os.path.join(home, "context_length_cache.yaml"), "w") as f:
f.write("group/sol@http://127.0.0.1:3425/v1: 500000\n"
"unrelated@https://other.example/v1: 123456\n")
json.dump(cache, f)
with open(os.path.join(home, "config.yaml"), "w") as f:
f.write("providers:\n solprov:\n base_url: http://127.0.0.1:3425\n")
win_model, win_provider = "group/sol", "solprov"
Expand Down Expand Up @@ -686,28 +689,30 @@ describe("python plugin runtime (subprocess)", () => {
// (origin-granularity match: cache keeps /v1, config omits it); the hook leg runs WITHOUT
// a provider kwarg — the known minimum hook surface — so it must work off the unambiguous
// single-entry fallback.
test("attach-window: host context_length_cache.yaml window rides header + runtime-info (#3261)", () => {
if (!PY) {
console.warn("skip: no python3/python interpreter on this machine (hermes requires Python 3.11+)");
return;
}
const { out, err } = runDriver("attach-window");
assert.ok(out, err);
assert.equal(out!.env_https_proxy, out!.origin, "attached");
if (!out!.yaml_ok) {
console.warn("skip positive leg: no YAML backend on this host — the cache is unreadable and the feature fails open");
return;
}
const h = out!.win_headers!;
assert.equal(h["x-bili-plugin"], "hermes");
assert.equal(h["x-bili-plugin-conversation"], "sess-w");
assert.equal(h["x-bili-plugin-model"], "group/sol");
assert.equal(h["x-bili-plugin-context-window"], "500000", "header channel: the host's own window value");
assert.equal(out!.req_unmutated, true);
assert.deepEqual(out!.runtime_info, [
{ agent: "hermes", model: "group/sol", maxOutput: 4096, contextWindow: 500000, source: "hermes-native" },
]);
});
for (const scenario of ["attach-window", "attach-window-flat"]) {
test(`${scenario}: host context_length_cache.yaml window rides header + runtime-info (#3261)`, () => {
if (!PY) {
console.warn("skip: no python3/python interpreter on this machine (hermes requires Python 3.11+)");
return;
}
const { out, err } = runDriver(scenario);
assert.ok(out, err);
assert.equal(out!.env_https_proxy, out!.origin, "attached");
if (!out!.yaml_ok) {
console.warn("skip positive leg: no YAML backend on this host — the cache is unreadable and the feature fails open");
return;
}
const h = out!.win_headers!;
assert.equal(h["x-bili-plugin"], "hermes");
assert.equal(h["x-bili-plugin-conversation"], "sess-w");
assert.equal(h["x-bili-plugin-model"], "group/sol");
assert.equal(h["x-bili-plugin-context-window"], "500000", "header channel: the host's own window value");
assert.equal(out!.req_unmutated, true);
assert.deepEqual(out!.runtime_info, [
{ agent: "hermes", model: "group/sol", maxOutput: 4096, contextWindow: 500000, source: "hermes-native" },
]);
});
}

test("attach-window-ambiguous: conflicting per-upstream windows are not guessed (#3261)", () => {
if (!PY) {
Expand Down
Loading