From 449a5766232303c72c7308b1f93c085fcade2d29 Mon Sep 17 00:00:00 2001 From: Zijian Zhang <35801754+futrime@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:32:04 +0000 Subject: [PATCH 1/5] fix(coganchor): read an argv entry as a command, not as a path Every entry of an intercepted argv was read with PATH_MAX as its ceiling. An argv entry is not a path: the kernel allows MAX_ARG_STRLEN, and a shell command is routinely longer than four kilobytes. Codex prefixes every `bash -lc` it runs with a preamble well past that, so what reached the target was the first 4096 bytes of the command -- which parses, runs, and means something else. It failed as a syntax error inside a truncated brace, and Codex went on reporting each tool call as done while nothing at all happened on the target. Truncating is worse than not reading: a prefix of a command is a different command. Co-Authored-By: Claude Opus 5 (1M context) --- src/hmz/coganchor/linux/procfs.py | 11 ++++++++++- tests/coganchor/test_smoke.py | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/hmz/coganchor/linux/procfs.py b/src/hmz/coganchor/linux/procfs.py index c8230f16..65ac1379 100644 --- a/src/hmz/coganchor/linux/procfs.py +++ b/src/hmz/coganchor/linux/procfs.py @@ -16,6 +16,7 @@ from hmz.coganchor.linux.syscalls import NR __all__ = [ + "MAX_ARG_STRLEN", "PATH_MAX", "TraceeGoneError", "fd_target", @@ -31,6 +32,10 @@ _PAGE_SIZE: Final = os.sysconf("SC_PAGESIZE") _MAX_ARGV_ENTRIES: Final = 65536 +#: The kernel's own ceiling on one ``argv`` or ``envp`` entry, ``MAX_ARG_STRLEN``: thirty-two +#: pages. What a command may be, as opposed to what a path may be. +MAX_ARG_STRLEN: Final = 32 * _PAGE_SIZE + _libc = ctypes.CDLL("libc.so.6", use_errno=True) @@ -137,7 +142,11 @@ def read_string_array( pointer = int.from_bytes(raw, "little") if pointer == 0: break - text = read_cstring(pid, pointer) + # An argv entry is not a path, so PATH_MAX is the wrong ceiling for it: the kernel + # lets one be MAX_ARG_STRLEN long, and a shell command is routinely longer than a + # path. Truncating one is worse than failing to read it -- what reaches the target + # is then a prefix of the command, which runs and means something else. + text = read_cstring(pid, pointer, MAX_ARG_STRLEN) values.append("" if text is None else text) address += word return values diff --git a/tests/coganchor/test_smoke.py b/tests/coganchor/test_smoke.py index dd2d2e13..e5230e87 100644 --- a/tests/coganchor/test_smoke.py +++ b/tests/coganchor/test_smoke.py @@ -94,3 +94,24 @@ def test_large_round_trip_is_byte_exact(anchorage: Anchorage) -> None: ) assert "1" in result.stdout assert anchorage.target_text("echoed.txt") == payload + + +def test_a_command_longer_than_a_path_crosses_whole(anchorage: Anchorage) -> None: + """An ``argv`` entry is not a path, so ``PATH_MAX`` is the wrong ceiling to read it at. + + A truncated one is worse than an unread one: what reaches the target is a prefix of the + command, which runs, and means something else. Codex prefixes every shell command it + runs with a preamble of several kilobytes, so this is the ordinary case for it rather + than an exotic one. + """ + payload = "y" * 8000 + + # `sh` rather than another `bash`: the agent here *is* bash, and an agent's own program + # runs on this machine wherever it turns up, so spawning one would never cross at all. + result = anchorage.shell(f"/bin/sh -c 'printf %s {payload} > long.txt'") + + assert result.returncode == 0, result.stderr + # Truncated, the command is `printf %s yyy...` with the redirect cut off, so the payload + # comes back on stdout and nothing is written anywhere. + assert result.stdout == "" + assert anchorage.target_text("long.txt") == payload From 7bb917d6f5ad60f4e102cf1ac002cb6bfe7109e4 Mon Sep 17 00:00:00 2001 From: Zijian Zhang <35801754+futrime@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:32:23 +0000 Subject: [PATCH 2/5] fix(coganchor): keep an npm agent's own runtime on this machine Two ways the agent itself ended up running on the target, which is the one thing the anchor exists to prevent. An `#!/usr/bin/env node` line kept only the `env` here. `env` then searches PATH for the interpreter, one execve per directory, and the first candidate names a path that does not exist here -- not the agent's own by name, so it was sent to the target, where the name resolves. Kimi's whole agent process ran there: it read the target's HOME, found none of its own configuration, and reported the model it was started with as unconfigured. So the whole search is claimed now rather than the one directory the interpreter is really in; a candidate kept here fails here, with the ENOENT that makes `env` try the next. And a program the agent keeps inside its own state directory was answered from this machine as a path but not as a program. grok installs its native binary under `~/.grok/bin` and re-execs it, so grok too ran on the target, where it reported `Not signed in`. Codex was spared both only because its runtime is listed by hand. Co-Authored-By: Claude Opus 5 (1M context) --- docs/features/anchor.md | 9 ++--- docs/guide/remote-execution.md | 7 ++-- docs/reference/remote-execution.md | 8 +++-- src/hmz/coganchor/statepaths.py | 58 +++++++++++++++++++++++++++++- tests/coganchor/test_statepaths.py | 36 +++++++++++++++++++ 5 files changed, 107 insertions(+), 11 deletions(-) diff --git a/docs/features/anchor.md b/docs/features/anchor.md index be296c9c..e161ea25 100644 --- a/docs/features/anchor.md +++ b/docs/features/anchor.md @@ -75,10 +75,11 @@ there kills its local counterpart the same way. ## What never leaves this machine -- the agent's own runtime executables and re-execs — for an npm-installed Codex, that includes - Node, the native CLI and its code-mode host -- its state directory — the known CLIs are known by name, and any other agent keeping state - inside the workspace has to be named +- the agent's own runtime executables and re-execs — for any CLI installed by npm that means + the interpreter its `#!/usr/bin/env` line names, wherever on `PATH` it is found, and for Codex + the native CLI and its code-mode host besides +- its state directory, and anything the agent runs from inside it — the known CLIs are known by + name, and any other agent keeping state inside the workspace has to be named - anything a path is answered with, and the paths that answer it: an agent run as somebody else's account reads those credentials from here, and a refreshed token lands here - any variable named as the agent's own, so that a credential it was given to reach its model diff --git a/docs/guide/remote-execution.md b/docs/guide/remote-execution.md index 57ec297b..b3ec099d 100644 --- a/docs/guide/remote-execution.md +++ b/docs/guide/remote-execution.md @@ -87,9 +87,10 @@ it adds nothing. Then check `python3 --version` there. See **Stays here** -- The agent's own runtime executables and re-execs. For an npm-installed Codex, that includes - Node, the native CLI and its code-mode host. -- Its state directory. humanize knows the eleven known CLIs by name — `agy`, `claude`, `codex`, +- The agent's own runtime executables and re-execs. For any CLI installed by npm that includes + the interpreter its `#!/usr/bin/env` line names, wherever on `PATH` it is found; for Codex, the + native CLI and its code-mode host besides. +- Its state directory, and anything the agent runs from inside it. humanize knows the eleven known CLIs by name — `agy`, `claude`, `codex`, `dsh`, `grok`, `kimi`, `mimo`, `opencode`, `pi`, `qwen`, `zcode` — and its own `~/.humanize`. Any other agent that keeps state inside the workspace has to be named with `--local-path`. - Anything named `--local-path` or `--local-exec`. diff --git a/docs/reference/remote-execution.md b/docs/reference/remote-execution.md index 02bdb5a2..123893f3 100644 --- a/docs/reference/remote-execution.md +++ b/docs/reference/remote-execution.md @@ -89,9 +89,11 @@ the agent once it exits, and when the session ends nothing it started is left ru ## What stays on this machine -- The agent's own runtime executables and re-execs. For an npm-installed Codex, that includes - Node, the native CLI and its code-mode host. -- Its state directory. All eleven known CLIs are known by name — `agy`, `claude`, `codex`, +- The agent's own runtime executables and re-execs. For any CLI installed by npm that includes + the interpreter its `#!/usr/bin/env` line names, at every path on `PATH` the search for it may + reach; for Codex, the native CLI and its code-mode host besides. +- Its state directory, and anything the agent runs from inside it -- grok keeps its native + binary under `~/.grok/bin` and re-execs it. All eleven known CLIs are known by name — `agy`, `claude`, `codex`, `dsh`, `grok`, `kimi`, `mimo`, `opencode`, `pi`, `qwen`, `zcode` — as is humanize's own `~/.humanize`; any other agent keeping state inside the workspace has to be named with `--local-path`. diff --git a/src/hmz/coganchor/statepaths.py b/src/hmz/coganchor/statepaths.py index bc96280a..d834bd4c 100644 --- a/src/hmz/coganchor/statepaths.py +++ b/src/hmz/coganchor/statepaths.py @@ -149,10 +149,20 @@ def resolve(command: list[str]) -> ResolvedAgent: # Only the agent's own runtime stays here. Work helpers such as ripgrep # deliberately go to the target: running them against the partly materialised # mirror would return quietly wrong answers, which is worse than a visible failure. - local_programs = [located, program] + # A program the agent keeps in its own state directory is its own runtime, not a helper + # for the work: grok installs its native binary under `~/.grok/bin` and re-execs it, and + # sending that to the target sends the agent there with it. Those directories are + # already answered from this machine as paths; this is the same claim about executing + # them. + local_programs = [ + located, + program, + *(_expand(path) for path in profile.state_paths), + ] shebang = _shebang(program) if shebang: local_programs.append(shebang[0]) + local_programs.extend(_interpreter(shebang)) if profile.name == "codex": local_programs.extend(_codex_runtime_programs(program, shebang)) @@ -181,6 +191,52 @@ def _shebang(program: str) -> tuple[str, ...]: return tuple(first[2:].decode("utf-8", "replace").strip().split()) +def _interpreter(shebang: tuple[str, ...]) -> list[str]: + """Every path an ``#!/usr/bin/env NAME`` line could reach its interpreter at. + + Every agent installed by npm starts with one, and keeping only the ``env`` keeps the + wrong program on this machine: ``env`` runs here and then searches ``PATH`` for the + interpreter, one ``execve`` per directory. The first of those names a path that does not + exist here -- which is not the agent's own by name, so it is sent to the target, where + the name resolves and the agent itself ends up running. It then reads the target's copy + of its state directory and cannot reach the account it was signed in with. Codex is + spared this only because its runtime is listed by hand. + + So the whole search is claimed, not just the directory the interpreter is really in: a + candidate kept here fails here, with the ``ENOENT`` that makes ``env`` try the next one. + + Args: + shebang: The command the script's first line names. + + Returns: + Each path the search may name, or nothing when the line names the interpreter + directly -- that one is already kept by its own path. + """ + if not shebang or os.path.basename(shebang[0]) != "env": + return [] + words: list[str] = [] + for word in shebang[1:]: + # `env -S "node --flag"` carries the whole command in one word, and + # `env NAME=VALUE prog` sets variables before naming one. + words.extend(word.split()) + name = next( + (word for word in words if not word.startswith("-") and "=" not in word), "" + ) + if not name: + return [] + if os.path.sep in name: + return [os.path.abspath(name), os.path.realpath(name)] + found = [ + os.path.join(directory, name) + for directory in os.environ.get("PATH", os.defpath).split(os.pathsep) + if directory + ] + resolved = shutil.which(name) + if resolved: + found.extend((os.path.abspath(resolved), os.path.realpath(resolved))) + return found + + def _codex_runtime_programs(program: str, shebang: tuple[str, ...]) -> list[str]: """Return the Node, native CLI and code-mode host that implement Codex.""" programs: list[str] = [] diff --git a/tests/coganchor/test_statepaths.py b/tests/coganchor/test_statepaths.py index ef320b8d..aff8113a 100644 --- a/tests/coganchor/test_statepaths.py +++ b/tests/coganchor/test_statepaths.py @@ -69,3 +69,39 @@ def test_codex_node_package_keeps_its_whole_runtime_local( assert str(host) in resolved.local_programs assert str(work_helper) not in resolved.local_programs assert str(launcher) in resolved.local_programs + + +def test_an_npm_shebang_keeps_the_whole_search_for_its_interpreter_local( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """``env`` searches ``PATH`` an ``execve`` at a time, and the first one decides. + + A candidate that does not exist here is not the agent's own by name, so it would be run + on the target -- where the name does resolve, and the agent itself ends up running. + """ + empty = tmp_path / "empty-bin" + empty.mkdir() + node = executable(tmp_path / "node-bin" / "node") + monkeypatch.setenv("PATH", f"{empty}:{node.parent}") + script = executable(tmp_path / "bin" / "kimi", b"#!/usr/bin/env node\n") + + resolved = resolve([str(script)]) + + assert str(node) in resolved.local_programs + assert ( + str(empty / "node") in resolved.local_programs + ) # tried first, and refused here + + +def test_a_program_the_agent_keeps_in_its_own_state_directory_runs_here( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Grok installs its native binary under ``~/.grok`` and re-execs it.""" + monkeypatch.setenv("HOME", str(tmp_path)) + native = executable(tmp_path / ".grok" / "bin" / "grok-1.0.13") + launcher = executable(tmp_path / "bin" / "grok", b"#!/usr/bin/env node\n") + + resolved = resolve([str(launcher)]) + + assert str(tmp_path / ".grok") in resolved.local_programs + assert any(str(native).startswith(one) for one in resolved.local_programs) From 9d86885b9881b69152d2e3c09f48954f59073c4a Mon Sep 17 00:00:00 2001 From: Zijian Zhang <35801754+futrime@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:32:37 +0000 Subject: [PATCH 3/5] test(coganchor): a rig for how many anchored agents a machine holds `docs/features/concurrency` says how wide a flow runs is a question about the machine and that nothing caps it. This measures the machine: real hmz, real coganchor, the five agent CLIs as installed, against a mock controlled end and a stand-in model provider that scripts the same three-command turn for every backend. Both stand-ins run outside the cgroup, since in production neither is part of the budget being sized, and the rig records the target's own CPU so a run can say whether the ceiling it found was its own. In 16 logical CPUs and 64 GiB: 16-24 agents run at full speed, 32-128 give the most turns per hour, and everything is still correct into the hundreds -- 640 for claude and 192 for kimi, both stopped by memory; 256 for codex, stopped by its own app-server; grok and dsh never stopped inside the rig's 832 slots. The first ceiling anybody meets is neither: hmz holds about three descriptors per concurrent agent, so a stock 1024 soft RLIMIT_NOFILE stops every backend at about 320. Co-Authored-By: Claude Opus 5 (1M context) --- bench/coganchor-concurrency/README.md | 79 ++ bench/coganchor-concurrency/RESULTS.md | 109 +++ bench/coganchor-concurrency/backdrop.sh | 119 +++ .../data/ladder-claude.jsonl | 20 + .../data/ladder-codex.jsonl | 15 + .../data/ladder-dsh.jsonl | 18 + .../data/ladder-grok.jsonl | 18 + .../data/ladder-kimi.jsonl | 15 + .../data/refine-codex.jsonl | 7 + .../data/stock-nofile/ladder-claude.jsonl | 13 + .../data/stock-nofile/ladder-codex.jsonl | 15 + .../data/stock-nofile/ladder-dsh.jsonl | 17 + .../data/stock-nofile/ladder-grok.jsonl | 15 + .../data/stock-nofile/ladder-kimi.jsonl | 13 + bench/coganchor-concurrency/ladder.sh | 37 + bench/coganchor-concurrency/ramp.py | 279 +++++++ bench/coganchor-concurrency/refine.sh | 37 + bench/coganchor-concurrency/run_one.sh | 61 ++ bench/coganchor-concurrency/standin_model.py | 724 ++++++++++++++++++ bench/coganchor-concurrency/summarise.py | 98 +++ 20 files changed, 1709 insertions(+) create mode 100644 bench/coganchor-concurrency/README.md create mode 100644 bench/coganchor-concurrency/RESULTS.md create mode 100644 bench/coganchor-concurrency/backdrop.sh create mode 100644 bench/coganchor-concurrency/data/ladder-claude.jsonl create mode 100644 bench/coganchor-concurrency/data/ladder-codex.jsonl create mode 100644 bench/coganchor-concurrency/data/ladder-dsh.jsonl create mode 100644 bench/coganchor-concurrency/data/ladder-grok.jsonl create mode 100644 bench/coganchor-concurrency/data/ladder-kimi.jsonl create mode 100644 bench/coganchor-concurrency/data/refine-codex.jsonl create mode 100644 bench/coganchor-concurrency/data/stock-nofile/ladder-claude.jsonl create mode 100644 bench/coganchor-concurrency/data/stock-nofile/ladder-codex.jsonl create mode 100644 bench/coganchor-concurrency/data/stock-nofile/ladder-dsh.jsonl create mode 100644 bench/coganchor-concurrency/data/stock-nofile/ladder-grok.jsonl create mode 100644 bench/coganchor-concurrency/data/stock-nofile/ladder-kimi.jsonl create mode 100644 bench/coganchor-concurrency/ladder.sh create mode 100644 bench/coganchor-concurrency/ramp.py create mode 100644 bench/coganchor-concurrency/refine.sh create mode 100644 bench/coganchor-concurrency/run_one.sh create mode 100644 bench/coganchor-concurrency/standin_model.py create mode 100644 bench/coganchor-concurrency/summarise.py diff --git a/bench/coganchor-concurrency/README.md b/bench/coganchor-concurrency/README.md new file mode 100644 index 00000000..072aa48d --- /dev/null +++ b/bench/coganchor-concurrency/README.md @@ -0,0 +1,79 @@ +# How many anchored agents fit on one machine + +A rig for the one question humanize deliberately refuses to answer for you. [Many turns at +once](../../docs/features/concurrency.md) says it plainly: + +> **How wide it runs is a question about the machine**, not about this library, so nothing +> caps it. + +This measures the machine. It runs real `hmz`, with real coganchor interception, against a +mock controlled end, and climbs the concurrency ladder until something stops behaving. + +## Table of Contents + +- [What is real and what stands in](#what-is-real-and-what-stands-in) +- [Install](#install) +- [Usage](#usage) +- [What a rung means](#what-a-rung-means) +- [Results](#results) + +## What is real and what stands in + +| | | +| --- | --- | +| **Real** | `hmz` itself: the backends, the argv they build, the anchor, the ptrace supervisor, the mirror, the protocol, and the agent CLIs — `claude`, `codex`, `grok`, `kimi`, `dsh` — as installed. | +| **Stood in for** | The target's *data*: `hmz anchor serve` really serves, over a real TCP channel, but the workspace it serves is synthetic. | +| **Stood in for** | The model provider: one local server answering the Anthropic Messages, OpenAI Responses and OpenAI Chat shapes with a scripted turn. | + +Both stand-ins run **outside** the constrained cgroup, deliberately. In production the target +is another machine and the model is somebody else's API, so neither belongs in the budget +being sized. `ramp.py` records the target's own CPU alongside the measurement, so a run can +say whether the stand-in was anywhere near its own limit — if it was, the ceiling found is a +fact about the rig rather than about the machine. + +Every backend is given the **same** scripted turn — three shell commands, each reading a +seeded file and appending to another — so the numbers compare backends rather than prompts. +A turn counts only if the agent said the sentence the script ends on *and* its work is on the +target, which is what distinguishes a turn that ran from one that merely reported. + +## Install + +Needs `uv`, the five agent CLIs on `PATH`, and `sudo` for the cgroup: + +```sh +npm install --global @xai-official/grok @moonshot-ai/kimi-code @deepseek-ai/dsh +``` + +## Usage + +```sh +bash backdrop.sh start # stand-ins up, outside the cgroup +bash ladder.sh claude # climb until two rungs in a row misbehave +RUNGS="256 320 384" APPEND=1 bash ladder.sh codex +bash refine.sh codex 208 224 240 # narrow a ceiling, keeping stderr +python3 summarise.py # the table +bash backdrop.sh stop +``` + +`run_one.sh ` is one rung on its own. The cgroup is the whole of the constraint: + +```sh +sudo systemd-run --scope -p AllowedCPUs=0-7,112-119 -p MemoryMax=64G -p MemorySwapMax=0 ... +``` + +`AllowedCPUs` names eight physical cores **and their hyperthread siblings** — sixteen logical +CPUs, which is what a 16-vCPU machine is, and what `nproc` reports inside the scope. Naming +`0-15` instead would quietly hand the benchmark sixteen *physical* cores. + +## What a rung means + +One rung is N agents in one `hmz` process, one session each, one turn each, all going at +once — the fan-out shape the concurrency guide describes, aimed at an anchor. Each agent gets +a workspace of its own, mirrored from a copy of its own on the mock target. + +A rung is **normal** when every agent finished, said the sentence, and left its work on the +target. The ladder stops after two consecutive rungs that are not. + +## Results + +See [RESULTS.md](RESULTS.md). diff --git a/bench/coganchor-concurrency/RESULTS.md b/bench/coganchor-concurrency/RESULTS.md new file mode 100644 index 00000000..c5418ea8 --- /dev/null +++ b/bench/coganchor-concurrency/RESULTS.md @@ -0,0 +1,109 @@ +# How many anchored agents fit in 16 CPUs and 64 GiB + +Measured 2026-08-29 on an AMD EPYC 7B13, confined to eight physical cores and their +hyperthread siblings (sixteen logical CPUs, which is what `nproc` reports inside the scope) +and 64 GiB with no swap. Real `hmz`, real coganchor, real agent CLIs; the target's data and +the model provider stood in for, both outside the cgroup. See [README](README.md) for what +that means exactly. + +Every backend ran the same turn: three shell commands, each reading a seeded file on the +target and appending to another. A turn counts only if the agent said the sentence the script +ends on **and** its work is on the target. + +## The short answer + +There are three different numbers, and conflating them is how this question gets answered +wrongly. + +| | claude | codex | grok | kimi | dsh | +| --- | ---: | ---: | ---: | ---: | ---: | +| **Runs at full speed** (p95 within 2× of one agent alone) | **24** | **24** | **16** | **16** | **16** | +| **Best throughput** (turns/minute peaks here) | **48** | **128** | **48** | **64** | **32** | +| — turns/min there | 336 | 583 | 274 | 116 | 444 | +| **Still all-correct** (everything lands, just slowly) | **640** | **256** | **>832** | **192** | **>832** | +| What stops it there | 64 GiB | its own app-server | not found | 64 GiB | not found | + +**Before any of that, on a stock machine, everything stops at about 320** — see +[The first ceiling is descriptors](#the-first-ceiling-is-descriptors). + +So: if you want turns to run at the speed they run at alone, **16–24 anchored agents** is the +honest number for a 16-CPU box. If you want the most work done per hour and do not care that +each turn takes longer, **32–128**. Past that you are only lengthening the queue: claude's +throughput peaks at 336 turns/min with 48 agents and has fallen to 222 with 640. + +## Cost per agent + +| | claude | codex | grok | kimi | dsh | +| --- | ---: | ---: | ---: | ---: | ---: | +| memory | 102 MiB | 27 MiB | 44 MiB | **323 MiB** | 68 MiB | +| processes | 23 | **45** | **60** | 13 | 16 | +| CPU per turn | 2.0 s | 0.9 s | 2.8 s | 3.6 s | 1.1 s | +| one turn, alone | 2.8 s | 1.3 s | 2.4 s | 4.5 s | 1.1 s | + +Memory is what decides kimi and claude; nothing else got near 64 GiB. Kimi's 323 MiB is its +`kimi web` daemon, which humanize starts one of per agent. + +## The first ceiling is descriptors + +**hmz holds about three file descriptors per concurrent anchored agent.** A stock login has +a soft `RLIMIT_NOFILE` of 1024, so the first wall anybody meets is at roughly **320 agents**, +whatever the backend and however much RAM is free. It arrives as `[Errno 24] Too many open +files` from dsh, and as a bare exit from codex, which is the same thing seen from further +away. + +With the soft limit raised to the hard one, the same rung goes from 329/384 to **384/384**. +Everything above is measured with it raised. + +```sh +ulimit -n 1048576 # or LimitNOFILE in the unit that runs hmz +``` + +## Where each one actually stops + +- **claude — 640, on memory.** 640 agents used 63.74 of the 64 GiB and every turn still + landed. That is the last rung measured; there is 0.26 GiB of headroom left at it, so the + next one up was not attempted rather than shown to fail. +- **kimi — 192, on memory.** 192 used 60.5 GiB; 208 pinned the cgroup at exactly 64.00 GiB + and the OOM killer took daemons out, which the agents saw as `Remote end closed connection + without response`. 42 of 208 failed. +- **codex — 256, on codex.** Not memory (6.9 GiB), not descriptors, not the target: at 320 it + loses 13 agents to `app server stopped mid-turn`, at 512 it loses 96, and the ones that die + die at ~22 s while the survivors take ~45 s. The codex app-server gives up when the machine + is oversubscribed. It is also the one backend sensitive to being run in a tight loop: 256 + passes cleanly on its own and loses one agent when it follows a 192-agent rung immediately, + because several thousand processes from the previous rung are still going away. +- **grok and dsh — no ceiling found.** Both did 832 of 832 with nothing failing, at 36 GiB + and 55 GiB. 832 was the rig's slot count, not the machine's limit. Turns take a long time + there — 9 minutes for grok, 10 for dsh — but they all land. + +## Was the stand-in the bottleneck? + +No. The mock controlled end never exceeded **3%** of the 208 CPUs it had to itself, and was +under 1% for most runs. The measured cgroup sat at 80–87% throughout. Every ceiling above is +a fact about the 16-CPU machine or about the agent, not about the rig. + +## Three things this found in coganchor + +None of codex, kimi or grok could take an anchored turn at all before these. Each is a +separate way for the agent's own runtime to end up on the wrong machine, and each looks like +a backend problem until you look. + +1. **Argv entries were read with `PATH_MAX` as the ceiling.** An argv entry is not a path; + the kernel allows `MAX_ARG_STRLEN`. Codex prefixes every `bash -lc` with a preamble longer + than 4 KiB, so what reached the target was the first 4096 bytes of the command — which + parses, runs, and means something else. Codex reported every tool call as successful while + nothing happened on the target. +2. **`#!/usr/bin/env node` sent the agent to the target.** `env` runs here and then searches + `PATH` for the interpreter, one `execve` per directory. The first candidate names a path + that does not exist here, which is not the agent's own by name — so it was run on the + target, where the name resolves. Kimi's entire agent process ran on the target, read the + target's `HOME`, and could not find the account it was signed in with. Codex escaped this + only because its runtime is listed by hand. +3. **A binary in the agent's own state directory was kept local as a path but not as a + program.** grok installs its native binary under `~/.grok/bin` and re-execs it; that exec + went to the target, and grok reported `Not signed in`. + +## Raw data + +`data/ladder-*.jsonl` and `data/refine-codex.jsonl` hold one line per rung. +`data/stock-nofile/` holds the first climb, before the descriptor limit was raised. diff --git a/bench/coganchor-concurrency/backdrop.sh b/bench/coganchor-concurrency/backdrop.sh new file mode 100644 index 00000000..d34ccc83 --- /dev/null +++ b/bench/coganchor-concurrency/backdrop.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# Everything the measurement needs that is NOT part of what is being measured: +# +# * the stand-in model provider, which in production is a vendor's API +# * `hmz anchor serve`, which in production is the target machine +# +# Both are started outside the constrained cgroup, deliberately: the question is how many +# anchored agents fit in 16 CPUs and 64 GiB, not how many agents plus the machine they are +# working on fit there. +set -u + +TMP=$(cd "$(dirname "$0")" && pwd) +LAB=${LAB:-$TMP/lab} +ROOT=$(cd "$TMP/../.." && pwd) +MODEL_PORT=${MODEL_PORT:-18081} +TARGET_PORT=${TARGET_PORT:-18090} +SLOTS=${SLOTS:-192} +AB=${AGENT_BIN:-$(dirname "$(command -v kimi)")} + +stop() { + for pid in $(pgrep -f "standin_model[.]py" || true); do kill "$pid" 2>/dev/null || true; done + for pid in $(pgrep -f "anchor serve --listen" || true); do kill "$pid" 2>/dev/null || true; done + sleep 1 +} + +case "${1:-start}" in +stop) stop; echo "backdrop down"; exit 0 ;; +esac + +stop +mkdir -p "$LAB/ws" "$LAB/tgt" "$LAB/homes" "$LAB/logs" "$LAB/runs" + +# ---------------------------------------------------------------- the stand-in provider +STANDIN_LOG="$LAB/logs/standin.log" nohup python3 \ + $TMP/standin_model.py "$MODEL_PORT" \ + > "$LAB/logs/standin.out" 2>&1 & +sleep 1 +curl -sS "http://127.0.0.1:$MODEL_PORT/v1/models" > /dev/null || { echo "standin failed"; exit 1; } + +# ---------------------------------------------------------------- the mock controlled end +# One listener serves every session; its export table is fixed at start, so a slot for each +# concurrency level this rig will ever reach is declared up front. The directories behind +# them are wiped and reseeded per run by ramp.py. +EXPORTS=() +for i in $(seq 0 $((SLOTS - 1))); do + EXPORTS+=(--export "$LAB/ws/$i:$LAB/tgt/$i") +done +cd "$ROOT" +# Several listeners rather than one. A listener is a Python process holding a thread per +# session, and one of those would eventually be the thing that runs out -- which would make +# the ceiling a fact about the stand-in rather than about the machine running hmz. +LISTENERS=${LISTENERS:-4} +for n in $(seq 0 $((LISTENERS - 1))); do + port=$((TARGET_PORT + n)) + nohup ./.venv/bin/python -m hmz anchor serve --listen "127.0.0.1:$port" "${EXPORTS[@]}" \ + > "$LAB/logs/target-$port.out" 2>&1 & +done +sleep 8 +for n in $(seq 0 $((LISTENERS - 1))); do + port=$((TARGET_PORT + n)) + grep -q "listening" "$LAB/logs/target-$port.out" || { + echo "target listener on $port failed:"; cat "$LAB/logs/target-$port.out"; exit 1; } +done + +# ---------------------------------------------------------------- one HOME per backend +# Shared across that backend's concurrent agents, which is how humanize runs on a real +# machine: many agents, one user, one state directory each CLI keeps its sessions in. +BASE="http://127.0.0.1:$MODEL_PORT" + +for backend in claude codex grok kimi dsh; do + H="$LAB/homes/$backend" + rm -rf "$H"; mkdir -p "$H" +done + +# codex: a provider in its own home, so humanize's `codex app-server` finds it. +mkdir -p "$LAB/homes/codex/.codex" +cat > "$LAB/homes/codex/.codex/config.toml" < "$LAB/homes/grok/.grok/config.toml" < "$LAB/logs/kimi-provider.log" 2>&1 +printf 'default_model = "standin/standin-1"\n' > "$LAB/homes/kimi/.kimi-code/head.toml" +cat "$LAB/homes/kimi/.kimi-code/head.toml" "$LAB/homes/kimi/.kimi-code/config.toml" \ + > "$LAB/homes/kimi/.kimi-code/config.new" +mv "$LAB/homes/kimi/.kimi-code/config.new" "$LAB/homes/kimi/.kimi-code/config.toml" +grep -q "standin" "$LAB/homes/kimi/.kimi-code/config.toml" || { echo "kimi provider import failed"; exit 1; } + +echo "backdrop up: model on $MODEL_PORT, $LISTENERS mock targets from $TARGET_PORT, $SLOTS slots" diff --git a/bench/coganchor-concurrency/data/ladder-claude.jsonl b/bench/coganchor-concurrency/data/ladder-claude.jsonl new file mode 100644 index 00000000..3094a0e0 --- /dev/null +++ b/bench/coganchor-concurrency/data/ladder-claude.jsonl @@ -0,0 +1,20 @@ +{"backend": "claude", "concurrency": 1, "ok": 1, "failed": 0, "wall_seconds": 2.9, "turn_p50": 2.83, "turn_p95": 2.83, "turn_max": 2.83, "peak_memory_gib": 0.14, "peak_pids": 32, "cpu_seconds": 1.3, "cpu_busy_ratio": 0.03, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 2, "ok": 2, "failed": 0, "wall_seconds": 3.09, "turn_p50": 2.96, "turn_p95": 2.89, "turn_max": 3.02, "peak_memory_gib": 0.27, "peak_pids": 65, "cpu_seconds": 3.0, "cpu_busy_ratio": 0.06, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 4, "ok": 4, "failed": 0, "wall_seconds": 3.13, "turn_p50": 2.95, "turn_p95": 2.98, "turn_max": 3.05, "peak_memory_gib": 0.53, "peak_pids": 124, "cpu_seconds": 6.3, "cpu_busy_ratio": 0.13, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 8, "ok": 8, "failed": 0, "wall_seconds": 3.39, "turn_p50": 3.28, "turn_p95": 3.32, "turn_max": 3.32, "peak_memory_gib": 1.04, "peak_pids": 229, "cpu_seconds": 13.5, "cpu_busy_ratio": 0.25, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 16, "ok": 16, "failed": 0, "wall_seconds": 4.07, "turn_p50": 3.91, "turn_p95": 3.97, "turn_max": 3.99, "peak_memory_gib": 2.05, "peak_pids": 469, "cpu_seconds": 32.0, "cpu_busy_ratio": 0.49, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 24, "ok": 24, "failed": 0, "wall_seconds": 5.06, "turn_p50": 4.79, "turn_p95": 4.93, "turn_max": 4.97, "peak_memory_gib": 3.01, "peak_pids": 665, "cpu_seconds": 49.8, "cpu_busy_ratio": 0.62, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 32, "ok": 32, "failed": 0, "wall_seconds": 6.18, "turn_p50": 5.72, "turn_p95": 6.06, "turn_max": 6.1, "peak_memory_gib": 3.99, "peak_pids": 897, "cpu_seconds": 68.8, "cpu_busy_ratio": 0.7, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 48, "ok": 48, "failed": 0, "wall_seconds": 8.56, "turn_p50": 7.72, "turn_p95": 8.36, "turn_max": 8.46, "peak_memory_gib": 5.83, "peak_pids": 1300, "cpu_seconds": 107.0, "cpu_busy_ratio": 0.78, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 64, "ok": 64, "failed": 0, "wall_seconds": 12.03, "turn_p50": 11.19, "turn_p95": 11.74, "turn_max": 11.92, "peak_memory_gib": 7.63, "peak_pids": 1667, "cpu_seconds": 150.9, "cpu_busy_ratio": 0.78, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 96, "ok": 96, "failed": 0, "wall_seconds": 21.96, "turn_p50": 20.21, "turn_p95": 21.47, "turn_max": 21.85, "peak_memory_gib": 10.5, "peak_pids": 2510, "cpu_seconds": 241.3, "cpu_busy_ratio": 0.69, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 128, "ok": 128, "failed": 0, "wall_seconds": 26.13, "turn_p50": 24.84, "turn_p95": 25.71, "turn_max": 26.0, "peak_memory_gib": 14.84, "peak_pids": 3346, "cpu_seconds": 339.1, "cpu_busy_ratio": 0.81, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 160, "ok": 160, "failed": 0, "wall_seconds": 34.47, "turn_p50": 33.18, "turn_p95": 33.91, "turn_max": 34.29, "peak_memory_gib": 18.14, "peak_pids": 4047, "cpu_seconds": 432.3, "cpu_busy_ratio": 0.78, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 192, "ok": 192, "failed": 0, "wall_seconds": 39.24, "turn_p50": 37.32, "turn_p95": 38.68, "turn_max": 39.07, "peak_memory_gib": 20.72, "peak_pids": 4689, "cpu_seconds": 512.4, "cpu_busy_ratio": 0.82, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 256, "ok": 256, "failed": 0, "wall_seconds": 53.63, "turn_p50": 51.39, "turn_p95": 53.17, "turn_max": 53.44, "peak_memory_gib": 28.42, "peak_pids": 6237, "cpu_seconds": 740.0, "cpu_busy_ratio": 0.86, "target_cpu_seconds": 115.5, "target_busy_ratio": 0.01, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 320, "ok": 320, "failed": 0, "wall_seconds": 67.39, "turn_p50": 64.7, "turn_p95": 66.92, "turn_max": 67.18, "peak_memory_gib": 35.21, "peak_pids": 7976, "cpu_seconds": 929.9, "cpu_busy_ratio": 0.86, "target_cpu_seconds": 140.5, "target_busy_ratio": 0.01, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 384, "ok": 384, "failed": 0, "wall_seconds": 84.84, "turn_p50": 81.44, "turn_p95": 84.44, "turn_max": 84.59, "peak_memory_gib": 41.32, "peak_pids": 9262, "cpu_seconds": 1174.9, "cpu_busy_ratio": 0.87, "target_cpu_seconds": 180.3, "target_busy_ratio": 0.01, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 448, "ok": 448, "failed": 0, "wall_seconds": 103.01, "turn_p50": 99.37, "turn_p95": 102.6, "turn_max": 102.77, "peak_memory_gib": 46.82, "peak_pids": 11150, "cpu_seconds": 1427.7, "cpu_busy_ratio": 0.87, "target_cpu_seconds": 222.5, "target_busy_ratio": 0.01, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 512, "ok": 512, "failed": 0, "wall_seconds": 121.43, "turn_p50": 117.08, "turn_p95": 120.82, "turn_max": 121.12, "peak_memory_gib": 51.42, "peak_pids": 11965, "cpu_seconds": 1673.9, "cpu_busy_ratio": 0.86, "target_cpu_seconds": 249.0, "target_busy_ratio": 0.01, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 576, "ok": 576, "failed": 0, "wall_seconds": 138.11, "turn_p50": 131.69, "turn_p95": 137.49, "turn_max": 137.8, "peak_memory_gib": 54.92, "peak_pids": 13105, "cpu_seconds": 1898.8, "cpu_busy_ratio": 0.86, "target_cpu_seconds": 289.2, "target_busy_ratio": 0.01, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 640, "ok": 640, "failed": 0, "wall_seconds": 172.69, "turn_p50": 164.31, "turn_p95": 171.99, "turn_max": 172.37, "peak_memory_gib": 63.74, "peak_pids": 14766, "cpu_seconds": 2219.4, "cpu_busy_ratio": 0.8, "target_cpu_seconds": 333.4, "target_busy_ratio": 0.009, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} diff --git a/bench/coganchor-concurrency/data/ladder-codex.jsonl b/bench/coganchor-concurrency/data/ladder-codex.jsonl new file mode 100644 index 00000000..65b8e2ba --- /dev/null +++ b/bench/coganchor-concurrency/data/ladder-codex.jsonl @@ -0,0 +1,15 @@ +{"backend": "codex", "concurrency": 1, "ok": 1, "failed": 0, "wall_seconds": 1.42, "turn_p50": 1.34, "turn_p95": 1.34, "turn_max": 1.34, "peak_memory_gib": 0.05, "peak_pids": 54, "cpu_seconds": 0.6, "cpu_busy_ratio": 0.02, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 2, "ok": 2, "failed": 0, "wall_seconds": 1.43, "turn_p50": 1.33, "turn_p95": 1.31, "turn_max": 1.36, "peak_memory_gib": 0.08, "peak_pids": 108, "cpu_seconds": 1.0, "cpu_busy_ratio": 0.05, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 4, "ok": 4, "failed": 0, "wall_seconds": 1.47, "turn_p50": 1.37, "turn_p95": 1.37, "turn_max": 1.4, "peak_memory_gib": 0.13, "peak_pids": 210, "cpu_seconds": 2.2, "cpu_busy_ratio": 0.09, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 8, "ok": 8, "failed": 0, "wall_seconds": 1.6, "turn_p50": 1.48, "turn_p95": 1.5, "turn_max": 1.52, "peak_memory_gib": 0.25, "peak_pids": 417, "cpu_seconds": 5.8, "cpu_busy_ratio": 0.23, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 16, "ok": 16, "failed": 0, "wall_seconds": 1.98, "turn_p50": 1.85, "turn_p95": 1.9, "turn_max": 1.9, "peak_memory_gib": 0.48, "peak_pids": 812, "cpu_seconds": 11.4, "cpu_busy_ratio": 0.36, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 24, "ok": 24, "failed": 0, "wall_seconds": 2.78, "turn_p50": 2.45, "turn_p95": 2.57, "turn_max": 2.69, "peak_memory_gib": 0.72, "peak_pids": 1221, "cpu_seconds": 22.1, "cpu_busy_ratio": 0.5, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 32, "ok": 32, "failed": 0, "wall_seconds": 3.95, "turn_p50": 3.64, "turn_p95": 3.82, "turn_max": 3.86, "peak_memory_gib": 0.91, "peak_pids": 1608, "cpu_seconds": 33.3, "cpu_busy_ratio": 0.53, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 48, "ok": 48, "failed": 0, "wall_seconds": 5.75, "turn_p50": 5.08, "turn_p95": 5.58, "turn_max": 5.63, "peak_memory_gib": 1.38, "peak_pids": 2514, "cpu_seconds": 53.5, "cpu_busy_ratio": 0.58, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 64, "ok": 64, "failed": 0, "wall_seconds": 7.79, "turn_p50": 6.95, "turn_p95": 7.58, "turn_max": 7.68, "peak_memory_gib": 1.82, "peak_pids": 3274, "cpu_seconds": 81.3, "cpu_busy_ratio": 0.65, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 96, "ok": 96, "failed": 0, "wall_seconds": 10.12, "turn_p50": 8.39, "turn_p95": 9.81, "turn_max": 9.98, "peak_memory_gib": 2.6, "peak_pids": 4659, "cpu_seconds": 118.7, "cpu_busy_ratio": 0.73, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 128, "ok": 128, "failed": 0, "wall_seconds": 13.18, "turn_p50": 10.57, "turn_p95": 12.99, "turn_max": 13.05, "peak_memory_gib": 3.45, "peak_pids": 6117, "cpu_seconds": 157.7, "cpu_busy_ratio": 0.75, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 160, "ok": 160, "failed": 0, "wall_seconds": 17.03, "turn_p50": 13.52, "turn_p95": 16.68, "turn_max": 16.9, "peak_memory_gib": 4.23, "peak_pids": 7380, "cpu_seconds": 205.5, "cpu_busy_ratio": 0.75, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 192, "ok": 192, "failed": 0, "wall_seconds": 21.31, "turn_p50": 16.19, "turn_p95": 20.83, "turn_max": 21.15, "peak_memory_gib": 5.07, "peak_pids": 8672, "cpu_seconds": 253.2, "cpu_busy_ratio": 0.74, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 256, "ok": 255, "failed": 1, "wall_seconds": 22.28, "turn_p50": 16.68, "turn_p95": 21.68, "turn_max": 22.1, "peak_memory_gib": 6.59, "peak_pids": 11012, "cpu_seconds": 289.6, "cpu_busy_ratio": 0.81, "target_cpu_seconds": 132.3, "target_busy_ratio": 0.029, "errors": ["Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18092', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/102', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/102', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn"], "no_done": 1, "no_landing": 1, "steps_landed": [0, 3]} +{"backend": "codex", "concurrency": 320, "ok": 298, "failed": 22, "wall_seconds": 35.05, "turn_p50": 24.9, "turn_p95": 34.46, "turn_max": 34.82, "peak_memory_gib": 7.83, "peak_pids": 13076, "cpu_seconds": 427.3, "cpu_busy_ratio": 0.76, "target_cpu_seconds": 159.9, "target_busy_ratio": 0.022, "errors": ["Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/132', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/132', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/164', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/164', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/232', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/232', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/268', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/268', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/48', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/48', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn"], "no_done": 22, "no_landing": 22, "steps_landed": [0, 3]} diff --git a/bench/coganchor-concurrency/data/ladder-dsh.jsonl b/bench/coganchor-concurrency/data/ladder-dsh.jsonl new file mode 100644 index 00000000..fcd0c209 --- /dev/null +++ b/bench/coganchor-concurrency/data/ladder-dsh.jsonl @@ -0,0 +1,18 @@ +{"backend": "dsh", "concurrency": 1, "ok": 1, "failed": 0, "wall_seconds": 1.21, "turn_p50": 1.13, "turn_p95": 1.13, "turn_max": 1.13, "peak_memory_gib": 0.11, "peak_pids": 20, "cpu_seconds": 0.9, "cpu_busy_ratio": 0.05, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 2, "ok": 2, "failed": 0, "wall_seconds": 1.16, "turn_p50": 1.08, "turn_p95": 1.07, "turn_max": 1.09, "peak_memory_gib": 0.17, "peak_pids": 34, "cpu_seconds": 1.9, "cpu_busy_ratio": 0.1, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 4, "ok": 4, "failed": 0, "wall_seconds": 1.33, "turn_p50": 1.15, "turn_p95": 1.16, "turn_max": 1.25, "peak_memory_gib": 0.32, "peak_pids": 74, "cpu_seconds": 3.7, "cpu_busy_ratio": 0.18, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 8, "ok": 8, "failed": 0, "wall_seconds": 1.54, "turn_p50": 1.43, "turn_p95": 1.46, "turn_max": 1.46, "peak_memory_gib": 0.58, "peak_pids": 130, "cpu_seconds": 11.0, "cpu_busy_ratio": 0.45, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 16, "ok": 16, "failed": 0, "wall_seconds": 2.27, "turn_p50": 2.0, "turn_p95": 2.11, "turn_max": 2.18, "peak_memory_gib": 1.2, "peak_pids": 265, "cpu_seconds": 23.2, "cpu_busy_ratio": 0.64, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 24, "ok": 24, "failed": 0, "wall_seconds": 3.26, "turn_p50": 2.8, "turn_p95": 3.08, "turn_max": 3.18, "peak_memory_gib": 1.8, "peak_pids": 390, "cpu_seconds": 36.9, "cpu_busy_ratio": 0.71, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 32, "ok": 32, "failed": 0, "wall_seconds": 4.32, "turn_p50": 3.75, "turn_p95": 4.11, "turn_max": 4.22, "peak_memory_gib": 2.39, "peak_pids": 524, "cpu_seconds": 50.7, "cpu_busy_ratio": 0.73, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 48, "ok": 48, "failed": 0, "wall_seconds": 6.73, "turn_p50": 5.93, "turn_p95": 6.55, "turn_max": 6.62, "peak_memory_gib": 3.35, "peak_pids": 772, "cpu_seconds": 85.5, "cpu_busy_ratio": 0.79, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 64, "ok": 64, "failed": 0, "wall_seconds": 9.42, "turn_p50": 8.76, "turn_p95": 9.25, "turn_max": 9.3, "peak_memory_gib": 4.46, "peak_pids": 1028, "cpu_seconds": 124.1, "cpu_busy_ratio": 0.82, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 96, "ok": 96, "failed": 0, "wall_seconds": 15.34, "turn_p50": 14.04, "turn_p95": 15.1, "turn_max": 15.21, "peak_memory_gib": 6.45, "peak_pids": 1540, "cpu_seconds": 202.9, "cpu_busy_ratio": 0.83, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 128, "ok": 128, "failed": 0, "wall_seconds": 23.89, "turn_p50": 21.14, "turn_p95": 23.59, "turn_max": 23.74, "peak_memory_gib": 8.09, "peak_pids": 2025, "cpu_seconds": 312.8, "cpu_busy_ratio": 0.82, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 160, "ok": 160, "failed": 0, "wall_seconds": 32.34, "turn_p50": 29.38, "turn_p95": 32.03, "turn_max": 32.2, "peak_memory_gib": 10.01, "peak_pids": 2533, "cpu_seconds": 429.1, "cpu_busy_ratio": 0.83, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 192, "ok": 192, "failed": 0, "wall_seconds": 46.72, "turn_p50": 43.14, "turn_p95": 46.31, "turn_max": 46.56, "peak_memory_gib": 13.08, "peak_pids": 3060, "cpu_seconds": 633.4, "cpu_busy_ratio": 0.85, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 448, "ok": 448, "failed": 0, "wall_seconds": 87.47, "turn_p50": 77.83, "turn_p95": 86.61, "turn_max": 87.25, "peak_memory_gib": 29.9, "peak_pids": 7117, "cpu_seconds": 1196.9, "cpu_busy_ratio": 0.86, "target_cpu_seconds": 25.4, "target_busy_ratio": 0.001, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 512, "ok": 512, "failed": 0, "wall_seconds": 206.71, "turn_p50": 193.15, "turn_p95": 205.7, "turn_max": 206.48, "peak_memory_gib": 37.16, "peak_pids": 8195, "cpu_seconds": 2861.3, "cpu_busy_ratio": 0.87, "target_cpu_seconds": 27.7, "target_busy_ratio": 0.001, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 640, "ok": 640, "failed": 0, "wall_seconds": 338.41, "turn_p50": 326.34, "turn_p95": 337.25, "turn_max": 338.12, "peak_memory_gib": 45.74, "peak_pids": 10244, "cpu_seconds": 4711.7, "cpu_busy_ratio": 0.87, "target_cpu_seconds": 36.2, "target_busy_ratio": 0.001, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 768, "ok": 768, "failed": 0, "wall_seconds": 476.93, "turn_p50": 456.3, "turn_p95": 475.51, "turn_max": 476.62, "peak_memory_gib": 54.88, "peak_pids": 12291, "cpu_seconds": 6648.3, "cpu_busy_ratio": 0.87, "target_cpu_seconds": 45.1, "target_busy_ratio": 0.0, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 832, "ok": 832, "failed": 0, "wall_seconds": 648.15, "turn_p50": 618.36, "turn_p95": 646.48, "turn_max": 647.83, "peak_memory_gib": 54.95, "peak_pids": 13315, "cpu_seconds": 8866.5, "cpu_busy_ratio": 0.85, "target_cpu_seconds": 47.5, "target_busy_ratio": 0.0, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} diff --git a/bench/coganchor-concurrency/data/ladder-grok.jsonl b/bench/coganchor-concurrency/data/ladder-grok.jsonl new file mode 100644 index 00000000..25595d22 --- /dev/null +++ b/bench/coganchor-concurrency/data/ladder-grok.jsonl @@ -0,0 +1,18 @@ +{"backend": "grok", "concurrency": 1, "ok": 1, "failed": 0, "wall_seconds": 2.47, "turn_p50": 2.39, "turn_p95": 2.39, "turn_max": 2.39, "peak_memory_gib": 0.06, "peak_pids": 71, "cpu_seconds": 1.8, "cpu_busy_ratio": 0.05, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 2, "ok": 2, "failed": 0, "wall_seconds": 2.45, "turn_p50": 2.33, "turn_p95": 2.27, "turn_max": 2.38, "peak_memory_gib": 0.11, "peak_pids": 140, "cpu_seconds": 3.4, "cpu_busy_ratio": 0.09, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 4, "ok": 4, "failed": 0, "wall_seconds": 2.77, "turn_p50": 2.6, "turn_p95": 2.63, "turn_max": 2.7, "peak_memory_gib": 0.21, "peak_pids": 273, "cpu_seconds": 8.6, "cpu_busy_ratio": 0.19, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 8, "ok": 8, "failed": 0, "wall_seconds": 3.34, "turn_p50": 3.24, "turn_p95": 3.26, "turn_max": 3.26, "peak_memory_gib": 0.41, "peak_pids": 552, "cpu_seconds": 19.5, "cpu_busy_ratio": 0.36, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 16, "ok": 16, "failed": 0, "wall_seconds": 3.81, "turn_p50": 3.46, "turn_p95": 3.72, "turn_max": 3.73, "peak_memory_gib": 0.78, "peak_pids": 1082, "cpu_seconds": 37.1, "cpu_busy_ratio": 0.61, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 24, "ok": 24, "failed": 0, "wall_seconds": 5.49, "turn_p50": 4.8, "turn_p95": 5.32, "turn_max": 5.4, "peak_memory_gib": 1.12, "peak_pids": 1611, "cpu_seconds": 59.5, "cpu_busy_ratio": 0.68, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 32, "ok": 32, "failed": 0, "wall_seconds": 7.5, "turn_p50": 6.95, "turn_p95": 7.28, "turn_max": 7.38, "peak_memory_gib": 1.43, "peak_pids": 2113, "cpu_seconds": 85.2, "cpu_busy_ratio": 0.71, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 48, "ok": 48, "failed": 0, "wall_seconds": 10.52, "turn_p50": 9.84, "turn_p95": 10.26, "turn_max": 10.42, "peak_memory_gib": 2.16, "peak_pids": 3219, "cpu_seconds": 135.4, "cpu_busy_ratio": 0.8, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 64, "ok": 64, "failed": 0, "wall_seconds": 14.89, "turn_p50": 13.96, "turn_p95": 14.58, "turn_max": 14.76, "peak_memory_gib": 2.78, "peak_pids": 4195, "cpu_seconds": 198.4, "cpu_busy_ratio": 0.83, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 96, "ok": 96, "failed": 0, "wall_seconds": 24.89, "turn_p50": 23.57, "turn_p95": 24.61, "turn_max": 24.74, "peak_memory_gib": 4.06, "peak_pids": 6192, "cpu_seconds": 336.0, "cpu_busy_ratio": 0.84, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 128, "ok": 128, "failed": 0, "wall_seconds": 38.55, "turn_p50": 36.87, "turn_p95": 38.12, "turn_max": 38.39, "peak_memory_gib": 5.38, "peak_pids": 8141, "cpu_seconds": 527.3, "cpu_busy_ratio": 0.85, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 160, "ok": 160, "failed": 0, "wall_seconds": 54.5, "turn_p50": 52.57, "turn_p95": 54.2, "turn_max": 54.34, "peak_memory_gib": 7.05, "peak_pids": 10237, "cpu_seconds": 750.3, "cpu_busy_ratio": 0.86, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 192, "ok": 192, "failed": 0, "wall_seconds": 75.96, "turn_p50": 73.43, "turn_p95": 75.55, "turn_max": 75.78, "peak_memory_gib": 8.42, "peak_pids": 12169, "cpu_seconds": 1050.3, "cpu_busy_ratio": 0.86, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 384, "ok": 384, "failed": 0, "wall_seconds": 131.29, "turn_p50": 128.14, "turn_p95": 130.95, "turn_max": 130.99, "peak_memory_gib": 16.2, "peak_pids": 23603, "cpu_seconds": 1708.4, "cpu_busy_ratio": 0.81, "target_cpu_seconds": 77.0, "target_busy_ratio": 0.003, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 512, "ok": 512, "failed": 0, "wall_seconds": 273.17, "turn_p50": 269.18, "turn_p95": 272.88, "turn_max": 272.92, "peak_memory_gib": 22.46, "peak_pids": 31309, "cpu_seconds": 3648.8, "cpu_busy_ratio": 0.83, "target_cpu_seconds": 104.8, "target_busy_ratio": 0.002, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 640, "ok": 640, "failed": 0, "wall_seconds": 401.37, "turn_p50": 389.94, "turn_p95": 399.69, "turn_max": 401.05, "peak_memory_gib": 28.23, "peak_pids": 39453, "cpu_seconds": 5582.2, "cpu_busy_ratio": 0.87, "target_cpu_seconds": 140.3, "target_busy_ratio": 0.002, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 768, "ok": 768, "failed": 0, "wall_seconds": 535.38, "turn_p50": 512.68, "turn_p95": 531.1, "turn_max": 535.01, "peak_memory_gib": 32.81, "peak_pids": 46399, "cpu_seconds": 7301.4, "cpu_busy_ratio": 0.85, "target_cpu_seconds": 175.2, "target_busy_ratio": 0.002, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 832, "ok": 832, "failed": 0, "wall_seconds": 559.15, "turn_p50": 546.41, "turn_p95": 555.17, "turn_max": 558.72, "peak_memory_gib": 35.69, "peak_pids": 50061, "cpu_seconds": 7652.5, "cpu_busy_ratio": 0.86, "target_cpu_seconds": 184.9, "target_busy_ratio": 0.002, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} diff --git a/bench/coganchor-concurrency/data/ladder-kimi.jsonl b/bench/coganchor-concurrency/data/ladder-kimi.jsonl new file mode 100644 index 00000000..4fa471a0 --- /dev/null +++ b/bench/coganchor-concurrency/data/ladder-kimi.jsonl @@ -0,0 +1,15 @@ +{"backend": "kimi", "concurrency": 1, "ok": 1, "failed": 0, "wall_seconds": 4.6, "turn_p50": 4.52, "turn_p95": 4.52, "turn_max": 4.52, "peak_memory_gib": 0.43, "peak_pids": 19, "cpu_seconds": 3.8, "cpu_busy_ratio": 0.05, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 2, "ok": 2, "failed": 0, "wall_seconds": 4.6, "turn_p50": 4.5, "turn_p95": 4.47, "turn_max": 4.53, "peak_memory_gib": 0.84, "peak_pids": 34, "cpu_seconds": 7.9, "cpu_busy_ratio": 0.11, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 4, "ok": 4, "failed": 0, "wall_seconds": 4.87, "turn_p50": 4.76, "turn_p95": 4.77, "turn_max": 4.79, "peak_memory_gib": 1.65, "peak_pids": 71, "cpu_seconds": 17.5, "cpu_busy_ratio": 0.22, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 8, "ok": 8, "failed": 0, "wall_seconds": 5.95, "turn_p50": 5.73, "turn_p95": 5.84, "turn_max": 5.87, "peak_memory_gib": 3.29, "peak_pids": 130, "cpu_seconds": 42.6, "cpu_busy_ratio": 0.45, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 16, "ok": 16, "failed": 0, "wall_seconds": 8.86, "turn_p50": 8.12, "turn_p95": 8.56, "turn_max": 8.78, "peak_memory_gib": 6.54, "peak_pids": 257, "cpu_seconds": 97.2, "cpu_busy_ratio": 0.69, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 24, "ok": 24, "failed": 0, "wall_seconds": 13.15, "turn_p50": 11.93, "turn_p95": 12.77, "turn_max": 13.07, "peak_memory_gib": 9.17, "peak_pids": 368, "cpu_seconds": 156.0, "cpu_busy_ratio": 0.74, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 32, "ok": 32, "failed": 0, "wall_seconds": 17.61, "turn_p50": 15.9, "turn_p95": 17.16, "turn_max": 17.52, "peak_memory_gib": 11.9, "peak_pids": 478, "cpu_seconds": 221.8, "cpu_busy_ratio": 0.79, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 48, "ok": 48, "failed": 0, "wall_seconds": 25.48, "turn_p50": 23.34, "turn_p95": 24.78, "turn_max": 25.38, "peak_memory_gib": 16.51, "peak_pids": 667, "cpu_seconds": 330.0, "cpu_busy_ratio": 0.81, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 64, "ok": 64, "failed": 0, "wall_seconds": 33.21, "turn_p50": 29.98, "turn_p95": 32.24, "turn_max": 33.11, "peak_memory_gib": 21.61, "peak_pids": 870, "cpu_seconds": 436.8, "cpu_busy_ratio": 0.82, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 96, "ok": 96, "failed": 0, "wall_seconds": 50.23, "turn_p50": 46.39, "turn_p95": 49.09, "turn_max": 50.1, "peak_memory_gib": 28.63, "peak_pids": 1171, "cpu_seconds": 667.7, "cpu_busy_ratio": 0.83, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 128, "ok": 128, "failed": 0, "wall_seconds": 66.39, "turn_p50": 62.52, "turn_p95": 65.84, "turn_max": 66.25, "peak_memory_gib": 43.73, "peak_pids": 1799, "cpu_seconds": 907.6, "cpu_busy_ratio": 0.85, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 160, "ok": 160, "failed": 0, "wall_seconds": 83.42, "turn_p50": 72.12, "turn_p95": 82.75, "turn_max": 83.27, "peak_memory_gib": 49.05, "peak_pids": 2046, "cpu_seconds": 1130.0, "cpu_busy_ratio": 0.85, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 192, "ok": 192, "failed": 0, "wall_seconds": 99.03, "turn_p50": 94.24, "turn_p95": 97.95, "turn_max": 98.86, "peak_memory_gib": 60.48, "peak_pids": 2455, "cpu_seconds": 1356.0, "cpu_busy_ratio": 0.86, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 208, "ok": 166, "failed": 42, "wall_seconds": 171.64, "turn_p50": 159.18, "turn_p95": 170.43, "turn_max": 171.43, "peak_memory_gib": 64.0, "peak_pids": 2656, "cpu_seconds": 2319.0, "cpu_busy_ratio": 0.84, "target_cpu_seconds": 11.5, "target_busy_ratio": 0.0, "errors": ["Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/108', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/108', '--force', 'kimi', 'web', '--no-open', '--port', '0', '--log-level', 'error']' returned non-zero exit status 1. [Errno 104] Connection reset by peer", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/116', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/116', '--force', 'kimi', 'web', '--no-open', '--port', '0', '--log-level', 'error']' returned non-zero exit status 1. Remote end closed connection without response", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/12', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/12', '--force', 'kimi', 'web', '--no-open', '--port', '0', '--log-level', 'error']' returned non-zero exit status 1. Remote end closed connection without response", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/144', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/144', '--force', 'kimi', 'web', '--no-open', '--port', '0', '--log-level', 'error']' returned non-zero exit status 1. Remote end closed connection without response", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/160', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/160', '--force', 'kimi', 'web', '--no-open', '--port', '0', '--log-level', 'error']' returned non-zero exit status 1. Remote end closed connection without response"], "no_done": 42, "no_landing": 38, "steps_landed": [0, 1, 2, 3]} +{"backend": "kimi", "concurrency": 224, "ok": 170, "failed": 54, "wall_seconds": 138.59, "turn_p50": 129.73, "turn_p95": 137.78, "turn_max": 138.43, "peak_memory_gib": 64.0, "peak_pids": 2678, "cpu_seconds": 1910.2, "cpu_busy_ratio": 0.86, "target_cpu_seconds": 11.0, "target_busy_ratio": 0.0, "errors": ["Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/100', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/100', '--force', 'kimi', 'web', '--no-open', '--port', '0', '--log-level', 'error']' returned non-zero exit status 137. /home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python stopped without listening", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/108', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/108', '--force', 'kimi', 'web', '--no-open', '--port', '0', '--log-level', 'error']' returned non-zero exit status 137. /home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python stopped without listening", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/216', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/216', '--force', 'kimi', 'web', '--no-open', '--port', '0', '--log-level', 'error']' returned non-zero exit status 1. Remote end closed connection without response", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/28', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/28', '--force', 'kimi', 'web', '--no-open', '--port', '0', '--log-level', 'error']' returned non-zero exit status 1. ", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/48', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/48', '--force', 'kimi', 'web', '--no-open', '--port', '0', '--log-level', 'error']' returned non-zero exit status 1. Remote end closed connection without response"], "no_done": 54, "no_landing": 50, "steps_landed": [0, 1, 2, 3]} diff --git a/bench/coganchor-concurrency/data/refine-codex.jsonl b/bench/coganchor-concurrency/data/refine-codex.jsonl new file mode 100644 index 00000000..7e74c45f --- /dev/null +++ b/bench/coganchor-concurrency/data/refine-codex.jsonl @@ -0,0 +1,7 @@ +{"backend": "codex", "concurrency": 208, "ok": 208, "failed": 0, "wall_seconds": 20.6, "turn_p50": 16.12, "turn_p95": 20.23, "turn_max": 20.42, "peak_memory_gib": 5.56, "peak_pids": 9022, "cpu_seconds": 274.2, "cpu_busy_ratio": 0.83, "target_cpu_seconds": 106.6, "target_busy_ratio": 0.025, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 224, "ok": 224, "failed": 0, "wall_seconds": 22.63, "turn_p50": 17.05, "turn_p95": 21.99, "turn_max": 22.46, "peak_memory_gib": 6.0, "peak_pids": 9750, "cpu_seconds": 297.9, "cpu_busy_ratio": 0.82, "target_cpu_seconds": 110.9, "target_busy_ratio": 0.024, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 240, "ok": 240, "failed": 0, "wall_seconds": 25.64, "turn_p50": 19.34, "turn_p95": 25.03, "turn_max": 25.46, "peak_memory_gib": 6.46, "peak_pids": 10486, "cpu_seconds": 340.3, "cpu_busy_ratio": 0.83, "target_cpu_seconds": 121.9, "target_busy_ratio": 0.023, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 256, "ok": 256, "failed": 0, "wall_seconds": 28.72, "turn_p50": 21.67, "turn_p95": 28.19, "turn_max": 28.53, "peak_memory_gib": 6.89, "peak_pids": 11120, "cpu_seconds": 379.8, "cpu_busy_ratio": 0.83, "target_cpu_seconds": 128.4, "target_busy_ratio": 0.021, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 320, "ok": 307, "failed": 13, "wall_seconds": 36.83, "turn_p50": 26.73, "turn_p95": 36.23, "turn_max": 36.59, "peak_memory_gib": 8.22, "peak_pids": 13517, "cpu_seconds": 496.2, "cpu_busy_ratio": 0.84, "target_cpu_seconds": 157.0, "target_busy_ratio": 0.02, "errors": ["Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/220', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/220', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/284', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/284', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18091', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/1', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/1', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18091', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/201', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/201', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18091', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/305', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/305', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn"], "no_done": 13, "no_landing": 13, "steps_landed": [0, 3]} +{"backend": "codex", "concurrency": 384, "ok": 340, "failed": 44, "wall_seconds": 45.92, "turn_p50": 30.51, "turn_p95": 45.08, "turn_max": 45.71, "peak_memory_gib": 9.2, "peak_pids": 15215, "cpu_seconds": 619.3, "cpu_busy_ratio": 0.84, "target_cpu_seconds": 176.7, "target_busy_ratio": 0.018, "errors": ["Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/228', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/228', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/304', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/304', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/32', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/32', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/328', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/328', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/356', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/356', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn"], "no_done": 44, "no_landing": 44, "steps_landed": [0, 3]} +{"backend": "codex", "concurrency": 512, "ok": 416, "failed": 96, "wall_seconds": 59.69, "turn_p50": 38.98, "turn_p95": 58.43, "turn_max": 59.41, "peak_memory_gib": 11.32, "peak_pids": 18155, "cpu_seconds": 808.0, "cpu_busy_ratio": 0.85, "target_cpu_seconds": 213.8, "target_busy_ratio": 0.017, "errors": ["Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/108', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/108', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/132', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/132', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/152', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/152', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/176', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/176', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/212', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/212', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn"], "no_done": 96, "no_landing": 96, "steps_landed": [0, 3]} diff --git a/bench/coganchor-concurrency/data/stock-nofile/ladder-claude.jsonl b/bench/coganchor-concurrency/data/stock-nofile/ladder-claude.jsonl new file mode 100644 index 00000000..ed00d602 --- /dev/null +++ b/bench/coganchor-concurrency/data/stock-nofile/ladder-claude.jsonl @@ -0,0 +1,13 @@ +{"backend": "claude", "concurrency": 1, "ok": 1, "failed": 0, "wall_seconds": 2.9, "turn_p50": 2.83, "turn_p95": 2.83, "turn_max": 2.83, "peak_memory_gib": 0.14, "peak_pids": 32, "cpu_seconds": 1.3, "cpu_busy_ratio": 0.03, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 2, "ok": 2, "failed": 0, "wall_seconds": 3.09, "turn_p50": 2.96, "turn_p95": 2.89, "turn_max": 3.02, "peak_memory_gib": 0.27, "peak_pids": 65, "cpu_seconds": 3.0, "cpu_busy_ratio": 0.06, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 4, "ok": 4, "failed": 0, "wall_seconds": 3.13, "turn_p50": 2.95, "turn_p95": 2.98, "turn_max": 3.05, "peak_memory_gib": 0.53, "peak_pids": 124, "cpu_seconds": 6.3, "cpu_busy_ratio": 0.13, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 8, "ok": 8, "failed": 0, "wall_seconds": 3.39, "turn_p50": 3.28, "turn_p95": 3.32, "turn_max": 3.32, "peak_memory_gib": 1.04, "peak_pids": 229, "cpu_seconds": 13.5, "cpu_busy_ratio": 0.25, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 16, "ok": 16, "failed": 0, "wall_seconds": 4.07, "turn_p50": 3.91, "turn_p95": 3.97, "turn_max": 3.99, "peak_memory_gib": 2.05, "peak_pids": 469, "cpu_seconds": 32.0, "cpu_busy_ratio": 0.49, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 24, "ok": 24, "failed": 0, "wall_seconds": 5.06, "turn_p50": 4.79, "turn_p95": 4.93, "turn_max": 4.97, "peak_memory_gib": 3.01, "peak_pids": 665, "cpu_seconds": 49.8, "cpu_busy_ratio": 0.62, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 32, "ok": 32, "failed": 0, "wall_seconds": 6.18, "turn_p50": 5.72, "turn_p95": 6.06, "turn_max": 6.1, "peak_memory_gib": 3.99, "peak_pids": 897, "cpu_seconds": 68.8, "cpu_busy_ratio": 0.7, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 48, "ok": 48, "failed": 0, "wall_seconds": 8.56, "turn_p50": 7.72, "turn_p95": 8.36, "turn_max": 8.46, "peak_memory_gib": 5.83, "peak_pids": 1300, "cpu_seconds": 107.0, "cpu_busy_ratio": 0.78, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 64, "ok": 64, "failed": 0, "wall_seconds": 12.03, "turn_p50": 11.19, "turn_p95": 11.74, "turn_max": 11.92, "peak_memory_gib": 7.63, "peak_pids": 1667, "cpu_seconds": 150.9, "cpu_busy_ratio": 0.78, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 96, "ok": 96, "failed": 0, "wall_seconds": 21.96, "turn_p50": 20.21, "turn_p95": 21.47, "turn_max": 21.85, "peak_memory_gib": 10.5, "peak_pids": 2510, "cpu_seconds": 241.3, "cpu_busy_ratio": 0.69, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 128, "ok": 128, "failed": 0, "wall_seconds": 26.13, "turn_p50": 24.84, "turn_p95": 25.71, "turn_max": 26.0, "peak_memory_gib": 14.84, "peak_pids": 3346, "cpu_seconds": 339.1, "cpu_busy_ratio": 0.81, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 160, "ok": 160, "failed": 0, "wall_seconds": 34.47, "turn_p50": 33.18, "turn_p95": 33.91, "turn_max": 34.29, "peak_memory_gib": 18.14, "peak_pids": 4047, "cpu_seconds": 432.3, "cpu_busy_ratio": 0.78, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "claude", "concurrency": 192, "ok": 192, "failed": 0, "wall_seconds": 39.24, "turn_p50": 37.32, "turn_p95": 38.68, "turn_max": 39.07, "peak_memory_gib": 20.72, "peak_pids": 4689, "cpu_seconds": 512.4, "cpu_busy_ratio": 0.82, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} diff --git a/bench/coganchor-concurrency/data/stock-nofile/ladder-codex.jsonl b/bench/coganchor-concurrency/data/stock-nofile/ladder-codex.jsonl new file mode 100644 index 00000000..5c48d81f --- /dev/null +++ b/bench/coganchor-concurrency/data/stock-nofile/ladder-codex.jsonl @@ -0,0 +1,15 @@ +{"backend": "codex", "concurrency": 1, "ok": 1, "failed": 0, "wall_seconds": 1.42, "turn_p50": 1.34, "turn_p95": 1.34, "turn_max": 1.34, "peak_memory_gib": 0.05, "peak_pids": 54, "cpu_seconds": 0.6, "cpu_busy_ratio": 0.02, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 2, "ok": 2, "failed": 0, "wall_seconds": 1.43, "turn_p50": 1.33, "turn_p95": 1.31, "turn_max": 1.36, "peak_memory_gib": 0.08, "peak_pids": 108, "cpu_seconds": 1.0, "cpu_busy_ratio": 0.05, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 4, "ok": 4, "failed": 0, "wall_seconds": 1.47, "turn_p50": 1.37, "turn_p95": 1.37, "turn_max": 1.4, "peak_memory_gib": 0.13, "peak_pids": 210, "cpu_seconds": 2.2, "cpu_busy_ratio": 0.09, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 8, "ok": 8, "failed": 0, "wall_seconds": 1.6, "turn_p50": 1.48, "turn_p95": 1.5, "turn_max": 1.52, "peak_memory_gib": 0.25, "peak_pids": 417, "cpu_seconds": 5.8, "cpu_busy_ratio": 0.23, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 16, "ok": 16, "failed": 0, "wall_seconds": 1.98, "turn_p50": 1.85, "turn_p95": 1.9, "turn_max": 1.9, "peak_memory_gib": 0.48, "peak_pids": 812, "cpu_seconds": 11.4, "cpu_busy_ratio": 0.36, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 24, "ok": 24, "failed": 0, "wall_seconds": 2.78, "turn_p50": 2.45, "turn_p95": 2.57, "turn_max": 2.69, "peak_memory_gib": 0.72, "peak_pids": 1221, "cpu_seconds": 22.1, "cpu_busy_ratio": 0.5, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 32, "ok": 32, "failed": 0, "wall_seconds": 3.95, "turn_p50": 3.64, "turn_p95": 3.82, "turn_max": 3.86, "peak_memory_gib": 0.91, "peak_pids": 1608, "cpu_seconds": 33.3, "cpu_busy_ratio": 0.53, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 48, "ok": 48, "failed": 0, "wall_seconds": 5.75, "turn_p50": 5.08, "turn_p95": 5.58, "turn_max": 5.63, "peak_memory_gib": 1.38, "peak_pids": 2514, "cpu_seconds": 53.5, "cpu_busy_ratio": 0.58, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 64, "ok": 64, "failed": 0, "wall_seconds": 7.79, "turn_p50": 6.95, "turn_p95": 7.58, "turn_max": 7.68, "peak_memory_gib": 1.82, "peak_pids": 3274, "cpu_seconds": 81.3, "cpu_busy_ratio": 0.65, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 96, "ok": 96, "failed": 0, "wall_seconds": 10.12, "turn_p50": 8.39, "turn_p95": 9.81, "turn_max": 9.98, "peak_memory_gib": 2.6, "peak_pids": 4659, "cpu_seconds": 118.7, "cpu_busy_ratio": 0.73, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 128, "ok": 128, "failed": 0, "wall_seconds": 13.18, "turn_p50": 10.57, "turn_p95": 12.99, "turn_max": 13.05, "peak_memory_gib": 3.45, "peak_pids": 6117, "cpu_seconds": 157.7, "cpu_busy_ratio": 0.75, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 160, "ok": 160, "failed": 0, "wall_seconds": 17.03, "turn_p50": 13.52, "turn_p95": 16.68, "turn_max": 16.9, "peak_memory_gib": 4.23, "peak_pids": 7380, "cpu_seconds": 205.5, "cpu_busy_ratio": 0.75, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 192, "ok": 192, "failed": 0, "wall_seconds": 21.31, "turn_p50": 16.19, "turn_p95": 20.83, "turn_max": 21.15, "peak_memory_gib": 5.07, "peak_pids": 8672, "cpu_seconds": 253.2, "cpu_busy_ratio": 0.74, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "codex", "concurrency": 256, "ok": 253, "failed": 3, "wall_seconds": 22.13, "turn_p50": 17.01, "turn_p95": 21.6, "turn_max": 21.97, "peak_memory_gib": 6.7, "peak_pids": 11196, "cpu_seconds": 289.5, "cpu_busy_ratio": 0.82, "target_cpu_seconds": 132.3, "target_busy_ratio": 0.029, "errors": ["Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/200', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/200', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18091', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/25', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/25', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18093', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/27', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/27', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn"], "no_done": 3, "no_landing": 3, "steps_landed": [0, 3]} +{"backend": "codex", "concurrency": 320, "ok": 291, "failed": 29, "wall_seconds": 28.2, "turn_p50": 20.67, "turn_p95": 27.5, "turn_max": 28.01, "peak_memory_gib": 7.55, "peak_pids": 12546, "cpu_seconds": 373.6, "cpu_busy_ratio": 0.83, "target_cpu_seconds": 148.0, "target_busy_ratio": 0.025, "errors": ["Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/108', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/108', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/152', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/152', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/164', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/164', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/176', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/176', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn", "Failed: Command '['/home/ubuntu/humanize2/.claude/worktrees/coganchor-concurrency/.venv/bin/python', '-m', 'hmz', 'anchor', '--target=tcp://127.0.0.1:18090', '--net=local', '--workspace=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/ws/232', '--remote-path=/home/ubuntu/.claude/jobs/2ce04de0/tmp/lab/tgt/232', '--force', 'codex', 'app-server', '-c', 'tools.web_search=true', '--stdio']' returned non-zero exit status 1. app server stopped mid-turn"], "no_done": 29, "no_landing": 29, "steps_landed": [0, 3]} diff --git a/bench/coganchor-concurrency/data/stock-nofile/ladder-dsh.jsonl b/bench/coganchor-concurrency/data/stock-nofile/ladder-dsh.jsonl new file mode 100644 index 00000000..da2bdf7c --- /dev/null +++ b/bench/coganchor-concurrency/data/stock-nofile/ladder-dsh.jsonl @@ -0,0 +1,17 @@ +{"backend": "dsh", "concurrency": 1, "ok": 1, "failed": 0, "wall_seconds": 1.21, "turn_p50": 1.13, "turn_p95": 1.13, "turn_max": 1.13, "peak_memory_gib": 0.11, "peak_pids": 20, "cpu_seconds": 0.9, "cpu_busy_ratio": 0.05, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 2, "ok": 2, "failed": 0, "wall_seconds": 1.16, "turn_p50": 1.08, "turn_p95": 1.07, "turn_max": 1.09, "peak_memory_gib": 0.17, "peak_pids": 34, "cpu_seconds": 1.9, "cpu_busy_ratio": 0.1, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 4, "ok": 4, "failed": 0, "wall_seconds": 1.33, "turn_p50": 1.15, "turn_p95": 1.16, "turn_max": 1.25, "peak_memory_gib": 0.32, "peak_pids": 74, "cpu_seconds": 3.7, "cpu_busy_ratio": 0.18, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 8, "ok": 8, "failed": 0, "wall_seconds": 1.54, "turn_p50": 1.43, "turn_p95": 1.46, "turn_max": 1.46, "peak_memory_gib": 0.58, "peak_pids": 130, "cpu_seconds": 11.0, "cpu_busy_ratio": 0.45, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 16, "ok": 16, "failed": 0, "wall_seconds": 2.27, "turn_p50": 2.0, "turn_p95": 2.11, "turn_max": 2.18, "peak_memory_gib": 1.2, "peak_pids": 265, "cpu_seconds": 23.2, "cpu_busy_ratio": 0.64, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 24, "ok": 24, "failed": 0, "wall_seconds": 3.26, "turn_p50": 2.8, "turn_p95": 3.08, "turn_max": 3.18, "peak_memory_gib": 1.8, "peak_pids": 390, "cpu_seconds": 36.9, "cpu_busy_ratio": 0.71, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 32, "ok": 32, "failed": 0, "wall_seconds": 4.32, "turn_p50": 3.75, "turn_p95": 4.11, "turn_max": 4.22, "peak_memory_gib": 2.39, "peak_pids": 524, "cpu_seconds": 50.7, "cpu_busy_ratio": 0.73, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 48, "ok": 48, "failed": 0, "wall_seconds": 6.73, "turn_p50": 5.93, "turn_p95": 6.55, "turn_max": 6.62, "peak_memory_gib": 3.35, "peak_pids": 772, "cpu_seconds": 85.5, "cpu_busy_ratio": 0.79, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 64, "ok": 64, "failed": 0, "wall_seconds": 9.42, "turn_p50": 8.76, "turn_p95": 9.25, "turn_max": 9.3, "peak_memory_gib": 4.46, "peak_pids": 1028, "cpu_seconds": 124.1, "cpu_busy_ratio": 0.82, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 96, "ok": 96, "failed": 0, "wall_seconds": 15.34, "turn_p50": 14.04, "turn_p95": 15.1, "turn_max": 15.21, "peak_memory_gib": 6.45, "peak_pids": 1540, "cpu_seconds": 202.9, "cpu_busy_ratio": 0.83, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 128, "ok": 128, "failed": 0, "wall_seconds": 23.89, "turn_p50": 21.14, "turn_p95": 23.59, "turn_max": 23.74, "peak_memory_gib": 8.09, "peak_pids": 2025, "cpu_seconds": 312.8, "cpu_busy_ratio": 0.82, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 160, "ok": 160, "failed": 0, "wall_seconds": 32.34, "turn_p50": 29.38, "turn_p95": 32.03, "turn_max": 32.2, "peak_memory_gib": 10.01, "peak_pids": 2533, "cpu_seconds": 429.1, "cpu_busy_ratio": 0.83, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 192, "ok": 192, "failed": 0, "wall_seconds": 46.72, "turn_p50": 43.14, "turn_p95": 46.31, "turn_max": 46.56, "peak_memory_gib": 13.08, "peak_pids": 3060, "cpu_seconds": 633.4, "cpu_busy_ratio": 0.85, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 256, "ok": 256, "failed": 0, "wall_seconds": 39.73, "turn_p50": 34.51, "turn_p95": 38.77, "turn_max": 39.54, "peak_memory_gib": 17.48, "peak_pids": 4084, "cpu_seconds": 535.0, "cpu_busy_ratio": 0.84, "target_cpu_seconds": 12.9, "target_busy_ratio": 0.002, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 320, "ok": 320, "failed": 0, "wall_seconds": 88.71, "turn_p50": 83.63, "turn_p95": 88.13, "turn_max": 88.5, "peak_memory_gib": 23.25, "peak_pids": 5123, "cpu_seconds": 1225.3, "cpu_busy_ratio": 0.86, "target_cpu_seconds": 16.0, "target_busy_ratio": 0.001, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "dsh", "concurrency": 384, "ok": 329, "failed": 55, "wall_seconds": 113.68, "turn_p50": 103.38, "turn_p95": 113.04, "turn_max": 113.46, "peak_memory_gib": 24.08, "peak_pids": 5323, "cpu_seconds": 1571.1, "cpu_busy_ratio": 0.86, "target_cpu_seconds": 17.6, "target_busy_ratio": 0.001, "errors": ["Failed: Command '['dsh', 'session-012a1caaf8e54ce78b8468374fc8a5b7']' returned non-zero exit status 1. [Errno 24] Too many open files", "Failed: Command '['dsh', 'session-01dd5bc0381f43429156a33a89174382']' returned non-zero exit status 1. [Errno 24] Too many open files", "Failed: Command '['dsh', 'session-03d5b853b13143a28753c95ae37e41a8']' returned non-zero exit status 1. [Errno 24] Too many open files", "Failed: Command '['dsh', 'session-043b2839e63b4969a403a6217bbc18fb']' returned non-zero exit status 1. [Errno 24] Too many open files", "Failed: Command '['dsh', 'session-09ada9edfa8f47dfa5ab1d7c5016cd21']' returned non-zero exit status 1. [Errno 24] Too many open files"], "no_done": 55, "no_landing": 55, "steps_landed": [0, 3]} +{"backend": "dsh", "concurrency": 512, "ok": 338, "failed": 174, "wall_seconds": 131.36, "turn_p50": 101.04, "turn_p95": 130.24, "turn_max": 131.05, "peak_memory_gib": 24.53, "peak_pids": 5586, "cpu_seconds": 1811.9, "cpu_busy_ratio": 0.86, "target_cpu_seconds": 17.8, "target_busy_ratio": 0.001, "errors": ["Failed: Command '['dsh', 'session-013ae1bc1dbf41fab329dc0866df562d']' returned non-zero exit status 1. [Errno 24] Too many open files", "Failed: Command '['dsh', 'session-013b21ae40c54b5f9799fe7c71e7801f']' returned non-zero exit status 1. [Errno 24] Too many open files", "Failed: Command '['dsh', 'session-037c6fad9ddf4fbc9a3b1277a1ccc1ac']' returned non-zero exit status 1. [Errno 24] Too many open files", "Failed: Command '['dsh', 'session-04371294ddcf464484421c4467130ca8']' returned non-zero exit status 1. [Errno 24] Too many open files", "Failed: Command '['dsh', 'session-05d377989ad64865b50d8148a9e7a72d']' returned non-zero exit status 1. [Errno 24] Too many open files"], "no_done": 174, "no_landing": 174, "steps_landed": [0, 3]} diff --git a/bench/coganchor-concurrency/data/stock-nofile/ladder-grok.jsonl b/bench/coganchor-concurrency/data/stock-nofile/ladder-grok.jsonl new file mode 100644 index 00000000..60b443d5 --- /dev/null +++ b/bench/coganchor-concurrency/data/stock-nofile/ladder-grok.jsonl @@ -0,0 +1,15 @@ +{"backend": "grok", "concurrency": 1, "ok": 1, "failed": 0, "wall_seconds": 2.47, "turn_p50": 2.39, "turn_p95": 2.39, "turn_max": 2.39, "peak_memory_gib": 0.06, "peak_pids": 71, "cpu_seconds": 1.8, "cpu_busy_ratio": 0.05, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 2, "ok": 2, "failed": 0, "wall_seconds": 2.45, "turn_p50": 2.33, "turn_p95": 2.27, "turn_max": 2.38, "peak_memory_gib": 0.11, "peak_pids": 140, "cpu_seconds": 3.4, "cpu_busy_ratio": 0.09, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 4, "ok": 4, "failed": 0, "wall_seconds": 2.77, "turn_p50": 2.6, "turn_p95": 2.63, "turn_max": 2.7, "peak_memory_gib": 0.21, "peak_pids": 273, "cpu_seconds": 8.6, "cpu_busy_ratio": 0.19, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 8, "ok": 8, "failed": 0, "wall_seconds": 3.34, "turn_p50": 3.24, "turn_p95": 3.26, "turn_max": 3.26, "peak_memory_gib": 0.41, "peak_pids": 552, "cpu_seconds": 19.5, "cpu_busy_ratio": 0.36, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 16, "ok": 16, "failed": 0, "wall_seconds": 3.81, "turn_p50": 3.46, "turn_p95": 3.72, "turn_max": 3.73, "peak_memory_gib": 0.78, "peak_pids": 1082, "cpu_seconds": 37.1, "cpu_busy_ratio": 0.61, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 24, "ok": 24, "failed": 0, "wall_seconds": 5.49, "turn_p50": 4.8, "turn_p95": 5.32, "turn_max": 5.4, "peak_memory_gib": 1.12, "peak_pids": 1611, "cpu_seconds": 59.5, "cpu_busy_ratio": 0.68, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 32, "ok": 32, "failed": 0, "wall_seconds": 7.5, "turn_p50": 6.95, "turn_p95": 7.28, "turn_max": 7.38, "peak_memory_gib": 1.43, "peak_pids": 2113, "cpu_seconds": 85.2, "cpu_busy_ratio": 0.71, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 48, "ok": 48, "failed": 0, "wall_seconds": 10.52, "turn_p50": 9.84, "turn_p95": 10.26, "turn_max": 10.42, "peak_memory_gib": 2.16, "peak_pids": 3219, "cpu_seconds": 135.4, "cpu_busy_ratio": 0.8, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 64, "ok": 64, "failed": 0, "wall_seconds": 14.89, "turn_p50": 13.96, "turn_p95": 14.58, "turn_max": 14.76, "peak_memory_gib": 2.78, "peak_pids": 4195, "cpu_seconds": 198.4, "cpu_busy_ratio": 0.83, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 96, "ok": 96, "failed": 0, "wall_seconds": 24.89, "turn_p50": 23.57, "turn_p95": 24.61, "turn_max": 24.74, "peak_memory_gib": 4.06, "peak_pids": 6192, "cpu_seconds": 336.0, "cpu_busy_ratio": 0.84, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 128, "ok": 128, "failed": 0, "wall_seconds": 38.55, "turn_p50": 36.87, "turn_p95": 38.12, "turn_max": 38.39, "peak_memory_gib": 5.38, "peak_pids": 8141, "cpu_seconds": 527.3, "cpu_busy_ratio": 0.85, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 160, "ok": 160, "failed": 0, "wall_seconds": 54.5, "turn_p50": 52.57, "turn_p95": 54.2, "turn_max": 54.34, "peak_memory_gib": 7.05, "peak_pids": 10237, "cpu_seconds": 750.3, "cpu_busy_ratio": 0.86, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 192, "ok": 192, "failed": 0, "wall_seconds": 75.96, "turn_p50": 73.43, "turn_p95": 75.55, "turn_max": 75.78, "peak_memory_gib": 8.42, "peak_pids": 12169, "cpu_seconds": 1050.3, "cpu_busy_ratio": 0.86, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 256, "ok": 256, "failed": 0, "wall_seconds": 72.81, "turn_p50": 70.07, "turn_p95": 72.46, "turn_max": 72.59, "peak_memory_gib": 10.86, "peak_pids": 15882, "cpu_seconds": 990.0, "cpu_busy_ratio": 0.85, "target_cpu_seconds": 48.6, "target_busy_ratio": 0.003, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "grok", "concurrency": 320, "ok": 320, "failed": 0, "wall_seconds": 116.59, "turn_p50": 113.28, "turn_p95": 116.25, "turn_max": 116.39, "peak_memory_gib": 13.64, "peak_pids": 19631, "cpu_seconds": 1597.5, "cpu_busy_ratio": 0.86, "target_cpu_seconds": 59.0, "target_busy_ratio": 0.002, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} diff --git a/bench/coganchor-concurrency/data/stock-nofile/ladder-kimi.jsonl b/bench/coganchor-concurrency/data/stock-nofile/ladder-kimi.jsonl new file mode 100644 index 00000000..93eeb435 --- /dev/null +++ b/bench/coganchor-concurrency/data/stock-nofile/ladder-kimi.jsonl @@ -0,0 +1,13 @@ +{"backend": "kimi", "concurrency": 1, "ok": 1, "failed": 0, "wall_seconds": 4.6, "turn_p50": 4.52, "turn_p95": 4.52, "turn_max": 4.52, "peak_memory_gib": 0.43, "peak_pids": 19, "cpu_seconds": 3.8, "cpu_busy_ratio": 0.05, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 2, "ok": 2, "failed": 0, "wall_seconds": 4.6, "turn_p50": 4.5, "turn_p95": 4.47, "turn_max": 4.53, "peak_memory_gib": 0.84, "peak_pids": 34, "cpu_seconds": 7.9, "cpu_busy_ratio": 0.11, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 4, "ok": 4, "failed": 0, "wall_seconds": 4.87, "turn_p50": 4.76, "turn_p95": 4.77, "turn_max": 4.79, "peak_memory_gib": 1.65, "peak_pids": 71, "cpu_seconds": 17.5, "cpu_busy_ratio": 0.22, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 8, "ok": 8, "failed": 0, "wall_seconds": 5.95, "turn_p50": 5.73, "turn_p95": 5.84, "turn_max": 5.87, "peak_memory_gib": 3.29, "peak_pids": 130, "cpu_seconds": 42.6, "cpu_busy_ratio": 0.45, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 16, "ok": 16, "failed": 0, "wall_seconds": 8.86, "turn_p50": 8.12, "turn_p95": 8.56, "turn_max": 8.78, "peak_memory_gib": 6.54, "peak_pids": 257, "cpu_seconds": 97.2, "cpu_busy_ratio": 0.69, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 24, "ok": 24, "failed": 0, "wall_seconds": 13.15, "turn_p50": 11.93, "turn_p95": 12.77, "turn_max": 13.07, "peak_memory_gib": 9.17, "peak_pids": 368, "cpu_seconds": 156.0, "cpu_busy_ratio": 0.74, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 32, "ok": 32, "failed": 0, "wall_seconds": 17.61, "turn_p50": 15.9, "turn_p95": 17.16, "turn_max": 17.52, "peak_memory_gib": 11.9, "peak_pids": 478, "cpu_seconds": 221.8, "cpu_busy_ratio": 0.79, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 48, "ok": 48, "failed": 0, "wall_seconds": 25.48, "turn_p50": 23.34, "turn_p95": 24.78, "turn_max": 25.38, "peak_memory_gib": 16.51, "peak_pids": 667, "cpu_seconds": 330.0, "cpu_busy_ratio": 0.81, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 64, "ok": 64, "failed": 0, "wall_seconds": 33.21, "turn_p50": 29.98, "turn_p95": 32.24, "turn_max": 33.11, "peak_memory_gib": 21.61, "peak_pids": 870, "cpu_seconds": 436.8, "cpu_busy_ratio": 0.82, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 96, "ok": 96, "failed": 0, "wall_seconds": 50.23, "turn_p50": 46.39, "turn_p95": 49.09, "turn_max": 50.1, "peak_memory_gib": 28.63, "peak_pids": 1171, "cpu_seconds": 667.7, "cpu_busy_ratio": 0.83, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 128, "ok": 128, "failed": 0, "wall_seconds": 66.39, "turn_p50": 62.52, "turn_p95": 65.84, "turn_max": 66.25, "peak_memory_gib": 43.73, "peak_pids": 1799, "cpu_seconds": 907.6, "cpu_busy_ratio": 0.85, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 160, "ok": 160, "failed": 0, "wall_seconds": 83.42, "turn_p50": 72.12, "turn_p95": 82.75, "turn_max": 83.27, "peak_memory_gib": 49.05, "peak_pids": 2046, "cpu_seconds": 1130.0, "cpu_busy_ratio": 0.85, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} +{"backend": "kimi", "concurrency": 192, "ok": 192, "failed": 0, "wall_seconds": 99.03, "turn_p50": 94.24, "turn_p95": 97.95, "turn_max": 98.86, "peak_memory_gib": 60.48, "peak_pids": 2455, "cpu_seconds": 1356.0, "cpu_busy_ratio": 0.86, "errors": [], "no_done": 0, "no_landing": 0, "steps_landed": [3]} diff --git a/bench/coganchor-concurrency/ladder.sh b/bench/coganchor-concurrency/ladder.sh new file mode 100644 index 00000000..780f3d3b --- /dev/null +++ b/bench/coganchor-concurrency/ladder.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Climb the concurrency ladder for one backend until it stops behaving. +# +# ladder.sh [max] +# +# A rung is "normal" when every agent on it finished its turn, said the sentence the scripted +# turn ends on, and left its work on the mock target. The climb stops at the first rung that +# is not, and at the one after it -- one bad rung can be a fluke, two in a row is a ceiling. +set -u + +BACKEND=$1 +MAX=${2:-192} +TMP=$(cd "$(dirname "$0")" && pwd) +LAB=${LAB:-$TMP/lab} +OUT=${OUT:-$LAB/ladder-$BACKEND.jsonl} +[ "${APPEND:-0}" = 1 ] || : > "$OUT" + +RUNGS=${RUNGS:-"1 2 4 8 16 24 32 48 64 96 128 160 192"} +bad=0 +for n in $RUNGS; do + [ "$n" -gt "$MAX" ] && break + line=$(timeout 1800 bash "$TMP/run_one.sh" "$BACKEND" "$n" 2>/dev/null | tail -1) + case "$line" in + '{'*) ;; + *) line="{\"backend\":\"$BACKEND\",\"concurrency\":$n,\"ok\":0,\"failed\":$n,\"errors\":[\"rung produced no summary\"]}" ;; + esac + echo "$line" >> "$OUT" + echo "$line" + failed=$(echo "$line" | python3 -c "import json,sys; print(json.load(sys.stdin).get('failed', -1))") + if [ "$failed" != "0" ]; then + bad=$((bad + 1)) + [ "$bad" -ge 2 ] && { echo "### $BACKEND: two bad rungs, stopping"; break; } + else + bad=0 + fi +done +echo "### $BACKEND ladder done -> $OUT" diff --git a/bench/coganchor-concurrency/ramp.py b/bench/coganchor-concurrency/ramp.py new file mode 100644 index 00000000..93d9433c --- /dev/null +++ b/bench/coganchor-concurrency/ramp.py @@ -0,0 +1,279 @@ +"""Run N anchored humanize agents at once and say whether all N behaved. + +Runs *inside* the constrained cgroup. One process, N agents, one session each, one turn +each, all going together -- the fan-out shape humanize documents, with every turn's work +landing on the mock target rather than here. + +Everything it needs from outside is already up when it starts: the stand-in model provider +and the ``hmz anchor serve`` standing in for the controlled end both run beyond the cgroup, +because neither is part of what is being sized. + +Prints one JSON object on stdout describing the run. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import statistics +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +PROMPT = "do the work" +DONE = "STANDIN-TURN-COMPLETE" + +#: Where the cgroup this is running in reports what it is using. +_CGROUP = Path("/sys/fs/cgroup") + + +def _own_cgroup() -> Path: + """The directory of the cgroup this process is in, for reading its own meters.""" + try: + line = Path("/proc/self/cgroup").read_text().strip().splitlines()[-1] + return _CGROUP / line.split(":")[-1].lstrip("/") + except (OSError, IndexError): + return _CGROUP + + +def _meter(where: Path) -> dict[str, float]: + def number(name: str, key: str | None = None) -> float: + try: + text = (where / name).read_text() + except OSError: + return -1.0 + if key is None: + return float(text.strip().split()[0]) if text.strip() else -1.0 + for line in text.splitlines(): + if line.startswith(key + " "): + return float(line.split()[1]) + return -1.0 + + return { + "memory_bytes": number("memory.current"), + "pids": number("pids.current"), + "cpu_usec": number("cpu.stat", "usage_usec"), + } + + +class Sampler(threading.Thread): + """Reads the cgroup's own meters while the run is going.""" + + def __init__(self, where: Path, every: float = 0.5) -> None: + super().__init__(daemon=True) + self.where = where + self.every = every + self.samples: list[dict[str, float]] = [] + self.stopped = threading.Event() + + def run(self) -> None: + while not self.stopped.wait(self.every): + sample = _meter(self.where) + sample["at"] = time.time() + self.samples.append(sample) + + +def _target_cpu() -> float: + """CPU seconds burnt so far by the mock controlled end, listeners and their children. + + Read so the report can say whether the stand-in was near its own limit: a ceiling found + while the target is idle is a fact about the machine running hmz, and one found while + the target is saturated is a fact about the stand-in. + """ + ticks = os.sysconf("SC_CLK_TCK") + total = 0.0 + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + command = (entry / "cmdline").read_bytes().replace(b"\0", b" ").decode() + if "anchor serve --listen" not in command: + continue + fields = (entry / "stat").read_text().rsplit(") ", 1)[-1].split() + # utime, stime, cutime, cstime -- the children matter most: every command a + # session runs on the target is one of them. + total += sum(int(fields[index]) for index in (11, 12, 13, 14)) / ticks + except (OSError, ValueError, IndexError): + continue + return total + + +def _prepare(lab: Path, index: int, seed_bytes: int) -> tuple[Path, Path]: + """A fresh mirror and a fresh copy on the mock target for one session.""" + workspace = lab / "ws" / str(index) + target = lab / "tgt" / str(index) + for path in (workspace, target): + shutil.rmtree(path, ignore_errors=True) + path.mkdir(parents=True, exist_ok=True) + # The target's data is the mocked part: a seeded file each turn reads, and a small tree + # around it so a listing is not trivially empty. + (target / "seed.txt").write_text("seeded on the target\n" + "x" * seed_bytes + "\n") + for name in ("README.md", "main.py", "notes.txt"): + (target / name).write_text(f"# {name}\n" + "line\n" * 40) + (target / "pkg").mkdir(exist_ok=True) + for number in range(8): + (target / "pkg" / f"mod{number}.py").write_text("def f():\n return %d\n" % number) + return workspace, target + + +#: The thinking level each backend is asked for. Uniform where it can be: dsh's ladder has +#: no bottom rung by that name, so it takes the lowest it has. +_EFFORT = {"dsh": "high"} + + +def _turn(backend: str, model: str, workspace: Path, target: Path, endpoint: str) -> dict[str, object]: + from hmz.agents import driver + from hmz.coganchor import AnchorConfig + from hmz.machines import AnchoredConfig + + record: dict[str, object] = {"workspace": str(workspace)} + started = time.time() + try: + agent_type, settings = driver(backend) + anchor = AnchorConfig( + target=endpoint, + workspace=str(workspace), + remote_path=str(target), + # The rig wipes and reseeds both sides of every slot before each rung, so the + # mirror is deliberately new each time -- and a slot that served a different + # listener on the previous rung would otherwise be refused as one that mirrors + # another target. + force=True, + ) + agent = agent_type( + settings( # type: ignore[call-arg] + model=model, + effort=_EFFORT.get(backend, "low"), + machine=AnchoredConfig(anchor=anchor), + permission="bypass", + # On for every backend rather than off: kimi and dsh have no way of being + # told not to search, and the scripted turn never reaches for it anyway, so + # this is the one setting all five can be given alike. + web_search=True, + ) + ) + session = agent.new(cwd=str(workspace)) + said = list(session.stream(PROMPT)) + record["answer"] = (said[-1].text if said else "")[-160:] + record["events"] = len(said) + try: + agent.stop() + except Exception: # noqa: BLE001,S110 -- a daemon that will not stop is not this turn's verdict + pass + except Exception as exc: # noqa: BLE001 -- every failure is a datum + record["error"] = f"{type(exc).__name__}: {exc}"[:1500] + record["seconds"] = round(time.time() - started, 2) + + landed = target / "touched.txt" + try: + record["steps_on_target"] = len( + [line for line in landed.read_text().splitlines() if line.strip()] + ) + except OSError: + record["steps_on_target"] = 0 + record["said_done"] = DONE in str(record.get("answer", "")) + record["ok"] = ( + bool(record["said_done"]) + and int(record["steps_on_target"]) >= 1 + and "error" not in record + ) + return record + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--backend", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--concurrency", type=int, required=True) + parser.add_argument("--lab", required=True) + parser.add_argument( + "--endpoint", + required=True, + help="tcp://HOST:PORT of the mock target; several, comma-separated, are spread over", + ) + parser.add_argument("--seed-bytes", type=int, default=4096) + args = parser.parse_args() + + lab = Path(args.lab) + slots = [_prepare(lab, index, args.seed_bytes) for index in range(args.concurrency)] + + where = _own_cgroup() + before = _meter(where) + sampler = Sampler(where) + sampler.start() + + # humanize echoes every turn's transcript. At two hundred agents that is a great deal + # of writing to one pipe, and it is not what is being measured, so it goes to a file for + # the length of the run and stdout is given back for the summary. + transcript = open(lab / "logs" / f"{args.backend}-{args.concurrency}.transcript", "w") + console, sys.stdout = sys.stdout, transcript + + endpoints = [one for one in args.endpoint.split(",") if one] + target_cpu_before = _target_cpu() + + started = time.time() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + futures = [ + pool.submit( + _turn, + args.backend, + args.model, + workspace, + target, + endpoints[index % len(endpoints)], + ) + for index, (workspace, target) in enumerate(slots) + ] + results = [future.result() for future in futures] + elapsed = time.time() - started + target_cpu = _target_cpu() - target_cpu_before + + sys.stdout = console + transcript.close() + + sampler.stopped.set() + sampler.join(timeout=2) + + times = sorted(float(one["seconds"]) for one in results) + good = [one for one in results if one["ok"]] + peak_memory = max((one["memory_bytes"] for one in sampler.samples), default=-1.0) + peak_pids = max((one["pids"] for one in sampler.samples), default=-1.0) + cpu_used = ( + (sampler.samples[-1]["cpu_usec"] - before["cpu_usec"]) / 1e6 + if sampler.samples and before["cpu_usec"] >= 0 + else -1.0 + ) + summary = { + "backend": args.backend, + "concurrency": args.concurrency, + "ok": len(good), + "failed": len(results) - len(good), + "wall_seconds": round(elapsed, 2), + "turn_p50": round(statistics.median(times), 2) if times else -1, + "turn_p95": round(times[max(0, int(len(times) * 0.95) - 1)], 2) if times else -1, + "turn_max": round(times[-1], 2) if times else -1, + "peak_memory_gib": round(peak_memory / (1 << 30), 2), + "peak_pids": int(peak_pids), + "cpu_seconds": round(cpu_used, 1), + "cpu_busy_ratio": round(cpu_used / elapsed / 16, 2) if elapsed > 0 and cpu_used >= 0 else -1, + "target_cpu_seconds": round(target_cpu, 1), + # Against the 208 CPUs the mock target has to itself, outside the cgroup. + "target_busy_ratio": round(target_cpu / elapsed / 208, 3) if elapsed > 0 else -1, + "errors": sorted({str(one.get("error", "")) for one in results if one.get("error")})[:5], + "no_done": sum(1 for one in results if not one["said_done"]), + "no_landing": sum(1 for one in results if not int(one["steps_on_target"])), + "steps_landed": sorted({int(one["steps_on_target"]) for one in results}), + } + print(json.dumps(summary), flush=True) + detail = lab / "runs" / f"{args.backend}-{args.concurrency}.json" + detail.parent.mkdir(parents=True, exist_ok=True) + detail.write_text(json.dumps({"summary": summary, "results": results}, indent=2)) + return 0 if summary["failed"] == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/coganchor-concurrency/refine.sh b/bench/coganchor-concurrency/refine.sh new file mode 100644 index 00000000..0fd178db --- /dev/null +++ b/bench/coganchor-concurrency/refine.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Narrow a backend's ceiling, keeping what the failures said. +# +# refine.sh [rung...] +# +# The ladder finds the ceiling to within a wide step; this walks the gap in small ones and +# keeps stderr, which the ladder throws away and which is where a dying agent explains +# itself. +set -u + +BACKEND=$1 +shift +TMP=$(cd "$(dirname "$0")" && pwd) +LAB=${LAB:-$TMP/lab} +OUT=$LAB/refine-$BACKEND.jsonl +: > "$OUT" +mkdir -p "$LAB/logs" + +for n in "$@"; do + err=$LAB/logs/refine-$BACKEND-$n.err + line=$(timeout 2400 bash "$TMP/run_one.sh" "$BACKEND" "$n" 2>"$err" | tail -1) + case "$line" in '{'*) ;; *) line="{\"backend\":\"$BACKEND\",\"concurrency\":$n,\"ok\":0,\"failed\":$n}" ;; esac + echo "$line" >> "$OUT" + echo "$line" | python3 -c " +import json, sys +one = json.load(sys.stdin) +print(f\"{one['backend']:7} N={one['concurrency']:<4} ok={one['ok']:<4} failed={one['failed']:<4} \" + f\"p95={one.get('turn_p95', -1)}s mem={one.get('peak_memory_gib', -1)}GiB \" + f\"pids={one.get('peak_pids', -1)} cpu={one.get('cpu_busy_ratio', -1)} \" + f\"target={one.get('target_busy_ratio', -1)}\") +" + if [ -s "$err" ]; then + echo " stderr (first distinct lines):" + sort -u "$err" | grep -viE "^\s*$" | head -4 | sed 's/^/ /' + fi +done +echo "### $BACKEND refined -> $OUT" diff --git a/bench/coganchor-concurrency/run_one.sh b/bench/coganchor-concurrency/run_one.sh new file mode 100644 index 00000000..68b571a7 --- /dev/null +++ b/bench/coganchor-concurrency/run_one.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# One rung of the ladder: N anchored agents of one backend, inside 16 CPUs and 64 GiB. +# +# run_one.sh +# +# The cgroup is the whole of the constraint. AllowedCPUs names eight physical cores and +# their siblings -- sixteen logical CPUs, which is what a 16-vCPU machine is -- so anything +# below that asks the kernel how wide it is gets the right answer. +set -u + +BACKEND=$1 +TMP=$(cd "$(dirname "$0")" && pwd) +N=$2 +LAB=${LAB:-$TMP/lab} +ROOT=$(cd "$TMP/../.." && pwd) +PY=$ROOT/.venv/bin/python +MODEL_PORT=${MODEL_PORT:-18081} +TARGET_PORT=${TARGET_PORT:-18090} +BASE="http://127.0.0.1:$MODEL_PORT" +LISTENERS=${LISTENERS:-4} +ENDPOINTS="" +for n in $(seq 0 $((LISTENERS - 1))); do + ENDPOINTS="$ENDPOINTS${ENDPOINTS:+,}tcp://127.0.0.1:$((TARGET_PORT + n))" +done +H="$LAB/homes/$BACKEND" +PATHS=${AGENT_PATH:-$HOME/.local/agents/bin:$HOME/.local/bin}:/usr/local/bin:/usr/bin:/bin + +case "$BACKEND" in +claude) + MODEL=claude-sonnet-4-5 + ENVS=(ANTHROPIC_BASE_URL="$BASE" ANTHROPIC_API_KEY=standin-key + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1) ;; +codex) + MODEL=standin-1 + ENVS=(CODEX_HOME="$H/.codex" OPENAI_API_KEY=standin-key OPENAI_BASE_URL="$BASE/v1") ;; +grok) + MODEL=standin-1 + ENVS=(XAI_API_KEY=standin-key) ;; +kimi) + MODEL=standin/standin-1 + ENVS=(STANDIN_API_KEY=standin-key) ;; +dsh) + MODEL=deepseek-chat + ENVS=(DEEPSEEK_API_KEY=standin-key DEEPSEEK_BASE_URL="$BASE/v1") ;; +*) echo "unknown backend $BACKEND" >&2; exit 2 ;; +esac + +# A scope takes no `LimitNOFILE`, and systemd hands it the stock 1024 soft limit against a +# 1048576 hard one. Three descriptors per concurrent agent means that alone stops hmz at +# about 320 -- a real limit, and the first one anybody meets, but a fact about the login +# defaults rather than about the machine. `NOFILE=1024` asks for it back. +exec sudo systemd-run --scope --quiet --uid="$(id -u)" --gid="$(id -g)" \ + -p AllowedCPUs=0-7,112-119 \ + -p MemoryMax=64G -p MemorySwapMax=0 -p TasksMax=infinity \ + -- /bin/sh -c 'ulimit -n "$1" || true; shift; exec "$@"' _ "${NOFILE:-1048576}" \ + env -i HOME="$H" PATH="$PATHS" TERM=dumb LANG=C.UTF-8 \ + HUMANIZE_SENTRY=off HUMANIZE_TELEMETRY=off \ + "${ENVS[@]}" \ + "$PY" $TMP/ramp.py \ + --backend "$BACKEND" --model "$MODEL" --concurrency "$N" \ + --lab "$LAB" --endpoint "$ENDPOINTS" diff --git a/bench/coganchor-concurrency/standin_model.py b/bench/coganchor-concurrency/standin_model.py new file mode 100644 index 00000000..f1aefc1f --- /dev/null +++ b/bench/coganchor-concurrency/standin_model.py @@ -0,0 +1,724 @@ +"""A model provider that scripts the same turn for every agent CLI under test. + +Five coding agents, three wire protocols. This answers all three, and answers them the +same way: a fixed number of shell tool calls, then a final sentence. What each agent does +under coganchor is therefore identical work, which is what makes the concurrency numbers +comparable across backends. + + POST /v1/messages Anthropic Messages (claude) + POST /v1/responses OpenAI Responses (codex, grok) + POST /v1/chat/completions OpenAI Chat (dsh, kimi) + GET /v1/models a catalogue (grok, others probing) + GET /api.json a models.dev registry (kimi provider add) + +It runs beside the harness rather than inside the measured cgroup: it stands in for a model +provider, which is not part of what is being sized. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +#: How many shell tool calls one scripted turn makes before it answers. +STEPS = int(os.environ.get("STANDIN_STEPS", "3")) + +#: The shell command each of those calls runs, formatted with the step number. It reads a +#: seeded file and writes one, so every step is a full coganchor round trip: a read +#: materialised from the target, a command run there, and a write pushed back. +COMMAND = os.environ.get( + "STANDIN_COMMAND", + "cat seed.txt && echo step-{step} >> touched.txt && ls -1 | wc -l", +) + +#: What the turn says when it is done, so a harness can tell a finished turn from a +#: truncated one. +FINAL = os.environ.get("STANDIN_FINAL", "STANDIN-TURN-COMPLETE") + +#: Names that mean "run this shell command", best first. Five CLIs spell it five ways -- +#: `Bash`, `shell`, `run_terminal_command`, `execute_command` -- so the pick is scored +#: rather than matched exactly, and a name no pattern claims leaves the turn tool-free. +_SHELL_PATTERNS = ( + re.compile(r"^(bash|shell|sh|local_shell)$", re.IGNORECASE), + re.compile(r"^(run|execute|exec)_?(terminal_?)?(command|shell|bash)s?$", re.IGNORECASE), + re.compile(r"terminal.*command|command.*terminal", re.IGNORECASE), + re.compile(r"^(run|execute|exec)[_-]?", re.IGNORECASE), + re.compile(r"(^|_)(bash|shell|terminal)(_|$)", re.IGNORECASE), +) + +_MODEL_IDS = ( + "standin-1", + "claude-sonnet-4-5", + "gpt-5-codex", + "grok-code", + "deepseek-chat", + "kimi-k2", +) + +_log_lock = threading.Lock() +LOG_PATH = os.environ.get("STANDIN_LOG", "") + + +def _log(kind: str, detail: object) -> None: + if not LOG_PATH: + return + with _log_lock, open(LOG_PATH, "a") as handle: + handle.write(json.dumps({"at": time.time(), "kind": kind, "detail": detail}) + "\n") + + +# --------------------------------------------------------------------------- tool choice + + +def _fill(schema: dict[str, Any], command: str) -> dict[str, Any]: + """Builds arguments for a tool out of the schema the agent declared it with. + + Reading the schema rather than hard-coding one shape per CLI is what keeps this working + across five different tool vocabularies. + """ + properties = schema.get("properties") + if not isinstance(properties, dict) or not properties: + return {"command": command} + required = schema.get("required") + required = list(required) if isinstance(required, list) else list(properties) + filled: dict[str, Any] = {} + for name in required: + spec = properties.get(name) + spec = spec if isinstance(spec, dict) else {} + kind = spec.get("type") + if isinstance(kind, list): + kind = next((one for one in kind if one != "null"), "string") + lowered = str(name).lower() + if lowered in ("command", "cmd", "script", "shell_command", "commandline"): + if kind == "array": + filled[name] = ["bash", "-lc", command] + else: + filled[name] = command + elif kind == "array": + filled[name] = [] + elif kind == "boolean": + filled[name] = False + elif kind in ("number", "integer"): + filled[name] = 60000 if "timeout" in lowered else 0 + elif kind == "object": + filled[name] = {} + else: + filled[name] = "sizing coganchor concurrency" + # A tool whose required list never named the command still has to be given one. + if not any( + str(key).lower() in ("command", "cmd", "script", "shell_command", "commandline") + for key in filled + ): + for name, spec in properties.items(): + if str(name).lower() in ("command", "cmd", "script"): + spec = spec if isinstance(spec, dict) else {} + filled[name] = ( + ["bash", "-lc", command] if spec.get("type") == "array" else command + ) + break + return filled + + +def _declared(body: dict[str, Any]) -> list[tuple[str, dict[str, Any], str]]: + """Every tool the request declared, as ``(name, schema, kind)``.""" + found: list[tuple[str, dict[str, Any], str]] = [] + tools = body.get("tools") + if not isinstance(tools, list): + return found + for tool in tools: + if not isinstance(tool, dict): + continue + kind = str(tool.get("type") or "function") + if kind == "local_shell": + found.append(("local_shell", {}, "local_shell")) + continue + inner = tool.get("function") + if isinstance(inner, dict): # chat completions + name = str(inner.get("name") or "") + schema = inner.get("parameters") + else: # responses / anthropic + name = str(tool.get("name") or "") + schema = tool.get("parameters") or tool.get("input_schema") + if name: + found.append((name, schema if isinstance(schema, dict) else {}, kind)) + return found + + +def _shell_tool(body: dict[str, Any]) -> tuple[str, dict[str, Any], str] | None: + declared = _declared(body) + for pattern in _SHELL_PATTERNS: + for entry in declared: + if pattern.search(entry[0]): + return entry + return None + + +def _steps_taken_anthropic(body: dict[str, Any]) -> int: + taken = 0 + for message in body.get("messages") or []: + if not isinstance(message, dict): + continue + content = message.get("content") + if isinstance(content, list): + taken += sum( + 1 + for block in content + if isinstance(block, dict) and block.get("type") == "tool_result" + ) + return taken + + +def _steps_taken_responses(body: dict[str, Any]) -> int: + taken = 0 + items = body.get("input") + if isinstance(items, list): + for item in items: + if isinstance(item, dict) and str(item.get("type", "")).endswith( + ("function_call_output", "local_shell_call_output", "custom_tool_call_output") + ): + taken += 1 + return taken + + +def _steps_taken_chat(body: dict[str, Any]) -> int: + return sum( + 1 + for message in body.get("messages") or [] + if isinstance(message, dict) and message.get("role") == "tool" + ) + + +# --------------------------------------------------------------------------- wire shapes + + +def _sse(handler: BaseHTTPRequestHandler, chunks: list[tuple[str | None, Any]]) -> None: + """Writes one scripted turn as an event stream, and ends it by closing the connection. + + Closing is the part that matters. A stream left open on keep-alive with no + content-length never ends as far as the client is concerned: Claude Code runs the + turn's tool call and then waits out its own timeout instead of asking for the next + step, which reads exactly like an agent that gave up after one tool. + """ + handler.send_response(200) + handler.send_header("content-type", "text/event-stream") + handler.send_header("cache-control", "no-cache") + handler.send_header("connection", "close") + handler.end_headers() + for event, payload in chunks: + blob = payload if isinstance(payload, str) else json.dumps(payload) + line = (f"event: {event}\n" if event else "") + f"data: {blob}\n\n" + handler.wfile.write(line.encode()) + handler.wfile.flush() + handler.close_connection = True + + +def _anthropic(body: dict[str, Any]) -> list[tuple[str | None, Any]]: + model = str(body.get("model") or "standin-1") + step = _steps_taken_anthropic(body) + tool = _shell_tool(body) + usage = { + "input_tokens": 1000 + 100 * step, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + start: list[tuple[str | None, Any]] = [ + ( + "message_start", + { + "type": "message_start", + "message": { + "id": f"msg_{uuid.uuid4().hex[:16]}", + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": usage, + }, + }, + ) + ] + if tool is None or step >= STEPS: + text = FINAL if step >= STEPS or tool is None else FINAL + return [ + *start, + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": text}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": usage, + }, + ), + ("message_stop", {"type": "message_stop"}), + ] + name, schema, _ = tool + arguments = json.dumps(_fill(schema, COMMAND.format(step=step + 1))) + return [ + *start, + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": f"step {step + 1}"}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "tool_use", + "id": f"toolu_{uuid.uuid4().hex[:16]}", + "name": name, + "input": {}, + }, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "input_json_delta", "partial_json": arguments}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 1}), + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": usage, + }, + ), + ("message_stop", {"type": "message_stop"}), + ] + + +def _responses(body: dict[str, Any]) -> list[tuple[str | None, Any]]: + model = str(body.get("model") or "standin-1") + step = _steps_taken_responses(body) + tool = _shell_tool(body) + response_id = f"resp_{uuid.uuid4().hex[:16]}" + usage = { + "input_tokens": 1000 + 100 * step, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 40, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 1040 + 100 * step, + } + counter = {"n": 0} + + def numbered(event: str, payload: dict[str, Any]) -> tuple[str, Any]: + payload["sequence_number"] = counter["n"] + counter["n"] += 1 + return (event, payload) + + shell: dict[str, Any] + if tool is None or step >= STEPS: + item = { + "type": "message", + "id": f"msg_{uuid.uuid4().hex[:16]}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": FINAL, "annotations": []}], + } + elif tool[2] == "local_shell": + item = { + "type": "local_shell_call", + "id": f"lsh_{uuid.uuid4().hex[:16]}", + "call_id": f"call_{uuid.uuid4().hex[:16]}", + "status": "completed", + "action": { + "type": "exec", + "command": ["bash", "-lc", COMMAND.format(step=step + 1)], + "timeout_ms": 60000, + }, + } + else: + name, schema, _ = tool + shell = _fill(schema, COMMAND.format(step=step + 1)) + item = { + "type": "function_call", + "id": f"fc_{uuid.uuid4().hex[:16]}", + "call_id": f"call_{uuid.uuid4().hex[:16]}", + "name": name, + "arguments": json.dumps(shell), + "status": "completed", + } + + envelope = { + "id": response_id, + "object": "response", + "created_at": int(time.time()), + "model": model, + "status": "in_progress", + "output": [], + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": {}, + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": None, + } + done = dict(envelope, status="completed", output=[item], usage=usage) + chunks: list[tuple[str | None, Any]] = [ + numbered("response.created", {"type": "response.created", "response": dict(envelope)}), + numbered( + "response.in_progress", + {"type": "response.in_progress", "response": dict(envelope)}, + ), + numbered( + "response.output_item.added", + {"type": "response.output_item.added", "output_index": 0, "item": dict(item)}, + ), + ] + if item["type"] == "message": + chunks.append( + numbered( + "response.output_text.delta", + { + "type": "response.output_text.delta", + "item_id": item["id"], + "output_index": 0, + "content_index": 0, + "delta": FINAL, + }, + ) + ) + chunks.append( + numbered( + "response.output_text.done", + { + "type": "response.output_text.done", + "item_id": item["id"], + "output_index": 0, + "content_index": 0, + "text": FINAL, + }, + ) + ) + elif item["type"] == "function_call": + chunks.append( + numbered( + "response.function_call_arguments.delta", + { + "type": "response.function_call_arguments.delta", + "item_id": item["id"], + "output_index": 0, + "delta": item["arguments"], + }, + ) + ) + chunks.append( + numbered( + "response.function_call_arguments.done", + { + "type": "response.function_call_arguments.done", + "item_id": item["id"], + "output_index": 0, + "arguments": item["arguments"], + }, + ) + ) + chunks.append( + numbered( + "response.output_item.done", + {"type": "response.output_item.done", "output_index": 0, "item": dict(item)}, + ) + ) + chunks.append( + numbered("response.completed", {"type": "response.completed", "response": done}) + ) + return chunks + + +def _chat_chunks(body: dict[str, Any]) -> tuple[dict[str, Any], list[tuple[str | None, Any]]]: + model = str(body.get("model") or "standin-1") + step = _steps_taken_chat(body) + tool = _shell_tool(body) + made = f"chatcmpl-{uuid.uuid4().hex[:16]}" + created = int(time.time()) + usage = { + "prompt_tokens": 1000 + 100 * step, + "completion_tokens": 40, + "total_tokens": 1040 + 100 * step, + "prompt_tokens_details": {"cached_tokens": 0}, + } + head = {"id": made, "object": "chat.completion.chunk", "created": created, "model": model} + + if tool is None or step >= STEPS: + message = {"role": "assistant", "content": FINAL} + finish = "stop" + deltas: list[dict[str, Any]] = [ + {"role": "assistant", "content": ""}, + {"content": FINAL}, + ] + else: + name, schema, _ = tool + arguments = json.dumps(_fill(schema, COMMAND.format(step=step + 1))) + call_id = f"call_{uuid.uuid4().hex[:16]}" + message = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": arguments}, + } + ], + } + finish = "tool_calls" + deltas = [ + {"role": "assistant", "content": None}, + { + "tool_calls": [ + { + "index": 0, + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": arguments}, + } + ] + }, + ] + + chunks: list[tuple[str | None, Any]] = [ + (None, dict(head, choices=[{"index": 0, "delta": delta, "finish_reason": None}])) + for delta in deltas + ] + chunks.append( + (None, dict(head, choices=[{"index": 0, "delta": {}, "finish_reason": finish}], usage=usage)) + ) + chunks.append((None, "[DONE]")) + whole = { + "id": made, + "object": "chat.completion", + "created": created, + "model": model, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": usage, + } + return whole, chunks + + +# --------------------------------------------------------------------------- the server + + +def _catalogue() -> dict[str, Any]: + return { + "object": "list", + "data": [ + {"id": name, "object": "model", "created": 1700000000, "owned_by": "standin"} + for name in _MODEL_IDS + ], + } + + +def _registry(port: int) -> dict[str, Any]: + """A models.dev-shaped api.json, which is what ``kimi provider add`` imports.""" + return { + "standin": { + "id": "standin", + "name": "Standin", + # `type` is Kimi's own addition to the models.dev shape, and an entry without + # one is skipped as invalid. + "type": "openai", + "npm": "@ai-sdk/openai-compatible", + "api": f"http://127.0.0.1:{port}/v1", + "env": ["STANDIN_API_KEY"], + "doc": "http://127.0.0.1/standin", + "models": { + "standin-1": { + "id": "standin-1", + "name": "Standin 1", + "attachment": False, + "reasoning": False, + "tool_call": True, + "temperature": True, + "knowledge": "2026-01", + "release_date": "2026-01-01", + "last_updated": "2026-01-01", + "modalities": {"input": ["text"], "output": ["text"]}, + "open_weights": False, + "cost": {"input": 0.0, "output": 0.0}, + "limit": {"context": 200000, "output": 32000}, + } + }, + } + } + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + port = 0 + + def log_message(self, *args: object) -> None: + pass + + def _json(self, payload: object, status: int = 200) -> None: + blob = json.dumps(payload).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(blob))) + self.end_headers() + self.wfile.write(blob) + + def do_GET(self) -> None: + path = self.path.split("?")[0] + if path.endswith("api.json"): + self._json(_registry(self.port)) + elif path.endswith("/models"): + self._json(_catalogue()) + elif path.endswith("/api-key"): + self._json( + { + "redacted_api_key": "standin", + "name": "standin", + "acls": ["api-key:model:*"], + "api_key_blocked": False, + "api_key_disabled": False, + "team_blocked": False, + } + ) + else: + self._json({"object": "ok"}) + + def do_POST(self) -> None: + length = int(self.headers.get("content-length") or 0) + raw = self.rfile.read(length) if length else b"{}" + try: + body = json.loads(raw or b"{}") + except ValueError: + body = {} + body = body if isinstance(body, dict) else {} + path = self.path.split("?")[0] + streaming = bool(body.get("stream", False)) + try: + if path.endswith("/messages"): + _log( + "messages", + { + "step": _steps_taken_anthropic(body), + "tools": [one[0] for one in _declared(body)], + "picked": (_shell_tool(body) or ("-",))[0], + }, + ) + chunks = _anthropic(body) + if streaming: + _sse(self, chunks) + else: + self._json(_anthropic_whole(body)) + elif path.endswith("/responses"): + _log( + "responses", + { + "step": _steps_taken_responses(body), + "tools": [one[0] for one in _declared(body)], + "picked": (_shell_tool(body) or ("-",))[0], + }, + ) + chunks = _responses(body) + if streaming: + _sse(self, chunks) + else: + self._json(json.loads(json.dumps(chunks[-1][1]))["response"]) + elif path.endswith("/chat/completions"): + _log( + "chat", + { + "step": _steps_taken_chat(body), + "tools": [one[0] for one in _declared(body)], + "picked": (_shell_tool(body) or ("-",))[0], + }, + ) + whole, chunks = _chat_chunks(body) + if streaming: + _sse(self, chunks) + else: + self._json(whole) + else: + self._json({"error": {"message": f"no route {path}", "type": "not_found"}}, 404) + except (BrokenPipeError, ConnectionResetError): + pass + + +def _anthropic_whole(body: dict[str, Any]) -> dict[str, Any]: + """The same scripted turn, for a client that asked for it unstreamed.""" + step = _steps_taken_anthropic(body) + tool = _shell_tool(body) + content: list[dict[str, Any]] + if tool is None or step >= STEPS: + content = [{"type": "text", "text": FINAL}] + stop = "end_turn" + else: + name, schema, _ = tool + content = [ + {"type": "text", "text": f"step {step + 1}"}, + { + "type": "tool_use", + "id": f"toolu_{uuid.uuid4().hex[:16]}", + "name": name, + "input": _fill(schema, COMMAND.format(step=step + 1)), + }, + ] + stop = "tool_use" + return { + "id": f"msg_{uuid.uuid4().hex[:16]}", + "type": "message", + "role": "assistant", + "model": str(body.get("model") or "standin-1"), + "content": content, + "stop_reason": stop, + "stop_sequence": None, + "usage": {"input_tokens": 1000, "output_tokens": 40}, + } + + +if __name__ == "__main__": + port = int(sys.argv[1]) + Handler.port = port + server = ThreadingHTTPServer(("127.0.0.1", port), Handler) + server.daemon_threads = True + print(f"standin model on {port}", flush=True) + server.serve_forever() diff --git a/bench/coganchor-concurrency/summarise.py b/bench/coganchor-concurrency/summarise.py new file mode 100644 index 00000000..dfc7f5f8 --- /dev/null +++ b/bench/coganchor-concurrency/summarise.py @@ -0,0 +1,98 @@ +"""Read the ladders and say, per backend, how many anchored agents 16 CPUs and 64 GiB hold. + +Two numbers per backend, because they answer different questions: + +``correct`` + The widest rung on which every agent finished its turn and left its work on the target. + The ceiling on what works at all. +``comfortable`` + The widest rung that is also within twice the turn latency of a single agent. Past it + the machine is still correct and simply slower, which for a flow of long turns may be + perfectly acceptable -- so it is reported beside the ceiling rather than instead of it. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +LAB = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).resolve().parent / "lab" +SLOWDOWN = 2.0 + +BACKENDS = ("claude", "codex", "grok", "kimi", "dsh") + + +def rungs(backend: str) -> list[dict[str, object]]: + path = LAB / f"ladder-{backend}.jsonl" + if not path.exists(): + return [] + out = [] + for line in path.read_text().splitlines(): + line = line.strip() + if line.startswith("{"): + out.append(json.loads(line)) + return out + + +print( + f"{'backend':8} {'N':>4} {'ok':>4} {'fail':>4} {'p50 s':>7} {'p95 s':>7} " + f"{'wall s':>7} {'turns/min':>10} {'mem GiB':>8} {'pids':>6} {'cpu%':>6} {'tgt%':>5}" +) +print("-" * 92) +verdicts: dict[str, dict[str, object]] = {} +for backend in BACKENDS: + ladder = rungs(backend) + if not ladder: + continue + base = next((r for r in ladder if r["concurrency"] == 1), None) + baseline = float(base["turn_p50"]) if base else 0.0 + correct = 0 + comfortable = 0 + peak_rate = 0.0 + for rung in ladder: + n = int(rung["concurrency"]) + wall = float(rung.get("wall_seconds", 0)) or 1.0 + rate = int(rung["ok"]) / wall * 60 + print( + f"{backend:8} {n:4d} {int(rung['ok']):4d} {int(rung['failed']):4d} " + f"{float(rung.get('turn_p50', -1)):7.2f} {float(rung.get('turn_p95', -1)):7.2f} " + f"{float(rung.get('wall_seconds', -1)):7.1f} {rate:10.0f} " + f"{float(rung.get('peak_memory_gib', -1)):8.2f} " + f"{int(rung.get('peak_pids', -1)):6d} " + f"{float(rung.get('cpu_busy_ratio', -1)) * 100:5.0f}% " + f"{float(rung.get('target_busy_ratio', 0)) * 100:4.1f}%" + ) + if int(rung["failed"]) == 0: + correct = max(correct, n) + peak_rate = max(peak_rate, rate) + if baseline and float(rung["turn_p95"]) <= SLOWDOWN * baseline: + comfortable = max(comfortable, n) + per = next((r for r in ladder if int(r["concurrency"]) == correct), None) + verdicts[backend] = { + "correct": correct, + "comfortable": comfortable, + "mem_per_agent_mib": ( + round(float(per["peak_memory_gib"]) * 1024 / correct, 1) + if per and correct + else -1 + ), + "pids_per_agent": ( + round(int(per["peak_pids"]) / correct, 1) if per and correct else -1 + ), + "errors": (per or {}).get("errors", []), + "peak_turns_per_min": round(peak_rate), + } + print("-" * 92) + +print() +print(f"{'backend':8} {'all correct up to':>18} {'and still quick to':>19} " + f"{'MiB/agent':>10} {'procs/agent':>12} {'peak turns/min':>15}") +for backend, verdict in verdicts.items(): + print( + f"{backend:8} {verdict['correct']:18d} {verdict['comfortable']:19d} " + f"{verdict['mem_per_agent_mib']:10} {verdict['pids_per_agent']:12} " + f"{verdict['peak_turns_per_min']:15}" + ) +print() +print(json.dumps(verdicts, indent=2)) From e3dda1f08e2518746f8f524fe2906ff622bf6e14 Mon Sep 17 00:00:00 2001 From: Zijian Zhang <35801754+futrime@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:36:40 +0000 Subject: [PATCH 4/5] fix(bench): find the agent CLIs when they are not already on PATH `command -v kimi` answers nothing in a shell that has not been given `~/.local/agents/bin`, and `dirname ""` is `.`, so the rig went looking for `./kimi` and reported the provider import as failed. $AGENT_BIN says where they are outright, the home default stands in, and a missing one is now said plainly rather than three lines later. Also ignores the `lab/` the rig writes beside itself. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 3 +++ bench/coganchor-concurrency/backdrop.sh | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b8439954..3cd0b6b1 100644 --- a/.gitignore +++ b/.gitignore @@ -223,3 +223,6 @@ __marimo__/ .humanize/ *.trace.json + +# The concurrency rig writes its workspaces, mock target and results here. +bench/*/lab/ diff --git a/bench/coganchor-concurrency/backdrop.sh b/bench/coganchor-concurrency/backdrop.sh index d34ccc83..7a8cfc51 100644 --- a/bench/coganchor-concurrency/backdrop.sh +++ b/bench/coganchor-concurrency/backdrop.sh @@ -15,7 +15,15 @@ ROOT=$(cd "$TMP/../.." && pwd) MODEL_PORT=${MODEL_PORT:-18081} TARGET_PORT=${TARGET_PORT:-18090} SLOTS=${SLOTS:-192} -AB=${AGENT_BIN:-$(dirname "$(command -v kimi)")} +# Where the agent CLIs are. `command -v` only finds them if this shell already has them on +# PATH, which it need not; $AGENT_BIN says so outright. +AB=${AGENT_BIN:-} +if [ -z "$AB" ]; then + _kimi=$(command -v kimi || true) + [ -n "$_kimi" ] && AB=$(dirname "$_kimi") +fi +AB=${AB:-$HOME/.local/agents/bin} +[ -x "$AB/kimi" ] || { echo "no kimi under $AB; set AGENT_BIN"; exit 1; } stop() { for pid in $(pgrep -f "standin_model[.]py" || true); do kill "$pid" 2>/dev/null || true; done From b8114bef073b5793cecceaa9e4ab3d2e2f15896c Mon Sep 17 00:00:00 2001 From: Zijian Zhang <35801754+futrime@users.noreply.github.com> Date: Sun, 30 Aug 2026 01:20:36 +0000 Subject: [PATCH 5/5] fix(bench): make the concurrency rig pass the first gate The rig's three scripts had never been through `ruff check`: `bench/` does not exist on main, so nothing linted them where they were written. Fixed rather than ignored, because each was a real thing to say: - `standin_model.py` picked its final text with `FINAL if ... else FINAL`, a conditional whose test was the enclosing `if` and whose arms were the same value. Inlined. - `_fill` narrowed a schema onto the loop variable it came from, so the declared property and the narrowed dict were one name. Now two. - `ramp.py` swapped `sys.stdout` for a file by hand and put it back at the far end of the body. `contextlib.redirect_stdout` is what the standard library has for that, and it puts stdout back however the block ends. - `%d` formatting, a `try`/`except`/`pass`, and `open()` where the module already holds a `Path`. - Docstrings on the three public names that had none. `INP001` and `T201` are ignored for `bench/**` instead: a script run by path inside the measured cgroup has nothing for an `__init__.py` to do, and what a measurement writes, it prints. Same reason `docs/tapes/stage.py` is already listed. --- bench/coganchor-concurrency/ramp.py | 74 ++++++++++-------- bench/coganchor-concurrency/standin_model.py | 80 +++++++++++++++----- bench/coganchor-concurrency/summarise.py | 19 ++--- pyproject.toml | 5 ++ 4 files changed, 120 insertions(+), 58 deletions(-) diff --git a/bench/coganchor-concurrency/ramp.py b/bench/coganchor-concurrency/ramp.py index 93d9433c..c5208587 100644 --- a/bench/coganchor-concurrency/ramp.py +++ b/bench/coganchor-concurrency/ramp.py @@ -14,6 +14,7 @@ from __future__ import annotations import argparse +import contextlib import json import os import shutil @@ -116,7 +117,9 @@ def _prepare(lab: Path, index: int, seed_bytes: int) -> tuple[Path, Path]: (target / name).write_text(f"# {name}\n" + "line\n" * 40) (target / "pkg").mkdir(exist_ok=True) for number in range(8): - (target / "pkg" / f"mod{number}.py").write_text("def f():\n return %d\n" % number) + (target / "pkg" / f"mod{number}.py").write_text( + f"def f():\n return {number}\n" + ) return workspace, target @@ -125,7 +128,9 @@ def _prepare(lab: Path, index: int, seed_bytes: int) -> tuple[Path, Path]: _EFFORT = {"dsh": "high"} -def _turn(backend: str, model: str, workspace: Path, target: Path, endpoint: str) -> dict[str, object]: +def _turn( + backend: str, model: str, workspace: Path, target: Path, endpoint: str +) -> dict[str, object]: from hmz.agents import driver from hmz.coganchor import AnchorConfig from hmz.machines import AnchoredConfig @@ -160,10 +165,9 @@ def _turn(backend: str, model: str, workspace: Path, target: Path, endpoint: str said = list(session.stream(PROMPT)) record["answer"] = (said[-1].text if said else "")[-160:] record["events"] = len(said) - try: + # A daemon that will not stop is not this turn's verdict. + with contextlib.suppress(Exception): agent.stop() - except Exception: # noqa: BLE001,S110 -- a daemon that will not stop is not this turn's verdict - pass except Exception as exc: # noqa: BLE001 -- every failure is a datum record["error"] = f"{type(exc).__name__}: {exc}"[:1500] record["seconds"] = round(time.time() - started, 2) @@ -185,6 +189,7 @@ def _turn(backend: str, model: str, workspace: Path, target: Path, endpoint: str def main() -> int: + """Run one rung -- N agents at once -- and print a JSON object describing it.""" parser = argparse.ArgumentParser() parser.add_argument("--backend", required=True) parser.add_argument("--model", required=True) @@ -209,31 +214,28 @@ def main() -> int: # humanize echoes every turn's transcript. At two hundred agents that is a great deal # of writing to one pipe, and it is not what is being measured, so it goes to a file for # the length of the run and stdout is given back for the summary. - transcript = open(lab / "logs" / f"{args.backend}-{args.concurrency}.transcript", "w") - console, sys.stdout = sys.stdout, transcript - endpoints = [one for one in args.endpoint.split(",") if one] - target_cpu_before = _target_cpu() - - started = time.time() - with ThreadPoolExecutor(max_workers=args.concurrency) as pool: - futures = [ - pool.submit( - _turn, - args.backend, - args.model, - workspace, - target, - endpoints[index % len(endpoints)], - ) - for index, (workspace, target) in enumerate(slots) - ] - results = [future.result() for future in futures] - elapsed = time.time() - started - target_cpu = _target_cpu() - target_cpu_before - sys.stdout = console - transcript.close() + transcript = lab / "logs" / f"{args.backend}-{args.concurrency}.transcript" + with transcript.open("w") as handle, contextlib.redirect_stdout(handle): + target_cpu_before = _target_cpu() + + started = time.time() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + futures = [ + pool.submit( + _turn, + args.backend, + args.model, + workspace, + target, + endpoints[index % len(endpoints)], + ) + for index, (workspace, target) in enumerate(slots) + ] + results = [future.result() for future in futures] + elapsed = time.time() - started + target_cpu = _target_cpu() - target_cpu_before sampler.stopped.set() sampler.join(timeout=2) @@ -254,16 +256,24 @@ def main() -> int: "failed": len(results) - len(good), "wall_seconds": round(elapsed, 2), "turn_p50": round(statistics.median(times), 2) if times else -1, - "turn_p95": round(times[max(0, int(len(times) * 0.95) - 1)], 2) if times else -1, + "turn_p95": round(times[max(0, int(len(times) * 0.95) - 1)], 2) + if times + else -1, "turn_max": round(times[-1], 2) if times else -1, "peak_memory_gib": round(peak_memory / (1 << 30), 2), "peak_pids": int(peak_pids), "cpu_seconds": round(cpu_used, 1), - "cpu_busy_ratio": round(cpu_used / elapsed / 16, 2) if elapsed > 0 and cpu_used >= 0 else -1, + "cpu_busy_ratio": round(cpu_used / elapsed / 16, 2) + if elapsed > 0 and cpu_used >= 0 + else -1, "target_cpu_seconds": round(target_cpu, 1), # Against the 208 CPUs the mock target has to itself, outside the cgroup. - "target_busy_ratio": round(target_cpu / elapsed / 208, 3) if elapsed > 0 else -1, - "errors": sorted({str(one.get("error", "")) for one in results if one.get("error")})[:5], + "target_busy_ratio": round(target_cpu / elapsed / 208, 3) + if elapsed > 0 + else -1, + "errors": sorted( + {str(one.get("error", "")) for one in results if one.get("error")} + )[:5], "no_done": sum(1 for one in results if not one["said_done"]), "no_landing": sum(1 for one in results if not int(one["steps_on_target"])), "steps_landed": sorted({int(one["steps_on_target"]) for one in results}), diff --git a/bench/coganchor-concurrency/standin_model.py b/bench/coganchor-concurrency/standin_model.py index f1aefc1f..629deef0 100644 --- a/bench/coganchor-concurrency/standin_model.py +++ b/bench/coganchor-concurrency/standin_model.py @@ -25,6 +25,7 @@ import time import uuid from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import Any #: How many shell tool calls one scripted turn makes before it answers. @@ -47,7 +48,9 @@ #: rather than matched exactly, and a name no pattern claims leaves the turn tool-free. _SHELL_PATTERNS = ( re.compile(r"^(bash|shell|sh|local_shell)$", re.IGNORECASE), - re.compile(r"^(run|execute|exec)_?(terminal_?)?(command|shell|bash)s?$", re.IGNORECASE), + re.compile( + r"^(run|execute|exec)_?(terminal_?)?(command|shell|bash)s?$", re.IGNORECASE + ), re.compile(r"terminal.*command|command.*terminal", re.IGNORECASE), re.compile(r"^(run|execute|exec)[_-]?", re.IGNORECASE), re.compile(r"(^|_)(bash|shell|terminal)(_|$)", re.IGNORECASE), @@ -69,8 +72,10 @@ def _log(kind: str, detail: object) -> None: if not LOG_PATH: return - with _log_lock, open(LOG_PATH, "a") as handle: - handle.write(json.dumps({"at": time.time(), "kind": kind, "detail": detail}) + "\n") + with _log_lock, Path(LOG_PATH).open("a") as handle: + handle.write( + json.dumps({"at": time.time(), "kind": kind, "detail": detail}) + "\n" + ) # --------------------------------------------------------------------------- tool choice @@ -117,9 +122,11 @@ def _fill(schema: dict[str, Any], command: str) -> dict[str, Any]: ): for name, spec in properties.items(): if str(name).lower() in ("command", "cmd", "script"): - spec = spec if isinstance(spec, dict) else {} + shape = spec if isinstance(spec, dict) else {} filled[name] = ( - ["bash", "-lc", command] if spec.get("type") == "array" else command + ["bash", "-lc", command] + if shape.get("type") == "array" + else command ) break return filled @@ -180,7 +187,11 @@ def _steps_taken_responses(body: dict[str, Any]) -> int: if isinstance(items, list): for item in items: if isinstance(item, dict) and str(item.get("type", "")).endswith( - ("function_call_output", "local_shell_call_output", "custom_tool_call_output") + ( + "function_call_output", + "local_shell_call_output", + "custom_tool_call_output", + ) ): taken += 1 return taken @@ -247,7 +258,6 @@ def _anthropic(body: dict[str, Any]) -> list[tuple[str | None, Any]]: ) ] if tool is None or step >= STEPS: - text = FINAL if step >= STEPS or tool is None else FINAL return [ *start, ( @@ -263,7 +273,7 @@ def _anthropic(body: dict[str, Any]) -> list[tuple[str | None, Any]]: { "type": "content_block_delta", "index": 0, - "delta": {"type": "text_delta", "text": text}, + "delta": {"type": "text_delta", "text": FINAL}, }, ), ("content_block_stop", {"type": "content_block_stop", "index": 0}), @@ -402,14 +412,20 @@ def numbered(event: str, payload: dict[str, Any]) -> tuple[str, Any]: } done = dict(envelope, status="completed", output=[item], usage=usage) chunks: list[tuple[str | None, Any]] = [ - numbered("response.created", {"type": "response.created", "response": dict(envelope)}), + numbered( + "response.created", {"type": "response.created", "response": dict(envelope)} + ), numbered( "response.in_progress", {"type": "response.in_progress", "response": dict(envelope)}, ), numbered( "response.output_item.added", - {"type": "response.output_item.added", "output_index": 0, "item": dict(item)}, + { + "type": "response.output_item.added", + "output_index": 0, + "item": dict(item), + }, ), ] if item["type"] == "message": @@ -463,7 +479,11 @@ def numbered(event: str, payload: dict[str, Any]) -> tuple[str, Any]: chunks.append( numbered( "response.output_item.done", - {"type": "response.output_item.done", "output_index": 0, "item": dict(item)}, + { + "type": "response.output_item.done", + "output_index": 0, + "item": dict(item), + }, ) ) chunks.append( @@ -472,7 +492,9 @@ def numbered(event: str, payload: dict[str, Any]) -> tuple[str, Any]: return chunks -def _chat_chunks(body: dict[str, Any]) -> tuple[dict[str, Any], list[tuple[str | None, Any]]]: +def _chat_chunks( + body: dict[str, Any], +) -> tuple[dict[str, Any], list[tuple[str | None, Any]]]: model = str(body.get("model") or "standin-1") step = _steps_taken_chat(body) tool = _shell_tool(body) @@ -484,7 +506,12 @@ def _chat_chunks(body: dict[str, Any]) -> tuple[dict[str, Any], list[tuple[str | "total_tokens": 1040 + 100 * step, "prompt_tokens_details": {"cached_tokens": 0}, } - head = {"id": made, "object": "chat.completion.chunk", "created": created, "model": model} + head = { + "id": made, + "object": "chat.completion.chunk", + "created": created, + "model": model, + } if tool is None or step >= STEPS: message = {"role": "assistant", "content": FINAL} @@ -524,11 +551,21 @@ def _chat_chunks(body: dict[str, Any]) -> tuple[dict[str, Any], list[tuple[str | ] chunks: list[tuple[str | None, Any]] = [ - (None, dict(head, choices=[{"index": 0, "delta": delta, "finish_reason": None}])) + ( + None, + dict(head, choices=[{"index": 0, "delta": delta, "finish_reason": None}]), + ) for delta in deltas ] chunks.append( - (None, dict(head, choices=[{"index": 0, "delta": {}, "finish_reason": finish}], usage=usage)) + ( + None, + dict( + head, + choices=[{"index": 0, "delta": {}, "finish_reason": finish}], + usage=usage, + ), + ) ) chunks.append((None, "[DONE]")) whole = { @@ -549,7 +586,12 @@ def _catalogue() -> dict[str, Any]: return { "object": "list", "data": [ - {"id": name, "object": "model", "created": 1700000000, "owned_by": "standin"} + { + "id": name, + "object": "model", + "created": 1700000000, + "owned_by": "standin", + } for name in _MODEL_IDS ], } @@ -590,6 +632,8 @@ def _registry(port: int) -> dict[str, Any]: class Handler(BaseHTTPRequestHandler): + """The one handler all three wire protocols are answered from, by path.""" + protocol_version = "HTTP/1.1" port = 0 @@ -678,7 +722,9 @@ def do_POST(self) -> None: else: self._json(whole) else: - self._json({"error": {"message": f"no route {path}", "type": "not_found"}}, 404) + self._json( + {"error": {"message": f"no route {path}", "type": "not_found"}}, 404 + ) except (BrokenPipeError, ConnectionResetError): pass diff --git a/bench/coganchor-concurrency/summarise.py b/bench/coganchor-concurrency/summarise.py index dfc7f5f8..f27c3af4 100644 --- a/bench/coganchor-concurrency/summarise.py +++ b/bench/coganchor-concurrency/summarise.py @@ -17,22 +17,21 @@ import sys from pathlib import Path -LAB = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).resolve().parent / "lab" +LAB = ( + Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).resolve().parent / "lab" +) SLOWDOWN = 2.0 BACKENDS = ("claude", "codex", "grok", "kimi", "dsh") def rungs(backend: str) -> list[dict[str, object]]: + """Every rung recorded for one backend, in the order the ladder ran them.""" path = LAB / f"ladder-{backend}.jsonl" if not path.exists(): return [] - out = [] - for line in path.read_text().splitlines(): - line = line.strip() - if line.startswith("{"): - out.append(json.loads(line)) - return out + lines = (line.strip() for line in path.read_text().splitlines()) + return [json.loads(line) for line in lines if line.startswith("{")] print( @@ -86,8 +85,10 @@ def rungs(backend: str) -> list[dict[str, object]]: print("-" * 92) print() -print(f"{'backend':8} {'all correct up to':>18} {'and still quick to':>19} " - f"{'MiB/agent':>10} {'procs/agent':>12} {'peak turns/min':>15}") +print( + f"{'backend':8} {'all correct up to':>18} {'and still quick to':>19} " + f"{'MiB/agent':>10} {'procs/agent':>12} {'peak turns/min':>15}" +) for backend, verdict in verdicts.items(): print( f"{backend:8} {verdict['correct']:18d} {verdict['comfortable']:19d} " diff --git a/pyproject.toml b/pyproject.toml index e58c51d9..f62db6f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,6 +123,11 @@ ignore = [ # recorded in. It is copied into a container and run by path rather than imported, so it # is a file rather than a package, and there is nothing for an `__init__.py` to do. "docs/tapes/stage.py" = ["INP001"] +# The concurrency rig, for the same reason: each script is run by path inside the cgroup +# being measured rather than imported, so there is nothing for an `__init__.py` to do. And +# what a measurement writes, it prints -- the ladder reads one JSON object per run off +# their stdout, and the summary is a table. +"bench/**" = ["INP001", "T201"] # The syscall layer's domain type is the string path: it reads them out of ptrace # registers and puts them on a wire, so `pathlib` would only convert them twice. "src/hmz/coganchor/**" = ["PTH"]