diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dab159a3..6ffed743 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,9 +69,11 @@ jobs: # else: no npm ci anywhere else gets slower, and a contributor who has not run this line gets # a skip rather than a failure, exactly as the pip line above behaves. - run: npm install --no-save @mastra/core @ai-sdk/openai - # From the tree, not PyPI: the `openai-agents-handoff` check exists to prove the tracing layer - # in *this* commit works, and installing a published copy would test the last release instead. + # From the tree, not PyPI: the `openai-agents-handoff` and `langgraph-nodes` checks exist to + # prove the adapters in *this* commit work, and installing a published copy would test the + # last release instead. - run: pip install -e python-openai-agents + - run: pip install -e python-langgraph # `--require-all`, because installing them above was never what enforced it. Two checks were # added without a line here and CI stayed green on the skips, while the README said "in CI" # about both. A skip is now a failure on this runner, so forgetting the line above fails @@ -110,6 +112,11 @@ jobs: # of its tests are about how `install()` registers with it. - run: pip install ./python-openai-agents[dev] - run: python -m pytest python-openai-agents/ -q + # The third. Its `[dev]` extra brings langgraph and langchain-core, because the half of its + # suite that matters builds real graphs and invokes them — the unit tests assert what + # langchain-core is believed to pass, and only a real graph keeps that belief honest. + - run: pip install ./python-langgraph[dev] + - run: python -m pytest python-langgraph/ -q plugin-api-neutrality: name: plugin API neutrality diff --git a/.github/workflows/release-langgraph.yml b/.github/workflows/release-langgraph.yml new file mode 100644 index 00000000..8b304cb2 --- /dev/null +++ b/.github/workflows/release-langgraph.yml @@ -0,0 +1,81 @@ +# `orcareplay-langgraph` records a LangGraph run's own structure — which node ran, in which +# superstep, and how it ended — into an OrcaReplay trace. It ships on its own tag for the same +# reason `orca-trace` and `orcareplay-openai-agents` do: a Python-only fix must not force a version +# bump across twelve npm packages, and the npm release must not fail because PyPI had a bad day. +# +# A *fourth* release workflow rather than another job beside the others, because the three Python +# packages are independent. They share a repository and nothing else: different sources, different +# versions, different reasons to cut a release. One tag that published several would make every fix +# to one reissue the rest, and PyPI does not allow reusing a version number. +# +# Publishing uses PyPI Trusted Publishing — GitHub's OIDC identity is exchanged for a short-lived +# upload token, so there is no PyPI secret in this repository to leak or rotate. It needs a +# one-time registration at pypi.org (Publishing -> Add a pending publisher) naming: +# +# PyPI project orcareplay-langgraph +# owner Continuum-AI-Corp +# repository OrcaReplay +# workflow release-langgraph.yml +# environment pypi +# +# Until that exists the upload step fails and nothing else is affected. +name: Release (orcareplay-langgraph) + +on: + push: + tags: ['langgraph-v*'] + workflow_dispatch: + inputs: + dry-run: + description: 'Build and check without uploading' + type: boolean + default: true + +jobs: + release-langgraph: + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # Trusted Publishing: this is the whole credential + contents: read + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + # A tag that disagrees with pyproject would publish a version nobody can reproduce from the + # tree. Same check the other three releases make, for the same reason. + - name: Tag matches package version + if: startsWith(github.ref, 'refs/tags/langgraph-v') + run: | + set -euo pipefail + tag="${GITHUB_REF#refs/tags/langgraph-v}" + pkg=$(python -c "import tomllib;print(tomllib.load(open('python-langgraph/pyproject.toml','rb'))['project']['version'])") + [ "$tag" = "$pkg" ] || { echo "tag langgraph-v$tag does not match pyproject $pkg"; exit 1; } + + # Neither langgraph nor langchain-core is a dependency — this package is inert without them, + # and `orca record` puts it in front of every Python process a recording starts, so importing + # it has to cost nothing. The `dev` extra brings them, because half the suite builds real + # graphs: the unit tests assert what langchain-core is believed to pass, and only a real graph + # keeps that belief honest. + - name: Test + working-directory: python-langgraph + run: | + python -m pip install --upgrade pip + pip install -e '.[dev]' + python -m pytest -q + + - name: Build + working-directory: python-langgraph + run: | + pip install --upgrade build twine + python -m build + twine check dist/* + + - name: Publish to PyPI + if: ${{ github.event.inputs.dry-run != 'true' }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: python-langgraph/dist diff --git a/README.md b/README.md index 8281ab58..a7e29a22 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,8 @@ Three more layers catch what the protocol cannot see: an exit code, a real durat byte came out of, a file written without telling anyone. A fifth exists for the agents that read no base-URL variable at all — see [which agents](#which-agents). A sixth reads the agent's own account of its structure, for a harness that has one: which sub-agent ran, which handed off to which, -whether a guardrail tripped — none of which reaches the wire. +whether a guardrail tripped, which graph node produced what and which of them called no model at +all — none of which reaches the wire. ```mermaid %%{init: {'theme':'neutral'}}%% @@ -474,7 +475,7 @@ whether orca understands the wire format it speaks once it arrives. | **OpenClaw** | `orca record openclaw` — the hook for the gateway, inherited variables for the agents it spawns | works | | **opencode** | `orca record opencode` | adapter shipped, both origins redirected | | **goose** (Block) | `orca record goose` — `OPENAI_HOST` **and** `OPENAI_BASE_URL`, `ANTHROPIC_HOST` → Responses API | works — driven end to end against goose 1.49.0, [what is different about it](#the-harness-that-reads-different-variables) | -| **LangGraph / LangChain** | `OPENAI_BASE_URL`, `ANTHROPIC_BASE_URL` | works — a two-node graph, streaming and with a tool, records and replays at `exact=2` and forks live, [in CI](test/integrations/) | +| **LangGraph / LangChain** | `OPENAI_BASE_URL`, `ANTHROPIC_BASE_URL` | works — a two-node graph, streaming and with a tool, records and replays at `exact=2` and forks live, [in CI](test/integrations/); `pip install orcareplay-langgraph` [also records the graph](python-langgraph/README.md) — which node ran, in which superstep, including the ones that call no model | | **OpenHands** | `orca record generic-openai -- python your_agent.py` — its SDK wraps LiteLLM and reads `OPENAI_API_BASE` | works — the SDK's own LLM layer records and replays at `exact=1`, [in CI](test/integrations/) | | **CrewAI** | `orca record generic-openai -- python your_crew.py` — since 1.x its own provider, reading `OPENAI_API_BASE` and `OPENAI_BASE_URL` | works — a real `Agent`, `Task` and `Crew` records and replays at `exact=1`, [in CI](test/integrations/); [what 1.x changed](docs/integrations.md#crewai) | | **Aider** | `orca record generic-openai -- python your_agent.py` — routes through LiteLLM, which reads `OPENAI_API_BASE` | works — the LiteLLM layer records and replays at `exact=1`, [in CI](test/integrations/) | diff --git a/RELEASING.md b/RELEASING.md index e4fac81d..95f8cdb0 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -9,7 +9,7 @@ sequence of `npm publish` calls run by hand at the end of a long day. - An **`NPM_TOKEN`** repository secret, from an npm account that owns the `orcareplay` name and the `@orcareplay` scope. An automation token, not a personal one. - **Actions enabled** for the repository, and Actions billing active on the org. -- For the two Python packages, a **pending publisher** registered at pypi.org under +- For the three Python packages, a **pending publisher** registered at pypi.org under *Publishing → Add a pending publisher*, and a repository **environment named `pypi`**. There is no PyPI secret: Trusted Publishing exchanges GitHub's OIDC identity for a short-lived upload token, so there is nothing here to leak or rotate. One registration per package, all four other @@ -19,6 +19,7 @@ sequence of `npm publish` calls run by hand at the end of a long day. | --- | --- | --- | --- | --- | | `orca-trace` | `Continuum-AI-Corp` | `OrcaReplay` | `release-python.yml` | `pypi` | | `orcareplay-openai-agents` | `Continuum-AI-Corp` | `OrcaReplay` | `release-openai-agents.yml` | `pypi` | + | `orcareplay-langgraph` | `Continuum-AI-Corp` | `OrcaReplay` | `release-langgraph.yml` | `pypi` | A pending publisher does not reserve the name — it is only honoured on the first upload, and another account registering that name first invalidates it. So publish reasonably soon after @@ -129,7 +130,7 @@ registry at a version the rest do not name — which is the one state npm will n To rehearse without sending anything: **Actions → Release → Run workflow**, leaving *dry run* checked. It packs and validates every tarball and publishes nothing. -## The two Python packages +## The three Python packages They are not part of the npm release and do not share its version. Each ships on its own tag: @@ -137,6 +138,7 @@ They are not part of the npm release and do not share its version. Each ships on | --- | --- | --- | --- | | `orca-trace` — the read-only reader for the trace format | `python/` | `py-v` | `release-python.yml` | | `orcareplay-openai-agents` — records the Agents SDK's own run structure | `python-openai-agents/` | `agents-v` | `release-openai-agents.yml` | +| `orcareplay-langgraph` — records which LangGraph node ran, and in which superstep | `python-langgraph/` | `langgraph-v` | `release-langgraph.yml` | ```console # edit the version in the package's pyproject.toml, merge that, then: @@ -148,11 +150,11 @@ builds, and `twine check`s before it uploads — the same gate the npm release a reason. **Actions → Release (…) → Run workflow** with *dry run* left checked does everything except the upload, which is how to find out whether a release would work without making one. -> **Why three workflows rather than one.** A Python-only fix must not force a version bump across -> twelve npm packages, and the npm publish must not fail because PyPI had a bad day. The two Python +> **Why four workflows rather than one.** A Python-only fix must not force a version bump across +> twelve npm packages, and the npm publish must not fail because PyPI had a bad day. The Python > packages are separated from each other for the same reason: they share a repository and nothing -> else, so one tag publishing both would make every `orca-trace` fix reissue the adapter — and PyPI -> does not allow reusing a version number. +> else, so one tag publishing several would make every `orca-trace` fix reissue the adapters — and +> PyPI does not allow reusing a version number. ## Why order matters diff --git a/docs/integrations.md b/docs/integrations.md index df2cf3f9..91bb84ac 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -59,6 +59,48 @@ orca's, but a recorded run will differ from an unrecorded one in exactly that fi a replay rebuilds it by re-running the graph. A database-backed checkpointer is a different matter and is not covered by the checks here. +### Recording the graph itself + +Everything above is about the traffic. The *graph* — which node ran, in which superstep, and how it +ended — is not on the wire at all, and one package puts it in the trace: + +```console +pip install orcareplay-langgraph +``` + +No code change: `orca record` attaches it to a graph you have not edited, the same way it attaches +the fetch hook and the shell shim. Measured on `plan → validate → answer` against a stub origin: + +``` +graph.node.start plan +model.request +model.response +graph.node.end plan +graph.node.start validate ← makes no request of any kind +graph.node.end validate +graph.node.start answer +graph.node.end answer +``` + +`validate` is the point. Validators, state reducers, routers and writers call no model, so without +this a proxy sees a graph with those nodes and a graph without them as the same run. The same goes +for a parallel fan-out, which reaches the wire as two ordinary consecutive turns, and for a node +that raised — `graph.node.end` carries the exception's class, which no request can. + +Two records per node and nothing else: `node`, `step`, the run ids that pair them, the instants, +and `error`. No state, no inputs, no outputs, and no exception message. The model exchanges stay +with the proxy, which already has them byte for byte. + +**One caveat, and it is orca's rather than the package's.** A node record carries the instant the +callback fired, measured exact against the agent's own clock. A `model.request` is stamped when +orca persists it — after the response and after a workspace snapshot, measured 95 ms late on one +call and 31 ms on the next. So the node boundaries are trustworthy, and a model call within about a +tenth of a second of one can sit on the wrong side of it. + +See [`python-langgraph/`](../python-langgraph/README.md) for how a node is told apart from an inner +runnable, a conditional edge and the graph itself — which is the whole design, and also what keeps a +`run_name` interpolated from user data out of the trace. + --- ## CrewAI diff --git a/examples/traces/run_9f2c14a03b71/events.jsonl b/examples/traces/run_9f2c14a03b71/events.jsonl index 6f947d65..9a921e30 100644 --- a/examples/traces/run_9f2c14a03b71/events.jsonl +++ b/examples/traces/run_9f2c14a03b71/events.jsonl @@ -26,4 +26,6 @@ {"seq":25,"ts":"2026-08-29T10:14:14.680Z","mono_us":36680000,"turn":5,"type":"agent.start","actor":"harness","attrs":{"name":"Triage","handoffs":"Billing Specialist","tools":0,"output_type":"str"}} {"seq":26,"ts":"2026-08-29T10:14:14.960Z","mono_us":36960000,"turn":5,"type":"agent.guardrail","actor":"harness","attrs":{"name":"not_empty","triggered":false}} {"seq":27,"ts":"2026-08-29T10:14:15.240Z","mono_us":37240000,"turn":5,"type":"agent.handoff","actor":"harness","attrs":{"from":"Triage","to":"Billing Specialist"}} -{"seq":28,"ts":"2026-08-29T10:14:15.800Z","mono_us":37800000,"turn":5,"type":"run.end","actor":"orca","attrs":{"exit_code":1}} +{"seq":28,"ts":"2026-08-29T10:14:15.430Z","mono_us":37430000,"turn":5,"type":"graph.node.start","actor":"harness","attrs":{"node":"summarise_invoice","step":1,"run_id":"3f6a1c8e-0b2d-4f51-9c77-1a5be2d40c93","parent_run_id":"9d4c7b12-5e83-4a06-b1f2-77c0ae35d918","started_at":"2026-08-29T10:14:15.430Z"}} +{"seq":29,"ts":"2026-08-29T10:14:15.610Z","mono_us":37610000,"turn":5,"type":"graph.node.end","actor":"harness","attrs":{"node":"summarise_invoice","step":1,"run_id":"3f6a1c8e-0b2d-4f51-9c77-1a5be2d40c93","ended_at":"2026-08-29T10:14:15.610Z"}} +{"seq":30,"ts":"2026-08-29T10:14:15.800Z","mono_us":37800000,"turn":5,"type":"run.end","actor":"orca","attrs":{"exit_code":1}} diff --git a/examples/traces/run_9f2c14a03b71/manifest.json b/examples/traces/run_9f2c14a03b71/manifest.json index 98d10855..857ae7ab 100644 --- a/examples/traces/run_9f2c14a03b71/manifest.json +++ b/examples/traces/run_9f2c14a03b71/manifest.json @@ -1,5 +1,5 @@ { - "schema_version": "0.2.0", + "schema_version": "0.3.0", "run_id": "run_9f2c14a03b71", "created_at": "2026-08-29T10:13:38.000Z", "ended_at": "2026-08-29T10:18:29.000Z", @@ -28,7 +28,7 @@ "node": "v22.22.2" }, "counts": { - "events": 29, + "events": 31, "blobs": 5 }, "redaction": { @@ -40,7 +40,7 @@ }, "exit_code": 1, "integrity": { - "events_sha256": "f5664c53bd40fcddf3ab763eef2f02247278cfccd41b77af9ad9a9b3df3f984e", + "events_sha256": "07aa293d6af485d1637a64e0004453282bcca43f2b4cb32970eda4cabbdbd0de", "blob_count": 5 } } diff --git a/packages/cli/src/agent-spans.ts b/packages/cli/src/agent-spans.ts index 0b4d9331..173d6223 100644 --- a/packages/cli/src/agent-spans.ts +++ b/packages/cli/src/agent-spans.ts @@ -34,13 +34,84 @@ export const SPANS_ENV = 'ORCA_AGENT_SPANS'; export const SPANS_FILENAME = 'agent-spans.jsonl'; export const SITECUSTOMIZE = 'sitecustomize.py'; +/** A Python framework whose own run structure orca can read, and the package that reads it. */ +export interface PythonAdapter { + /** The framework, as `importlib.metadata` names the distribution that provides it. */ + distribution: string; + /** The top-level module the agent imports to use it, which is how the bootstrap notices it. */ + root: string; + /** The adapter's import name, for the bootstrap's `from … import install`. */ + module: string; + /** The adapter's distribution name, as `pip install` takes it. */ + package: string; + /** What the trace goes without when the framework is used and the adapter is not there. */ + lost: string; +} + +/** + * Every adapter the bootstrap installs — one declaration, three consumers. + * + * The bootstrap's Python is generated from this, {@link agentSpanLosses} reports against it, and + * the drain's warning reads its prose from it. Adding a row is the whole of adding an adapter to + * the record path, and nothing can be added to one of the three and forgotten in the others. + * + * They share a transport file on purpose. The reader recovers a record from a torn line by finding + * `{"kind":` in it, which is a property of the writers rather than of the file, so a second writer + * costs nothing; a second file would cost a second environment variable, a second drain and a + * second entry in the stale-transport sweep. + */ +export const PYTHON_ADAPTERS: readonly PythonAdapter[] = [ + { + distribution: 'openai-agents', + root: 'agents', + module: 'orcareplay_openai_agents', + package: 'orcareplay-openai-agents', + lost: 'the agents, handoffs and guardrails only it can see', + }, + { + distribution: 'langgraph', + root: 'langgraph', + module: 'orcareplay_langgraph', + package: 'orcareplay-langgraph', + lost: 'which node produced which call, and the nodes that call no model at all', + }, +]; + +/** + * One adapter's stanza inside the bootstrap's `_install`. + * + * Written out per adapter rather than looped over a list, because the import has to be a literal + * `from … import install` statement: `__import__(name)` would work, and would leave the file with + * no readable statement of what it attaches to. This runs on someone else's machine, in front of + * their interpreter, and it should be possible to read it and see exactly that. + * + * `else` rather than a bare sequence, so an adapter whose `install` raises is not reported as one + * that is missing — a package that is present and broken sends the operator somewhere different + * from one that was never installed. + */ +function installBlock({ module, root, distribution, package: pkg }: PythonAdapter): string { + return ` try: + from ${module} import install as _${module} + except Exception: + missing["${root}"] = ("${distribution}", "${pkg}") + else: + try: + _${module}() + except Exception: + pass +`; +} + /** * The bootstrap, as it is written into the run directory. * * Every path through it ends in a return rather than a traceback. It runs before the agent's first * statement, in *every* Python process the recording starts — including `python --version` — so a - * failure here is a failure of the run rather than of the capture. Absence of the package, absence - * of the SDK and an SDK whose interface moved are all the same non-event. + * failure here is a failure of the run rather than of the capture. Absence of an adapter, absence + * of the framework and a framework whose interface moved are all the same non-event. + * + * The adapters come from {@link PYTHON_ADAPTERS}, so adding one is a row in that list rather than + * an edit to this string. * * It also chains to whatever `sitecustomize` it displaced. Ours arrives via `PYTHONPATH` and so wins * over a site's or a virtualenv's own; silently disabling someone's startup hook to add a debugging @@ -48,9 +119,10 @@ export const SITECUSTOMIZE = 'sitecustomize.py'; */ export const SITECUSTOMIZE_SOURCE = `# Written by \`orca record\`. Deleted with the run directory. # -# Attaches OrcaReplay's tracing processor to the OpenAI Agents SDK without editing the agent. -# Inert unless ORCA_AGENT_SPANS is set, which only \`orca record\` does. +# Attaches OrcaReplay's adapters to the agent frameworks on this interpreter, without editing the +# agent. Inert unless ORCA_AGENT_SPANS is set, which only \`orca record\` does. import os +import sys def _chain(): @@ -76,52 +148,89 @@ def _chain(): return -def _unavailable(path): - """Record that the SDK is installed and the adapter is not — the one case worth reporting. +def _unavailable(path, distribution, package): + """Record that a framework was used and its adapter was not there — the case worth reporting. - Without this, a run whose agent uses the Agents SDK on a machine without - \`orcareplay-openai-agents\` is byte-identical to one that never used the SDK: the same - \`recorded ... exit=0\`, no agent events, no warning. + Without this, a run whose agent uses the framework on a machine without the adapter is + byte-identical to one that never used it: the same \`recorded ... exit=0\`, no structural + events, no warning. - **The question is about the distribution, not the import name.** \`find_spec("agents")\` is - true of *anything* called that, and \`agents.py\` is an ordinary name for an ordinary module — + **The confirmation is about the distribution, not the import name.** The caller has only seen + a module *name* being imported, and \`agents.py\` is an ordinary name for an ordinary module — measured: a project with its own two-line \`agents.py\` and \`PYTHONPATH=.\`, on a machine with no SDK at all, was told "the agent imported the OpenAI Agents SDK" and sent to install a - package it has no use for. \`importlib.metadata\` asks which *distribution* provides it, which - is what "the SDK is installed" actually means. + package it has no use for. \`importlib.metadata\` asks which *distribution* provides that name, + which is what "the framework is here" actually means. Metadata rather than importing, and rather than probing a submodule. Measured on this machine: - \`distribution("openai-agents")\` 2.3ms, \`find_spec("agents")\` 0.5ms but wrong, and - \`find_spec("agents.tracing")\` **2066ms** — resolving a submodule spec imports the parent, so - the specific-looking option costs the same as the import it was avoiding. This bootstrap runs - in every Python process the recording starts, on an interpreter that starts in 50ms. + \`distribution("openai-agents")\` 2.3ms, and \`find_spec("agents.tracing")\` **2066ms** — + resolving a submodule spec imports the parent, so the specific-looking option costs the same as + the import it was avoiding. """ import importlib.metadata try: - importlib.metadata.distribution("openai-agents") + importlib.metadata.distribution(distribution) except Exception: - return # not installed, or metadata unreadable: either way, nothing was lost here + return # a module that merely shares the name: nothing was lost here try: with open(path, "a", encoding="utf-8") as f: - f.write('{"kind": "unavailable", "package": "orcareplay-openai-agents"}\\n') + f.write('{"kind": "unavailable", "package": "' + package + '"}\\n') except Exception: return +class _WhenImported: + """Report a missing adapter if, and only if, the agent imports the framework it is for. + + Asking \`importlib.metadata\` at startup instead would answer a different question — is the + framework *installed* — and on an ordinary machine that is yes for frameworks the run never + touches. One venv here has both langgraph and openai-agents, so every recorded run in it would + be told to install two adapters it has no use for, and the count grows with every adapter orca + ships. A warning that fires when nothing is wrong is how warnings stop being read. + + Installed only for the adapters that were *not* importable, so a fully equipped machine carries + nothing, and it takes itself off \`sys.meta_path\` once there is nothing left to watch for. + Every path returns None: it never claims a module, and the ordinary finders load the framework + exactly as they would have. + """ + + def __init__(self, path, missing): + self._path = path + self._missing = missing + + def find_spec(self, fullname, path=None, target=None): + try: + found = self._missing.pop(fullname.partition(".")[0], None) + if not self._missing: + try: + sys.meta_path.remove(self) + except ValueError: + pass + if found is not None: + _unavailable(self._path, found[0], found[1]) + except Exception: + pass + return None + + def _install(): + """Attach every adapter that is here, and watch for the frameworks whose adapter is not. + + One environment read for all of them, and each guarded on its own: an adapter that is absent, + or whose \`install\` raises, must not stop the next from attaching. \`else\` rather than a bare + sequence, so an adapter that is present and broken is not reported as one that is missing — + those send the operator somewhere different. + """ path = os.environ.get("ORCA_AGENT_SPANS") if not path: return - try: - from orcareplay_openai_agents import install - except Exception: - _unavailable(path) - return - try: - install() - except Exception: - return + missing = {} +${PYTHON_ADAPTERS.map(installBlock).join('\n')} if missing: + try: + sys.meta_path.insert(0, _WhenImported(path, missing)) + except Exception: + pass _chain() @@ -427,16 +536,17 @@ export async function readAgentSpans(path: string): Promise { for (const parsed of raw.split('\n').flatMap((line) => objectsOnLine(line, SPAN_START))) { const span = parsed as unknown as AgentSpan; // And the third rule the other two transports apply, for the same reason and with the same - // consequence. The drain builds `new Date(Date.parse(String(span.started_at ?? '')))` and hands - // it to `TraceWriter.append` as `occurredAt`; `Date.parse` accepts instants a `date-time` - // cannot write down, and `assertEvent` then throws where the trace is being sealed — after - // `discardAgentSpanTransport` has already taken the file, so the run ends with no `run.end` - // and every span it recorded is gone. Exactly the drain's own expression, so the two cannot - // disagree about what parses. + // consequence. The drain builds `new Date(Date.parse(String(span.started_at ?? span.ended_at ?? + // '')))` and hands it to `TraceWriter.append` as `occurredAt`; `Date.parse` accepts instants a + // `date-time` cannot write down, and `assertEvent` then throws where the trace is being sealed + // — after `discardAgentSpanTransport` has already taken the file, so the run ends with no + // `run.end` and every span it recorded is gone. Exactly the drain's own expression, so the two + // cannot disagree about what parses — including the fallback, which is what a record that + // closes something carries instead. // // Absent or unparseable is kept, as in the siblings: the drain drops `occurredAt` for those // and stamps from its own clock, which degrades a field rather than losing a handoff. - const startedMs = Date.parse(String(span.started_at ?? '')); + const startedMs = Date.parse(String(span.started_at ?? span.ended_at ?? '')); if (!Number.isNaN(startedMs) && (startedMs < EARLIEST_TS_MS || startedMs > LATEST_TS_MS)) { continue; } @@ -465,14 +575,24 @@ const SPAN_START = '{"kind":'; * as an event. They are for the operator, now, while the run is still on screen. */ export interface AgentSpanLosses { - /** The SDK was importable and the adapter was not, so nothing structural was captured at all. */ + /** The framework was used and its adapter was not there, so nothing structural was captured. */ unavailable: string[]; - /** Records the processor held and could not write: unserialisable, or the append failed. */ + /** Records an adapter held and could not write: unserialisable, or the append failed. */ dropped: number; + /** + * Which adapters reported those drops, for the records that said. + * + * The count alone stopped being actionable when there was more than one adapter: several write + * to one transport, so a total says how much was lost and nothing about where to look. Read + * rather than merely written — `_dropped` in the first adapter was counted from the start and + * read by nothing, which is how a field becomes payload. + */ + droppedBy: string[]; } export function agentSpanLosses(spans: AgentSpan[]): AgentSpanLosses { const unavailable = new Set(); + const droppedBy = new Set(); let dropped = 0; for (const span of spans) { if (span.kind === 'unavailable') { @@ -480,15 +600,24 @@ export function agentSpanLosses(spans: AgentSpan[]): AgentSpanLosses { // starts, so an agent that shells out to `python` writes this once per child. Six lines // saying the same thing is one fact, and reporting it six times reads like six failures. const pkg = (span as { package?: unknown }).package; - unavailable.add(typeof pkg === 'string' && pkg !== '' ? pkg : 'orcareplay-openai-agents'); + // The fallback is the first adapter rather than a name spelled out here, because a record + // with no `package` can only have come from a bootstrap older than the field — which is the + // version that had exactly one adapter. + unavailable.add( + typeof pkg === 'string' && pkg !== '' ? pkg : (PYTHON_ADAPTERS[0]?.package ?? 'unknown'), + ); } else if (span.kind === 'dropped') { // Summed rather than taken: one `dropped` line per process, same as above, and here the // counts are of different records so they add rather than collapse. const count = (span as { count?: unknown }).count; if (typeof count === 'number' && Number.isFinite(count) && count > 0) dropped += count; + // Optional: the first adapter's records predate the field, and a loss reported without a + // name is still a loss worth counting. + const pkg = (span as { package?: unknown }).package; + if (typeof pkg === 'string' && pkg !== '') droppedBy.add(pkg); } } - return { unavailable: [...unavailable].sort(), dropped }; + return { unavailable: [...unavailable].sort(), dropped, droppedBy: [...droppedBy].sort() }; } /** A trace event, or undefined for a span that carries nothing the proxy lacks. */ @@ -531,6 +660,35 @@ export function eventForSpan( ...at, }, }; + // A LangGraph superstep, which the wire has no representation of at all. A node's name is not + // in any request, a node that calls no model makes no request, and two nodes in one parallel + // superstep are indistinguishable from two consecutive turns. + case 'LangGraphNodeStart': + return { + type: 'graph.node.start', + attrs: { + node: String(data['node'] ?? 'unknown'), + // The superstep is per graph, so two records can share one without being concurrent. + // `parent` is what separates those, and it is the only reason the pair is useful. + step: typeof data['step'] === 'number' ? data['step'] : undefined, + run_id: span.span_id, + parent_run_id: span.parent_id, + ...at, + }, + }; + case 'LangGraphNodeEnd': + return { + type: 'graph.node.end', + attrs: { + node: String(data['node'] ?? 'unknown'), + step: typeof data['step'] === 'number' ? data['step'] : undefined, + run_id: span.span_id, + // The exception's class, never its message: the adapter has no redactor in front of it, + // and a node's exception text is whatever it was handed. + error: data['error'] === undefined ? undefined : String(data['error']), + ended_at: span.ended_at, + }, + }; default: // Turn, Task, Custom, Speech and the rest describe structure the timeline already shows, or // belong to pipelines this layer does not claim to cover. Dropping them keeps the trace to diff --git a/packages/cli/src/commands/record.ts b/packages/cli/src/commands/record.ts index bca807c1..581e29ec 100644 --- a/packages/cli/src/commands/record.ts +++ b/packages/cli/src/commands/record.ts @@ -24,6 +24,7 @@ import { pythonPathWith, agentSpanLosses, readAgentSpans, + PYTHON_ADAPTERS, SPANS_ENV, type AgentSpanCapture, } from '../agent-spans.js'; @@ -596,32 +597,46 @@ async function runRecording( // Before the events, because both say the trace is about to be less complete than it looks, // and the operator reads the top of the drain rather than the bottom. const losses = agentSpanLosses(spans); - // The second gate, and the one the bootstrap cannot apply: it decides before the agent's first - // statement, so all it can know is what is installed. A run that made no model call did not - // lose an agent structure — there was no agent turn to belong to one — and `capture.empty` - // below already says the larger thing that went wrong. Naming a missing package on top of it - // would send someone to `pip install` over a problem that is not about a package. + // The second gate. The bootstrap now applies the first one itself — it reports a package only + // once the agent actually imports the framework — so what is left here is the case where a run + // captured no model traffic at all. `capture.empty` below already says that larger thing, and + // naming a missing package on top of it would send someone to `pip install` over a problem + // that is not about a package. for (const pkg of modelExchanges > 0 ? losses.unavailable : []) { + // The prose comes from the adapter table, so a package cannot be reported with another + // package's explanation — which is exactly what happened when this text was a constant and a + // second adapter arrived. + const adapter = PYTHON_ADAPTERS.find((entry) => entry.package === pkg); out.warn('agent_spans.unavailable', { package: pkg, - // What was actually checked, rather than what it suggests. The bootstrap asks - // `importlib.metadata` which distribution provides the SDK; it does not — and before the - // agent runs, cannot — know whether the agent went on to import it. - cause: 'openai-agents is installed and this package is not', - effect: 'the run is recorded, without the agents, handoffs and guardrails only it can see', + // What was actually observed, rather than what it suggests. The bootstrap saw the + // framework's own module being imported and confirmed with `importlib.metadata` that a + // distribution provides that name; it does not know what the agent then did with it. + cause: `${adapter?.distribution ?? 'the framework'} was imported and this package is not installed`, + effect: `the run is recorded, without ${adapter?.lost ?? 'the structure only it can see'}`, next: `pip install ${pkg}`, }); } if (losses.dropped > 0) { out.warn('agent_spans.dropped', { count: losses.dropped, - cause: 'the processor could not serialise or could not write these records', + // Which adapter, where it said. Several write to one transport, so the total on its own + // says how much was lost and nothing about where to look. + ...(losses.droppedBy.length > 0 ? { package: losses.droppedBy.join(', ') } : {}), + cause: 'the adapter could not serialise or could not write these records', }); } for (const span of spans) { const derived = eventForSpan(span); if (derived === undefined) continue; - const startedAt = Date.parse(String(span.started_at ?? '')); + // When the span happened, from whichever end it carries. A record that closes something has + // only `ended_at`, and reading `started_at` alone stamped every one of them from the drain's + // own clock — which runs after the agent has exited. Measured on a three-node graph: all + // three `graph.node.end` events landed at the same instant, 600ms after the run, so the + // first node appeared to finish after the second had started and the timeline said the + // opposite of what happened. `occurredAt` is also what `turnAt` reads, so the same mistake + // filed them under the wrong turn. + const startedAt = Date.parse(String(span.started_at ?? span.ended_at ?? '')); const at = Number.isNaN(startedAt) ? undefined : new Date(startedAt); await writer.append({ type: derived.type as 'agent.start', diff --git a/packages/cli/test/agent-spans-drain.test.ts b/packages/cli/test/agent-spans-drain.test.ts index 13406027..65f50b8f 100644 --- a/packages/cli/test/agent-spans-drain.test.ts +++ b/packages/cli/test/agent-spans-drain.test.ts @@ -113,4 +113,45 @@ describe('a span line the drain cannot stamp', () => { expect(handoff, 'the span never reached the trace').toBeDefined(); expect(handoff!.ts.startsWith('2026-09-12'), `stamped ${handoff!.ts}`).toBe(true); }, 60_000); + + /** + * A record that closes something carries `ended_at` and no `started_at`. + * + * Reading only `started_at` left every one of them stamped from the drain's own clock, which + * runs after the agent has exited. Measured on a three-node graph: all three `graph.node.end` + * events landed within a millisecond of each other, 600ms after the last node actually finished, + * so the first node appeared to close after the second had opened — a timeline saying the + * opposite of what happened, in the one layer whose entire purpose is ordering. + */ + it('stamps a closing record from the instant it does carry', async () => { + planted.line = JSON.stringify({ + kind: 'span', + type: 'LangGraphNodeEnd', + span_id: 'r1', + ended_at: '2026-09-12T00:00:00.000Z', + data: { node: 'validate', step: 2 }, + }); + + const rows = await record(); + const end = rows.find((row) => row.type === 'graph.node.end'); + expect(end, 'the span never reached the trace').toBeDefined(); + expect(end!.ts.startsWith('2026-09-12'), `stamped ${end!.ts}`).toBe(true); + }, 60_000); + + /** And the bound has to cover that field too, or it is a hole the size of the one it closed. */ + it('does not take the run down with an end it cannot stamp either', async () => { + planted.line = JSON.stringify({ + kind: 'span', + type: 'LangGraphNodeEnd', + span_id: 'r1', + ended_at: '+275760-09-13T00:00:00+00:00', + data: { node: 'validate' }, + }); + + const rows = await record(); + expect( + rows.some((row) => row.type === 'run.end'), + 'the run was left unsealed by one closing span it could not stamp', + ).toBe(true); + }, 60_000); }); diff --git a/packages/cli/test/agent-spans.test.ts b/packages/cli/test/agent-spans.test.ts index f9153330..eecb457a 100644 --- a/packages/cli/test/agent-spans.test.ts +++ b/packages/cli/test/agent-spans.test.ts @@ -15,9 +15,11 @@ import { eventForSpan, installAgentSpans, pythonPathWith, + PYTHON_ADAPTERS, readAgentSpans, SITECUSTOMIZE, SITECUSTOMIZE_SOURCE, + type PythonAdapter, } from '../src/agent-spans.js'; import { parseArgs } from '../src/args.js'; import { recordCommand } from '../src/commands/record.js'; @@ -86,6 +88,52 @@ describe('turning SDK spans into trace events', () => { }); }); + it('keeps a graph node, with the superstep and the ids that pair it', () => { + const event = eventForSpan({ + kind: 'span', + type: 'LangGraphNodeStart', + span_id: 'run-1', + parent_id: 'graph-1', + started_at: '2026-09-16T03:46:03.483810+00:00', + data: { node: 'validate_only', step: 2 }, + }); + expect(event?.type).toBe('graph.node.start'); + expect(event?.attrs).toMatchObject({ + node: 'validate_only', + step: 2, + run_id: 'run-1', + parent_run_id: 'graph-1', + }); + }); + + it('keeps which node raised, and only the class', () => { + // The node that failed is the first thing anyone asks of a broken graph, and the wire cannot + // say: a node that dies before calling a model leaves no request at all. The message is not + // carried, because the adapter has no redactor in front of it. + const event = eventForSpan({ + kind: 'span', + type: 'LangGraphNodeEnd', + span_id: 'run-2', + ended_at: '2026-09-16T03:46:04.000000+00:00', + data: { node: 'lookup', step: 3, error: 'ValueError' }, + }); + expect(event?.type).toBe('graph.node.end'); + expect(event?.attrs).toMatchObject({ node: 'lookup', step: 3, error: 'ValueError' }); + expect(event?.attrs['ended_at']).toBe('2026-09-16T03:46:04.000000+00:00'); + }); + + it('leaves the superstep out rather than guessing when it is not a number', () => { + // `attrs` values are scalars the reader prints. A step that arrived as a string would render + // as a superstep that never existed. + const event = eventForSpan({ + kind: 'span', + type: 'LangGraphNodeStart', + data: { node: 'n', step: '2' }, + }); + expect(event?.attrs).not.toHaveProperty('step', '2'); + expect(event?.attrs['step']).toBeUndefined(); + }); + it('drops the model exchanges, which the proxy already has byte for byte', () => { // Not an optimisation. Writing them again would put a second, worse copy of the conversation in // the trace, and replay matches on the recorded bytes. @@ -105,6 +153,14 @@ describe('turning SDK spans into trace events', () => { { kind: 'span', type: 'AgentSpanData', data: { name: 'A', handoffs: [], tools: [] } }, { kind: 'span', type: 'HandoffSpanData', data: { from_agent: 'A', to_agent: 'B' } }, { kind: 'span', type: 'GuardrailSpanData', data: { name: 'g', triggered: true } }, + { + kind: 'span', + type: 'LangGraphNodeStart', + span_id: 'r', + parent_id: 'p', + data: { node: 'n', step: 1 }, + }, + { kind: 'span', type: 'LangGraphNodeEnd', span_id: 'r', data: { node: 'n', error: 'E' } }, ]; for (const span of spans) { const derived = eventForSpan(span)!; @@ -195,10 +251,22 @@ describe('the bootstrap orca writes', () => { it('imports the published package, never this repository', () => { // It runs in the agent's interpreter, which resolves imports against its own environment. The // fetch hook and the shell shim are written out standalone for exactly this reason. - expect(SITECUSTOMIZE_SOURCE).toContain('from orcareplay_openai_agents import install'); + for (const { module } of PYTHON_ADAPTERS) { + expect(SITECUSTOMIZE_SOURCE).toContain(`from ${module} import install`); + } expect(SITECUSTOMIZE_SOURCE).not.toMatch(/packages[/\\]/); }); + it('attaches every adapter in the table, and each independently', () => { + // One adapter that is absent, or whose `install` raises, must not stop the next from + // attaching. `else` is what separates those two outcomes: a package that is present and broken + // is not reported as one that was never installed. + for (const { module, root, distribution, package: pkg } of PYTHON_ADAPTERS) { + expect(SITECUSTOMIZE_SOURCE).toContain(`from ${module} import install as _${module}`); + expect(SITECUSTOMIZE_SOURCE).toContain(`missing["${root}"] = ("${distribution}", "${pkg}")`); + } + }); + it('does nothing at all unless orca is recording', () => { // It is imported by *every* Python process the run starts, including `python --version`. expect(SITECUSTOMIZE_SOURCE).toContain('path = os.environ.get("ORCA_AGENT_SPANS")'); @@ -208,19 +276,24 @@ describe('the bootstrap orca writes', () => { /** * Asserted against the code rather than the file, because the file explains itself. * - * `_unavailable`'s docstring names `find_spec("agents")` and `find_spec("agents.tracing")` — - * they are the two options it rejects, with the measurements that rejected them. A test that - * searched the whole source would trip on the prose describing what the code does not do. + * `_unavailable`'s docstring names `find_spec("agents.tracing")` — the option it rejects, with + * the measurement that rejected it. A test that searched the whole source would trip on the + * prose describing what the code does not do. */ const bootstrapCode = (): string => SITECUSTOMIZE_SOURCE.replace(/"""[\s\S]*?"""/g, '').replace(/^\s*#.*$/gm, ''); - it('asks which distribution provides the SDK, not what a module is called', () => { - // `find_spec("agents")` is true of anything called that, and `agents.py` is an ordinary name - // for an ordinary module — measured, a project with its own two-line `agents.py` and - // `PYTHONPATH=.` on a machine with no SDK was told to install the adapter. - expect(bootstrapCode()).toContain('importlib.metadata.distribution("openai-agents")'); - expect(bootstrapCode()).not.toContain('find_spec'); + it('confirms with distribution metadata before it reports anything', () => { + // Seeing a module name imported is where the question starts, not where it is answered: + // `agents.py` is an ordinary name for an ordinary module, and measured, a project with its own + // two-line `agents.py` and `PYTHONPATH=.` on a machine with no SDK was told to install the + // adapter. `importlib.metadata` asks which distribution provides that name. + expect(bootstrapCode()).toContain('importlib.metadata.distribution(distribution)'); + // And it never *asks* whether a name resolves. It defines `find_spec` — that is how a finder + // is told an import is happening — but calling one on a name would be the rejected design, and + // the expensive one: `find_spec("agents.tracing")` costs 2066ms because resolving a submodule + // spec imports the parent. + expect(bootstrapCode()).not.toMatch(/find_spec\s*\(\s*["']/); }); it('never imports what it is asking about', () => { @@ -228,15 +301,38 @@ describe('the bootstrap orca writes', () => { // and `find_spec("agents.tracing")`, the specific-looking middle option, **2066ms**, because // resolving a submodule spec imports the parent. This file is in front of every Python process // the recording starts, on an interpreter that starts in 50ms. - expect(bootstrapCode()).not.toMatch(/^\s*import agents\b/m); + for (const { root } of PYTHON_ADAPTERS) { + expect(bootstrapCode()).not.toMatch(new RegExp(String.raw`^\s*import ${root}\b`, 'm')); + } expect(bootstrapCode()).not.toContain('agents.tracing'); }); + it('reports a framework only once the agent imports it', () => { + // The whole reason the watcher exists. Installed is not used, and a machine with several + // frameworks lying around would otherwise be told to install an adapter for each of them on + // every recorded run. + expect(bootstrapCode()).toContain('class _WhenImported:'); + expect(bootstrapCode()).toContain('sys.meta_path.insert(0, _WhenImported(path, missing))'); + // Armed only for what is actually missing, so a fully equipped machine carries nothing. + expect(bootstrapCode()).toContain('if missing:'); + }); + + it('never claims a module it is asked about', () => { + // The watcher observes imports; it must not participate in them. Every path through + // `find_spec` returns None, so the ordinary finders load the framework exactly as they would + // have — a debugging aid that changed how a package resolves would be a far worse trade than + // one that captured nothing. + const body = bootstrapCode().slice(bootstrapCode().indexOf('def find_spec')); + const returns = body.slice(0, body.indexOf('\ndef ') + 1).match(/^\s+return .*/gm) ?? []; + expect(returns.length).toBeGreaterThan(0); + for (const line of returns) expect(line.trim()).toBe('return None'); + }); + it('lets no failure of its own reach the agent', () => { // Every import and call is guarded. A capture layer that can break the run it is capturing is // worse than one that captures nothing. const guarded = SITECUSTOMIZE_SOURCE.split('except Exception:').length - 1; - expect(guarded).toBeGreaterThanOrEqual(3); + expect(guarded).toBeGreaterThanOrEqual(3 + PYTHON_ADAPTERS.length); }); }); @@ -498,26 +594,52 @@ describe('the spans transport is not in the trace', () => { } } - /** The adapter absent, deterministically, whatever this machine has installed. */ - async function withoutAdapter(dir: string): Promise { + /** + * Every adapter absent, deterministically, whatever this machine has installed. + * + * All of them rather than the one under test: the bootstrap installs each adapter in + * `PYTHON_ADAPTERS`, and a machine that happens to have one of the others would otherwise make + * this suite's assertions depend on what is in its site-packages. The venv here has both + * langgraph and openai-agents, which is how that stopped being hypothetical. + */ + async function withoutAdapters(dir: string): Promise { await mkdir(dir, { recursive: true }); - await writeFile( - join(dir, 'orcareplay_openai_agents.py'), - 'raise ImportError("stands in for a machine without the package")\n', - ); + for (const { module } of PYTHON_ADAPTERS) { + await writeFile( + join(dir, `${module}.py`), + 'raise ImportError("stands in for a machine without the package")\n', + ); + } return dir; } - /** Distribution metadata, because that is the question the bootstrap asks. */ - async function withSdkInstalled(dir: string): Promise { - const distInfo = join(dir, 'openai_agents-0.20.0.dist-info'); + /** + * A framework the agent can import, and the distribution metadata that confirms it. + * + * Both halves matter, and they are the two questions the bootstrap asks in order: it notices the + * *module* being imported, then asks `importlib.metadata` which *distribution* provides that + * name. A stub module with no metadata is the `agents.py` false positive; metadata with no + * module is a framework nobody used. + * + * Shadowed rather than relying on the real package, so the test says the same thing on a machine + * that has the framework and one that does not — and so it does not pay a two-second SDK import. + */ + async function frameworkPresent(dir: string, adapter: PythonAdapter): Promise { + await writeFile(join(dir, `${adapter.root}.py`), 'VALUE = 1\n'); + const distInfo = join(dir, `${adapter.distribution.replace(/-/g, '_')}-9.9.9.dist-info`); await mkdir(distInfo, { recursive: true }); await writeFile( join(distInfo, 'METADATA'), - 'Metadata-Version: 2.1\nName: openai-agents\nVersion: 0.20.0\n', + `Metadata-Version: 2.1\nName: ${adapter.distribution}\nVersion: 9.9.9\n`, ); } + const ADAPTER_BY_PACKAGE = (pkg: string): PythonAdapter => { + const found = PYTHON_ADAPTERS.find((entry) => entry.package === pkg); + if (found === undefined) throw new Error(`no adapter named ${pkg}`); + return found; + }; + /** One model call through whatever base URL orca set, needing no SDK installed. */ const CALLS_A_MODEL = [ 'import json, os, urllib.request', @@ -531,31 +653,67 @@ describe('the spans transport is not in the trace', () => { 'print("GOT: done")', ].join('\n'); - it('says so when the SDK is installed and the adapter is not', async () => { - const shadows = await withoutAdapter(join(workspace, 'shadows')); - await withSdkInstalled(shadows); + it.each(PYTHON_ADAPTERS.map((adapter) => [adapter.package, adapter] as const))( + 'says so when the agent uses the framework and %s is not there', + async (pkg, adapter) => { + const shadows = await withoutAdapters(join(workspace, 'shadows')); + await frameworkPresent(shadows, adapter); - const { said, exitCode, exchanges } = await recordWith(CALLS_A_MODEL, shadows); + const { said, exitCode, exchanges } = await recordWith( + `import ${adapter.root}\n${CALLS_A_MODEL}`, + shadows, + ); - expect(exchanges, 'the run made no model call, so the gate below is untested').toBeGreaterThan( - 0, - ); - const warning = said.find((line) => line.includes('agent_spans.unavailable')); - expect(warning, `no warning in:\n${said.join('')}`).toBeTruthy(); - expect(warning).toContain('orcareplay-openai-agents'); - // A warning, not a failure: the run is still a run, and the model traffic in it is still real. - expect(exitCode).toBe(0); + expect( + exchanges, + 'the run made no model call, so the gate below is untested', + ).toBeGreaterThan(0); + const warning = said.find( + (line) => line.includes('agent_spans.unavailable') && line.includes(pkg), + ); + expect(warning, `no warning for ${pkg} in:\n${said.join('')}`).toBeTruthy(); + // Its own explanation, not the first adapter's. When this text was one constant, adding a + // second adapter produced a warning that named langgraph and described the Agents SDK. + expect(warning).toContain(adapter.distribution); + expect(warning).toContain(adapter.lost); + expect(warning).toContain(`pip install ${pkg}`); + // A warning, not a failure: the run is still a run, and the model traffic in it is real. + expect(exitCode).toBe(0); + }, + 60_000, + ); + + it('says nothing about a framework the agent never imported', async () => { + // Installed is not used. An ordinary machine has frameworks lying around that a given run has + // no relationship to — this venv has two — and asking `importlib.metadata` at startup would + // tell every recorded run on it to install every adapter orca ships. A warning that fires when + // nothing is wrong is how warnings stop being read. + const shadows = await withoutAdapters(join(workspace, 'shadows')); + for (const adapter of PYTHON_ADAPTERS) await frameworkPresent(shadows, adapter); + + const { said, exchanges } = await recordWith(CALLS_A_MODEL, shadows); + + expect(exchanges).toBeGreaterThan(0); + expect(said.join('')).not.toContain('agent_spans.unavailable'); }, 60_000); - it('stays quiet for a run that made no model call', async () => { - // The bootstrap decides before the agent's first statement, so all it can know is what is - // installed. A run with no model exchange lost no agent structure, and `capture.empty` already - // says the larger thing that went wrong — naming a package on top of that sends someone to - // `pip install` over a problem that is not about a package. - const shadows = await withoutAdapter(join(workspace, 'shadows')); - await withSdkInstalled(shadows); + // The lookalike case — a project's own `agents.py` — is asserted in the `-S` suite below and not + // here, for the reason that suite gives: this level keeps site-packages, so on a machine that + // really has the framework installed a warning is correct however the module resolved, and the + // test would be asserting a property of the machine. - const { said, exchanges } = await recordWith('print("GOT: done")\n', shadows); + it('stays quiet for a run that made no model call', async () => { + // A run with no model exchange lost no structure worth a `pip install`, and `capture.empty` + // already says the larger thing that went wrong — naming a package on top of that sends + // someone to a package manager over a problem that is not about a package. + const shadows = await withoutAdapters(join(workspace, 'shadows')); + const adapter = ADAPTER_BY_PACKAGE('orcareplay-openai-agents'); + await frameworkPresent(shadows, adapter); + + const { said, exchanges } = await recordWith( + `import ${adapter.root}\nprint("GOT: done")\n`, + shadows, + ); expect(exchanges).toBe(0); expect(said.join('')).not.toContain('agent_spans.unavailable'); @@ -579,7 +737,21 @@ describe('agentSpanLosses', () => { { kind: 'span', type: 'HandoffSpanData' }, { kind: 'trace.end' }, ]), - ).toEqual({ unavailable: [], dropped: 0 }); + ).toEqual({ unavailable: [], dropped: 0, droppedBy: [] }); + }); + + it('says which adapter lost records, where the record says', () => { + // Several adapters write to one transport, so a total says how much was lost and nothing about + // where to look. The field is read here rather than merely written: `_dropped` in the first + // adapter was counted from the start and consumed by nothing, which is how a field becomes + // payload. + const losses = agentSpanLosses([ + { kind: 'dropped', count: 2, package: 'orcareplay-langgraph' }, + { kind: 'dropped', count: 1, package: 'orcareplay-openai-agents' }, + { kind: 'dropped', count: 4 }, + ] as never[]); + expect(losses.dropped).toBe(7); + expect(losses.droppedBy).toEqual(['orcareplay-langgraph', 'orcareplay-openai-agents']); }); it('reports a missing adapter once however many processes said so', () => { @@ -678,56 +850,106 @@ describe('the bootstrap reports a missing adapter', () => { * correctly, whatever else is on the path. What `-S` gives up is `site` importing `sitecustomize` * by itself, and that is covered by the end-to-end record above. */ - type Fixture = 'sdk-installed' | 'lookalike-module' | 'nothing'; + /** + * What the interpreter is given: a module it can import, metadata that names a distribution + * providing it, and what the process then imports. The three are independent because the three + * cases that matter differ in exactly one of them. + */ + interface Fixture { + name: string; + /** Adapters whose framework module exists on the path. */ + modules?: readonly PythonAdapter[]; + /** Adapters whose framework distribution metadata exists on the path. */ + metadata?: readonly PythonAdapter[]; + /** What the process imports after the bootstrap has run. */ + imports?: readonly string[]; + } async function runBootstrap(fixture: Fixture): Promise { - const boot = join(bootRoot, fixture); - const spans = join(bootRoot, `${fixture}.jsonl`); + const boot = join(bootRoot, fixture.name); + const spans = join(bootRoot, `${fixture.name}.jsonl`); await mkdir(boot, { recursive: true }); await writeFile(join(boot, 'sitecustomize.py'), SITECUSTOMIZE_SOURCE); - if (fixture === 'sdk-installed') { - // Distribution metadata, because that is the question the bootstrap asks. - const distInfo = join(boot, 'openai_agents-0.20.0.dist-info'); + for (const { root } of fixture.modules ?? []) { + await writeFile(join(boot, `${root}.py`), 'VALUE = 1\n'); + } + for (const { distribution } of fixture.metadata ?? []) { + const distInfo = join(boot, `${distribution.replace(/-/g, '_')}-9.9.9.dist-info`); await mkdir(distInfo, { recursive: true }); await writeFile( join(distInfo, 'METADATA'), - 'Metadata-Version: 2.1\nName: openai-agents\nVersion: 0.20.0\n', + `Metadata-Version: 2.1\nName: ${distribution}\nVersion: 9.9.9\n`, ); } - if (fixture === 'lookalike-module') { - // Somebody's own module. Importable, called `agents`, and nothing to do with OpenAI. - await writeFile(join(boot, 'agents.py'), 'ROSTER = ["alice", "bob"]\n'); - } - await exec(python as string, ['-S', '-c', 'import sitecustomize'], { + const program = ['import sitecustomize', ...(fixture.imports ?? []).map((m) => `import ${m}`)]; + await exec(python as string, ['-S', '-c', program.join('; ')], { env: { ...process.env, PYTHONPATH: boot, ORCA_AGENT_SPANS: spans }, }); return await readFile(spans, 'utf8').catch(() => ''); } - it('says so when the SDK is installed and the adapter is not', async () => { - if (!python) return; // no interpreter here; CI has one and asserts this - const written = await runBootstrap('sdk-installed'); - expect(agentSpanLosses(readSpansText(written)).unavailable).toEqual([ - 'orcareplay-openai-agents', - ]); + it.each(PYTHON_ADAPTERS.map((adapter) => [adapter.package, adapter] as const))( + 'says so when the agent imports the framework and %s is not there', + async (pkg, adapter) => { + if (!python) return; // no interpreter here; CI has one and asserts this + const written = await runBootstrap({ + name: `used-${adapter.root}`, + modules: [adapter], + metadata: [adapter], + imports: [adapter.root], + }); + expect(agentSpanLosses(readSpansText(written)).unavailable).toEqual([pkg]); + }, + ); + + it('stays quiet about a framework that is installed and never imported', async () => { + if (!python) return; + // Installed is not used, and this is the case that made the distinction worth drawing: this + // venv has both frameworks, so reporting on what is installed meant every recorded run on it + // was told to install two adapters it had no relationship to. The count grows with every + // adapter orca ships, and a warning that fires when nothing is wrong is how warnings stop + // being read. + expect( + await runBootstrap({ + name: 'installed-unused', + modules: PYTHON_ADAPTERS, + metadata: PYTHON_ADAPTERS, + }), + ).toBe(''); }); it('stays quiet for a project whose own module happens to be called agents', async () => { if (!python) return; - // The false positive the first version had, and the reason the question is about the - // distribution rather than the import name. `find_spec("agents")` is true of anything called - // that; `agents.py` is an ordinary name for an ordinary module. Reproduced before the fix: a - // project with a two-line `agents.py` and `PYTHONPATH=.`, on a machine with no SDK at all, was - // told "the agent imported the OpenAI Agents SDK" and sent to `pip install` a package it has - // no use for. - expect(await runBootstrap('lookalike-module')).toBe(''); + // The false positive #83 was about, still guarded. Noticing the import is where the question + // starts; `importlib.metadata` is what answers it. A project with a two-line `agents.py` and + // `PYTHONPATH=.`, on a machine with no SDK at all, was told "the agent imported the OpenAI + // Agents SDK" and sent to `pip install` a package it has no use for. + const agents = PYTHON_ADAPTERS.find((entry) => entry.root === 'agents')!; + expect( + await runBootstrap({ name: 'lookalike-module', modules: [agents], imports: ['agents'] }), + ).toBe(''); }); it('stays quiet for a Python process that is not an agent', async () => { if (!python) return; // `orca record` exports ORCA_AGENT_SPANS to every child. Without this guard a run whose agent // shells out to `python` would warn about a package that process had no use for. - expect(await runBootstrap('nothing')).toBe(''); + expect(await runBootstrap({ name: 'nothing' })).toBe(''); + }); + + it('reports a framework once, however many of its modules are imported', async () => { + if (!python) return; + // The watcher is asked for every submodule of an ordinary framework import — measured, 164 + // times for one `from langgraph.graph import StateGraph`. Reporting per `find_spec` would turn + // one fact into a screenful. + const adapter = PYTHON_ADAPTERS.find((entry) => entry.root === 'langgraph')!; + const written = await runBootstrap({ + name: 'many-submodules', + modules: [adapter], + metadata: [adapter], + imports: ['langgraph', 'langgraph', 'langgraph'], + }); + expect(readSpansText(written)).toHaveLength(1); }); it('never fails the process it is loaded into', async () => { diff --git a/packages/schema/schema/event.schema.json b/packages/schema/schema/event.schema.json index 73b84c8a..273a698a 100644 --- a/packages/schema/schema/event.schema.json +++ b/packages/schema/schema/event.schema.json @@ -35,6 +35,8 @@ "agent.start", "agent.handoff", "agent.guardrail", + "graph.node.start", + "graph.node.end", "route.decision", "session.snapshot", "note" diff --git a/packages/schema/src/constants.ts b/packages/schema/src/constants.ts index d6e732c5..b08b8f75 100644 --- a/packages/schema/src/constants.ts +++ b/packages/schema/src/constants.ts @@ -6,7 +6,7 @@ * same drift protection as codegen without a build step that can silently break. */ -export const SCHEMA_VERSION = '0.2.0'; +export const SCHEMA_VERSION = '0.3.0'; export const EVENT_TYPES = [ 'run.start', @@ -31,6 +31,8 @@ export const EVENT_TYPES = [ 'agent.start', 'agent.handoff', 'agent.guardrail', + 'graph.node.start', + 'graph.node.end', 'route.decision', 'session.snapshot', 'note', diff --git a/packages/viewer/src/render.ts b/packages/viewer/src/render.ts index 8e07a643..2c297cd8 100644 --- a/packages/viewer/src/render.ts +++ b/packages/viewer/src/render.ts @@ -88,6 +88,10 @@ const KIND_BY_TYPE: Record = { divergence: 'DIVERGE', note: 'NOTE', 'route.decision': 'ROUTE', + // Not the derived token, which would be `GRAPH` for both — the rows are about a node, and a + // reader scanning the chip column is looking for where in the graph they are. + 'graph.node.start': 'NODE', + 'graph.node.end': 'NODE', 'run.start': 'RUN', 'run.end': 'RUN', checkpoint: 'CKPT', @@ -373,6 +377,29 @@ function parts(event: TraceEvent): RowParts { tone: tripped ? 'attention' : 'normal', }; } + // The graph's own shape, which the proxy has no representation of. Both rows carry the step, + // because a reader following a run wants to know which superstep they are in and the answer is + // not derivable from the row's position — a parallel superstep emits several starts before any + // of their ends. + case 'graph.node.start': { + const step = num(a['step']); + return { + label: pick(a, 'node'), + detail: step === undefined ? 'enter' : `enter · step ${step}`, + }; + } + case 'graph.node.end': { + const error = pick(a, 'error'); + const step = num(a['step']); + return { + label: pick(a, 'node'), + // The class name the node raised. It is the first thing anyone asks of a failed graph and + // the one thing the wire cannot say — a node that dies before calling a model leaves no + // request behind at all. + detail: error ? `raised ${error}` : step === undefined ? 'exit' : `exit · step ${step}`, + tone: error ? 'attention' : 'normal', + }; + } case 'route.decision': return { label: pick(a, 'model', 'target'), detail: pick(a, 'reason', 'rule') }; case 'note': diff --git a/packages/viewer/test/render.test.ts b/packages/viewer/test/render.test.ts index 0daffb83..0e759b68 100644 --- a/packages/viewer/test/render.test.ts +++ b/packages/viewer/test/render.test.ts @@ -628,9 +628,65 @@ describe('the events a proxy cannot produce', () => { it('never leaves one of them without a label', () => { // `label` is the one thing worth reading at a glance, and the contract says it is never empty. - for (const type of ['agent.start', 'agent.handoff', 'agent.guardrail']) { + for (const type of [ + 'agent.start', + 'agent.handoff', + 'agent.guardrail', + 'graph.node.start', + 'graph.node.end', + ]) { const rows = buildTimeline([ev({ type, actor: 'harness', attrs: {} })]); expect(rows[0]!.label, type).not.toBe(''); } }); }); + +/** + * A graph's own shape, which the proxy has no representation of at all. + * + * A node's name is in no request; a node that calls no model makes no request; and two nodes of one + * parallel superstep are indistinguishable on the wire from two consecutive turns. Without these + * rows the timeline had nothing to say about any of it — they fell to `default:`, which looks for + * `message`, `name` or `summary` and would have rendered every one of them blank. + */ +describe('the graph rows', () => { + it('names the node and the superstep it belonged to', () => { + const rows = buildTimeline([ + ev({ type: 'graph.node.start', actor: 'harness', attrs: { node: 'validate_only', step: 2 } }), + ]); + expect(rows[0]!.kind).toBe('NODE'); + expect(rows[0]!.label).toBe('validate_only'); + expect(rows[0]!.detail).toBe('enter · step 2'); + }); + + it('says which node raised, and marks the row', () => { + const rows = buildTimeline([ + ev({ + type: 'graph.node.end', + actor: 'harness', + attrs: { node: 'lookup', step: 3, error: 'ValueError' }, + }), + ]); + expect(rows[0]!.label).toBe('lookup'); + expect(rows[0]!.detail).toBe('raised ValueError'); + expect(rows[0]!.tone).toBe('attention'); + }); + + it('closes a node that did not raise without shouting about it', () => { + const rows = buildTimeline([ + ev({ type: 'graph.node.end', actor: 'harness', attrs: { node: 'first', step: 1 } }), + ]); + expect(rows[0]!.detail).toBe('exit · step 1'); + expect(rows[0]!.tone).not.toBe('attention'); + }); + + it('keeps the two nodes of a parallel superstep apart', () => { + // The row a proxy cannot produce. Both are step 1 under one parent; on the wire this is two + // turns. + const rows = buildTimeline([ + ev({ type: 'graph.node.start', actor: 'harness', attrs: { node: 'worker', step: 1 } }), + ev({ type: 'graph.node.start', actor: 'harness', attrs: { node: 'worker', step: 1 } }), + ]); + expect(rows.map((row) => row.detail)).toEqual(['enter · step 1', 'enter · step 1']); + }); +}); diff --git a/python-langgraph/README.md b/python-langgraph/README.md new file mode 100644 index 00000000..9aa14861 --- /dev/null +++ b/python-langgraph/README.md @@ -0,0 +1,159 @@ +# orcareplay-langgraph + +Record a LangGraph run's own structure — which node ran, in which superstep, and how it ended — +into an [OrcaReplay](https://github.com/Continuum-AI-Corp/OrcaReplay) trace. + +```console +pip install orcareplay-langgraph +``` + +That is the whole setup. `orca record` attaches it to a graph you have not edited; there is no +callback to register, no `with_config`, and no import in your code. + +```console +orca record generic-openai -- python your_graph.py +``` + +## What it adds, and what it does not + +orca records model traffic at a proxy, and every claim it makes about capture comes from there. +This package is not needed for that: a LangGraph run recorded without it is complete in the sense +the rest of the project means. What it adds is the part a proxy structurally cannot see. + +The wire carries the conversation and not the graph. Searching a recorded three-node run for the +node names, the graph's name, `langgraph_node` and `langgraph_step` finds none of them in any +request: + +| the trace can answer | without | with | +| --- | --- | --- | +| which nodes ran, and in which superstep | no | yes | +| that a node ran at all, when it called no model | no | yes | +| that two calls were one parallel superstep, not two turns | no | yes | +| when each node began and ended | no | yes | + +The second row is the common shape rather than an edge case. Validators, state reducers, routers +and writers make no model call, so a proxy sees a graph with those nodes and a graph without them +as the same run. Recorded with this package, a graph of `plan → validate → answer` against a stub +origin produces: + +``` +03:24:45.102 graph.node.start plan +03:24:45.233 model.request +03:24:45.235 model.response +03:24:45.239 graph.node.end plan +03:24:45.240 graph.node.start validate ← no request of any kind +03:24:45.241 graph.node.end validate +03:24:45.241 graph.node.start answer +03:24:45.254 graph.node.end answer +``` + +**What it does not do is attribute a model call to a node by timestamp alone**, and the reason is +on orca's side rather than here. A node record carries the instant the callback fired — measured +against the agent's own clock, exactly. A `model.request` is stamped when orca *persists* it, which +is after the response and after a workspace snapshot: measured 95 ms late on one call and 31 ms on +the next. The node boundaries are trustworthy; a call within about a tenth of a second of one can +fall on the wrong side of it. + +## What is written + +Two record types, and from each only the fields the reader consumes: + +| record | fields | +| --- | --- | +| `LangGraphNodeStart` | `node`, `step`, `run_id`, `parent_run_id`, `started_at` | +| `LangGraphNodeEnd` | `node`, `step`, `run_id`, `ended_at`, and `error` when the node raised | + +Nothing else. No state, no inputs, no outputs, no messages, no tool arguments, and no exception +*message* — only the exception's class name. The model exchanges are left to the proxy, which +already holds them byte for byte; a second copy in a file orca does not redact on the way in would +be a lossier duplicate in a place nothing scrubs. + +`step` is the superstep, which is per graph rather than per run: a subgraph's node reports its own, +so two records can share a step without being concurrent. `parent_run_id` is what separates those, +and it is how two nodes of one parallel fan-out are told apart from two consecutive turns. + +## How a node is recognised + +This is the whole design, and it is also what keeps the package safe. + +`on_chain_start` fires for everything a graph runs. At that moment a node, a runnable *inside* a +node, a subgraph and a conditional-edge function all look alike — same inherited `langgraph_node`, +same `langgraph_step`, and an arbitrary `name`. Measured on one graph: `name='INNER-RUNNABLE'` +carrying `langgraph_node='first'`, `name='router'` carrying `langgraph_node='validate_only'`, and +`name='the_graph'` carrying no `langgraph_node` at all. Reporting on `name` alone reports all four. + +So a node is reported only when the `name` kwarg **equals** `metadata['langgraph_node']` — compared, +never copied. `langgraph_node` is the name the graph's author wrote in `add_node(...)`, while `name` +is whatever the caller supplied, including + +```python +node.with_config({"run_name": f"lookup for {customer.email}"}) +``` + +The two are equal only for an author-defined node, so a runtime string cannot reach the transport +through this path. LangGraph's own `__start__` and `__end__` markers are filtered as well: they are +reported exactly like nodes, and nobody wrote them. + +## Inert unless orca is recording + +Everything keys off `ORCA_AGENT_SPANS`, which only `orca record` sets. With it unset the package +registers nothing, opens nothing and costs nothing, which is what makes it safe to leave installed. + +It declares **no dependencies** — not langgraph, not langchain-core. Installing a framework as a +side effect of installing a debugging aid is the kind of thing that makes people uninstall the +debugging aid, and `orca record` puts this package's bootstrap in front of every Python process a +recording starts, `python --version` included. Importing the module that holds langchain-core's +hook registry costs **617 ms** against an interpreter that starts in about 50, so `install()` arms +a `sys.meta_path` finder and returns; the registration happens the first time anything under +`langgraph` is imported, in a process already paying a second for it. + +## Using it explicitly + +Supported, for anyone who would rather be. `install()` returns whether it did anything, and is +false — quietly — when orca is not recording or the hook is already registered. + +```python +from orcareplay_langgraph import install + +install() +``` + +To supply your own handler instance, set the context variable langchain-core offers: + +```python +from orcareplay_langgraph import HANDLER_VAR, OrcaCallbackHandler + +HANDLER_VAR.set(OrcaCallbackHandler()) +``` + +## The langchain-core interface it implements + +`OrcaCallbackHandler` is **not** a subclass of `BaseCallbackHandler`, deliberately: importing +langchain-core to define the class would make this module unimportable without it, and the point is +that it is inert when langchain-core is absent. langchain-core checks the attributes, not the base +class, and `_configure` type-checks with `isinstance(handler, handler_class)` against the class it +was handed — which is this one. + +| attribute | value | why | +| --- | --- | --- | +| `on_chain_start` / `on_chain_end` / `on_chain_error` | implemented | node boundaries, and how one ended | +| `ignore_chain` | `False` | the only callbacks this wants | +| `ignore_llm`, `ignore_chat_model`, `ignore_retriever`, `ignore_agent`, `ignore_retry`, `ignore_custom_event` | `True` | keeps it out of the token and tool hot path | +| `run_inline` | `True` | without it an `ainvoke` runs the callback on the default executor, and the timestamps say when it was scheduled | +| `raise_error` | `False` | a callback must never take the graph down | + +Every `ignore_*` the base class defines is present, including the ones set to `False`: the +dispatcher reads them with a bare `getattr`, so a missing one is an `AttributeError` per callback, +which langchain-core logs and swallows. Measured, omitting `ignore_chain` gave a graph that ran to +completion, printed four `Error in ... callback` lines, and recorded nothing — a silent miss. A test +compares the two sets so a langchain-core release that adds an eighth is caught by a test rather +than by an empty trace. + +`__init__` does nothing at all, and that is load-bearing: `_configure` constructs a handler on every +configure call, unguarded, before deciding whether one is already present — measured, one trivial +`invoke()` constructs two. A handler whose `__init__` raised made an ordinary `app.invoke()` raise +the same exception, so nothing here opens a file, reads the environment or joins a path. + +## Licence + +Apache-2.0. diff --git a/python-langgraph/orcareplay_langgraph/__init__.py b/python-langgraph/orcareplay_langgraph/__init__.py new file mode 100644 index 00000000..d148d0c3 --- /dev/null +++ b/python-langgraph/orcareplay_langgraph/__init__.py @@ -0,0 +1,21 @@ +"""Record which LangGraph node did what, into an OrcaReplay trace. + +Inert unless orca is recording: everything here keys off the `ORCA_AGENT_SPANS` environment +variable, which only `orca record` sets. See `handler` for what it captures and why it leaves the +model exchanges to the proxy, and `hook` for how it attaches to a graph nobody edited. +""" + +from .handler import PACKAGE, SPAN_END, SPAN_START, SPANS_ENV, OrcaCallbackHandler +from .hook import HANDLER_VAR, install, uninstall + +__all__ = [ + "HANDLER_VAR", + "PACKAGE", + "SPANS_ENV", + "SPAN_END", + "SPAN_START", + "OrcaCallbackHandler", + "install", + "uninstall", +] +__version__ = "0.1.0" diff --git a/python-langgraph/orcareplay_langgraph/handler.py b/python-langgraph/orcareplay_langgraph/handler.py new file mode 100644 index 00000000..c8b7f83a --- /dev/null +++ b/python-langgraph/orcareplay_langgraph/handler.py @@ -0,0 +1,317 @@ +"""A LangChain callback handler that records which LangGraph node did what. + +## What this is for, and what it is not for + +orca records model traffic at a proxy, and every claim it makes about capture comes from there. +This package is not needed for that: a LangGraph run recorded without it is complete in the sense +the rest of the project means. What it adds is the part a proxy structurally cannot see. + +The wire carries the conversation and not the graph. Searching a recorded three-node run for the +node names, the graph's name, `langgraph_node` and `langgraph_step` finds none of them in any +request: + +| the trace can answer | without | with | +|-----------------------------------------------------------|---------|------| +| which nodes ran, and in which superstep | no | yes | +| that a node ran at all, when it called no model | no | yes | +| that two calls were one parallel superstep, not two turns | no | yes | +| when each node began and ended | no | yes | + +The second row is the common shape rather than an edge case -- validators, state reducers, routers +and writers make no model call, so a proxy sees a graph with those nodes and a graph without them +as the same run. + +**What this does not do is attribute a model call to a node by timestamp alone**, and the reason is +on orca's side rather than here. A node record carries the instant the callback fired -- measured +against the agent's own clock, exactly: node start 54.843 against `invoke` at 54.843. A +`model.request` is stamped when orca *persists* it, which is after the response and after a +workspace snapshot: measured 95 ms late on one call and 31 ms on the next, enough to put a call +just outside the node that blocked on it. So the node boundaries are trustworthy, and a call within +about a tenth of a second of one can fall on the wrong side. + +## The discriminator, which is the whole design + +`on_chain_start` fires for everything a graph runs, and at that moment a node, a runnable *inside* +a node, a subgraph and a conditional-edge function all look alike: same inherited +`langgraph_node`, same `langgraph_step`, and an arbitrary `name`. Reporting on `name` alone reports +all four. Measured on one graph: `name='INNER-RUNNABLE'` carrying `langgraph_node='first'`, +`name='router'` carrying `langgraph_node='validate_only'`, and `name='probe_graph'` carrying no +`langgraph_node` at all. + +So a node is reported only when the `name` kwarg **equals** `metadata['langgraph_node']` -- +compared, never copied. That rule has a second effect worth stating, because it is what keeps this +file safe: `langgraph_node` is the name the graph's author wrote in `add_node(...)`, while `name` +is whatever the caller supplied, including a `run_name` interpolated from a user record. The two +are equal only for an author-defined node, so a runtime string cannot reach the transport here. +""" + +from __future__ import annotations + +import datetime +import json +import os +import threading +from typing import Any + +# The path orca hands us. Absent means orca is not recording this run, and then this package does +# nothing at all -- which is what makes it safe to leave installed. +SPANS_ENV = "ORCA_AGENT_SPANS" + +#: The name orca reports when this adapter is the one that lost something. +PACKAGE = "orcareplay-langgraph" + +#: The two record types, named the way the reader switches on them. +SPAN_START = "LangGraphNodeStart" +SPAN_END = "LangGraphNodeEnd" + +#: LangGraph's own entry and exit markers. They are reported exactly like a node -- measured, +#: `__start__` arrives with `langgraph_step` 0 -- and they are not nodes anyone wrote. +SYNTHETIC = frozenset({"__start__", "__end__"}) + +#: A ceiling on how many node starts may be open at once. +#: +#: The open map exists to pair an end with a start, since `on_chain_end` receives **no metadata** -- +#: measured, its `kwargs` are empty -- so the discriminator cannot run there and the run id is the +#: only link. Entries are removed on end or error, so the map is bounded by concurrent nodes in any +#: ordinary graph. This bounds the other case: a long-lived process in which nodes stop closing. +#: Over the ceiling the start is still written and the end is not, which loses a pairing rather +#: than growing without limit -- and `dropped` says it happened. +MAX_OPEN = 10_000 + + +class _Transport: + """One file per process, shared by every handler instance, because there are many. + + `_configure` constructs `handler_class()` **per configure call**, not per run, and it does so + before the `isinstance` check that decides whether to keep it: measured, one trivial `invoke()` + produced two instances. Per-instance state would therefore mean several file objects appending + to one path from one process, which is the interleaving the reader on the other side has to + recover from. Module-level state means one writer and one lock. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._open: dict[str, tuple[str, Any]] = {} + self.dropped = 0 + self.reported = 0 + + @property + def path(self) -> str: + """Where to append, or empty when orca is not recording. + + Read per write rather than cached at import: this module is imported during interpreter + startup, and a value captured then would be whatever the parent happened to export -- for + a nested `orca record`, the outer run's transport. + """ + return os.environ.get(SPANS_ENV, "") + + def write(self, record: dict[str, Any]) -> None: + path = self.path + # The one place that decides there is no transport. An empty value has to land here rather + # than reach `open`, which would raise and be counted as a dropped record -- a run claiming + # capture lost something when orca was not recording at all. + if not path: + return + try: + # `default=str` so a value JSON cannot take costs that one field and nothing else. + # Every field written here is a string or an int today; this is about what a future + # langchain-core puts in `metadata`, which is not ours to choose. + line = json.dumps(record, ensure_ascii=False, default=str) + except Exception: # noqa: BLE001 - a record that will not serialise must not end the run + self.dropped += 1 + return + try: + # Opened per write, like the sibling adapter: the transport may be swept out from under + # a long run, and holding a handle to a deleted file would discard every later record + # in silence. A failed append is counted instead. + with self._lock, open(path, "a", encoding="utf-8") as fh: + fh.write(line + "\n") + except Exception: # noqa: BLE001 - a debugger must not be able to fail a run + self.dropped += 1 + + def opened(self, run_id: str, node: str, step: Any) -> bool: + """Remember a node start, or refuse to when the map is already at its ceiling.""" + with self._lock: + if len(self._open) >= MAX_OPEN: + self.dropped += 1 + return False + self._open[run_id] = (node, step) + self.reported += 1 + return True + + def closing(self, run_id: str) -> tuple[str, Any] | None: + """What this run id was reported as, or None if it was not one of ours. + + Removing here is what makes an end written once. It is also the only reason the node name + can appear on an end record at all: `on_chain_end` is handed a run id and an output, and + nothing that says which node it belonged to. + """ + with self._lock: + return self._open.pop(run_id, None) + + def losses(self) -> dict[str, Any] | None: + """The `dropped` record, or None when there is nothing to say.""" + with self._lock: + unclosed = len(self._open) + total = self.dropped + unclosed + if total <= 0: + return None + return {"kind": "dropped", "count": total, "package": PACKAGE} + + def reset(self) -> None: + """Only for tests; nothing in the package calls it.""" + with self._lock: + self._open.clear() + self.dropped = 0 + self.reported = 0 + + +TRANSPORT = _Transport() + + +def _now() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def report_losses() -> None: + """Write what was not captured, at exit. Registered by `install`, never on import. + + A node that neither ended nor errored is counted with the drops rather than reported as a node + that ran forever: from here the two are the same observation, and the honest summary is that + the pairing is missing. + """ + record = TRANSPORT.losses() + if record is not None: + TRANSPORT.write(record) + + +class OrcaCallbackHandler: + """Reports LangGraph node boundaries, and nothing else. + + Not a subclass of `langchain_core.callbacks.BaseCallbackHandler`, deliberately: importing + langchain-core to define the class would make this module unimportable without it, and the + point is that it is inert when langchain-core is absent. Measured, importing the module that + holds the hook registry costs **617 ms** against an interpreter that starts in about 50 -- and + `orca record` puts this package's bootstrap in front of every Python process a recording + starts, `python --version` included. langchain-core checks the attributes, not the base class; + `_configure` type-checks with `isinstance(handler, handler_class)` against the class it was + handed, which is this one. + """ + + #: Without this, an `ainvoke` runs the handler on the default executor rather than on the + #: caller's thread -- measured, thread `asyncio_0` against `MainThread` -- which reorders the + #: records and makes the timestamps say when the callback was scheduled rather than when the + #: node ran. langchain-core supplies no timestamp of its own, so that would be the only one. + run_inline = True + + #: An exception from a callback must never reach the graph. False is langchain-core's default; + #: it is stated because this handler's whole contract is that it cannot fail a run. + raise_error = False + + #: What this handler listens to. **Every** `ignore_*` that `BaseCallbackHandler` defines has to + #: be here, not just the ones set to True: the dispatcher reads them with a bare + #: `getattr(handler, ignore_condition_name)`, so a missing one is an `AttributeError` per + #: callback -- which langchain-core logs and swallows. Measured, omitting `ignore_chain` gave a + #: graph that ran to completion, printed four `Error in ... callback` lines, and recorded + #: nothing. A silent miss, arriving through the one attribute that had to be False. + #: + #: `test_the_ignore_set_matches_langchain_core` compares this list against the installed + #: `BaseCallbackHandler` so that a version which adds an eighth is caught by a test rather than + #: by an empty trace. + ignore_chain = False + ignore_llm = True + ignore_chat_model = True + ignore_retriever = True + ignore_agent = True + ignore_retry = True + ignore_custom_event = True + + def __init__(self) -> None: + """Nothing. Deliberately nothing. + + `_configure` calls `handler_class()` directly, unguarded, and does it on every configure + call before deciding whether the handler is already present -- measured, a handler whose + `__init__` raised made an ordinary `app.invoke()` raise the same `RuntimeError`. So this + opens no file, reads no environment variable and joins no path; everything that can fail is + deferred to a write, where it is caught. + """ + + # -- langchain-core's interface ------------------------------------------------------------- + def on_chain_start( + self, + serialized: Any, + inputs: Any, + *, + run_id: Any = None, + parent_run_id: Any = None, + tags: Any = None, + metadata: Any = None, + **kwargs: Any, + ) -> None: + try: + meta = metadata if isinstance(metadata, dict) else {} + node = meta.get("langgraph_node") + name = kwargs.get("name") + # Compared, not copied. See the module docstring. + if name is None or node is None or name != node or node in SYNTHETIC: + return + node = str(node) + step = meta.get("langgraph_step") + # An int or nothing. `isinstance(True, int)` is also true, and a bool here would be a + # sign the field had changed meaning, so it is excluded rather than written as 1. + step = step if isinstance(step, int) and not isinstance(step, bool) else None + key = str(run_id) + if not TRANSPORT.opened(key, node, step): + return + TRANSPORT.write( + { + "kind": "span", + "type": SPAN_START, + "span_id": key, + "parent_id": None if parent_run_id is None else str(parent_run_id), + "started_at": _now(), + # The superstep is per graph, not per run: a subgraph's node reports its own, + # so two records can share a step without being concurrent unless they also + # share a parent. The reader is told both and can tell them apart. + "data": {"node": node, "step": step}, + } + ) + except Exception: # noqa: BLE001 - never raise into the graph + return + + def on_chain_end( + self, outputs: Any, *, run_id: Any = None, parent_run_id: Any = None, **kwargs: Any + ) -> None: + self._close(run_id, None) + + def on_chain_error( + self, error: BaseException, *, run_id: Any = None, parent_run_id: Any = None, **kwargs: Any + ) -> None: + # The class name, never the message. A node's exception text is whatever it was given -- a + # row it could not find, an argument it was called with -- and nothing redacts this file on + # the way in. + self._close(run_id, type(error).__name__) + + def _close(self, run_id: Any, error: str | None) -> None: + try: + opened = TRANSPORT.closing(str(run_id)) + # An end for something never reported as a node -- an inner runnable, a conditional + # edge, the graph itself. The discriminator cannot run here, so this is what stands in + # for it. + if opened is None: + return + node, step = opened + data: dict[str, Any] = {"node": node, "step": step} + if error is not None: + data["error"] = error + TRANSPORT.write( + { + "kind": "span", + "type": SPAN_END, + "span_id": str(run_id), + "ended_at": _now(), + "data": data, + } + ) + except Exception: # noqa: BLE001 - never raise into the graph + return diff --git a/python-langgraph/orcareplay_langgraph/hook.py b/python-langgraph/orcareplay_langgraph/hook.py new file mode 100644 index 00000000..5c457783 --- /dev/null +++ b/python-langgraph/orcareplay_langgraph/hook.py @@ -0,0 +1,168 @@ +"""Attaching the handler to a graph nobody edited, without paying for langchain in every process. + +## How it attaches + +`langchain_core.tracers.context.register_configure_hook(var, inheritable, cls, env_var)` appends to +a list that `CallbackManager._configure` walks on every run. For each entry it does, in effect: + + create_one = env_var is not None and env_var_is_set(env_var) and handler_class is not None + if var.get() is not None or create_one: + handler = var.get() or handler_class() + if not any(isinstance(h, handler_class) for h in manager.handlers): + manager.add_handler(handler, inheritable) + +Three things follow, and the package is shaped around them: + + - **`env_var` is the gate, and it is read per run.** `env_var_is_set` is true for any value that + is not `""`, `"0"`, `"false"` or `"False"` -- measured, true for a Windows path -- so + `ORCA_AGENT_SPANS` can be the gate directly. Nothing is added to a manager in a process orca is + not recording, and the decision is made per `_configure` rather than frozen at registration. + - **`handler_class()` runs before the `isinstance` check.** An instance is constructed on every + configure call and usually thrown away, which is why `OrcaCallbackHandler.__init__` does + nothing and why the state it would have held is module-level instead. + - **Registering twice cannot double-report.** The `isinstance` check is against the class, so a + second hook for the same class adds no second handler. `install` still guards, because a + pointless second instance per configure is still waste. + +`inheritable=True` because a node runs as a child of the graph's own run; a non-inheritable handler +would see the graph and none of its nodes. + +## Why it is installed lazily + +Importing `langchain_core.tracers.context` costs **617 ms** on this machine against an interpreter +that starts in about 50. `orca record` puts the bootstrap that calls `install()` in front of *every* +Python process a recording starts -- `python --version` included -- so doing that import eagerly +would tax processes that will never touch a graph. An agent that shells out to `python` twenty +times would pay twelve seconds for a package it did not use. + +So `install` arms a `sys.meta_path` finder and returns. The finder registers the hook the first time +anything under `langgraph` is imported, and a process that imports langgraph is a process already +paying a second for it -- measured, `import langgraph.graph` is 1037 ms, most of it langchain_core. + +**Only the `langgraph` root, and this is a correctness constraint rather than a preference.** The +finder runs inside `_find_and_load_unlocked`, *after* that function's `sys.modules` check. Importing +`langchain_core.tracers.context` from a `find_spec` for a `langchain_core.*` name would re-enter the +load of `langchain_core` itself, and for a top-level name CPython does not re-check `sys.modules` +before executing -- so the package would be executed twice, leaving two module objects and +submodules bound to the older one. Triggering on `langgraph` has no such loop: langchain_core does +not import langgraph. + +Waiting for `langchain_core.tracers.context` to appear in `sys.modules` on its own was the first +design and it does not work: measured across the 164 `find_spec` calls of one +`from langgraph.graph import StateGraph`, it is loaded for none of them. `_configure` imports it at +the first invoke, which is already too late to register in. +""" + +from __future__ import annotations + +import atexit +import os +import sys +from contextvars import ContextVar +from typing import Any + +from .handler import SPANS_ENV, OrcaCallbackHandler, report_losses + +#: langchain-core's own escape hatch: setting this to a handler makes `_configure` use that instance +#: instead of constructing one. Exposed for tests and for anyone who wants a handler of their own. +HANDLER_VAR: ContextVar[Any] = ContextVar("orca_langgraph_handler", default=None) + +#: The import root that triggers the lazy registration. See the module docstring for why it is this +#: one and not `langchain_core`. +TRIGGER = "langgraph" + +_registered = False +_finder: _LazyInstaller | None = None + + +def _register() -> bool: + """Append the hook. Returns whether it is now in place.""" + global _registered + if _registered: + return True + try: + from langchain_core.tracers.context import register_configure_hook + except Exception: # noqa: BLE001 - langchain-core is not here; that is not a failure + return False + try: + register_configure_hook(HANDLER_VAR, True, OrcaCallbackHandler, SPANS_ENV) + except Exception: # noqa: BLE001 - a langchain-core whose signature moved + return False + _registered = True + # Only in a process that got this far, so a `python --version` under a recording registers + # nothing. `report_losses` writes at most one line and only when something was actually lost. + atexit.register(report_losses) + return True + + +class _LazyInstaller: + """A meta-path finder that registers the hook and then gets out of the way. + + It never claims a module: `find_spec` returns None on every path, so the ordinary finders load + langgraph exactly as they would have. The return value is not the point -- being *asked* is. + """ + + def find_spec(self, fullname: str, path: Any = None, target: Any = None) -> None: + try: + if fullname != TRIGGER and not fullname.startswith(TRIGGER + "."): + return None + # Before the import below, so a nested `find_spec` for a langgraph name cannot start a + # second registration. Also what makes uninstalling safe to do first. + _uninstall() + _register() + except Exception: # noqa: BLE001 - an import must not fail because of a debugging aid + _uninstall() + return None + + +def _uninstall() -> None: + """Take the finder off `sys.meta_path`. Idempotent, and never raises.""" + global _finder + finder, _finder = _finder, None + if finder is None: + return + try: + sys.meta_path.remove(finder) + except ValueError: + pass # somebody else rebuilt meta_path; nothing to do + + +def install() -> bool: + """Arrange for the handler to attach to LangGraph runs. Returns whether anything was done. + + False, quietly, in the cases that are not errors: orca is not recording, the hook is already + registered, or langchain-core is absent from a process that has already imported langgraph. + Those are the normal state of a machine that merely has this package on it. + + Registers immediately when langgraph is already imported -- the cost is spent, and a caller who + imports the graph before calling `install` would otherwise never be armed, because the trigger + has already fired. Otherwise it arms the finder and returns; see the module docstring. + """ + global _finder + if _registered: + return False + # The same variable langchain-core will gate on. Checked here too so that a process orca is not + # recording does not even carry the finder. + if not os.environ.get(SPANS_ENV): + return False + if TRIGGER in sys.modules: + return _register() + if _finder is not None: + return False + _finder = _LazyInstaller() + # In front, so the registration happens before any other finder can resolve langgraph -- a + # finder that loads it from a zip or a bundle would otherwise satisfy the import first and the + # trigger would never be asked. + sys.meta_path.insert(0, _finder) + return True + + +def uninstall() -> bool: + """Undo `install` as far as it can be undone. Returns whether the finder was still armed. + + The registered hook itself stays: langchain-core offers no way to remove one, and the entry is + inert without `ORCA_AGENT_SPANS` anyway. This exists so a test can arm the finder twice. + """ + armed = _finder is not None + _uninstall() + return armed diff --git a/python-langgraph/pyproject.toml b/python-langgraph/pyproject.toml new file mode 100644 index 00000000..1e8f73bc --- /dev/null +++ b/python-langgraph/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "orcareplay-langgraph" +version = "0.1.0" +description = "Record a LangGraph run's own structure — which node ran, in which superstep, and how it ended — into an OrcaReplay trace." +readme = "README.md" +requires-python = ">=3.10" +license = { text = "Apache-2.0" } +keywords = ["langgraph", "langchain", "tracing", "replay", "debugging", "orcareplay"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Debuggers", +] + +# Neither `langgraph` nor `langchain-core` is a dependency, on purpose. This package is inert +# without them, and installing a framework as a side effect of installing a debugging aid is the +# kind of thing that makes people uninstall the debugging aid. It is also what lets `orca record` +# leave the bootstrap in front of every Python process in a run: importing this costs nothing. +dependencies = [] + +[project.optional-dependencies] +dev = ["pytest>=7.0", "langgraph>=0.2", "langchain-core>=0.3"] + +[project.urls] +Homepage = "https://github.com/Continuum-AI-Corp/OrcaReplay" +Source = "https://github.com/Continuum-AI-Corp/OrcaReplay/tree/main/python-langgraph" + +[tool.setuptools] +packages = ["orcareplay_langgraph"] diff --git a/python-langgraph/tests/test_handler.py b/python-langgraph/tests/test_handler.py new file mode 100644 index 00000000..1225019e --- /dev/null +++ b/python-langgraph/tests/test_handler.py @@ -0,0 +1,370 @@ +"""The handler's three obligations: report only nodes, be inert when it should be, never raise. + +All three are load-bearing rather than defensive. `orca record` installs this into *every* Python +process a recording starts, so "does nothing" is the common case and has to be the reliable one; +and a callback that raises takes the user's graph down with it, which is the one thing a debugging +aid must never do. + +These tests drive the handler directly with the kwargs langchain-core passes, rather than through a +graph. The shapes they use are not invented: they are what a real `StateGraph` produced under a +probe, and `test_integration.py` is what keeps them honest against the installed version. +""" + +from __future__ import annotations + +import json +import os +import uuid + +import pytest + +from orcareplay_langgraph import PACKAGE, SPAN_END, SPAN_START, SPANS_ENV, OrcaCallbackHandler +from orcareplay_langgraph.handler import MAX_OPEN, TRANSPORT, report_losses + + +@pytest.fixture +def spans(tmp_path, monkeypatch): + """A transport orca would have handed us, reset around every test.""" + path = tmp_path / "agent-spans.jsonl" + monkeypatch.setenv(SPANS_ENV, str(path)) + TRANSPORT.reset() + yield path + TRANSPORT.reset() + + +def records(path): + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] + + +#: Distinct from `None`, which is a `name` a test may want to pass deliberately. +SAME = object() + + +def start(handler, node, *, name=SAME, step=1, run_id=None, parent=None, extra=None): + """One `on_chain_start`, shaped the way langchain-core shapes it. + + `name` defaults to `node` — the case that should be reported. Passing them apart is how the + tests express an inner runnable, a conditional edge or a subgraph. + """ + run_id = run_id or uuid.uuid4() + metadata = {"langgraph_step": step, "thread_id": "1"} + if node is not None: + metadata["langgraph_node"] = node + metadata.update(extra or {}) + handler.on_chain_start( + {"name": "whatever"}, + {"state": "in"}, + run_id=run_id, + parent_run_id=parent, + tags=["graph:step:1"], + metadata=metadata, + name=node if name is SAME else name, + ) + return run_id + + +# -- the discriminator --------------------------------------------------------------------------- +def test_reports_a_node(spans): + handler = OrcaCallbackHandler() + run_id = start(handler, "validate_only", step=2) + + (record,) = records(spans) + assert record["kind"] == "span" + assert record["type"] == SPAN_START + assert record["span_id"] == str(run_id) + assert record["data"] == {"node": "validate_only", "step": 2} + assert record["started_at"].endswith("+00:00") + + +@pytest.mark.parametrize( + ("node", "name", "why"), + [ + ("first", "INNER-RUNNABLE", "a runnable inside a node inherits langgraph_node"), + ("validate_only", "router", "a conditional edge inherits the node it left"), + (None, "probe_graph", "the graph itself has no langgraph_node"), + ("first", None, "no name at all"), + ("__start__", "__start__", "LangGraph's own entry marker is not a node anyone wrote"), + ("__end__", "__end__", "nor its exit marker"), + ], +) +def test_reports_nothing_else(spans, node, name, why): + start(OrcaCallbackHandler(), node, name=name) + assert records(spans) == [], why + + +def test_a_run_name_cannot_reach_the_transport(spans): + """The safety property the discriminator buys, stated as a test. + + `.with_config({"run_name": ...})` puts caller-controlled text in `name`. It is only ever + compared against the author-written node name, so an interpolated secret is a mismatch and a + mismatch is a drop. + """ + start(OrcaCallbackHandler(), "lookup", name="lookup for ada@example.com (sk-live-abcd1234)") + assert records(spans) == [] + + +def test_step_must_be_an_int(spans): + """A step that is not an int is written as null rather than passed through. + + `isinstance(True, int)` is true, so a bool would otherwise arrive as 1 and read as superstep 1. + """ + start(OrcaCallbackHandler(), "n", step="2") + start(OrcaCallbackHandler(), "n", step=True) + start(OrcaCallbackHandler(), "n", step=0) + assert [r["data"]["step"] for r in records(spans)] == [None, None, 0] + + +def test_metadata_that_is_not_a_dict(spans): + handler = OrcaCallbackHandler() + handler.on_chain_start({}, {}, run_id=uuid.uuid4(), metadata="not a dict", name="first") + handler.on_chain_start({}, {}, run_id=uuid.uuid4(), metadata=None, name="first") + assert records(spans) == [] + + +# -- pairing a start with an end ----------------------------------------------------------------- +def test_end_carries_the_node_it_was_told_about(spans): + """`on_chain_end` gets no metadata, so the node name can only come from the remembered start.""" + handler = OrcaCallbackHandler() + run_id = start(handler, "second", step=3) + handler.on_chain_end({"state": "out"}, run_id=run_id) + + _, end = records(spans) + assert end["type"] == SPAN_END + assert end["span_id"] == str(run_id) + assert end["data"] == {"node": "second", "step": 3} + assert "error" not in end["data"] + assert "started_at" not in end + + +def test_end_for_something_never_reported_writes_nothing(spans): + """The stand-in for the discriminator, which cannot run on an end. + + Every inner runnable and conditional edge also ends, and each one arrives here indistinguishable + from a node's end except that its start was never remembered. + """ + OrcaCallbackHandler().on_chain_end({}, run_id=uuid.uuid4()) + assert records(spans) == [] + + +def test_an_end_is_written_once(spans): + handler = OrcaCallbackHandler() + run_id = start(handler, "first") + handler.on_chain_end({}, run_id=run_id) + handler.on_chain_end({}, run_id=run_id) + assert [r["type"] for r in records(spans)] == [SPAN_START, SPAN_END] + + +def test_error_records_the_class_and_not_the_message(spans): + handler = OrcaCallbackHandler() + run_id = start(handler, "lookup") + handler.on_chain_error(ValueError("no row for ada@example.com, token sk-live-abcd1234"), run_id=run_id) + + _, end = records(spans) + assert end["data"]["error"] == "ValueError" + assert "ada@example.com" not in json.dumps(end) + assert "sk-live" not in json.dumps(end) + + +def test_instances_share_state(spans): + """Started by one instance, ended by another — because langchain-core makes several. + + Measured: one `invoke()` constructs the handler class twice. If the open map were per instance, + the end would arrive at a handler that had never seen the start and be dropped. + """ + run_id = start(OrcaCallbackHandler(), "first") + OrcaCallbackHandler().on_chain_end({}, run_id=run_id) + assert [r["type"] for r in records(spans)] == [SPAN_START, SPAN_END] + + +def test_parallel_nodes_are_distinguishable(spans): + """Two workers in one superstep: same step, same parent, different run ids. + + This is the row the proxy cannot answer — two model calls in one superstep look exactly like + two turns on the wire. + """ + parent = uuid.uuid4() + handler = OrcaCallbackHandler() + a = start(handler, "worker", step=1, parent=parent) + b = start(handler, "worker", step=1, parent=parent) + + first, second = records(spans) + assert first["data"] == second["data"] == {"node": "worker", "step": 1} + assert first["parent_id"] == second["parent_id"] == str(parent) + assert first["span_id"] != second["span_id"] + assert {first["span_id"], second["span_id"]} == {str(a), str(b)} + + +def test_parent_is_null_when_there_is_none(spans): + start(OrcaCallbackHandler(), "first", parent=None) + assert records(spans)[0]["parent_id"] is None + + +# -- being inert --------------------------------------------------------------------------------- +def test_writes_nothing_without_the_variable(tmp_path, monkeypatch): + monkeypatch.delenv(SPANS_ENV, raising=False) + TRANSPORT.reset() + handler = OrcaCallbackHandler() + run_id = start(handler, "first") + handler.on_chain_end({}, run_id=run_id) + assert list(tmp_path.iterdir()) == [] + + +def test_an_empty_variable_is_the_same_as_none(tmp_path, monkeypatch): + """Not merely "writes nothing": it must not look like a loss either. + + Without the `or None`, an empty value is a path, `open("")` raises, and the failure is counted + as a dropped record. The run would then report that capture lost something when there was + nothing to capture and orca was not recording at all — the sort of false alarm that teaches + people to ignore the real ones. + """ + monkeypatch.setenv(SPANS_ENV, "") + TRANSPORT.reset() + handler = OrcaCallbackHandler() + run_id = start(handler, "first") + handler.on_chain_end({}, run_id=run_id) + assert list(tmp_path.iterdir()) == [] + assert TRANSPORT.dropped == 0, "an absent transport is not a loss" + + +def test_the_variable_is_read_per_write(spans, tmp_path, monkeypatch): + """Not cached at import: this module is imported during interpreter startup. + + A path captured then would be whatever the parent happened to have exported, which for a nested + `orca record` is the outer run's transport. + """ + handler = OrcaCallbackHandler() + start(handler, "first") + moved = tmp_path / "moved.jsonl" + monkeypatch.setenv(SPANS_ENV, str(moved)) + start(handler, "second") + + assert [r["data"]["node"] for r in records(spans)] == ["first"] + assert [r["data"]["node"] for r in records(moved)] == ["second"] + + +def test_constructing_the_handler_touches_nothing(tmp_path, monkeypatch): + """`_configure` constructs one on every configure call, unguarded, and usually discards it. + + Measured: a handler whose `__init__` raised made an ordinary `app.invoke()` raise the same + exception. So `__init__` must not read the environment, join a path, or open a file. + """ + monkeypatch.setenv(SPANS_ENV, str(tmp_path / "nested" / "does-not-exist.jsonl")) + for _ in range(100): + OrcaCallbackHandler() + assert list(tmp_path.iterdir()) == [] + + +def test_the_handler_declines_the_callbacks_it_does_not_implement(): + handler = OrcaCallbackHandler() + assert handler.run_inline is True + assert handler.raise_error is False + assert handler.ignore_chain is False, "chain callbacks are the only ones this wants" + for attr in ( + "ignore_llm", + "ignore_chat_model", + "ignore_retriever", + "ignore_agent", + "ignore_retry", + "ignore_custom_event", + ): + assert getattr(handler, attr) is True, attr + + +# -- never raising ------------------------------------------------------------------------------- +def test_an_unwritable_path_is_counted_not_raised(tmp_path, monkeypatch): + monkeypatch.setenv(SPANS_ENV, str(tmp_path / "no" / "such" / "dir" / "spans.jsonl")) + TRANSPORT.reset() + handler = OrcaCallbackHandler() + run_id = start(handler, "first") + handler.on_chain_end({}, run_id=run_id) + assert TRANSPORT.dropped == 2 + TRANSPORT.reset() + + +def test_a_node_name_that_will_not_serialise_is_counted_not_raised(spans): + class Unserialisable: + def __str__(self): + raise RuntimeError("not even str() works") + + def __eq__(self, other): + return True # equal to the `name` kwarg, so it passes the discriminator + + __hash__ = None + + handler = OrcaCallbackHandler() + handler.on_chain_start( + {}, {}, run_id=uuid.uuid4(), metadata={"langgraph_node": Unserialisable()}, name="x" + ) + assert records(spans) == [] + assert TRANSPORT.dropped == 0 # it never reached the write; str() raised first + + +def test_the_open_map_has_a_ceiling(spans): + handler = OrcaCallbackHandler() + kept = [start(handler, "n", run_id=uuid.uuid4()) for _ in range(MAX_OPEN)] + assert TRANSPORT.reported == MAX_OPEN + + over = start(handler, "n") + handler.on_chain_end({}, run_id=over) + assert TRANSPORT.dropped == 1 + assert TRANSPORT.reported == MAX_OPEN + + written = records(spans) + assert len(written) == MAX_OPEN, "the start over the ceiling is refused, not written" + handler.on_chain_end({}, run_id=kept[0]) + assert len(records(spans)) == MAX_OPEN + 1, "a remembered node still closes" + + +# -- what was lost ------------------------------------------------------------------------------- +def test_losses_says_nothing_when_nothing_was_lost(spans): + handler = OrcaCallbackHandler() + run_id = start(handler, "first") + handler.on_chain_end({}, run_id=run_id) + report_losses() + assert [r["type"] for r in records(spans)] == [SPAN_START, SPAN_END] + + +def test_losses_counts_a_node_that_never_closed(spans): + start(OrcaCallbackHandler(), "first") + report_losses() + loss = records(spans)[-1] + assert loss == {"kind": "dropped", "count": 1, "package": PACKAGE} + + +def test_losses_names_this_package(spans): + """Two adapters share one transport file, so a count that does not say whose is unactionable.""" + start(OrcaCallbackHandler(), "first") + report_losses() + assert records(spans)[-1]["package"] == "orcareplay-langgraph" + + +# -- the shape the reader depends on --------------------------------------------------------------- +def test_kind_is_the_first_key(spans): + """`readAgentSpans` recovers a torn line by searching for `{"kind":`. + + Every Python process in a recording appends to one file, so a short write leaves a fragment with + the next process's bytes on the end of it. That recovery only works while `kind` is written + first, which is a property of this file rather than of JSON. + """ + handler = OrcaCallbackHandler() + run_id = start(handler, "first") + handler.on_chain_end({}, run_id=run_id) + report_losses() + TRANSPORT.reset() + start(handler, "orphan") + report_losses() + + for line in spans.read_text(encoding="utf-8").splitlines(): + assert line.startswith('{"kind": '), line + + +def test_every_line_is_one_json_object(spans): + handler = OrcaCallbackHandler() + for i in range(20): + run_id = start(handler, f"n{i}") + handler.on_chain_end({}, run_id=run_id) + text = spans.read_text(encoding="utf-8") + assert text.endswith("\n") + assert len(records(spans)) == 40 diff --git a/python-langgraph/tests/test_integration.py b/python-langgraph/tests/test_integration.py new file mode 100644 index 00000000..ac790f15 --- /dev/null +++ b/python-langgraph/tests/test_integration.py @@ -0,0 +1,484 @@ +"""The claims that only a real LangGraph can settle, against the installed version. + +`test_handler.py` drives the handler with the kwargs langchain-core is believed to pass. This file +is what keeps that belief honest: it builds ordinary graphs, invokes them, and reads the transport. +Nothing here imports the handler into the test process — registration, `sys.meta_path` and the hook +registry are all process-wide, so each case is a subprocess with its own interpreter. + +Skipped rather than failed when langgraph is absent, because the package is meant to be installable +and inert on a machine that has never seen it. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap + +import pytest + +from orcareplay_langgraph import PACKAGE + +langgraph = pytest.importorskip("langgraph", reason="the adapter is inert without it") + +#: The repository's package directory, so a subprocess imports the source under test rather than +#: whatever `pip` may have left on the machine. +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def run(body, tmp_path, *, spans=True, env=None, expect_rc=0): + """Run a snippet in a fresh interpreter and return (stdout, the records it produced). + + `body` may be a list of parts, each dedented on its own before they are joined. Dedenting the + concatenation instead would strip only the prefix they share, which silently mis-indents every + part written at a deeper level than the shallowest one. + """ + parts = [body] if isinstance(body, str) else list(body) + script = tmp_path / "case.py" + script.write_text("\n".join(textwrap.dedent(p) for p in parts), encoding="utf-8") + path = tmp_path / "agent-spans.jsonl" + + child = dict(os.environ) + child["PYTHONPATH"] = ROOT + os.pathsep + child.get("PYTHONPATH", "") + child.pop("ORCA_AGENT_SPANS", None) + if spans: + child["ORCA_AGENT_SPANS"] = str(path) + child.update(env or {}) + + done = subprocess.run( + [sys.executable, str(script)], capture_output=True, text=True, env=child, timeout=300 + ) + assert done.returncode == expect_rc, f"rc={done.returncode}\n{done.stdout}\n{done.stderr}" + written = [] + if path.exists(): + written = [json.loads(l) for l in path.read_text(encoding="utf-8").splitlines() if l] + return done.stdout, written + + +GRAPH = """ + from typing import Annotated, TypedDict + from langchain_core.runnables import RunnableLambda + from langgraph.graph import END, START, StateGraph + + class S(TypedDict): + seen: Annotated[list, lambda a, b: a + b] + + def first(state): + # A runnable inside a node: inherits langgraph_node, and must not be reported as one. + RunnableLambda(lambda x: x).with_config({"run_name": "INNER-RUNNABLE"}).invoke({}) + return {"seen": ["first"]} + + def validate_only(state): + # A node that makes no model call at all: invisible to a proxy by construction. + return {"seen": []} + + def second(state): + return {"seen": ["second"]} + + def router(state): + return "second" + + g = StateGraph(S) + g.add_node("first", first) + g.add_node("validate_only", validate_only) + g.add_node("second", second) + g.add_edge(START, "first") + g.add_edge("first", "validate_only") + g.add_conditional_edges("validate_only", router, {"second": "second"}) + g.add_edge("second", END) + app = g.compile(name="the_graph") +""" + + +def nodes(records, type_): + return [r["data"]["node"] for r in records if r["type"] == type_] + + +def test_the_ignore_set_matches_langchain_core(): + """Every `ignore_*` the base class defines must exist on ours, because we do not subclass. + + The dispatcher reads them with a bare `getattr(handler, ignore_condition_name)`. A missing one + raises `AttributeError` inside the callback, which langchain-core logs and swallows — measured, + omitting `ignore_chain` produced a graph that ran to completion, printed four + `Error in ... callback` lines, and recorded nothing at all. + + So this compares the sets rather than spot-checking, and it is an integration test rather than + a unit test on purpose: the thing it guards against is a langchain-core release adding an + eighth, which only the installed package can tell us. + """ + from langchain_core.callbacks.base import BaseCallbackHandler + + from orcareplay_langgraph import OrcaCallbackHandler + + theirs = {n for n in dir(BaseCallbackHandler) if n.startswith("ignore_")} + ours = {n for n in dir(OrcaCallbackHandler) if n.startswith("ignore_")} + assert theirs <= ours, f"langchain-core reads these and we do not define them: {theirs - ours}" + + +def test_the_duck_typed_surface_is_complete(): + """Beyond `ignore_*`: everything the manager reads off a handler that is not tracer-guarded. + + `order_map`, `run_map`, `_external_run_ids` and `copy_with_metadata_defaults` are read only + behind `isinstance(handler, LangChainTracer)`, so they are not ours to supply. What is left is + this. + """ + from orcareplay_langgraph import OrcaCallbackHandler + + handler = OrcaCallbackHandler() + assert handler.raise_error is False + assert handler.run_inline is True + + +def test_a_graph_nobody_edited_is_recorded(tmp_path): + """The headline claim: `install()` first, the user's own graph after, no edit anywhere. + + This is the order `orca record` produces — the bootstrap runs before the agent's first + statement — and it is the order the lazy finder exists for. + """ + out, records = run( + [ + """ + from orcareplay_langgraph import install + assert install() is True + """, + GRAPH, + """ + print(app.invoke({"seen": []})) + """, + ], + tmp_path, + ) + + assert nodes(records, "LangGraphNodeStart") == ["first", "validate_only", "second"] + assert nodes(records, "LangGraphNodeEnd") == ["first", "validate_only", "second"] + assert [r["data"]["step"] for r in records if r["type"] == "LangGraphNodeStart"] == [1, 2, 3] + assert "INNER-RUNNABLE" not in json.dumps(records) + assert "the_graph" not in json.dumps(records) + assert "router" not in json.dumps(records) + + +def test_the_node_that_calls_no_model_is_the_point(tmp_path): + """`validate_only` makes no request. A proxy cannot know it ran; this is where it comes from.""" + _, records = run( + ["from orcareplay_langgraph import install\ninstall()", GRAPH, "app.invoke({'seen': []})"], + tmp_path, + ) + assert "validate_only" in nodes(records, "LangGraphNodeStart") + + +def test_installing_after_langgraph_is_imported_still_works(tmp_path): + """A user calling `install()` in their own script, below their imports. + + The trigger has already fired by then, so the finder would never be asked; `install` has to + notice langgraph in `sys.modules` and register on the spot. + """ + _, records = run( + [ + GRAPH, + """ + from orcareplay_langgraph import install + assert install() is True + app.invoke({"seen": []}) + """, + ], + tmp_path, + ) + assert nodes(records, "LangGraphNodeStart") == ["first", "validate_only", "second"] + + +def test_a_second_install_does_not_double_report(tmp_path): + _, records = run( + [ + """ + from orcareplay_langgraph import install + assert install() is True + assert install() is False + """, + GRAPH, + """ + from orcareplay_langgraph import install as again + assert again() is False + app.invoke({"seen": []}) + """, + ], + tmp_path, + ) + assert nodes(records, "LangGraphNodeStart") == ["first", "validate_only", "second"] + + +def test_without_the_variable_nothing_is_written_and_the_graph_still_runs(tmp_path): + out, records = run( + [ + """ + import sys + from orcareplay_langgraph import install + assert install() is False, "orca is not recording; nothing should be armed" + before = len(sys.meta_path) + """, + GRAPH, + """ + print(app.invoke({"seen": []})["seen"]) + assert len(sys.meta_path) == before, "no finder should have been left behind" + """, + ], + tmp_path, + spans=False, + ) + assert "['first', 'second']" in out + assert records == [] + + +def test_a_failing_node_is_named(tmp_path): + """Which node raised is not on the wire, and it is the first thing anyone asks.""" + _, records = run( + """ + from typing import TypedDict + from orcareplay_langgraph import install + install() + from langgraph.graph import END, START, StateGraph + + class S(TypedDict): + n: int + + def ok(state): + return {"n": 1} + + def boom(state): + raise ValueError("no row for ada@example.com") + + g = StateGraph(S) + g.add_node("ok", ok) + g.add_node("boom", boom) + g.add_edge(START, "ok") + g.add_edge("ok", "boom") + g.add_edge("boom", END) + try: + g.compile().invoke({"n": 0}) + except ValueError: + print("raised, as the graph should") + """, + tmp_path, + ) + + ends = {r["data"]["node"]: r["data"] for r in records if r["type"] == "LangGraphNodeEnd"} + assert ends["ok"].get("error") is None + assert ends["boom"]["error"] == "ValueError" + assert "ada@example.com" not in json.dumps(records), "the message must never be written" + + +def test_parallel_workers_share_a_superstep(tmp_path): + """One superstep, two nodes. On the wire this is indistinguishable from two turns.""" + _, records = run( + """ + from typing import Annotated, TypedDict + from orcareplay_langgraph import install + install() + from langgraph.graph import END, START, StateGraph + from langgraph.types import Send + + class S(TypedDict): + out: Annotated[list, lambda a, b: a + b] + + g = StateGraph(S) + g.add_node("worker", lambda s: {"out": ["w"]}) + g.add_conditional_edges(START, lambda s: [Send("worker", {"out": []})] * 2, ["worker"]) + g.add_edge("worker", END) + g.compile().invoke({"out": []}) + """, + tmp_path, + ) + + starts = [r for r in records if r["type"] == "LangGraphNodeStart"] + assert [r["data"] for r in starts] == [{"node": "worker", "step": 1}] * 2 + assert starts[0]["span_id"] != starts[1]["span_id"] + assert starts[0]["parent_id"] == starts[1]["parent_id"] + assert len([r for r in records if r["type"] == "LangGraphNodeEnd"]) == 2 + + +def test_ainvoke_records_on_the_calling_thread(tmp_path): + """`run_inline`, checked where it matters: the timestamps are the only ones the trace gets.""" + out, records = run( + """ + import asyncio, threading + from typing import TypedDict + from orcareplay_langgraph import install + install() + from langgraph.graph import END, START, StateGraph + import orcareplay_langgraph.handler as H + + seen = set() + real = H.TRANSPORT.write + H.TRANSPORT.write = lambda r: (seen.add(threading.current_thread().name), real(r))[1] + + class S(TypedDict): + n: int + g = StateGraph(S) + g.add_node("a", lambda s: {"n": 1}) + g.add_edge(START, "a") + g.add_edge("a", END) + asyncio.run(g.compile().ainvoke({"n": 0})) + print("threads:", sorted(seen)) + """, + tmp_path, + ) + assert "threads: ['MainThread']" in out + assert nodes(records, "LangGraphNodeStart") == ["a"] + + +def test_a_python_process_that_never_touches_langgraph_pays_nothing(tmp_path): + """The justification for the whole lazy design, as a test. + + `orca record` puts the bootstrap in front of every Python process a recording starts. Importing + the module that holds the hook registry costs 617 ms on this machine, so a process that only + prints its version must not import it — and must not import langchain_core or langgraph either. + """ + out, records = run( + """ + import sys, time + t = time.perf_counter() + from orcareplay_langgraph import install + install() + cost = (time.perf_counter() - t) * 1000 + leaked = sorted(m for m in sys.modules if m.split(".")[0] in ("langchain_core", "langgraph", "langchain")) + print("cost_ms:", round(cost, 1)) + print("leaked:", leaked) + """, + tmp_path, + ) + assert "leaked: []" in out, out + cost = float(out.split("cost_ms:")[1].split()[0]) + assert cost < 100, f"arming should be nearly free, took {cost} ms" + + +def test_the_finder_removes_itself(tmp_path): + """It is asked once. Leaving it on `sys.meta_path` would tax every later import in the run.""" + out, _ = run( + """ + import sys + from orcareplay_langgraph import install + install() + armed = sum(1 for f in sys.meta_path if type(f).__name__ == "_LazyInstaller") + import langgraph.graph # noqa: F401 + after = sum(1 for f in sys.meta_path if type(f).__name__ == "_LazyInstaller") + print("armed:", armed, "after:", after) + """, + tmp_path, + ) + assert "armed: 1 after: 0" in out + + +def test_langchain_core_is_executed_once(tmp_path): + """The re-entrancy the finder's trigger choice exists to avoid. + + The finder runs inside `_find_and_load_unlocked`, past its `sys.modules` check. Importing + langchain-core from a `find_spec` for a `langchain_core.*` name would re-enter that load and + execute the package twice, leaving two module objects. Triggering on `langgraph` cannot. + """ + out, _ = run( + """ + import sys + seen = [] + real_exec = None + import importlib.machinery as M + original = M.SourceFileLoader.exec_module + def counting(self, module): + if module.__name__ == "langchain_core": + seen.append(id(module)) + return original(self, module) + M.SourceFileLoader.exec_module = counting + + from orcareplay_langgraph import install + install() + import langgraph.graph # noqa: F401 + import langchain_core + print("executions:", len(seen), "identity_ok:", id(langchain_core) in seen) + """, + tmp_path, + ) + assert "executions: 1 identity_ok: True" in out, out + + +def test_langchain_core_alone_is_executed_once(tmp_path): + """The same hazard, reached the only way that actually reaches it. + + Importing langgraph fires the trigger before any `langchain_core` name is looked up, so a + finder that *also* watched `langchain_core` would behave identically there and the bug would + hide. It shows up when langchain-core is imported first and langgraph never: measured against a + finder widened to watch both, `langchain_core` was executed **twice**, leaving one stale module + object behind. That is why `TRIGGER` is a single root. + """ + out, _ = run( + """ + import importlib.machinery as M + seen = [] + original = M.SourceFileLoader.exec_module + def counting(self, module): + if module.__name__ == "langchain_core": + seen.append(id(module)) + return original(self, module) + M.SourceFileLoader.exec_module = counting + + from orcareplay_langgraph import install + install() + import langchain_core # first, and langgraph never + print("stale objects:", len(seen) - 1) + """, + tmp_path, + ) + assert "stale objects: 0" in out, out + + +def test_losses_are_reported_at_exit(tmp_path): + """A node the process never saw the end of is counted, and says which adapter counted it.""" + _, records = run( + """ + from typing import TypedDict + from orcareplay_langgraph import install + install() + from langgraph.graph import END, START, StateGraph + import orcareplay_langgraph.handler as H + + class S(TypedDict): + n: int + g = StateGraph(S) + g.add_node("a", lambda s: {"n": 1}) + g.add_edge(START, "a") + g.add_edge("a", END) + H.OrcaCallbackHandler.on_chain_end = lambda *a, **k: None # the end never arrives + g.compile().invoke({"n": 0}) + """, + tmp_path, + ) + assert records[-1] == {"kind": "dropped", "count": 1, "package": PACKAGE} + + +def test_the_user_keeps_their_own_callbacks(tmp_path): + """Attaching must not displace anything: a handler passed in `config` still fires.""" + out, records = run( + """ + from typing import TypedDict + from orcareplay_langgraph import install + install() + from langchain_core.callbacks.base import BaseCallbackHandler + from langgraph.graph import END, START, StateGraph + + theirs = [] + class Theirs(BaseCallbackHandler): + def on_chain_start(self, *a, **k): + theirs.append(k.get("name")) + + class S(TypedDict): + n: int + g = StateGraph(S) + g.add_node("a", lambda s: {"n": 1}) + g.add_edge(START, "a") + g.add_edge("a", END) + g.compile().invoke({"n": 0}, config={"callbacks": [Theirs()]}) + print("theirs saw:", "a" in theirs) + """, + tmp_path, + ) + assert "theirs saw: True" in out + assert nodes(records, "LangGraphNodeStart") == ["a"] diff --git a/python/orca_trace/models.py b/python/orca_trace/models.py index b5c3ec7c..98646ce5 100644 --- a/python/orca_trace/models.py +++ b/python/orca_trace/models.py @@ -31,7 +31,7 @@ "TraceFormatError", ] -SCHEMA_VERSION: Final = "0.2.0" +SCHEMA_VERSION: Final = "0.3.0" #: Spec §2.3. Adding a type is a MINOR bump, so a reader that meets an unknown one skips the #: event rather than failing the trace — see `TraceReader.problems`. @@ -60,6 +60,8 @@ "agent.start", "agent.handoff", "agent.guardrail", + "graph.node.start", + "graph.node.end", "route.decision", "note", } diff --git a/python/tests/test_conformance.py b/python/tests/test_conformance.py index e1fcea55..cc816ca5 100644 --- a/python/tests/test_conformance.py +++ b/python/tests/test_conformance.py @@ -41,15 +41,15 @@ # the sums, which is the one thing these constants exist to prevent. # #: Every seq that satisfies spec §3 for this trace, per deriveCheckpoints() in TypeScript. -GOLDEN_CHECKPOINT_SEQS = [1, 13, 14, 15, 16, 17, 20, 23, 24, 25, 26, 27, 28] +GOLDEN_CHECKPOINT_SEQS = [1, 13, 14, 15, 16, 17, 20, 23, 24, 25, 26, 27, 28, 29, 30] #: (turn, startSeq, endSeq) per turnsOf() in TypeScript. -GOLDEN_TURN_SPANS = [(0, 0, 1), (1, 2, 4), (2, 5, 8), (3, 9, 17), (4, 18, 20), (5, 21, 28)] +GOLDEN_TURN_SPANS = [(0, 0, 1), (1, 2, 4), (2, 5, 8), (3, 9, 17), (4, 18, 20), (5, 21, 30)] #: causalChain(events, 17) in TypeScript — the failing test, traced back to its first request. GOLDEN_CHAIN_TO_17 = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 16, 17] -GOLDEN_EVENT_COUNT = 29 +GOLDEN_EVENT_COUNT = 31 @pytest.fixture diff --git a/spec/orca-trace-v0.md b/spec/orca-trace-v0.md index 4b63f9a0..aef86a2d 100644 --- a/spec/orca-trace-v0.md +++ b/spec/orca-trace-v0.md @@ -76,6 +76,7 @@ Because every model turn resends the whole conversation, content addressing is w | `agent.start` | An agent began a turn: its name, the tools and handoffs it was given. From a harness's own tracing, not from the wire. | | `agent.handoff` | One agent handed control to another, naming both. A proxy sees the transfer as an ordinary tool call and cannot say which agent it came *from*. | | `agent.guardrail` | A guardrail ran, and whether it tripped. Guardrails need not make any request, so this can have no trace on the wire at all. | +| `graph.node.start` / `graph.node.end` | A node of an agent graph ran: its name, the superstep it belonged to, the run ids that pair the two and place it under its parent, and on the end the class of the exception it raised, if any. From the framework's own callbacks. A node's name appears in no request, a node that calls no model makes no request, and two nodes of one parallel superstep are indistinguishable on the wire from two consecutive turns. | | `session.snapshot` | The harness's own transcript, captured at the point the run ended. Carries what the run was *asked*, which a hand-driven run leaves nowhere on the wire. | | `error` | A failure derived from another event or reported by the harness. | | `divergence` | Replay matched inexactly. See §4. | @@ -84,11 +85,19 @@ Because every model turn resends the whole conversation, content addressing is w | `route.decision` | A gateway chose a model. **Generic** — any gateway may emit it. | | `note` | Derived annotation from an analyzer (e.g. loop detection). | -The three `agent.*` types are the first that cannot come from the proxy at all. Every other type -above is something orca observed itself; these are reported by the harness through its own tracing -interface, and a trace that has none of them is not missing anything — it is a run whose harness -either has no such interface or was not asked to use it. Readers must treat them as optional, like -any other type they do not find. +The `agent.*` and `graph.*` types are the ones that cannot come from the proxy at all. Every other +type above is something orca observed itself; these are reported by the harness through its own +tracing or callback interface, and a trace that has none of them is not missing anything — it is a +run whose harness either has no such interface or was not asked to use it. Readers must treat them +as optional, like any other type they do not find. + +They also differ from the rest in who wrote them. A `model.request` was assembled by orca out of +bytes it saw; a `graph.node.start` was handed to orca by code running inside the agent's own +interpreter. An implementation that writes these MUST keep them to structure the harness names for +itself — a node's declared name, an agent's name, whether a guardrail tripped — and MUST NOT carry +prompts, model output, tool arguments, tool results or exception messages in them. Those either +already exist elsewhere in the trace, byte for byte, or are values the harness never intended to +export. Adding a type is a MINOR version bump. Removing or changing the meaning of one is MAJOR. diff --git a/test/integrations/README.md b/test/integrations/README.md index 2693aee9..a6dea2d2 100644 --- a/test/integrations/README.md +++ b/test/integrations/README.md @@ -45,12 +45,17 @@ a matrix that only exercised a plain completion would have said nothing about ei | `openai-agents-handoff` | two agents, a handoff and a guardrail — reported by the SDK, not seen on the wire | the sixth capture layer | | `langgraph-stream` | LangChain's own `OPENAI_API_BASE`, over SSE | LangGraph, LangChain | | `langgraph-tools` | the same, with a bound tool | tool-calling graphs | +| `langgraph-nodes` | which node ran, in which superstep — including one that calls no model | the sixth capture layer, for graphs | +| `llama-index` | LlamaIndex's own OpenAI LLM, which reads the older base-URL variable and not the new one | LlamaIndex | | `browser-use` | its own `ChatOpenAI` passes an unset `base_url` through | browser-use, and the pattern any wrapper using the official SDK follows | +| `vision-repaint` | a screenshot recorded intact, and replayed against a different one | agents with eyes, and what a match means when the pixels move | +| `mastra` | a model provider that takes its origin in code rather than from the environment | Mastra, and any JS agent that never reads a variable | +| `mcp-stdio` | an MCP server launched from a config, recorded and then taken away | the MCP shim, and replay with the server gone | | `fetch-hook` | `NODE_OPTIONS` preload on `globalThis.fetch` | the Vercel AI SDK, and any JS agent with its origin compiled in | | `rag-index` | a concurrent index build, an embedding batch, and an answer over retrieved context | IndexRAG, LlamaIndex, GraphRAG, LightRAG — every pipeline that indexes before it answers | | `rag-split-origin` | the same run with embeddings at a **second origin of the same wire dialect** | any stack whose chat and embeddings do not share a provider | -Fourteen checks. Covering the layer underneath still covers what stands on it — that is what the +Nineteen checks. Covering the layer underneath still covers what stands on it — that is what the `litellm` row is for — but three of these exist because that stopped being enough, and each of the three was added after running the framework itself said something the layer could not. diff --git a/test/integrations/agents/langgraph_nodes.py b/test/integrations/agents/langgraph_nodes.py new file mode 100644 index 00000000..00d93678 --- /dev/null +++ b/test/integrations/agents/langgraph_nodes.py @@ -0,0 +1,62 @@ +"""A graph whose shape the proxy cannot see, for the adapter that reports it. + +`langgraph_agent.py` next door proves the traffic is captured. This one is about what the traffic +does not contain, so every node here is chosen for that: + + - `plan` calls the model, and runs an inner runnable that must *not* be reported as a node + - `validate` calls no model at all, so a proxy has no evidence it ran + - `route` is a conditional edge, which arrives at the callback looking exactly like a node + - `answer` calls the model again, in a later superstep + +Nothing here imports orcareplay_langgraph. The point of the check is that `orca record` attaches it +to a graph nobody edited. +""" + +from typing import Annotated, TypedDict + +from langchain_core.messages import HumanMessage +from langchain_core.runnables import RunnableLambda +from langchain_openai import ChatOpenAI +from langgraph.graph import END, START, StateGraph +from langgraph.graph.message import add_messages + +llm = ChatOpenAI(model="stub-1") + + +class State(TypedDict): + messages: Annotated[list, add_messages] + checked: bool + + +def plan(state: State): + # An inner runnable inside a node. It inherits `langgraph_node`, so anything keying off that + # alone reports it as a second `plan`. + RunnableLambda(lambda x: x).with_config({"run_name": "INNER-RUNNABLE"}).invoke({}) + return {"messages": [llm.invoke(state["messages"])]} + + +def validate(state: State): + """No model call. The whole reason this file exists.""" + return {"checked": True} + + +def route(state: State): + return "answer" + + +def answer(state: State): + return {"messages": [llm.invoke(state["messages"] + [HumanMessage("again")])]} + + +graph = StateGraph(State) +graph.add_node("plan", plan) +graph.add_node("validate", validate) +graph.add_node("answer", answer) +graph.add_edge(START, "plan") +graph.add_edge("plan", "validate") +graph.add_conditional_edges("validate", route, {"answer": "answer"}) +graph.add_edge("answer", END) + +out = graph.compile(name="the_graph").invoke({"messages": [HumanMessage("hello")], "checked": False}) +print("TURNS:", len(out["messages"])) +print("GOT:", out["messages"][-1].content or "(tool call)") diff --git a/test/integrations/run.mjs b/test/integrations/run.mjs index ddd9dedb..03e145a3 100644 --- a/test/integrations/run.mjs +++ b/test/integrations/run.mjs @@ -111,6 +111,29 @@ const CHECKS = [ needs: 'langgraph', exchanges: 2, }, + { + id: 'langgraph-nodes', + what: "the graph's own shape: which node ran, including the one that calls no model", + run: ['python', 'agents/langgraph_nodes.py'], + // Both, for the reason the array exists: with only langgraph this runs, captures its two + // exchanges, and fails with "no graph.node.start in the trace" — which reads as a broken layer + // rather than as a missing install. + needs: ['langgraph', 'orcareplay_langgraph'], + exchanges: 2, + expectEvents: ['graph.node.start', 'graph.node.end'], + /** + * The claim the event types alone do not make. + * + * `validate` calls no model, so nothing about it reaches the wire — a trace that has it got it + * from the adapter and nowhere else. The other two are the discriminator: an inner runnable + * carries `langgraph_node` and a conditional edge carries the node it left, so both arrive at + * the callback looking like nodes, and the graph itself arrives with a name and no node at all. + */ + expectNodes: { + present: ['plan', 'validate', 'answer'], + absent: ['INNER-RUNNABLE', 'route', 'the_graph'], + }, + }, { id: 'browser-use', what: "browser-use's own ChatOpenAI, which passes an unset base_url straight through", @@ -566,13 +589,29 @@ async function runCheck(check) { // Event types a check insists on. Counting exchanges says the traffic was captured; it says // nothing about a layer whose whole purpose is what the traffic does not contain. - if (check.expectEvents) { + if (check.expectEvents || check.expectNodes) { const listed = await orca(['events', '--json', runId], dir); const line = listed.out.split(/\r?\n/).find((l) => l.startsWith('[')); const events = JSON.parse(line ?? '[]'); const seen = new Set(events.map((e) => e.type)); - const missing = check.expectEvents.filter((t) => !seen.has(t)); + const missing = (check.expectEvents ?? []).filter((t) => !seen.has(t)); if (missing.length > 0) throw new Error(`no ${missing.join(', ')} in the trace`); + + // Which nodes, not just that there were nodes. A layer that reported every callback would + // satisfy the types above while saying something false about the graph. + if (check.expectNodes) { + const nodes = new Set( + events.filter((e) => e.type.startsWith('graph.node.')).map((e) => e.attrs?.node), + ); + const absentNodes = check.expectNodes.present.filter((n) => !nodes.has(n)); + if (absentNodes.length > 0) { + throw new Error(`no node named ${absentNodes.join(', ')} in [${[...nodes].join(', ')}]`); + } + const extra = check.expectNodes.absent.filter((n) => nodes.has(n)); + if (extra.length > 0) { + throw new Error(`${extra.join(', ')} reported as a node, and none of them is one`); + } + } } // Retrieval is a separate axis and asserted separately, for the reason it is reported