From 8da299bde3ffb5cf8c611845a60df76d65470d1b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 11 Sep 2026 17:55:34 +0000 Subject: [PATCH] =?UTF-8?q?feat(taxonomy):=20add=20Step.agent=5Fid=20?= =?UTF-8?q?=E2=80=94=20Phase=201=20of=20the=20MAST=20multi-agent=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of docs/concepts/multi-agent-failures.md's recommended phasing. Step.agent_id: str | None = None — records which agent produced a step; optional, None by default, zero behavior change for existing single-agent callers. triage.observability.otel_ingest. trajectory_from_spans() populates it from a span's gen_ai.agent.id (preferred) or gen_ai.agent.name attribute when present — the same extraction pattern already used for tool_called/metadata["http_status"]. The finding worth calling out, and why this commit corrects the scoping doc's own earlier estimate rather than just implementing it: RulesClassifier's LOOP_DETECTED needed ZERO changes to catch a step repeated across two different agents (MAST's "Step Repetition"), not the small extension originally estimated. _is_loop_window() was already agent-identity-agnostic — it only ever compared tool_called/tool_input, because there was no agent field to look at — so adding Step.agent_id made the existing single-agent matching logic correct for the multi-agent case for free. Verified, not assumed: see test_loop_detected_across_different_agent_ids in tests/test_classifier_rules.py. Documented as deliberate in rules.py's comments and RulesClassifier's docstring — same-agent-only matching would be the wrong default, since it would silently break single-agent loop detection for any caller who happens to tag steps with an agent id, and the whole point of this phase is that a loop spanning a handoff is still a loop. Also wires agent_id through triage/suspension.py's serialize_run/deserialize_run so a suspended run's trajectory round-trips it correctly — caught by the existing test_serialize_covers_all_step_fields regression guard, which failed until this was added. triage/checkpoint/base.py's checkpoint serialization has a pre-existing, separate gap here (it already didn't round-trip idempotent/partial either, before this change, and has no completeness test guarding it) — left as-is since fixing it is unrelated to this phase, not a regression introduced by it. Updates docs/concepts/multi-agent-failures.md (phase 1 marked done, its own "small code change" estimate corrected to "zero code change" for the loop-matching logic specifically), docs/known-limitations.md's "Multi-agent systems" section, and CLAUDE.md (repo layout, rule priority table, and a new design-decision entry explaining why agent-agnostic loop matching is intentional). Verification: ruff check/format clean, pytest 806 passed (was 801 — +5 new tests, 0 skipped), mypy --strict clean, mkdocs build --strict clean. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01M4WNEkbnKSx9mTg5Q1jX39 --- CHANGELOG.md | 24 +++++++++ CLAUDE.md | 18 ++++++- docs/concepts/multi-agent-failures.md | 14 +++--- docs/known-limitations.md | 6 ++- tests/test_classifier_rules.py | 17 +++++++ tests/test_observability_otel_ingest.py | 66 +++++++++++++++++++++++++ tests/test_suspension.py | 2 + triage/classifier/rules.py | 12 ++++- triage/observability/otel_ingest.py | 13 +++++ triage/suspension.py | 2 + triage/taxonomy.py | 11 +++++ 11 files changed, 173 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62a174a..f760c30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,30 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **`Step.agent_id: str | None = None`** — Phase 1 of the MAST multi-agent alignment scoped in + `docs/concepts/multi-agent-failures.md`. Records which agent produced a step; optional, + `None` by default, zero behavior change for existing single-agent callers. + `triage.observability.otel_ingest.trajectory_from_spans()` now populates it from a span's + `gen_ai.agent.id` (preferred) or `gen_ai.agent.name` attribute when present — the same + extraction pattern already used for `tool_called`/`metadata["http_status"]`. + + The finding worth calling out: `RulesClassifier`'s `LOOP_DETECTED` needed **zero changes** + to catch a step repeated across two different agents (MAST's "Step Repetition"), not the + small extension originally estimated. `_is_loop_window()` was already agent-identity-agnostic + — it only ever compared `tool_called`/`tool_input`, because there was no agent field to look + at — so adding `Step.agent_id` made the existing single-agent matching logic correct for the + multi-agent case for free. Pinned by + `test_loop_detected_across_different_agent_ids` in `tests/test_classifier_rules.py`, and + documented as deliberate (not an oversight to "fix" later) in `rules.py`'s comments and + `RulesClassifier`'s docstring — see `CLAUDE.md`'s design-decisions section for the full + rationale on why same-agent-only matching would be the wrong default. + + Also wires `agent_id` through `triage/suspension.py`'s `serialize_run`/`deserialize_run` so + a suspended run's trajectory round-trips it correctly (`triage/checkpoint/base.py`'s + checkpoint serialization has a pre-existing, separate gap here — it already didn't round-trip + `idempotent`/`partial` either, before this change — left as-is since fixing it is unrelated + to this phase). + - **`docs/concepts/multi-agent-failures.md`** — a design proposal, not implemented, scoping what it would take to detect multi-agent failures against the published [MAST taxonomy](https://github.com/multi-agent-systems-failure-taxonomy/MAST) (14 failure diff --git a/CLAUDE.md b/CLAUDE.md index 70a8e3b..491959c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ No framework imports anywhere in `triage/` core — adapters live in `triage/ada ``` triage/ — importable package - taxonomy.py — FailureType enum (9 members), Step (with idempotent field), FailureContext + taxonomy.py — FailureType enum (9 members), Step (with idempotent, agent_id fields), FailureContext trajectory.py — Trajectory class (append / replay_from / last_n_steps) checkpoint/ — Checkpoint package __init__.py — re-exports Checkpoint, CheckpointStore, InMemoryCheckpointStore, make_checkpoint @@ -126,7 +126,7 @@ without a labeled corpus. ## RulesClassifier rule priority -1. LOOP_DETECTED — last `loop_window` steps (default 3, configurable): identical `tool_called` + canonical `tool_input` +1. LOOP_DETECTED — last `loop_window` steps (default 3, configurable): identical `tool_called` + canonical `tool_input`, regardless of `Step.agent_id` — catches a step repeated across two different agents, not just one agent looping on itself (MAST "Step Repetition") 2. WRONG_TOOL_CALLED — error matches tool-not-found patterns across OpenAI/Anthropic/generic SDKs, OR `metadata["json_rpc_code"] == -32601` 3. SCHEMA_MISMATCH — error matches `validation error|json.*parse|jsondecodeerror|invalid json|unexpected token`, OR `metadata["json_rpc_code"] == -32700` 4. EXTERNAL_FAULT — error contains `\b(429|500|502|503)\b` (word-boundary, avoids false positives), OR `metadata["http_status"] in (429, 500, 502, 503)`, OR `metadata["json_rpc_code"] == -32603` @@ -321,6 +321,20 @@ attributes between spec revisions (see the module docstring for the exact key sp `Step.metadata["http_status"]` is extracted from the stable HTTP semconv and feeds directly into `RulesClassifier`'s structured-error-code matching above — a real HTTP client span with a 429/500/502/503/408/504 status classifies correctly with zero code from the caller. +`Step.agent_id` is extracted from a span's `gen_ai.agent.id` (preferred) or `gen_ai.agent.name` +attribute the same way, feeding MAST-alignment phase 1 below. + +**`Step.agent_id` and `LOOP_DETECTED`'s agent-agnostic matching (MAST phase 1).** `Step.agent_id: +str | None = None` records which agent produced a step — optional, `None` by default, zero +behavior change for single-agent callers. `RulesClassifier`'s `_is_loop_window()` matches purely +on `tool_called`/`tool_input` equality and has never looked at agent identity, so it already +catches a step repeated across two different `agent_id`s (MAST's "Step Repetition" — see +`docs/concepts/multi-agent-failures.md`) with zero changes to the matching logic itself; adding +the field was the entire change. Don't "fix" this into requiring same-`agent_id` matching by +default — that would silently break single-agent loop detection for any caller who happens to +tag steps with an agent id, and the whole point of MAST phase 1 is that a loop spanning a +handoff is still a loop. Regression guard: +`tests/test_classifier_rules.py::test_loop_detected_across_different_agent_ids`. ## Classifier accuracy measurement diff --git a/docs/concepts/multi-agent-failures.md b/docs/concepts/multi-agent-failures.md index aebcdbc..790cb70 100644 --- a/docs/concepts/multi-agent-failures.md +++ b/docs/concepts/multi-agent-failures.md @@ -1,6 +1,6 @@ # Multi-Agent Failures: MAST Scoping -**Status: a design proposal, not yet implemented.** Nothing in this document has shipped. It exists to answer one question concretely before any code is written: what would it actually take for `triage` to detect multi-agent failure modes, and how much of that can be done honestly with the same rigor the rest of this project holds itself to — measured, not assumed, and never a new `FailureType` without real disambiguation logic behind it. +**Status: phase 1 shipped (see "Recommended phasing" below); phases 2-3 are still a design proposal, not implemented.** This document exists to answer one question concretely before writing code: what would it actually take for `triage` to detect multi-agent failure modes, and how much of that can be done honestly with the same rigor the rest of this project holds itself to — measured, not assumed, and never a new `FailureType` without real disambiguation logic behind it. ## Why this exists @@ -45,7 +45,7 @@ Single-agent failure modes — wrong tool, bad schema, a timeout — are increas Before any classification question, there's a data-model question that blocks all of it. `Step`/`Trajectory` today represent one flat, single-actor sequence — there is no field recording *which agent* produced a given step. None of MAST's category 2 (inter-agent) failure modes are even representable in triage's current model, independent of how good a classifier gets — "ignored other agent's input" requires knowing there *was* another agent and what it said. -The minimal, additive fix: `Step.agent_id: str | None = None`, the same low-risk pattern `Step.metadata` already established — an optional field with zero effect on any existing caller. It connects directly to work already shipped: the OTel GenAI semantic conventions already define `gen_ai.agent.id`/`gen_ai.agent.name` on `invoke_agent` spans (verified directly against the spec), so `triage.observability.otel_ingest.trajectory_from_spans()` could capture agent identity the same way it already captures `tool_called`/`http_status` — real frameworks that emit per-agent spans would populate this for free, the same story as the HTTP-status connection to `RulesClassifier`'s structured-code matching. +**✅ Done (phase 1):** `Step.agent_id: str | None = None`, the same low-risk pattern `Step.metadata` already established — an optional field with zero effect on any existing caller. It connects directly to work already shipped: the OTel GenAI semantic conventions already define `gen_ai.agent.id`/`gen_ai.agent.name` on `invoke_agent` spans (verified directly against the spec), so `triage.observability.otel_ingest.trajectory_from_spans()` now captures agent identity the same way it already captured `tool_called`/`http_status` — real frameworks that emit per-agent spans populate this for free, the same story as the HTTP-status connection to `RulesClassifier`'s structured-code matching. ## Mapping the 14 modes @@ -55,7 +55,7 @@ Honesty matters more here than coverage. Three groups, by how they'd actually ge | MAST mode | Existing mapping | |---|---| -| 1.3 Step Repetition | `LOOP_DETECTED` — triage already detects this for one agent (`_is_loop_window()` in `rules.py`). Once `agent_id` exists, the same mechanism extends to catch a step repeated *across* agents — a small, concrete code change, not a new concept. | +| 1.3 Step Repetition | `LOOP_DETECTED` — ✅ done (phase 1, see below): `_is_loop_window()` in `rules.py` was already agent-identity-agnostic, so adding `Step.agent_id` made it catch a step repeated *across* agents with zero matching-logic changes — not the "small code change" this doc originally estimated, but no change at all. | | 3.1 Premature Termination | `PLAN_INCOMPLETE` — CLAUDE.md's own taxonomy table already defines this as "agent declared success but not all required sub-goals were completed," which *is* MAST's 3.1 definition, just framed for one agent. Needs a docs update connecting the two, not a new type. | Finding two direct hits here — including one where triage already has *working, tested code* for the single-agent case — is the strongest evidence this taxonomy is worth aligning with rather than inventing categories from scratch. @@ -81,10 +81,10 @@ CLAUDE.md documents exactly this failure mode already happening once: `HALLUCINA ## Recommended phasing -1. **`Step.agent_id: str | None = None`** — pure additive field, zero behavior change for existing callers. Extend `otel_ingest.trajectory_from_spans()` to populate it from `gen_ai.agent.id`/`gen_ai.agent.name` when present. Extend `LOOP_DETECTED`'s matching to also catch identical steps across different `agent_id`s. This alone ships something real: cross-agent loop detection, using code that already exists and is already tested. -2. **Prototype 3.3 (verification mismatch) and 2.1 (conversation reset)** as candidate `RulesClassifier` rules, validated against a constructed corpus of real multi-agent framework traces (AutoGen, CrewAI, LangGraph multi-agent) — same sourcing discipline as `tests/data/error_corpus_*.json`, not synthetic examples built to make the rule look good. -3. **Only after (1) and (2) ship and are measured:** extend `LLMClassifier`'s prompt to recognize the ten semantic-only modes, evaluate it against real multi-agent traces, and only then consider whether any of them earn a stable `FailureType` member — each one needs its own answer to "how does a classifier actually tell this apart from the others," not just a taxonomy citation. +1. ✅ **Done.** `Step.agent_id: str | None = None` — pure additive field, zero behavior change for existing callers. `otel_ingest.trajectory_from_spans()` populates it from `gen_ai.agent.id`/`gen_ai.agent.name` when present (`id` preferred when both are set). Cross-agent loop detection ships too — but the honest correction to this doc's own earlier estimate: it needed **zero changes** to `_is_loop_window()`'s matching logic, not an extension of it. That function was already agent-agnostic (it only ever compared `tool_called`/`tool_input`, never looked at agent identity, because the field didn't exist) — adding `agent_id` to `Step` made the existing single-agent test suite's matching logic correct for the multi-agent case for free. Pinned by `test_loop_detected_across_different_agent_ids` in `tests/test_classifier_rules.py`, so a future change can't accidentally narrow it back to same-agent-only without a test failing. `RulesClassifier`'s docstring and the `classify()` comment above the loop check now document this as deliberate, not unnoticed. +2. **Not started.** Prototype 3.3 (verification mismatch) and 2.1 (conversation reset) as candidate `RulesClassifier` rules, validated against a constructed corpus of real multi-agent framework traces (AutoGen, CrewAI, LangGraph multi-agent) — same sourcing discipline as `tests/data/error_corpus_*.json`, not synthetic examples built to make the rule look good. +3. **Not started, blocked on (2).** Only after (1) and (2) ship and are measured: extend `LLMClassifier`'s prompt to recognize the ten semantic-only modes, evaluate it against real multi-agent traces, and only then consider whether any of them earn a stable `FailureType` member — each one needs its own answer to "how does a classifier actually tell this apart from the others," not just a taxonomy citation. ## Non-goals -This is not a plan to add 14 new `FailureType` members. It is not a plan to build a multi-agent orchestration framework — triage still wraps whatever callable you give it, single- or multi-agent. It is not started — `git blame` on this file should show it arriving with no corresponding change to `taxonomy.py`. +This is not a plan to add 14 new `FailureType` members. It is not a plan to build a multi-agent orchestration framework — triage still wraps whatever callable you give it, single- or multi-agent. Phase 1 is done (see above); phases 2 and 3 are not. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 8dd269a..ef2b2b2 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -437,9 +437,11 @@ The conditional-import fallback pattern (`Tracer = Any`, `_otel_trace = None`) d ## Multi-agent systems -`Step`/`Trajectory` represent one flat, single-actor sequence — there is no field recording *which agent* produced a given step. Failures specific to multi-agent coordination (a handoff losing context, one agent ignoring another's output, a verifier claiming success on a broken result) aren't representable in triage's data model today, independent of classifier sophistication — you can't detect "agent A ignored agent B's input" without knowing there were two agents and what each said. +`Step.agent_id: str | None = None` records which agent produced a given step (optional, `None` by default — no effect on single-agent callers), and `triage.observability.otel_ingest.trajectory_from_spans()` populates it automatically from a span's `gen_ai.agent.id`/`gen_ai.agent.name` attribute when present. `RulesClassifier`'s `LOOP_DETECTED` already catches a step repeated across two different agents, not just the same agent looping on itself — its matching was always agent-identity-agnostic, so this needed no new matching logic once the field existed. -See [`docs/concepts/multi-agent-failures.md`](concepts/multi-agent-failures.md) for a full scoping against the published [MAST taxonomy](https://github.com/multi-agent-systems-failure-taxonomy/MAST) (14 failure modes, 3 categories) — what maps onto `triage`'s *existing* `FailureType`s with no new code, what's a plausible new `RulesClassifier` rule worth measuring, and what's semantic-only and would need an `LLMClassifier` prompt extension. This is a design proposal, not implemented — `Step.agent_id` doesn't exist yet. +That's as far as multi-agent detection goes today. Failures that need semantic understanding of what multiple agents actually said to each other (a handoff losing context, one agent ignoring another's output, a verifier claiming success on a broken result) aren't detectable yet — RulesClassifier has no way to reach them, and LLMClassifier's prompt doesn't yet know to look for them. + +See [`docs/concepts/multi-agent-failures.md`](concepts/multi-agent-failures.md) for the full scoping against the published [MAST taxonomy](https://github.com/multi-agent-systems-failure-taxonomy/MAST) (14 failure modes, 3 categories): what's shipped (phase 1, above), what's a plausible new `RulesClassifier` rule worth measuring next, and what's semantic-only and would need an `LLMClassifier` prompt extension — deliberately not built yet. ## Comparison with framework-native error handling diff --git a/tests/test_classifier_rules.py b/tests/test_classifier_rules.py index b6dcc95..2bcddb5 100644 --- a/tests/test_classifier_rules.py +++ b/tests/test_classifier_rules.py @@ -15,6 +15,7 @@ def make_step( llm_output: str | None = None, exception_type: str | None = None, metadata: dict | None = None, + agent_id: str | None = None, ) -> Step: return Step( index=index, @@ -25,6 +26,7 @@ def make_step( llm_output=llm_output, exception_type=exception_type, metadata=metadata or {}, + agent_id=agent_id, ) @@ -48,6 +50,21 @@ def test_loop_detected(): assert RulesClassifier().classify(t, "task") == FailureType.LOOP_DETECTED +def test_loop_detected_across_different_agent_ids(): + """MAST's "Step Repetition" (see docs/concepts/multi-agent-failures.md): + a handoff causes a second agent to unnecessarily redo work a first agent + already completed. Loop matching is deliberately agent_id-agnostic — + tool_called/tool_input equality alone is enough, regardless of who made + the call — so this fires with zero code change beyond Step.agent_id + existing as a field. Pins that as a verified fact, not an assumption.""" + t = traj( + make_step(0, tool_called="search", tool_input={"q": "hello"}, agent_id="agent-a"), + make_step(1, tool_called="search", tool_input={"q": "hello"}, agent_id="agent-b"), + make_step(2, tool_called="search", tool_input={"q": "hello"}, agent_id="agent-a"), + ) + assert RulesClassifier().classify(t, "task") == FailureType.LOOP_DETECTED + + def test_loop_not_detected_two_steps(): t = traj( make_step(0, tool_called="search", tool_input={"q": "hello"}), diff --git a/tests/test_observability_otel_ingest.py b/tests/test_observability_otel_ingest.py index cc9c5d5..bae5d86 100644 --- a/tests/test_observability_otel_ingest.py +++ b/tests/test_observability_otel_ingest.py @@ -292,6 +292,72 @@ def test_spans_sorted_by_end_time_regardless_of_input_order(): assert [s.tool_called for s in traj.steps] == ["first", "second"] +# ── agent identity -> Step.agent_id (feeds MAST-alignment phase 1) ────────── + + +@pytestmark_otel +def test_agent_id_extracted_from_gen_ai_agent_id(): + from triage.observability.otel_ingest import trajectory_from_spans + + tracer, exporter = _make_exporter() + with tracer.start_as_current_span( + "invoke_agent Researcher", + attributes={"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.id": "agent-123"}, + ): + pass + + traj = trajectory_from_spans(exporter.get_finished_spans()) + assert traj[0].agent_id == "agent-123" + + +@pytestmark_otel +def test_agent_id_falls_back_to_gen_ai_agent_name(): + from triage.observability.otel_ingest import trajectory_from_spans + + tracer, exporter = _make_exporter() + with tracer.start_as_current_span( + "invoke_agent Researcher", + attributes={"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "Researcher"}, + ): + pass + + traj = trajectory_from_spans(exporter.get_finished_spans()) + assert traj[0].agent_id == "Researcher" + + +@pytestmark_otel +def test_agent_id_prefers_id_over_name_when_both_present(): + from triage.observability.otel_ingest import trajectory_from_spans + + tracer, exporter = _make_exporter() + with tracer.start_as_current_span( + "invoke_agent Researcher", + attributes={ + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.id": "agent-123", + "gen_ai.agent.name": "Researcher", + }, + ): + pass + + traj = trajectory_from_spans(exporter.get_finished_spans()) + assert traj[0].agent_id == "agent-123" + + +@pytestmark_otel +def test_agent_id_absent_when_span_carries_no_agent_attribute(): + from triage.observability.otel_ingest import trajectory_from_spans + + tracer, exporter = _make_exporter() + with tracer.start_as_current_span( + "execute_tool search", attributes={"gen_ai.tool.name": "search"} + ): + pass + + traj = trajectory_from_spans(exporter.get_finished_spans()) + assert traj[0].agent_id is None + + # ── action field ────────────────────────────────────────────────────────────── diff --git a/tests/test_suspension.py b/tests/test_suspension.py index 81bd929..4b14e64 100644 --- a/tests/test_suspension.py +++ b/tests/test_suspension.py @@ -386,6 +386,7 @@ def _make_suspended_run(*, kwargs: dict[str, Any] | None = None) -> SuspendedRun metadata={"source": "unit-test"}, idempotent=True, partial=False, + agent_id="agent-researcher", ) ctx = FailureContext( failure_type=FailureType.EXTERNAL_FAULT, @@ -460,6 +461,7 @@ def test_serialize_deserialize_round_trip(): assert s.metadata == {"source": "unit-test"} assert s.idempotent is True assert s.partial is False + assert s.agent_id == "agent-researcher" def test_serialize_non_primitive_tool_output_coerced_to_str(): diff --git a/triage/classifier/rules.py b/triage/classifier/rules.py index 31e5bd5..97e0825 100644 --- a/triage/classifier/rules.py +++ b/triage/classifier/rules.py @@ -307,6 +307,10 @@ def _is_loop_window(steps: list[Step], threshold: float | None) -> bool: Consecutive comparison (not all-vs-first) so a loop where the query drifts gradually across the window is still caught — e.g. step 1 vs step 2 close, step 2 vs step 3 close, even if step 1 vs step 3 has drifted further apart. + + Does not look at ``Step.agent_id`` at all — a loop is the same tool call + repeated, regardless of which agent(s) made it. This is intentional, not + an oversight: see the caller's comment in ``classify()``. """ if steps[0].tool_called is None: return False @@ -399,7 +403,13 @@ def classify(self, trajectory: Trajectory, task: str) -> FailureType: # noqa: A # 1. LOOP_DETECTED — last loop_window steps share identical tool_called, # and either identical (default) or fuzzy-similar (loop_similarity_threshold) - # tool_input. + # tool_input. Deliberately agent_id-agnostic: matching is purely on + # tool_called/tool_input, so a step repeated across two different agents + # (MAST's "Step Repetition" — a handoff-caused re-do of already-completed + # work, not just one agent looping on itself) is caught by the exact same + # check, with no separate cross-agent code path. See + # test_loop_detected_across_different_agent_ids and + # docs/concepts/multi-agent-failures.md. if len(steps) >= self.loop_window: window = steps[-self.loop_window :] if _is_loop_window(window, self.loop_similarity_threshold): diff --git a/triage/observability/otel_ingest.py b/triage/observability/otel_ingest.py index 81e6d9b..bc45e73 100644 --- a/triage/observability/otel_ingest.py +++ b/triage/observability/otel_ingest.py @@ -39,6 +39,13 @@ message-text or exception-type support at all, with zero extra code from the caller. +Multi-agent systems: a span's gen_ai.agent.id (preferred) or gen_ai.agent.name +attribute — set on invoke_agent spans per the GenAI conventions — is copied +into Step.agent_id, the same field docs/concepts/multi-agent-failures.md +scopes MAST alignment around. This is the connection that proposal's phase 1 +relies on: a real multi-agent framework that emits per-agent spans populates +agent_id here for free, no extra caller code. + Usage:: from opentelemetry.sdk.trace import TracerProvider @@ -96,6 +103,10 @@ async def my_agent(task: str, *, record_step, **kwargs): _OPERATION_NAME_KEYS = ("gen_ai.operation.name",) _HTTP_STATUS_KEYS = ("http.response.status_code", "http.status_code") _ERROR_TYPE_FALLBACK_KEYS = ("error.type",) +# gen_ai.agent.id is the spec's "stable unique identifier"; gen_ai.agent.name +# is only "human-readable" — id preferred when both are present, name as a +# fallback for instrumentation that only sets the name. +_AGENT_ID_KEYS = ("gen_ai.agent.id", "gen_ai.agent.name") _GEN_AI_PREFIX = "gen_ai." @@ -150,6 +161,7 @@ def trajectory_from_spans(spans: Sequence[Any], *, include_all: bool = False) -> continue error, exception_type = _extract_error(span, attrs, is_error) + agent_id = _first_present(attrs, _AGENT_ID_KEYS) step = Step( index=i, action=_first_present(attrs, _OPERATION_NAME_KEYS) or getattr(span, "name", "") or "", @@ -159,6 +171,7 @@ def trajectory_from_spans(spans: Sequence[Any], *, include_all: bool = False) -> error=error, exception_type=exception_type, metadata=_extract_metadata(attrs), + agent_id=str(agent_id) if agent_id is not None else None, ) timestamp = getattr(span, "end_time", None) or getattr(span, "start_time", None) if timestamp is not None: diff --git a/triage/suspension.py b/triage/suspension.py index eb4b572..e7185ef 100644 --- a/triage/suspension.py +++ b/triage/suspension.py @@ -212,6 +212,7 @@ def serialize_run(run: SuspendedRun) -> str: "metadata": s.metadata, "idempotent": s.idempotent, "partial": s.partial, + "agent_id": s.agent_id, } for s in ctx.trajectory ], @@ -239,6 +240,7 @@ def deserialize_run(data: str) -> SuspendedRun: metadata=s.get("metadata") or {}, idempotent=s.get("idempotent", False), partial=s.get("partial", False), + agent_id=s.get("agent_id"), ) for s in ctx_d["trajectory"] ] diff --git a/triage/taxonomy.py b/triage/taxonomy.py index 3b0b85b..7989706 100644 --- a/triage/taxonomy.py +++ b/triage/taxonomy.py @@ -68,6 +68,16 @@ class Step: ``docs/concepts/classifiers.md``'s "Structured error codes" section. Populating them is the caller's responsibility: extract the code from the real exception object and pass it via ``record_step(Step(..., metadata={"http_status": 429}))``. + + ``agent_id`` is caller-supplied and optional — which agent produced this step, + for multi-agent systems where a single ``Trajectory`` interleaves steps from + more than one agent. ``None`` (the default) means either a single-agent system + or an agent identity the caller didn't track; existing single-agent callers + need no changes. ``RulesClassifier``'s loop detection does not currently key on + it — see ``docs/concepts/multi-agent-failures.md`` for why that turned out to + already be correct rather than a gap to close. + ``triage.observability.otel_ingest.trajectory_from_spans()`` populates it from + an OTel span's ``gen_ai.agent.id``/``gen_ai.agent.name`` attribute when present. """ index: int @@ -83,6 +93,7 @@ class Step: metadata: dict[str, Any] = field(default_factory=dict) idempotent: bool = False partial: bool = False + agent_id: str | None = None @dataclass