Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ The project follows [Keep a Changelog](https://keepachangelog.com/) conventions.
- Made the stdio adapter's terminal `complete` frame certify successful engine
shutdown. Close failures now emit `ADAPTER_ERROR`, omit `complete`, and exit
`2`; an earlier protocol error remains the single terminal error frame.
- Tightened trace reduction to preserve operational status and divergence
category while preventing a specific classified failure from broadening. A
generic execution drift may still narrow to the queue-priority defect exposed
by removing causal noise.

## 0.6.0 - 2026-07-27

Expand Down
6 changes: 6 additions & 0 deletions docs/conformance.md
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,12 @@ Minimization reports use
`artifact_type = "tracebook.conformance.minimization"` and include original and
minimized counts, run count, reduction percentage, minimality/budget status,
minimized trace hash, target failure category, and the final conformance report.
Reduction preserves one internal failure signature: operational status,
divergence category, and classified failure label. A specific label cannot
broaden during reduction; a queue-priority failure, for example, cannot become
generic execution drift merely because both use the `trades` category. Removing
causal noise may narrow generic execution drift to the queue-priority defect it
exposes.

Suite reports use `artifact_type = "tracebook.conformance.suite_report"` and
retain the suite hash plus each case's tags, fixture hash, and full report.
Expand Down
14 changes: 11 additions & 3 deletions src/tracebook/conformance/campaign.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@
from decimal import Decimal
from numbers import Integral
from pathlib import Path
from typing import Dict, List, Mapping, Optional, Sequence, Tuple, TypeVar
from typing import Dict, List, Mapping, Optional, Sequence, Tuple, TypeVar, cast

from ..core.order import OrderSide, OrderType
from ..events import MarketEvent
from .classification import classify_failure
from .classification import FailureSignature, failure_signature, preserves_failure_signature
from .compare import ConformanceReport, run_conformance
from .minimize import MinimizationResult, minimize_failing_trace
from .model import (
Expand Down Expand Up @@ -788,6 +788,9 @@ def run_campaign(
trace_hash=trace_sha256(original_events),
event_count=len(original_events),
)
target_signature = cast(
FailureSignature, failure_signature(original_events, original_report)
)
minimization = minimize_failing_trace(
original_events,
candidate_factory,
Expand All @@ -800,12 +803,17 @@ def run_campaign(
raise ConformanceError(
"candidate engine metadata changed during campaign minimization"
)
if not preserves_failure_signature(
target_signature,
failure_signature(minimization.events, minimization.report),
):
raise ConformanceError("campaign minimization changed the failure signature")
failure = CampaignFailure(
trace=trace_result,
original_events=original_events,
original_report=original_report,
minimization=minimization,
failure_class=classify_failure(original_events, original_report),
failure_class=target_signature.failure_class,
)
break

Expand Down
37 changes: 37 additions & 0 deletions src/tracebook/conformance/classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

from dataclasses import dataclass
from typing import TYPE_CHECKING, Sequence

from ..core.order import OrderSide
Expand All @@ -13,6 +14,15 @@
QUEUE_PRIORITY_DRIFT = "queue-priority drift"


@dataclass(frozen=True)
class FailureSignature:
"""Failure identity that a reduction must preserve."""

operational: bool
category: str
failure_class: str


def is_operational_divergence(
category: object,
*,
Expand Down Expand Up @@ -118,3 +128,30 @@ def classify_failure(events: Sequence[MarketEvent], report: ConformanceReport) -
"book_state": "book-state drift",
"protocol": "adapter protocol failure",
}.get(divergence.category, "semantic drift")


def failure_signature(
events: Sequence[MarketEvent], report: ConformanceReport
) -> FailureSignature | None:
"""Return the stable failure identity for one conformance result."""
divergence = report.divergence
if divergence is None:
return None
return FailureSignature(
operational=report.operational_failure,
category=divergence.category,
failure_class=classify_failure(events, report),
)


def preserves_failure_signature(
target: FailureSignature, observed: FailureSignature | None
) -> bool:
"""Return whether a reduction preserves or narrows the original failure."""
if observed is None:
return False
if target.operational != observed.operational or target.category != observed.category:
return False
return target.failure_class == observed.failure_class or (
target.failure_class == "execution drift" and observed.failure_class == QUEUE_PRIORITY_DRIFT
)
38 changes: 20 additions & 18 deletions src/tracebook/conformance/minimize.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,14 @@
import math
from dataclasses import dataclass, replace
from numbers import Integral
from typing import Iterable, Optional, Tuple
from typing import Iterable, Optional, Tuple, cast

from ..events import MarketEvent
from .classification import (
FailureSignature,
failure_signature,
preserves_failure_signature,
)
from .compare import ConformanceReport, run_conformance
from .model import (
ARTIFACT_SCHEMA_VERSION,
Expand Down Expand Up @@ -55,12 +60,15 @@ def to_dict(self) -> dict:

def _reject_protocol_failure_during_semantic_minimization(
report: ConformanceReport,
target_category: str,
target: FailureSignature,
observed: FailureSignature | None,
run_number: int,
) -> None:
"""Do not mistake an operational trial failure for a removable semantic subset."""
divergence = report.divergence
if target_category != "protocol" and divergence is not None and report.operational_failure:
if not target.operational and observed is not None and observed.operational:
if divergence is None:
raise ConformanceError("operational failure has no divergence")
kind = "adapter_close_error" if divergence.close_error is not None else divergence.kind
message = divergence.close_error or divergence.message
raise ConformanceError(
Expand Down Expand Up @@ -91,8 +99,8 @@ def minimize_failing_trace(
if initial.candidate_engine != candidate_engine:
raise ConformanceError("candidate engine metadata changed during minimization")

target = initial.divergence.category
_reject_protocol_failure_during_semantic_minimization(initial, target, runs)
target = cast(FailureSignature, failure_signature(original, initial))
_reject_protocol_failure_during_semantic_minimization(initial, target, target, runs)
prefix_length = initial.divergence.event_index
current = original[:prefix_length] if prefix_length > 0 else original
current_report = (
Expand Down Expand Up @@ -123,12 +131,9 @@ def minimize_failing_trace(
runs += 1
if report.candidate_engine != candidate_engine:
raise ConformanceError("candidate engine metadata changed during minimization")
_reject_protocol_failure_during_semantic_minimization(report, target, runs)
if (
not report.conformant
and report.divergence is not None
and report.divergence.category == target
):
observed = failure_signature(trial, report)
_reject_protocol_failure_during_semantic_minimization(report, target, observed, runs)
if preserves_failure_signature(target, observed):
current = trial
current_report = report
granularity = max(2, granularity - 1)
Expand All @@ -149,12 +154,9 @@ def minimize_failing_trace(
runs += 1
if empty_report.candidate_engine != candidate_engine:
raise ConformanceError("candidate engine metadata changed during minimization")
_reject_protocol_failure_during_semantic_minimization(empty_report, target, runs)
if (
not empty_report.conformant
and empty_report.divergence is not None
and empty_report.divergence.category == target
):
observed = failure_signature((), empty_report)
_reject_protocol_failure_during_semantic_minimization(empty_report, target, observed, runs)
if preserves_failure_signature(target, observed):
current = ()
current_report = empty_report
one_minimal = True
Expand All @@ -163,7 +165,7 @@ def minimize_failing_trace(
events=current,
original_event_count=len(original),
runs=runs,
target_category=target,
target_category=target.category,
one_minimal=one_minimal,
budget_exhausted=runs >= max_runs and not one_minimal,
report=current_report,
Expand Down
22 changes: 18 additions & 4 deletions src/tracebook/conformance/reproduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from typing import Any, Mapping, Optional, Sequence

from ..events import MarketEvent
from .classification import classify_failure
from .classification import FailureSignature, failure_signature, is_operational_divergence
from .compare import ConformanceReport, run_conformance
from .model import ARTIFACT_SCHEMA_VERSION, ConformanceConfig, ConformanceError, trace_sha256
from .protocol import AdapterFactory
Expand Down Expand Up @@ -107,18 +107,32 @@ def run_reproduction(
if expected_count != len(events):
raise ConformanceError("reduced event count does not match failure metadata")
report = run_conformance(events, candidate_factory, config=config, trace_name=trace_name)
failure_class = classify_failure(events, report)
observed_signature = failure_signature(events, report)
failure_class = (
observed_signature.failure_class if observed_signature is not None else "conformant"
)
divergence = report.divergence.to_dict() if report.divergence else None
if expected is None:
reproduced = divergence is not None and not report.operational_failure
reproduced = observed_signature is not None and not observed_signature.operational
else:
expected_class = expected.get("failure_class")
expected_divergence = expected.get("expected_reduced_divergence")
if not isinstance(expected_class, str) or not expected_class:
raise ConformanceError("failure metadata requires failure_class")
if not isinstance(expected_divergence, Mapping):
raise ConformanceError("failure metadata requires expected_reduced_divergence")
reproduced = failure_class == expected_class and divergence == dict(expected_divergence)
expected_signature = FailureSignature(
operational=is_operational_divergence(
expected_divergence.get("category"),
snapshot_failed=bool(expected_divergence.get("snapshot_error")),
close_failed=bool(expected_divergence.get("close_error")),
),
category=str(expected_divergence.get("category")),
failure_class=expected_class,
)
reproduced = observed_signature == expected_signature and divergence == dict(
expected_divergence
)
return ReproductionResult(
events=tuple(events),
report=report,
Expand Down
59 changes: 59 additions & 0 deletions tests/test_conformance.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,20 @@
ReferenceEngineAdapter,
TradeFill,
canonical_decimal,
classify_failure,
copy_bundled_conformance_suite,
generate_campaign_trace,
load_conformance_suite,
minimize_failing_trace,
run_conformance,
run_conformance_suite,
serve_stdio,
)
from tracebook.conformance.cli import main
from tracebook.conformance.classification import (
FailureSignature,
preserves_failure_signature,
)
from tracebook.conformance.exit_codes import exit_code_for_artifact
from tracebook.events import MarketEvent, load_market_events

Expand Down Expand Up @@ -95,6 +101,40 @@ def close(self):
self._inner.close()


class _TradeMismatchAdapter:
"""In-process fault that changes the first fill without changing state."""

def __init__(self, config):
self._inner = ReferenceEngineAdapter(config)
self.metadata = EngineMetadata("trade-mismatch", "1", "Python")

def apply(self, event, index):
observation = self._inner.apply(event, index)
if not observation.trades:
return observation
first = observation.trades[0]
mismatched = TradeFill(
first.symbol,
first.buy_order_id,
first.sell_order_id + 1000,
first.price,
first.quantity,
)
return Observation(
observation.index,
observation.outcome,
(mismatched,) + observation.trades[1:],
observation.state_hash,
observation.resting_order_count,
)

def snapshot(self):
return self._inner.snapshot()

def close(self):
self._inner.close()


class _ObservationFaultAdapter:
def __init__(self, config, fault):
self._inner = ReferenceEngineAdapter(config)
Expand Down Expand Up @@ -278,6 +318,25 @@ def test_delta_debugging_reduces_to_a_one_event_reproducer():
assert budget_limited.report.trace_hash == budget_limited.to_dict()["minimized_trace_sha256"]


def test_minimizer_preserves_queue_priority_failure_class():
events = generate_campaign_trace("fifo-limit-v1", seed=0, event_count=5)

initial = run_conformance(events, _TradeMismatchAdapter)
minimized = minimize_failing_trace(events, _TradeMismatchAdapter, max_runs=50)

assert classify_failure(events, initial) == "queue-priority drift"
assert minimized.events == events
assert classify_failure(minimized.events, minimized.report) == "queue-priority drift"


def test_failure_signature_can_narrow_but_not_broaden():
generic = FailureSignature(False, "trades", "execution drift")
queue_priority = FailureSignature(False, "trades", "queue-priority drift")

assert preserves_failure_signature(generic, queue_priority)
assert not preserves_failure_signature(queue_priority, generic)


def test_minimizer_rejects_candidate_metadata_changes_between_runs():
class ChangingFactory:
def __init__(self):
Expand Down