diff --git a/CHANGELOG.md b/CHANGELOG.md index 81deff7..d1175d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ The project follows [Keep a Changelog](https://keepachangelog.com/) conventions. - Made the stdio adapter's terminal `complete` frame certify successful engine shutdown. Close failures now emit `ADAPTER_ERROR`, omit `complete`, and exit `2`; an earlier protocol error remains the single terminal error frame. +- Tightened trace reduction to preserve operational status and divergence + category while preventing a specific classified failure from broadening. A + generic execution drift may still narrow to the queue-priority defect exposed + by removing causal noise. ## 0.6.0 - 2026-07-27 diff --git a/docs/conformance.md b/docs/conformance.md index 5c21525..5b830dd 100644 --- a/docs/conformance.md +++ b/docs/conformance.md @@ -570,6 +570,12 @@ Minimization reports use `artifact_type = "tracebook.conformance.minimization"` and include original and minimized counts, run count, reduction percentage, minimality/budget status, minimized trace hash, target failure category, and the final conformance report. +Reduction preserves one internal failure signature: operational status, +divergence category, and classified failure label. A specific label cannot +broaden during reduction; a queue-priority failure, for example, cannot become +generic execution drift merely because both use the `trades` category. Removing +causal noise may narrow generic execution drift to the queue-priority defect it +exposes. Suite reports use `artifact_type = "tracebook.conformance.suite_report"` and retain the suite hash plus each case's tags, fixture hash, and full report. diff --git a/src/tracebook/conformance/campaign.py b/src/tracebook/conformance/campaign.py index 0a7af35..bd9c3a5 100644 --- a/src/tracebook/conformance/campaign.py +++ b/src/tracebook/conformance/campaign.py @@ -12,11 +12,11 @@ from decimal import Decimal from numbers import Integral from pathlib import Path -from typing import Dict, List, Mapping, Optional, Sequence, Tuple, TypeVar +from typing import Dict, List, Mapping, Optional, Sequence, Tuple, TypeVar, cast from ..core.order import OrderSide, OrderType from ..events import MarketEvent -from .classification import classify_failure +from .classification import FailureSignature, failure_signature, preserves_failure_signature from .compare import ConformanceReport, run_conformance from .minimize import MinimizationResult, minimize_failing_trace from .model import ( @@ -788,6 +788,9 @@ def run_campaign( trace_hash=trace_sha256(original_events), event_count=len(original_events), ) + target_signature = cast( + FailureSignature, failure_signature(original_events, original_report) + ) minimization = minimize_failing_trace( original_events, candidate_factory, @@ -800,12 +803,17 @@ def run_campaign( raise ConformanceError( "candidate engine metadata changed during campaign minimization" ) + if not preserves_failure_signature( + target_signature, + failure_signature(minimization.events, minimization.report), + ): + raise ConformanceError("campaign minimization changed the failure signature") failure = CampaignFailure( trace=trace_result, original_events=original_events, original_report=original_report, minimization=minimization, - failure_class=classify_failure(original_events, original_report), + failure_class=target_signature.failure_class, ) break diff --git a/src/tracebook/conformance/classification.py b/src/tracebook/conformance/classification.py index 0bd882a..5987edc 100644 --- a/src/tracebook/conformance/classification.py +++ b/src/tracebook/conformance/classification.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import dataclass from typing import TYPE_CHECKING, Sequence from ..core.order import OrderSide @@ -13,6 +14,15 @@ QUEUE_PRIORITY_DRIFT = "queue-priority drift" +@dataclass(frozen=True) +class FailureSignature: + """Failure identity that a reduction must preserve.""" + + operational: bool + category: str + failure_class: str + + def is_operational_divergence( category: object, *, @@ -118,3 +128,30 @@ def classify_failure(events: Sequence[MarketEvent], report: ConformanceReport) - "book_state": "book-state drift", "protocol": "adapter protocol failure", }.get(divergence.category, "semantic drift") + + +def failure_signature( + events: Sequence[MarketEvent], report: ConformanceReport +) -> FailureSignature | None: + """Return the stable failure identity for one conformance result.""" + divergence = report.divergence + if divergence is None: + return None + return FailureSignature( + operational=report.operational_failure, + category=divergence.category, + failure_class=classify_failure(events, report), + ) + + +def preserves_failure_signature( + target: FailureSignature, observed: FailureSignature | None +) -> bool: + """Return whether a reduction preserves or narrows the original failure.""" + if observed is None: + return False + if target.operational != observed.operational or target.category != observed.category: + return False + return target.failure_class == observed.failure_class or ( + target.failure_class == "execution drift" and observed.failure_class == QUEUE_PRIORITY_DRIFT + ) diff --git a/src/tracebook/conformance/minimize.py b/src/tracebook/conformance/minimize.py index 146d778..3e51614 100644 --- a/src/tracebook/conformance/minimize.py +++ b/src/tracebook/conformance/minimize.py @@ -5,9 +5,14 @@ import math from dataclasses import dataclass, replace from numbers import Integral -from typing import Iterable, Optional, Tuple +from typing import Iterable, Optional, Tuple, cast from ..events import MarketEvent +from .classification import ( + FailureSignature, + failure_signature, + preserves_failure_signature, +) from .compare import ConformanceReport, run_conformance from .model import ( ARTIFACT_SCHEMA_VERSION, @@ -55,12 +60,15 @@ def to_dict(self) -> dict: def _reject_protocol_failure_during_semantic_minimization( report: ConformanceReport, - target_category: str, + target: FailureSignature, + observed: FailureSignature | None, run_number: int, ) -> None: """Do not mistake an operational trial failure for a removable semantic subset.""" divergence = report.divergence - if target_category != "protocol" and divergence is not None and report.operational_failure: + if not target.operational and observed is not None and observed.operational: + if divergence is None: + raise ConformanceError("operational failure has no divergence") kind = "adapter_close_error" if divergence.close_error is not None else divergence.kind message = divergence.close_error or divergence.message raise ConformanceError( @@ -91,8 +99,8 @@ def minimize_failing_trace( if initial.candidate_engine != candidate_engine: raise ConformanceError("candidate engine metadata changed during minimization") - target = initial.divergence.category - _reject_protocol_failure_during_semantic_minimization(initial, target, runs) + target = cast(FailureSignature, failure_signature(original, initial)) + _reject_protocol_failure_during_semantic_minimization(initial, target, target, runs) prefix_length = initial.divergence.event_index current = original[:prefix_length] if prefix_length > 0 else original current_report = ( @@ -123,12 +131,9 @@ def minimize_failing_trace( runs += 1 if report.candidate_engine != candidate_engine: raise ConformanceError("candidate engine metadata changed during minimization") - _reject_protocol_failure_during_semantic_minimization(report, target, runs) - if ( - not report.conformant - and report.divergence is not None - and report.divergence.category == target - ): + observed = failure_signature(trial, report) + _reject_protocol_failure_during_semantic_minimization(report, target, observed, runs) + if preserves_failure_signature(target, observed): current = trial current_report = report granularity = max(2, granularity - 1) @@ -149,12 +154,9 @@ def minimize_failing_trace( runs += 1 if empty_report.candidate_engine != candidate_engine: raise ConformanceError("candidate engine metadata changed during minimization") - _reject_protocol_failure_during_semantic_minimization(empty_report, target, runs) - if ( - not empty_report.conformant - and empty_report.divergence is not None - and empty_report.divergence.category == target - ): + observed = failure_signature((), empty_report) + _reject_protocol_failure_during_semantic_minimization(empty_report, target, observed, runs) + if preserves_failure_signature(target, observed): current = () current_report = empty_report one_minimal = True @@ -163,7 +165,7 @@ def minimize_failing_trace( events=current, original_event_count=len(original), runs=runs, - target_category=target, + target_category=target.category, one_minimal=one_minimal, budget_exhausted=runs >= max_runs and not one_minimal, report=current_report, diff --git a/src/tracebook/conformance/reproduce.py b/src/tracebook/conformance/reproduce.py index 1caa3aa..3d9062c 100644 --- a/src/tracebook/conformance/reproduce.py +++ b/src/tracebook/conformance/reproduce.py @@ -8,7 +8,7 @@ from typing import Any, Mapping, Optional, Sequence from ..events import MarketEvent -from .classification import classify_failure +from .classification import FailureSignature, failure_signature, is_operational_divergence from .compare import ConformanceReport, run_conformance from .model import ARTIFACT_SCHEMA_VERSION, ConformanceConfig, ConformanceError, trace_sha256 from .protocol import AdapterFactory @@ -107,10 +107,13 @@ def run_reproduction( if expected_count != len(events): raise ConformanceError("reduced event count does not match failure metadata") report = run_conformance(events, candidate_factory, config=config, trace_name=trace_name) - failure_class = classify_failure(events, report) + observed_signature = failure_signature(events, report) + failure_class = ( + observed_signature.failure_class if observed_signature is not None else "conformant" + ) divergence = report.divergence.to_dict() if report.divergence else None if expected is None: - reproduced = divergence is not None and not report.operational_failure + reproduced = observed_signature is not None and not observed_signature.operational else: expected_class = expected.get("failure_class") expected_divergence = expected.get("expected_reduced_divergence") @@ -118,7 +121,18 @@ def run_reproduction( raise ConformanceError("failure metadata requires failure_class") if not isinstance(expected_divergence, Mapping): raise ConformanceError("failure metadata requires expected_reduced_divergence") - reproduced = failure_class == expected_class and divergence == dict(expected_divergence) + expected_signature = FailureSignature( + operational=is_operational_divergence( + expected_divergence.get("category"), + snapshot_failed=bool(expected_divergence.get("snapshot_error")), + close_failed=bool(expected_divergence.get("close_error")), + ), + category=str(expected_divergence.get("category")), + failure_class=expected_class, + ) + reproduced = observed_signature == expected_signature and divergence == dict( + expected_divergence + ) return ReproductionResult( events=tuple(events), report=report, diff --git a/tests/test_conformance.py b/tests/test_conformance.py index ba3cfb3..77cefe8 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -23,7 +23,9 @@ ReferenceEngineAdapter, TradeFill, canonical_decimal, + classify_failure, copy_bundled_conformance_suite, + generate_campaign_trace, load_conformance_suite, minimize_failing_trace, run_conformance, @@ -31,6 +33,10 @@ serve_stdio, ) from tracebook.conformance.cli import main +from tracebook.conformance.classification import ( + FailureSignature, + preserves_failure_signature, +) from tracebook.conformance.exit_codes import exit_code_for_artifact from tracebook.events import MarketEvent, load_market_events @@ -95,6 +101,40 @@ def close(self): self._inner.close() +class _TradeMismatchAdapter: + """In-process fault that changes the first fill without changing state.""" + + def __init__(self, config): + self._inner = ReferenceEngineAdapter(config) + self.metadata = EngineMetadata("trade-mismatch", "1", "Python") + + def apply(self, event, index): + observation = self._inner.apply(event, index) + if not observation.trades: + return observation + first = observation.trades[0] + mismatched = TradeFill( + first.symbol, + first.buy_order_id, + first.sell_order_id + 1000, + first.price, + first.quantity, + ) + return Observation( + observation.index, + observation.outcome, + (mismatched,) + observation.trades[1:], + observation.state_hash, + observation.resting_order_count, + ) + + def snapshot(self): + return self._inner.snapshot() + + def close(self): + self._inner.close() + + class _ObservationFaultAdapter: def __init__(self, config, fault): self._inner = ReferenceEngineAdapter(config) @@ -278,6 +318,25 @@ def test_delta_debugging_reduces_to_a_one_event_reproducer(): assert budget_limited.report.trace_hash == budget_limited.to_dict()["minimized_trace_sha256"] +def test_minimizer_preserves_queue_priority_failure_class(): + events = generate_campaign_trace("fifo-limit-v1", seed=0, event_count=5) + + initial = run_conformance(events, _TradeMismatchAdapter) + minimized = minimize_failing_trace(events, _TradeMismatchAdapter, max_runs=50) + + assert classify_failure(events, initial) == "queue-priority drift" + assert minimized.events == events + assert classify_failure(minimized.events, minimized.report) == "queue-priority drift" + + +def test_failure_signature_can_narrow_but_not_broaden(): + generic = FailureSignature(False, "trades", "execution drift") + queue_priority = FailureSignature(False, "trades", "queue-priority drift") + + assert preserves_failure_signature(generic, queue_priority) + assert not preserves_failure_signature(queue_priority, generic) + + def test_minimizer_rejects_candidate_metadata_changes_between_runs(): class ChangingFactory: def __init__(self):